diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index b821700b49..a2646e3eae 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -1103,31 +1103,21 @@ jobs: echo "path=${notes_path}" >> "${GITHUB_OUTPUT}" - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - name: Download target artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist + pattern: "{aarch64,x86_64}-{apple-darwin{,-app-server},unknown-linux-musl{,-app-server},pc-windows-msvc}" + + - name: Download supplemental release artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: dist + pattern: "{*-symbols,argument-comment-lint-*,codex-zsh-*,python-runtime-wheel-*}" - name: List run: ls -R dist/ - - name: Delete entries from dist/ that should not go in the release - run: | - rm -rf dist/windows-binaries* - rm -rf dist/*-apple-darwin*-signed-binaries - rm -rf dist/*-apple-darwin*-packaged - rm -rf dist/*-apple-darwin*-unsigned-dmg - rm -rf dist/*-apple-darwin*-signed-dmg - rm -rf dist/*-apple-darwin*-binary-signing-verification - rm -rf dist/*-apple-darwin*-dmg-signing-verification - rm -rf dist/*-apple-darwin*-unsigned - # cargo-timing.html appears under multiple target-specific directories. - # If included in files: dist/**, release upload races on duplicate - # asset names and can fail with 404s. - find dist -type f -name 'cargo-timing.html' -delete - find dist -type d -empty -delete - - ls -R dist/ - - name: Add Codex package checksum manifest run: | set -euo pipefail @@ -1204,6 +1194,7 @@ jobs: ./scripts/stage_npm_packages.py \ --release-version "$RELEASE_VERSION" \ --workflow-url "$workflow_url" \ + --artifacts-dir "${GITHUB_WORKSPACE}/dist" \ --package codex \ --package codex-responses-api-proxy \ --package codex-sdk @@ -1226,6 +1217,18 @@ jobs: # (e.g. -alpha, -beta). Otherwise publish a normal release. prerelease: ${{ contains(steps.release_name.outputs.name, '-') }} + publish-dotslash: + name: publish-dotslash + needs: release + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: facebook/dotslash-publish-release@9c9ec027515c34db9282a09a25a9cab5880b2c52 # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1354,14 +1357,15 @@ jobs: other_tarballs+=("${tarball}") done - # Publish the platform packages before the root CLI wrapper. The root - # wrapper advances @openai/codex@latest, so it should only publish - # after the optional dependency versions it references exist. + # npm returns HTTP 409 when concurrent publishes update the same + # packument. Every platform tarball is a version of @openai/codex, + # so publish all tarballs serially. tarballs=( "${platform_tarballs[@]}" "${other_tarballs[@]}" "${root_tarball}" ) + # The SDK depends on this exact root package version. if [[ -f "${sdk_tarball}" ]]; then tarballs+=("${sdk_tarball}") fi diff --git a/AGENTS.md b/AGENTS.md index 46d2626e32..0118ef77b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,7 @@ In the codex-rs folder where the rust code lives: - Avoid bool or ambiguous `Option` parameters that force callers to write hard-to-read code such as `foo(false)` or `bar(None)`. Prefer enums, named methods, newtypes, or other idiomatic Rust API shapes when they keep the callsite self-documenting. - When you cannot make that API change and still need a small positional-literal callsite in Rust, follow the `argument_comment_lint` convention: - Use an exact `/*param_name*/` comment before opaque literal arguments such as `None`, booleans, and numeric literals when passing them by position. + - A method's sole non-self argument is exempt when the method and parameter names match, such as `.enabled(false)` for `fn enabled(&self, enabled: bool)`. - Do not add these comments for string or char literals unless the comment adds real clarity; those literals are intentionally exempt from the lint. - The parameter name in the comment must exactly match the callee signature. - You can run `just argument-comment-lint` to run the lint check locally. This is powered by Bazel, so running it the first time can be slow if Bazel is not warmed up, though incremental invocations should take <15s. Most of the time, it is best to update the PR and let CI take responsibility for checking this (or run it asynchronously in the background after submitting the PR). Note CI checks all three platforms, which the local run does not. diff --git a/MODULE.bazel b/MODULE.bazel index c4ef9500a7..559516bd1a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -11,6 +11,7 @@ single_version_override( patch_strip = 1, patches = [ "//patches:llvm_rusty_v8_custom_libcxx.patch", + "//patches:llvm_windows_arm64_powl.patch", "//patches:llvm_windows_symlink_extract.patch", ], ) @@ -94,6 +95,7 @@ single_version_override( module_name = "rules_rs", patch_strip = 1, patches = [ + "//patches:rules_rs_build_script_deps_annotation.patch", "//patches:rules_rs_windows_gnullvm_exec.patch", "//patches:rules_rs_windows_exec_linker.patch", ], @@ -295,6 +297,11 @@ bazel_dep(name = "openssl", version = "3.5.4.bcr.0") inject_repo(crate, "xz") crate.annotation( + # Build scripts compile in Bazel's exec configuration, so target-specific + # optional build deps are otherwise dropped for the musl release platforms. + build_script_deps = [ + "@crates//:openssl-src-300.5.5+3.5.5", + ], build_script_data = [ "@openssl//:gen_dir", ], diff --git a/bazel/platforms/BUILD.bazel b/bazel/platforms/BUILD.bazel new file mode 100644 index 0000000000..398d5d8083 --- /dev/null +++ b/bazel/platforms/BUILD.bazel @@ -0,0 +1 @@ +# Release platform rules live in this package. diff --git a/bazel/platforms/release_binaries.bzl b/bazel/platforms/release_binaries.bzl new file mode 100644 index 0000000000..2e2f445036 --- /dev/null +++ b/bazel/platforms/release_binaries.bzl @@ -0,0 +1,27 @@ +"""Rules for building release binaries across supported target platforms.""" + +load("@rules_platform//platform_data:defs.bzl", "platform_data") + +PLATFORMS = [ + "linux_arm64_musl", + "linux_amd64_musl", + "macos_amd64", + "macos_arm64", + "windows_amd64", + "windows_arm64", +] + +def multiplatform_binaries(name, platforms = PLATFORMS): + for platform in platforms: + platform_data( + name = name + "_" + platform, + platform = "@llvm//platforms:" + platform, + target = name, + tags = ["manual"], + ) + + native.filegroup( + name = "release_binaries", + srcs = [name + "_" + platform for platform in platforms], + tags = ["manual"], + ) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index d1538d2bd9..125d8e19e9 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1979,6 +1979,7 @@ dependencies = [ "codex-git-utils", "codex-goal-extension", "codex-guardian", + "codex-home", "codex-hooks", "codex-image-generation-extension", "codex-login", @@ -2004,6 +2005,7 @@ dependencies = [ "codex-utils-cargo-bin", "codex-utils-cli", "codex-utils-json-to-toml", + "codex-utils-path-uri", "codex-utils-pty", "codex-web-search-extension", "codex-windows-sandbox", @@ -2183,6 +2185,7 @@ dependencies = [ "codex-exec-server", "codex-utils-absolute-path", "codex-utils-cargo-bin", + "codex-utils-path-uri", "pretty_assertions", "similar", "tempfile", @@ -2319,6 +2322,7 @@ dependencies = [ "codex-execpolicy", "codex-features", "codex-git-utils", + "codex-home", "codex-install-context", "codex-login", "codex-mcp", @@ -2517,6 +2521,7 @@ dependencies = [ "codex-protocol", "codex-utils-absolute-path", "codex-utils-path", + "codex-utils-path-uri", "core-foundation 0.9.4", "dns-lookup", "dunce", @@ -2603,6 +2608,7 @@ dependencies = [ "codex-features", "codex-feedback", "codex-git-utils", + "codex-home", "codex-hooks", "codex-image-generation-extension", "codex-install-context", @@ -2636,6 +2642,7 @@ dependencies = [ "codex-utils-image", "codex-utils-output-truncation", "codex-utils-path", + "codex-utils-path-uri", "codex-utils-plugins", "codex-utils-pty", "codex-utils-stream-parser", @@ -2706,6 +2713,7 @@ dependencies = [ "codex-exec-server", "codex-extension-api", "codex-features", + "codex-home", "codex-login", "codex-model-provider-info", "codex-models-manager", @@ -2770,6 +2778,7 @@ dependencies = [ "codex-skills", "codex-utils-absolute-path", "codex-utils-output-truncation", + "codex-utils-path-uri", "codex-utils-plugins", "dirs", "dunce", @@ -2850,6 +2859,7 @@ dependencies = [ "codex-shell-command", "codex-test-binary-support", "codex-utils-absolute-path", + "codex-utils-path-uri", "codex-utils-pty", "codex-utils-rustls-provider", "ctor 0.6.3", @@ -2927,6 +2937,7 @@ dependencies = [ "codex-context-fragments", "codex-protocol", "codex-tools", + "codex-utils-absolute-path", "pretty_assertions", "tokio", ] @@ -3006,6 +3017,7 @@ dependencies = [ "async-trait", "codex-protocol", "codex-utils-absolute-path", + "codex-utils-path-uri", "serde", ] @@ -3029,6 +3041,7 @@ dependencies = [ "codex-file-system", "codex-protocol", "codex-utils-absolute-path", + "codex-utils-path-uri", "futures", "gix", "once_cell", @@ -3075,6 +3088,17 @@ dependencies = [ "codex-protocol", ] +[[package]] +name = "codex-home" +version = "0.0.0" +dependencies = [ + "codex-extension-api", + "codex-utils-absolute-path", + "pretty_assertions", + "tempfile", + "tokio", +] + [[package]] name = "codex-hooks" version = "0.0.0" @@ -3222,6 +3246,7 @@ name = "codex-mcp" version = "0.0.0" dependencies = [ "anyhow", + "arc-swap", "async-channel", "codex-api", "codex-async-utils", @@ -3275,6 +3300,7 @@ dependencies = [ "codex-core", "codex-exec-server", "codex-extension-api", + "codex-home", "codex-login", "codex-protocol", "codex-shell-command", @@ -3816,15 +3842,22 @@ name = "codex-skills-extension" version = "0.0.0" dependencies = [ "async-trait", - "codex-core", "codex-core-skills", "codex-exec-server", "codex-extension-api", + "codex-mcp", "codex-protocol", + "codex-tools", "codex-utils-absolute-path", + "codex-utils-path-uri", "codex-utils-string", "pretty_assertions", + "schemars 0.8.22", + "serde", + "serde_json", "tokio", + "tracing", + "url", ] [[package]] @@ -4198,6 +4231,7 @@ dependencies = [ "codex-exec-server", "codex-login", "codex-utils-absolute-path", + "codex-utils-path-uri", "serde", "serde_json", "tempfile", @@ -4580,6 +4614,7 @@ dependencies = [ "codex-exec-server", "codex-extension-api", "codex-features", + "codex-home", "codex-hooks", "codex-login", "codex-model-provider-info", @@ -4587,6 +4622,7 @@ dependencies = [ "codex-protocol", "codex-utils-absolute-path", "codex-utils-cargo-bin", + "codex-utils-path-uri", "ctor 0.6.3", "futures", "notify", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index b87b0cf39f..dab5e82855 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -21,6 +21,7 @@ members = [ "install-context", "codex-backend-openapi-models", "code-mode", + "codex-home", "cloud-config", "cloud-tasks", "cloud-tasks-client", @@ -157,6 +158,7 @@ codex-cloud-config = { path = "cloud-config" } codex-cloud-tasks-client = { path = "cloud-tasks-client" } codex-cloud-tasks-mock-client = { path = "cloud-tasks-mock-client" } codex-code-mode = { path = "code-mode" } +codex-home = { path = "codex-home" } codex-config = { path = "config" } codex-connectors = { path = "connectors" } codex-context-fragments = { path = "context-fragments" } @@ -237,6 +239,7 @@ codex-utils-json-to-toml = { path = "utils/json-to-toml" } codex-utils-oss = { path = "utils/oss" } codex-utils-output-truncation = { path = "utils/output-truncation" } codex-utils-path = { path = "utils/path-utils" } +codex-utils-path-uri = { path = "utils/path-uri" } codex-utils-plugins = { path = "utils/plugins" } codex-utils-pty = { path = "utils/pty" } codex-utils-rustls-provider = { path = "utils/rustls-provider" } diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 02779c6609..c0983a3dba 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -43,6 +43,7 @@ use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CodexCompactionEvent; +use crate::facts::CodexErrKind; use crate::facts::CompactionImplementation; use crate::facts::CompactionPhase; use crate::facts::CompactionReason; @@ -778,6 +779,33 @@ fn sample_initialize_fact(connection_id: u64) -> AnalyticsFact { } } +async fn ingest_complete_child_turn( + reducer: &mut AnalyticsReducer, + events: &mut Vec, + thread_id: &str, + turn_id: &str, +) { + for fact in [ + AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( + sample_turn_resolved_config(thread_id, turn_id), + ))), + AnalyticsFact::Custom(CustomAnalyticsFact::TurnProfile(Box::new( + TurnProfileFact { + turn_id: turn_id.to_string(), + profile: sample_turn_profile(), + }, + ))), + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + thread_id, + turn_id, + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + ] { + reducer.ingest(fact, events).await; + } +} + fn sample_command_execution_item( status: CommandExecutionStatus, exit_code: Option, @@ -1251,11 +1279,13 @@ fn compaction_event_serializes_expected_shape() { phase: CompactionPhase::MidTurn, strategy: CompactionStrategy::Memento, status: CompactionStatus::Completed, - error: None, + codex_error_kind: None, + codex_error_http_status_code: None, active_context_tokens_before: 120_000, active_context_tokens_after: 18_000, retained_image_count: None, compaction_summary_tokens: None, + cached_input_tokens: None, started_at: 100, completed_at: 106, duration_ms: Some(6543), @@ -1301,11 +1331,13 @@ fn compaction_event_serializes_expected_shape() { "phase": "mid_turn", "strategy": "memento", "status": "completed", - "error": null, + "codex_error_kind": null, + "codex_error_http_status_code": null, "active_context_tokens_before": 120000, "active_context_tokens_after": 18000, "retained_image_count": null, "compaction_summary_tokens": null, + "cached_input_tokens": null, "started_at": 100, "completed_at": 106, "duration_ms": 6543 @@ -1823,11 +1855,13 @@ async fn compaction_event_ingests_custom_fact() { phase: CompactionPhase::StandaloneTurn, strategy: CompactionStrategy::Memento, status: CompactionStatus::Failed, - error: Some("context limit exceeded".to_string()), + codex_error_kind: Some(CodexErrKind::ContextWindowExceeded), + codex_error_http_status_code: None, active_context_tokens_before: 131_000, active_context_tokens_after: 131_000, retained_image_count: None, compaction_summary_tokens: None, + cached_input_tokens: None, started_at: 100, completed_at: 101, duration_ms: Some(1200), @@ -1843,6 +1877,14 @@ async fn compaction_event_ingests_custom_fact() { assert_eq!(payload[0]["event_params"]["session_id"], "session-thread-1"); assert_eq!(payload[0]["event_params"]["thread_id"], "thread-1"); assert_eq!(payload[0]["event_params"]["turn_id"], "turn-compact"); + assert_eq!( + payload[0]["event_params"]["codex_error_kind"], + json!("context_window_exceeded") + ); + assert_eq!( + payload[0]["event_params"]["codex_error_http_status_code"], + json!(null) + ); assert_eq!( payload[0]["event_params"]["app_server_client"]["product_client_id"], DEFAULT_ORIGINATOR @@ -1937,6 +1979,7 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() { decision: GuardianReviewDecision::Denied, terminal_status: GuardianReviewTerminalStatus::TimedOut, failure_reason: Some(GuardianReviewFailureReason::Timeout), + attempt_count: 1, risk_level: None, user_authorization: None, outcome: None, @@ -2008,6 +2051,7 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() { ); assert_eq!(payload[0]["event_params"]["terminal_status"], "timed_out"); assert_eq!(payload[0]["event_params"]["failure_reason"], "timeout"); + assert_eq!(payload[0]["event_params"]["attempt_count"], 1); assert_eq!(payload[0]["event_params"]["review_timeout_ms"], 90_000); } @@ -2665,7 +2709,7 @@ async fn subagent_thread_started_publishes_without_initialize() { } #[tokio::test] -async fn subagent_thread_started_inherits_parent_connection_for_new_thread() { +async fn subagent_events_use_inherited_connection_unless_turn_connection_is_explicit() { let mut reducer = AnalyticsReducer::default(); let mut events = Vec::new(); let parent_thread_id = @@ -2747,11 +2791,13 @@ async fn subagent_thread_started_inherits_parent_connection_for_new_thread() { phase: CompactionPhase::StandaloneTurn, strategy: CompactionStrategy::Memento, status: CompactionStatus::Completed, - error: None, + codex_error_kind: None, + codex_error_http_status_code: None, active_context_tokens_before: 131_000, active_context_tokens_after: 64_000, retained_image_count: None, compaction_summary_tokens: None, + cached_input_tokens: None, started_at: 100, completed_at: 101, duration_ms: Some(1200), @@ -2772,6 +2818,68 @@ async fn subagent_thread_started_inherits_parent_connection_for_new_thread() { payload[0]["event_params"]["parent_thread_id"], "44444444-4444-4444-4444-444444444444" ); + + events.clear(); + ingest_complete_child_turn(&mut reducer, &mut events, "thread-review", "turn-inherited").await; + let [TrackEventRequest::TurnEvent(event)] = events.as_slice() else { + panic!("expected one turn event"); + }; + let params = &event.event_params; + assert_eq!(params.session_id, "session-root"); + assert_eq!(params.thread_source, Some(ThreadSource::Subagent)); + assert_eq!(params.subagent_source.as_deref(), Some("thread_spawn")); + assert_eq!( + params.parent_thread_id.as_deref(), + Some("44444444-4444-4444-4444-444444444444") + ); + assert_eq!(params.app_server_client.product_client_id, "parent-client"); + assert_eq!(params.runtime.codex_rs_version, "0.1.0"); + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnTokenUsage(Box::new( + sample_turn_token_usage_fact("thread-review", "turn-inherited"), + ))), + &mut events, + ) + .await; + assert_eq!(events.len(), 1); + + events.clear(); + reducer + .ingest(sample_initialize_fact(/*connection_id*/ 8), &mut events) + .await; + reducer + .ingest( + AnalyticsFact::ClientRequest { + connection_id: 8, + request_id: RequestId::Integer(3), + request: Box::new(sample_turn_start_request( + "thread-review", + /*request_id*/ 3, + )), + }, + &mut events, + ) + .await; + reducer + .ingest( + AnalyticsFact::ClientResponse { + connection_id: 8, + request_id: RequestId::Integer(3), + response: Box::new(sample_turn_start_response("turn-explicit")), + }, + &mut events, + ) + .await; + ingest_complete_child_turn(&mut reducer, &mut events, "thread-review", "turn-explicit").await; + let [TrackEventRequest::TurnEvent(event)] = events.as_slice() else { + panic!("expected one turn event"); + }; + assert_eq!( + event.event_params.app_server_client.product_client_id, + DEFAULT_ORIGINATOR + ); } #[tokio::test] @@ -3755,6 +3863,32 @@ async fn turn_event_counts_completed_tool_items() { ) .await; + let mcp_tool_call_item = |status, duration_ms| ThreadItem::McpToolCall { + id: "mcp-1".to_string(), + server: "server".to_string(), + tool: "search".to_string(), + status, + arguments: json!({}), + mcp_app_resource_uri: None, + plugin_id: Some("sample@test".to_string()), + result: None, + error: None, + duration_ms, + }; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted( + ItemStartedNotification { + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + started_at_ms: 998, + item: mcp_tool_call_item(McpToolCallStatus::InProgress, None), + }, + ))), + &mut out, + ) + .await; + let completed_tool_items = vec![ sample_command_execution_item(CommandExecutionStatus::Completed, Some(0), Some(1)), ThreadItem::FileChange { @@ -3762,18 +3896,7 @@ async fn turn_event_counts_completed_tool_items() { changes: Vec::new(), status: PatchApplyStatus::Completed, }, - ThreadItem::McpToolCall { - id: "mcp-1".to_string(), - server: "server".to_string(), - tool: "search".to_string(), - status: McpToolCallStatus::Completed, - arguments: json!({}), - mcp_app_resource_uri: None, - plugin_id: None, - result: None, - error: None, - duration_ms: Some(2), - }, + mcp_tool_call_item(McpToolCallStatus::Completed, Some(2)), ThreadItem::DynamicToolCall { id: "dynamic-1".to_string(), namespace: None, @@ -3831,6 +3954,13 @@ async fn turn_event_counts_completed_tool_items() { .await; } + let mcp_tool_call_event = out + .iter() + .find(|event| matches!(event, TrackEventRequest::McpToolCall(_))) + .expect("MCP tool call event should be emitted"); + let payload = serde_json::to_value(mcp_tool_call_event).expect("serialize MCP tool call event"); + assert_eq!(payload["event_params"]["plugin_id"], json!("sample@test")); + reducer .ingest( AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 8a25215ec8..2df113a43a 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -266,6 +266,7 @@ pub struct GuardianReviewEventParams { pub decision: GuardianReviewDecision, pub terminal_status: GuardianReviewTerminalStatus, pub failure_reason: Option, + pub attempt_count: i64, pub risk_level: Option, pub user_authorization: Option, pub outcome: Option, @@ -338,6 +339,7 @@ impl GuardianReviewTrackContext { decision: result.decision, terminal_status: result.terminal_status, failure_reason: result.failure_reason, + attempt_count: result.attempt_count, risk_level: result.risk_level, user_authorization: result.user_authorization, outcome: result.outcome, @@ -373,6 +375,7 @@ pub struct GuardianReviewAnalyticsResult { pub decision: GuardianReviewDecision, pub terminal_status: GuardianReviewTerminalStatus, pub failure_reason: Option, + pub attempt_count: i64, pub risk_level: Option, pub user_authorization: Option, pub outcome: Option, @@ -392,6 +395,7 @@ impl GuardianReviewAnalyticsResult { decision: GuardianReviewDecision::Denied, terminal_status: GuardianReviewTerminalStatus::FailedClosed, failure_reason: None, + attempt_count: 1, risk_level: None, user_authorization: None, outcome: None, @@ -629,6 +633,7 @@ pub(crate) struct CodexMcpToolCallEventParams { pub(crate) mcp_server_name: String, pub(crate) mcp_tool_name: String, pub(crate) mcp_error_present: bool, + pub(crate) plugin_id: Option, } #[derive(Serialize)] @@ -758,11 +763,13 @@ pub(crate) struct CodexCompactionEventParams { pub(crate) phase: CompactionPhase, pub(crate) strategy: CompactionStrategy, pub(crate) status: CompactionStatus, - pub(crate) error: Option, + pub(crate) codex_error_kind: Option, + pub(crate) codex_error_http_status_code: Option, pub(crate) active_context_tokens_before: i64, pub(crate) active_context_tokens_after: i64, pub(crate) retained_image_count: Option, pub(crate) compaction_summary_tokens: Option, + pub(crate) cached_input_tokens: Option, pub(crate) started_at: u64, pub(crate) completed_at: u64, pub(crate) duration_ms: Option, @@ -995,11 +1002,13 @@ pub(crate) fn codex_compaction_event_params( phase: input.phase, strategy: input.strategy, status: input.status, - error: input.error, + codex_error_kind: input.codex_error_kind, + codex_error_http_status_code: input.codex_error_http_status_code, active_context_tokens_before: input.active_context_tokens_before, active_context_tokens_after: input.active_context_tokens_after, retained_image_count: input.retained_image_count, compaction_summary_tokens: input.compaction_summary_tokens, + cached_input_tokens: input.cached_input_tokens, started_at: input.started_at, completed_at: input.completed_at, duration_ms: input.duration_ms, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index a19fb05d6b..ab278e223c 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -137,7 +137,7 @@ impl TurnCodexErrorFact { #[derive(Clone, Copy, Debug, Serialize)] #[serde(rename_all = "snake_case")] -pub(crate) enum CodexErrKind { +pub enum CodexErrKind { TurnAborted, Stream, ContextWindowExceeded, @@ -364,6 +364,7 @@ pub enum CompactionReason { UserRequested, ContextLimit, ModelDownshift, + CompHashChanged, } #[derive(Clone, Copy, Debug, Serialize)] @@ -407,11 +408,13 @@ pub struct CodexCompactionEvent { pub phase: CompactionPhase, pub strategy: CompactionStrategy, pub status: CompactionStatus, - pub error: Option, + pub codex_error_kind: Option, + pub codex_error_http_status_code: Option, pub active_context_tokens_before: i64, pub active_context_tokens_after: i64, pub retained_image_count: Option, pub compaction_summary_tokens: Option, + pub cached_input_tokens: Option, pub started_at: u64, pub completed_at: u64, pub duration_ms: Option, diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 35bd539cd4..fd236cbce9 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -24,6 +24,7 @@ pub use facts::AcceptedLineFingerprint; pub use facts::AnalyticsJsonRpcError; pub use facts::AppInvocation; pub use facts::CodexCompactionEvent; +pub use facts::CodexErrKind; pub use facts::CodexGoalEvent; pub use facts::CodexTurnSteerEvent; pub use facts::CompactionImplementation; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 1be9f624f1..a621051fb5 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -1497,17 +1497,23 @@ impl AnalyticsReducer { let Some(thread_id) = turn_state.thread_id.as_ref() else { return; }; - let Some(connection_id) = turn_state.connection_id else { + let drop_site = AnalyticsDropSite::turn(thread_id, turn_id); + let connection_id = turn_state.connection_id.or_else(|| { + self.threads + .get(drop_site.thread_id) + .and_then(|thread| thread.connection_id) + }); + let Some(connection_id) = connection_id else { + warn_missing_analytics_context(&drop_site, MissingAnalyticsContext::ThreadConnection); return; }; let Some(connection_state) = self.connections.get(&connection_id) else { warn_missing_analytics_context( - &AnalyticsDropSite::turn(thread_id, turn_id), + &drop_site, MissingAnalyticsContext::Connection { connection_id }, ); return; }; - let drop_site = AnalyticsDropSite::turn(thread_id, turn_id); let Some(thread_metadata) = self .threads .get(drop_site.thread_id) @@ -1745,6 +1751,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { status, error, duration_ms, + plugin_id, .. } => { let (terminal_status, failure_kind) = mcp_tool_call_outcome(status)?; @@ -1774,6 +1781,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { mcp_server_name: server.clone(), mcp_tool_name: tool.clone(), mcp_error_present: error.is_some(), + plugin_id: plugin_id.clone(), }, }, )) diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index 11684c6360..1faae917eb 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -22,6 +22,7 @@ use std::fmt; use std::io::Error as IoError; use std::io::ErrorKind; use std::io::Result as IoResult; +use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -48,7 +49,11 @@ use codex_config::LoaderOverrides; use codex_config::NoopThreadConfigLoader; use codex_config::RemoteThreadConfigLoader; use codex_config::ThreadConfigLoader; +use codex_config::config_toml::ConfigToml; use codex_core::config::Config; +pub use codex_core::otel_init::build_provider as build_otel_provider; +use codex_core::personality_migration::PersonalityMigrationStatus; +use codex_core::personality_migration::maybe_migrate_personality; pub use codex_exec_server::EnvironmentManager; pub use codex_exec_server::ExecServerRuntimePaths; use codex_feedback::CodexFeedback; @@ -72,7 +77,6 @@ pub use crate::remote::RemoteAppServerEndpoint; pub mod legacy_core { pub use codex_core::check_execpolicy_for_warnings; pub use codex_core::format_exec_policy_error_with_source; - pub use codex_core::grant_read_root_non_elevated; pub mod config { pub use codex_core::config::*; @@ -81,26 +85,27 @@ pub mod legacy_core { pub use codex_core::config::edit::*; } } - - pub mod otel_init { - pub use codex_core::otel_init::*; - } - - pub mod personality_migration { - pub use codex_core::personality_migration::*; - } - - pub mod test_support { - pub use codex_core::test_support::*; - } - - pub mod windows_sandbox { - pub use codex_core::windows_sandbox::*; - } } const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); +/// Runs the embedded app-server personality migration. +/// +/// Returns `true` when the migration changed config and the caller should reload it. +pub async fn migrate_personality_if_needed( + codex_home: &Path, + config_toml: &ConfigToml, + state_db: Option, +) -> IoResult { + let status = maybe_migrate_personality(codex_home, config_toml, state_db).await?; + match status { + PersonalityMigrationStatus::Applied => Ok(true), + PersonalityMigrationStatus::SkippedMarker + | PersonalityMigrationStatus::SkippedExplicitPersonality + | PersonalityMigrationStatus::SkippedNoSessions => Ok(false), + } +} + /// Raw app-server request result for typed in-process requests. /// /// Even on the in-process path, successful responses still travel back through @@ -158,6 +163,7 @@ pub(crate) fn server_notification_requires_delivery(notification: &ServerNotific ServerNotification::TurnCompleted(_) | ServerNotification::ThreadSettingsUpdated(_) | ServerNotification::ItemCompleted(_) + | ServerNotification::ExternalAgentConfigImportCompleted(_) | ServerNotification::AgentMessageDelta(_) | ServerNotification::PlanDelta(_) | ServerNotification::ReasoningSummaryTextDelta(_) @@ -2139,6 +2145,13 @@ mod tests { ) ) )); + assert!(event_requires_delivery( + &InProcessServerEvent::ServerNotification( + codex_app_server_protocol::ServerNotification::ExternalAgentConfigImportCompleted( + codex_app_server_protocol::ExternalAgentConfigImportCompletedNotification {}, + ) + ) + )); assert!(!event_requires_delivery(&InProcessServerEvent::Lagged { skipped: 1 })); diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 2701dfb34d..6bdf195361 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -3203,6 +3203,17 @@ ], "type": "object" }, + "ThreadDeleteParams": { + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "type": "object" + }, "ThreadForkParams": { "description": "There are two ways to fork a thread: 1. By thread_id: load the thread from disk by thread_id and fork it into a new thread. 2. By path: load the thread from disk by path and fork it into a new thread.\n\nIf using a non-empty path, the thread_id param will be ignored. Empty string path values are treated as absent.\n\nPrefer using thread_id whenever possible.", "properties": { @@ -4514,6 +4525,30 @@ "title": "Thread/archiveRequest", "type": "object" }, + { + "properties": { + "id": { + "$ref": "#/definitions/RequestId" + }, + "method": { + "enum": [ + "thread/delete" + ], + "title": "Thread/deleteRequestMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadDeleteParams" + } + }, + "required": [ + "id", + "method", + "params" + ], + "title": "Thread/deleteRequest", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index e0651d1727..1c37ea6f33 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -532,6 +532,13 @@ "personalAccessToken" ], "type": "string" + }, + { + "description": "Amazon Bedrock bearer token managed by Codex.", + "enum": [ + "bedrockApiKey" + ], + "type": "string" } ] }, @@ -3523,6 +3530,17 @@ ], "type": "object" }, + "ThreadDeletedNotification": { + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "type": "object" + }, "ThreadGoal": { "properties": { "createdAt": { @@ -5447,6 +5465,26 @@ "title": "Thread/archivedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/deleted" + ], + "title": "Thread/deletedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadDeletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/deletedNotification", + "type": "object" + }, { "properties": { "method": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index d65c043a7d..62b0717691 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -324,6 +324,30 @@ "title": "Thread/archiveRequest", "type": "object" }, + { + "properties": { + "id": { + "$ref": "#/definitions/v2/RequestId" + }, + "method": { + "enum": [ + "thread/delete" + ], + "title": "Thread/deleteRequestMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadDeleteParams" + } + }, + "required": [ + "id", + "method", + "params" + ], + "title": "Thread/deleteRequest", + "type": "object" + }, { "properties": { "id": { @@ -4061,6 +4085,26 @@ "title": "Thread/archivedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/deleted" + ], + "title": "Thread/deletedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadDeletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/deletedNotification", + "type": "object" + }, { "properties": { "method": { @@ -6372,6 +6416,12 @@ "AppSummary": { "description": "EXPERIMENTAL - app metadata summary for plugin responses.", "properties": { + "category": { + "type": [ + "string", + "null" + ] + }, "description": { "type": [ "string", @@ -6389,15 +6439,11 @@ }, "name": { "type": "string" - }, - "needsAuth": { - "type": "boolean" } }, "required": [ "id", - "name", - "needsAuth" + "name" ], "type": "object" }, @@ -6409,6 +6455,12 @@ "null" ] }, + "category": { + "type": [ + "string", + "null" + ] + }, "description": { "type": [ "string", @@ -6684,6 +6736,13 @@ "personalAccessToken" ], "type": "string" + }, + { + "description": "Amazon Bedrock bearer token managed by Codex.", + "enum": [ + "bedrockApiKey" + ], + "type": "string" } ] }, @@ -16053,6 +16112,37 @@ "title": "ThreadCompactStartResponse", "type": "object" }, + "ThreadDeleteParams": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeleteParams", + "type": "object" + }, + "ThreadDeleteResponse": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ThreadDeleteResponse", + "type": "object" + }, + "ThreadDeletedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeletedNotification", + "type": "object" + }, "ThreadForkParams": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "There are two ways to fork a thread: 1. By thread_id: load the thread from disk by thread_id and fork it into a new thread. 2. By path: load the thread from disk by path and fork it into a new thread.\n\nIf using a non-empty path, the thread_id param will be ignored. Empty string path values are treated as absent.\n\nPrefer using thread_id whenever possible.", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 8271dc485d..d9f0b91798 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -694,6 +694,12 @@ "AppSummary": { "description": "EXPERIMENTAL - app metadata summary for plugin responses.", "properties": { + "category": { + "type": [ + "string", + "null" + ] + }, "description": { "type": [ "string", @@ -711,15 +717,11 @@ }, "name": { "type": "string" - }, - "needsAuth": { - "type": "boolean" } }, "required": [ "id", - "name", - "needsAuth" + "name" ], "type": "object" }, @@ -731,6 +733,12 @@ "null" ] }, + "category": { + "type": [ + "string", + "null" + ] + }, "description": { "type": [ "string", @@ -1006,6 +1014,13 @@ "personalAccessToken" ], "type": "string" + }, + { + "description": "Amazon Bedrock bearer token managed by Codex.", + "enum": [ + "bedrockApiKey" + ], + "type": "string" } ] }, @@ -1263,6 +1278,30 @@ "title": "Thread/archiveRequest", "type": "object" }, + { + "properties": { + "id": { + "$ref": "#/definitions/RequestId" + }, + "method": { + "enum": [ + "thread/delete" + ], + "title": "Thread/deleteRequestMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadDeleteParams" + } + }, + "required": [ + "id", + "method", + "params" + ], + "title": "Thread/deleteRequest", + "type": "object" + }, { "properties": { "id": { @@ -11726,6 +11765,26 @@ "title": "Thread/archivedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/deleted" + ], + "title": "Thread/deletedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadDeletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/deletedNotification", + "type": "object" + }, { "properties": { "method": { @@ -13870,6 +13929,37 @@ "title": "ThreadCompactStartResponse", "type": "object" }, + "ThreadDeleteParams": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeleteParams", + "type": "object" + }, + "ThreadDeleteResponse": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ThreadDeleteResponse", + "type": "object" + }, + "ThreadDeletedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeletedNotification", + "type": "object" + }, "ThreadForkParams": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "There are two ways to fork a thread: 1. By thread_id: load the thread from disk by thread_id and fork it into a new thread. 2. By path: load the thread from disk by path and fork it into a new thread.\n\nIf using a non-empty path, the thread_id param will be ignored. Empty string path values are treated as absent.\n\nPrefer using thread_id whenever possible.", diff --git a/codex-rs/app-server-protocol/schema/json/v2/AccountUpdatedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/AccountUpdatedNotification.json index 013592c9ed..5c1847c56c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/AccountUpdatedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/AccountUpdatedNotification.json @@ -38,6 +38,13 @@ "personalAccessToken" ], "type": "string" + }, + { + "description": "Amazon Bedrock bearer token managed by Codex.", + "enum": [ + "bedrockApiKey" + ], + "type": "string" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/PluginInstallResponse.json b/codex-rs/app-server-protocol/schema/json/v2/PluginInstallResponse.json index 2ca7fda461..c9b4f6caf0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/PluginInstallResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/PluginInstallResponse.json @@ -4,6 +4,12 @@ "AppSummary": { "description": "EXPERIMENTAL - app metadata summary for plugin responses.", "properties": { + "category": { + "type": [ + "string", + "null" + ] + }, "description": { "type": [ "string", @@ -21,15 +27,11 @@ }, "name": { "type": "string" - }, - "needsAuth": { - "type": "boolean" } }, "required": [ "id", - "name", - "needsAuth" + "name" ], "type": "object" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/PluginReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/PluginReadResponse.json index 5a4c2e5b09..04919f8592 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/PluginReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/PluginReadResponse.json @@ -8,6 +8,12 @@ "AppSummary": { "description": "EXPERIMENTAL - app metadata summary for plugin responses.", "properties": { + "category": { + "type": [ + "string", + "null" + ] + }, "description": { "type": [ "string", @@ -25,15 +31,11 @@ }, "name": { "type": "string" - }, - "needsAuth": { - "type": "boolean" } }, "required": [ "id", - "name", - "needsAuth" + "name" ], "type": "object" }, @@ -45,6 +47,12 @@ "null" ] }, + "category": { + "type": [ + "string", + "null" + ] + }, "description": { "type": [ "string", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteParams.json new file mode 100644 index 0000000000..1711e11a2f --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteParams.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeleteParams", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteResponse.json new file mode 100644 index 0000000000..ff9f485338 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteResponse.json @@ -0,0 +1,5 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ThreadDeleteResponse", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadDeletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeletedNotification.json new file mode 100644 index 0000000000..53011ea014 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeletedNotification.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeletedNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/typescript/AuthMode.ts b/codex-rs/app-server-protocol/schema/typescript/AuthMode.ts index 1cb6ccb6b3..77bc037ebe 100644 --- a/codex-rs/app-server-protocol/schema/typescript/AuthMode.ts +++ b/codex-rs/app-server-protocol/schema/typescript/AuthMode.ts @@ -5,4 +5,4 @@ /** * Authentication mode for OpenAI-backed providers. */ -export type AuthMode = "apikey" | "chatgpt" | "chatgptAuthTokens" | "agentIdentity" | "personalAccessToken"; +export type AuthMode = "apikey" | "chatgpt" | "chatgptAuthTokens" | "agentIdentity" | "personalAccessToken" | "bedrockApiKey"; diff --git a/codex-rs/app-server-protocol/schema/typescript/ClientRequest.ts b/codex-rs/app-server-protocol/schema/typescript/ClientRequest.ts index c91792c726..b6a13fe307 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ClientRequest.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ClientRequest.ts @@ -62,6 +62,7 @@ import type { SkillsListParams } from "./v2/SkillsListParams"; import type { ThreadApproveGuardianDeniedActionParams } from "./v2/ThreadApproveGuardianDeniedActionParams"; import type { ThreadArchiveParams } from "./v2/ThreadArchiveParams"; import type { ThreadCompactStartParams } from "./v2/ThreadCompactStartParams"; +import type { ThreadDeleteParams } from "./v2/ThreadDeleteParams"; import type { ThreadForkParams } from "./v2/ThreadForkParams"; import type { ThreadGoalClearParams } from "./v2/ThreadGoalClearParams"; import type { ThreadGoalGetParams } from "./v2/ThreadGoalGetParams"; @@ -86,4 +87,4 @@ import type { WindowsSandboxSetupStartParams } from "./v2/WindowsSandboxSetupSta /** * Request from the client to the server. */ -export type ClientRequest ={ "method": "initialize", id: RequestId, params: InitializeParams, } | { "method": "thread/start", id: RequestId, params: ThreadStartParams, } | { "method": "thread/resume", id: RequestId, params: ThreadResumeParams, } | { "method": "thread/fork", id: RequestId, params: ThreadForkParams, } | { "method": "thread/archive", id: RequestId, params: ThreadArchiveParams, } | { "method": "thread/unsubscribe", id: RequestId, params: ThreadUnsubscribeParams, } | { "method": "thread/name/set", id: RequestId, params: ThreadSetNameParams, } | { "method": "thread/goal/set", id: RequestId, params: ThreadGoalSetParams, } | { "method": "thread/goal/get", id: RequestId, params: ThreadGoalGetParams, } | { "method": "thread/goal/clear", id: RequestId, params: ThreadGoalClearParams, } | { "method": "thread/metadata/update", id: RequestId, params: ThreadMetadataUpdateParams, } | { "method": "thread/unarchive", id: RequestId, params: ThreadUnarchiveParams, } | { "method": "thread/compact/start", id: RequestId, params: ThreadCompactStartParams, } | { "method": "thread/shellCommand", id: RequestId, params: ThreadShellCommandParams, } | { "method": "thread/approveGuardianDeniedAction", id: RequestId, params: ThreadApproveGuardianDeniedActionParams, } | { "method": "thread/rollback", id: RequestId, params: ThreadRollbackParams, } | { "method": "thread/list", id: RequestId, params: ThreadListParams, } | { "method": "thread/loaded/list", id: RequestId, params: ThreadLoadedListParams, } | { "method": "thread/read", id: RequestId, params: ThreadReadParams, } | { "method": "thread/inject_items", id: RequestId, params: ThreadInjectItemsParams, } | { "method": "skills/list", id: RequestId, params: SkillsListParams, } | { "method": "skills/extraRoots/set", id: RequestId, params: SkillsExtraRootsSetParams, } | { "method": "hooks/list", id: RequestId, params: HooksListParams, } | { "method": "marketplace/add", id: RequestId, params: MarketplaceAddParams, } | { "method": "marketplace/remove", id: RequestId, params: MarketplaceRemoveParams, } | { "method": "marketplace/upgrade", id: RequestId, params: MarketplaceUpgradeParams, } | { "method": "plugin/list", id: RequestId, params: PluginListParams, } | { "method": "plugin/installed", id: RequestId, params: PluginInstalledParams, } | { "method": "plugin/read", id: RequestId, params: PluginReadParams, } | { "method": "plugin/skill/read", id: RequestId, params: PluginSkillReadParams, } | { "method": "plugin/share/save", id: RequestId, params: PluginShareSaveParams, } | { "method": "plugin/share/updateTargets", id: RequestId, params: PluginShareUpdateTargetsParams, } | { "method": "plugin/share/list", id: RequestId, params: PluginShareListParams, } | { "method": "plugin/share/checkout", id: RequestId, params: PluginShareCheckoutParams, } | { "method": "plugin/share/delete", id: RequestId, params: PluginShareDeleteParams, } | { "method": "app/list", id: RequestId, params: AppsListParams, } | { "method": "fs/readFile", id: RequestId, params: FsReadFileParams, } | { "method": "fs/writeFile", id: RequestId, params: FsWriteFileParams, } | { "method": "fs/createDirectory", id: RequestId, params: FsCreateDirectoryParams, } | { "method": "fs/getMetadata", id: RequestId, params: FsGetMetadataParams, } | { "method": "fs/readDirectory", id: RequestId, params: FsReadDirectoryParams, } | { "method": "fs/remove", id: RequestId, params: FsRemoveParams, } | { "method": "fs/copy", id: RequestId, params: FsCopyParams, } | { "method": "fs/watch", id: RequestId, params: FsWatchParams, } | { "method": "fs/unwatch", id: RequestId, params: FsUnwatchParams, } | { "method": "skills/config/write", id: RequestId, params: SkillsConfigWriteParams, } | { "method": "plugin/install", id: RequestId, params: PluginInstallParams, } | { "method": "plugin/uninstall", id: RequestId, params: PluginUninstallParams, } | { "method": "turn/start", id: RequestId, params: TurnStartParams, } | { "method": "turn/steer", id: RequestId, params: TurnSteerParams, } | { "method": "turn/interrupt", id: RequestId, params: TurnInterruptParams, } | { "method": "review/start", id: RequestId, params: ReviewStartParams, } | { "method": "model/list", id: RequestId, params: ModelListParams, } | { "method": "modelProvider/capabilities/read", id: RequestId, params: ModelProviderCapabilitiesReadParams, } | { "method": "experimentalFeature/list", id: RequestId, params: ExperimentalFeatureListParams, } | { "method": "permissionProfile/list", id: RequestId, params: PermissionProfileListParams, } | { "method": "experimentalFeature/enablement/set", id: RequestId, params: ExperimentalFeatureEnablementSetParams, } | { "method": "mcpServer/oauth/login", id: RequestId, params: McpServerOauthLoginParams, } | { "method": "config/mcpServer/reload", id: RequestId, params: undefined, } | { "method": "mcpServerStatus/list", id: RequestId, params: ListMcpServerStatusParams, } | { "method": "mcpServer/resource/read", id: RequestId, params: McpResourceReadParams, } | { "method": "mcpServer/tool/call", id: RequestId, params: McpServerToolCallParams, } | { "method": "windowsSandbox/setupStart", id: RequestId, params: WindowsSandboxSetupStartParams, } | { "method": "windowsSandbox/readiness", id: RequestId, params: undefined, } | { "method": "account/login/start", id: RequestId, params: LoginAccountParams, } | { "method": "account/login/cancel", id: RequestId, params: CancelLoginAccountParams, } | { "method": "account/logout", id: RequestId, params: undefined, } | { "method": "account/rateLimits/read", id: RequestId, params: undefined, } | { "method": "account/usage/read", id: RequestId, params: undefined, } | { "method": "account/sendAddCreditsNudgeEmail", id: RequestId, params: SendAddCreditsNudgeEmailParams, } | { "method": "feedback/upload", id: RequestId, params: FeedbackUploadParams, } | { "method": "command/exec", id: RequestId, params: CommandExecParams, } | { "method": "command/exec/write", id: RequestId, params: CommandExecWriteParams, } | { "method": "command/exec/terminate", id: RequestId, params: CommandExecTerminateParams, } | { "method": "command/exec/resize", id: RequestId, params: CommandExecResizeParams, } | { "method": "config/read", id: RequestId, params: ConfigReadParams, } | { "method": "externalAgentConfig/detect", id: RequestId, params: ExternalAgentConfigDetectParams, } | { "method": "externalAgentConfig/import", id: RequestId, params: ExternalAgentConfigImportParams, } | { "method": "config/value/write", id: RequestId, params: ConfigValueWriteParams, } | { "method": "config/batchWrite", id: RequestId, params: ConfigBatchWriteParams, } | { "method": "configRequirements/read", id: RequestId, params: undefined, } | { "method": "account/read", id: RequestId, params: GetAccountParams, } | { "method": "getConversationSummary", id: RequestId, params: GetConversationSummaryParams, } | { "method": "gitDiffToRemote", id: RequestId, params: GitDiffToRemoteParams, } | { "method": "getAuthStatus", id: RequestId, params: GetAuthStatusParams, } | { "method": "fuzzyFileSearch", id: RequestId, params: FuzzyFileSearchParams, }; +export type ClientRequest ={ "method": "initialize", id: RequestId, params: InitializeParams, } | { "method": "thread/start", id: RequestId, params: ThreadStartParams, } | { "method": "thread/resume", id: RequestId, params: ThreadResumeParams, } | { "method": "thread/fork", id: RequestId, params: ThreadForkParams, } | { "method": "thread/archive", id: RequestId, params: ThreadArchiveParams, } | { "method": "thread/delete", id: RequestId, params: ThreadDeleteParams, } | { "method": "thread/unsubscribe", id: RequestId, params: ThreadUnsubscribeParams, } | { "method": "thread/name/set", id: RequestId, params: ThreadSetNameParams, } | { "method": "thread/goal/set", id: RequestId, params: ThreadGoalSetParams, } | { "method": "thread/goal/get", id: RequestId, params: ThreadGoalGetParams, } | { "method": "thread/goal/clear", id: RequestId, params: ThreadGoalClearParams, } | { "method": "thread/metadata/update", id: RequestId, params: ThreadMetadataUpdateParams, } | { "method": "thread/unarchive", id: RequestId, params: ThreadUnarchiveParams, } | { "method": "thread/compact/start", id: RequestId, params: ThreadCompactStartParams, } | { "method": "thread/shellCommand", id: RequestId, params: ThreadShellCommandParams, } | { "method": "thread/approveGuardianDeniedAction", id: RequestId, params: ThreadApproveGuardianDeniedActionParams, } | { "method": "thread/rollback", id: RequestId, params: ThreadRollbackParams, } | { "method": "thread/list", id: RequestId, params: ThreadListParams, } | { "method": "thread/loaded/list", id: RequestId, params: ThreadLoadedListParams, } | { "method": "thread/read", id: RequestId, params: ThreadReadParams, } | { "method": "thread/inject_items", id: RequestId, params: ThreadInjectItemsParams, } | { "method": "skills/list", id: RequestId, params: SkillsListParams, } | { "method": "skills/extraRoots/set", id: RequestId, params: SkillsExtraRootsSetParams, } | { "method": "hooks/list", id: RequestId, params: HooksListParams, } | { "method": "marketplace/add", id: RequestId, params: MarketplaceAddParams, } | { "method": "marketplace/remove", id: RequestId, params: MarketplaceRemoveParams, } | { "method": "marketplace/upgrade", id: RequestId, params: MarketplaceUpgradeParams, } | { "method": "plugin/list", id: RequestId, params: PluginListParams, } | { "method": "plugin/installed", id: RequestId, params: PluginInstalledParams, } | { "method": "plugin/read", id: RequestId, params: PluginReadParams, } | { "method": "plugin/skill/read", id: RequestId, params: PluginSkillReadParams, } | { "method": "plugin/share/save", id: RequestId, params: PluginShareSaveParams, } | { "method": "plugin/share/updateTargets", id: RequestId, params: PluginShareUpdateTargetsParams, } | { "method": "plugin/share/list", id: RequestId, params: PluginShareListParams, } | { "method": "plugin/share/checkout", id: RequestId, params: PluginShareCheckoutParams, } | { "method": "plugin/share/delete", id: RequestId, params: PluginShareDeleteParams, } | { "method": "app/list", id: RequestId, params: AppsListParams, } | { "method": "fs/readFile", id: RequestId, params: FsReadFileParams, } | { "method": "fs/writeFile", id: RequestId, params: FsWriteFileParams, } | { "method": "fs/createDirectory", id: RequestId, params: FsCreateDirectoryParams, } | { "method": "fs/getMetadata", id: RequestId, params: FsGetMetadataParams, } | { "method": "fs/readDirectory", id: RequestId, params: FsReadDirectoryParams, } | { "method": "fs/remove", id: RequestId, params: FsRemoveParams, } | { "method": "fs/copy", id: RequestId, params: FsCopyParams, } | { "method": "fs/watch", id: RequestId, params: FsWatchParams, } | { "method": "fs/unwatch", id: RequestId, params: FsUnwatchParams, } | { "method": "skills/config/write", id: RequestId, params: SkillsConfigWriteParams, } | { "method": "plugin/install", id: RequestId, params: PluginInstallParams, } | { "method": "plugin/uninstall", id: RequestId, params: PluginUninstallParams, } | { "method": "turn/start", id: RequestId, params: TurnStartParams, } | { "method": "turn/steer", id: RequestId, params: TurnSteerParams, } | { "method": "turn/interrupt", id: RequestId, params: TurnInterruptParams, } | { "method": "review/start", id: RequestId, params: ReviewStartParams, } | { "method": "model/list", id: RequestId, params: ModelListParams, } | { "method": "modelProvider/capabilities/read", id: RequestId, params: ModelProviderCapabilitiesReadParams, } | { "method": "experimentalFeature/list", id: RequestId, params: ExperimentalFeatureListParams, } | { "method": "permissionProfile/list", id: RequestId, params: PermissionProfileListParams, } | { "method": "experimentalFeature/enablement/set", id: RequestId, params: ExperimentalFeatureEnablementSetParams, } | { "method": "mcpServer/oauth/login", id: RequestId, params: McpServerOauthLoginParams, } | { "method": "config/mcpServer/reload", id: RequestId, params: undefined, } | { "method": "mcpServerStatus/list", id: RequestId, params: ListMcpServerStatusParams, } | { "method": "mcpServer/resource/read", id: RequestId, params: McpResourceReadParams, } | { "method": "mcpServer/tool/call", id: RequestId, params: McpServerToolCallParams, } | { "method": "windowsSandbox/setupStart", id: RequestId, params: WindowsSandboxSetupStartParams, } | { "method": "windowsSandbox/readiness", id: RequestId, params: undefined, } | { "method": "account/login/start", id: RequestId, params: LoginAccountParams, } | { "method": "account/login/cancel", id: RequestId, params: CancelLoginAccountParams, } | { "method": "account/logout", id: RequestId, params: undefined, } | { "method": "account/rateLimits/read", id: RequestId, params: undefined, } | { "method": "account/usage/read", id: RequestId, params: undefined, } | { "method": "account/sendAddCreditsNudgeEmail", id: RequestId, params: SendAddCreditsNudgeEmailParams, } | { "method": "feedback/upload", id: RequestId, params: FeedbackUploadParams, } | { "method": "command/exec", id: RequestId, params: CommandExecParams, } | { "method": "command/exec/write", id: RequestId, params: CommandExecWriteParams, } | { "method": "command/exec/terminate", id: RequestId, params: CommandExecTerminateParams, } | { "method": "command/exec/resize", id: RequestId, params: CommandExecResizeParams, } | { "method": "config/read", id: RequestId, params: ConfigReadParams, } | { "method": "externalAgentConfig/detect", id: RequestId, params: ExternalAgentConfigDetectParams, } | { "method": "externalAgentConfig/import", id: RequestId, params: ExternalAgentConfigImportParams, } | { "method": "config/value/write", id: RequestId, params: ConfigValueWriteParams, } | { "method": "config/batchWrite", id: RequestId, params: ConfigBatchWriteParams, } | { "method": "configRequirements/read", id: RequestId, params: undefined, } | { "method": "account/read", id: RequestId, params: GetAccountParams, } | { "method": "getConversationSummary", id: RequestId, params: GetConversationSummaryParams, } | { "method": "gitDiffToRemote", id: RequestId, params: GitDiffToRemoteParams, } | { "method": "getAuthStatus", id: RequestId, params: GetAuthStatusParams, } | { "method": "fuzzyFileSearch", id: RequestId, params: FuzzyFileSearchParams, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts index 2520c71c3c..7cae63a553 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts @@ -43,6 +43,7 @@ import type { SkillsChangedNotification } from "./v2/SkillsChangedNotification"; import type { TerminalInteractionNotification } from "./v2/TerminalInteractionNotification"; import type { ThreadArchivedNotification } from "./v2/ThreadArchivedNotification"; import type { ThreadClosedNotification } from "./v2/ThreadClosedNotification"; +import type { ThreadDeletedNotification } from "./v2/ThreadDeletedNotification"; import type { ThreadGoalClearedNotification } from "./v2/ThreadGoalClearedNotification"; import type { ThreadGoalUpdatedNotification } from "./v2/ThreadGoalUpdatedNotification"; import type { ThreadNameUpdatedNotification } from "./v2/ThreadNameUpdatedNotification"; @@ -71,4 +72,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW /** * Notification sent from the server to the client. */ -export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/goal/updated", "params": ThreadGoalUpdatedNotification } | { "method": "thread/goal/cleared", "params": ThreadGoalClearedNotification } | { "method": "thread/settings/updated", "params": ThreadSettingsUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "process/outputDelta", "params": ProcessOutputDeltaNotification } | { "method": "process/exited", "params": ProcessExitedNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/fileChange/patchUpdated", "params": FileChangePatchUpdatedNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "remoteControl/status/changed", "params": RemoteControlStatusChangedNotification } | { "method": "externalAgentConfig/import/completed", "params": ExternalAgentConfigImportCompletedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "model/verification", "params": ModelVerificationNotification } | { "method": "turn/moderationMetadata", "params": TurnModerationMetadataNotification } | { "method": "warning", "params": WarningNotification } | { "method": "guardianWarning", "params": GuardianWarningNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcript/delta", "params": ThreadRealtimeTranscriptDeltaNotification } | { "method": "thread/realtime/transcript/done", "params": ThreadRealtimeTranscriptDoneNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; +export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/deleted", "params": ThreadDeletedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/goal/updated", "params": ThreadGoalUpdatedNotification } | { "method": "thread/goal/cleared", "params": ThreadGoalClearedNotification } | { "method": "thread/settings/updated", "params": ThreadSettingsUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "process/outputDelta", "params": ProcessOutputDeltaNotification } | { "method": "process/exited", "params": ProcessExitedNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/fileChange/patchUpdated", "params": FileChangePatchUpdatedNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "remoteControl/status/changed", "params": RemoteControlStatusChangedNotification } | { "method": "externalAgentConfig/import/completed", "params": ExternalAgentConfigImportCompletedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "model/verification", "params": ModelVerificationNotification } | { "method": "turn/moderationMetadata", "params": TurnModerationMetadataNotification } | { "method": "warning", "params": WarningNotification } | { "method": "guardianWarning", "params": GuardianWarningNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcript/delta", "params": ThreadRealtimeTranscriptDeltaNotification } | { "method": "thread/realtime/transcript/done", "params": ThreadRealtimeTranscriptDoneNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/AppSummary.ts b/codex-rs/app-server-protocol/schema/typescript/v2/AppSummary.ts index 586c76f8f7..f295009a0d 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/AppSummary.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/AppSummary.ts @@ -5,4 +5,4 @@ /** * EXPERIMENTAL - app metadata summary for plugin responses. */ -export type AppSummary = { id: string, name: string, description: string | null, installUrl: string | null, needsAuth: boolean, }; +export type AppSummary = { id: string, name: string, description: string | null, installUrl: string | null, category: string | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/AppTemplateSummary.ts b/codex-rs/app-server-protocol/schema/typescript/v2/AppTemplateSummary.ts index dd5f76229f..65d22beb7e 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/AppTemplateSummary.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/AppTemplateSummary.ts @@ -3,4 +3,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { AppTemplateUnavailableReason } from "./AppTemplateUnavailableReason"; -export type AppTemplateSummary = { templateId: string, name: string, description: string | null, canonicalConnectorId: string | null, logoUrl: string | null, logoUrlDark: string | null, materializedAppIds: Array, reason: AppTemplateUnavailableReason | null, }; +export type AppTemplateSummary = { templateId: string, name: string, description: string | null, category: string | null, canonicalConnectorId: string | null, logoUrl: string | null, logoUrlDark: string | null, materializedAppIds: Array, reason: AppTemplateUnavailableReason | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteParams.ts new file mode 100644 index 0000000000..909ccda715 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteParams.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadDeleteParams = { threadId: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteResponse.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteResponse.ts new file mode 100644 index 0000000000..1af1c3077f --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteResponse.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadDeleteResponse = Record; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeletedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeletedNotification.ts new file mode 100644 index 0000000000..5122a2229c --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeletedNotification.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadDeletedNotification = { threadId: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index 71f4c67cd0..b4cd88aadf 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -374,6 +374,9 @@ export type { ThreadArchivedNotification } from "./ThreadArchivedNotification"; export type { ThreadClosedNotification } from "./ThreadClosedNotification"; export type { ThreadCompactStartParams } from "./ThreadCompactStartParams"; export type { ThreadCompactStartResponse } from "./ThreadCompactStartResponse"; +export type { ThreadDeleteParams } from "./ThreadDeleteParams"; +export type { ThreadDeleteResponse } from "./ThreadDeleteResponse"; +export type { ThreadDeletedNotification } from "./ThreadDeletedNotification"; export type { ThreadForkParams } from "./ThreadForkParams"; export type { ThreadForkResponse } from "./ThreadForkResponse"; export type { ThreadGoal } from "./ThreadGoal"; diff --git a/codex-rs/app-server-protocol/src/export.rs b/codex-rs/app-server-protocol/src/export.rs index cd6c7e47b6..5e6c2ad015 100644 --- a/codex-rs/app-server-protocol/src/export.rs +++ b/codex-rs/app-server-protocol/src/export.rs @@ -39,8 +39,11 @@ use ts_rs::TS; pub(crate) const GENERATED_TS_HEADER: &str = "// GENERATED CODE! DO NOT MODIFY BY HAND!\n\n"; const IGNORED_DEFINITIONS: &[&str] = &["Option<()>"]; const JSON_V1_ALLOWLIST: &[&str] = &["InitializeParams", "InitializeResponse"]; -const EXPERIMENTAL_CLIENT_METHOD_DEPENDENCY_TYPES: &[&str] = - &["RemoteControlClient", "RemoteControlClientsListOrder"]; +const EXPERIMENTAL_CLIENT_METHOD_DEPENDENCY_TYPES: &[&str] = &[ + "RemoteControlClient", + "RemoteControlClientsListOrder", + "ThreadBackgroundTerminal", +]; const SPECIAL_DEFINITIONS: &[&str] = &[ "ClientNotification", "ClientRequest", diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index d3245091a9..9311fe9058 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -41,6 +41,11 @@ pub enum AuthMode { #[ts(rename = "personalAccessToken")] #[strum(serialize = "personalAccessToken")] PersonalAccessToken, + /// Amazon Bedrock bearer token managed by Codex. + #[serde(rename = "bedrockApiKey")] + #[ts(rename = "bedrockApiKey")] + #[strum(serialize = "bedrockApiKey")] + BedrockApiKey, } impl AuthMode { @@ -48,7 +53,7 @@ impl AuthMode { pub fn has_chatgpt_account(self) -> bool { match self { Self::Chatgpt | Self::ChatgptAuthTokens | Self::PersonalAccessToken => true, - Self::ApiKey | Self::AgentIdentity => false, + Self::ApiKey | Self::AgentIdentity | Self::BedrockApiKey => false, } } } @@ -480,6 +485,11 @@ client_request_definitions! { serialization: thread_id(params.thread_id), response: v2::ThreadArchiveResponse, }, + ThreadDelete => "thread/delete" { + params: v2::ThreadDeleteParams, + serialization: thread_id(params.thread_id), + response: v2::ThreadDeleteResponse, + }, ThreadUnsubscribe => "thread/unsubscribe" { params: v2::ThreadUnsubscribeParams, serialization: thread_id(params.thread_id), @@ -574,6 +584,18 @@ client_request_definitions! { serialization: thread_id(params.thread_id), response: v2::ThreadBackgroundTerminalsCleanResponse, }, + #[experimental("thread/backgroundTerminals/list")] + ThreadBackgroundTerminalsList => "thread/backgroundTerminals/list" { + params: v2::ThreadBackgroundTerminalsListParams, + serialization: thread_id(params.thread_id), + response: v2::ThreadBackgroundTerminalsListResponse, + }, + #[experimental("thread/backgroundTerminals/terminate")] + ThreadBackgroundTerminalsTerminate => "thread/backgroundTerminals/terminate" { + params: v2::ThreadBackgroundTerminalsTerminateParams, + serialization: thread_id(params.thread_id), + response: v2::ThreadBackgroundTerminalsTerminateResponse, + }, ThreadRollback => "thread/rollback" { params: v2::ThreadRollbackParams, serialization: thread_id(params.thread_id), @@ -1522,6 +1544,7 @@ server_notification_definitions! { ThreadStarted => "thread/started" (v2::ThreadStartedNotification), ThreadStatusChanged => "thread/status/changed" (v2::ThreadStatusChangedNotification), ThreadArchived => "thread/archived" (v2::ThreadArchivedNotification), + ThreadDeleted => "thread/deleted" (v2::ThreadDeletedNotification), ThreadUnarchived => "thread/unarchived" (v2::ThreadUnarchivedNotification), ThreadClosed => "thread/closed" (v2::ThreadClosedNotification), SkillsChanged => "skills/changed" (v2::SkillsChangedNotification), @@ -2936,6 +2959,54 @@ mod tests { Ok(()) } + #[test] + fn serialize_thread_background_terminals_list() -> Result<()> { + let request = ClientRequest::ThreadBackgroundTerminalsList { + request_id: RequestId::Integer(8), + params: v2::ThreadBackgroundTerminalsListParams { + thread_id: "thr_123".to_string(), + cursor: None, + limit: None, + }, + }; + assert_eq!( + json!({ + "method": "thread/backgroundTerminals/list", + "id": 8, + "params": { + "threadId": "thr_123", + "cursor": null, + "limit": null + } + }), + serde_json::to_value(&request)?, + ); + Ok(()) + } + + #[test] + fn serialize_thread_background_terminals_terminate() -> Result<()> { + let request = ClientRequest::ThreadBackgroundTerminalsTerminate { + request_id: RequestId::Integer(8), + params: v2::ThreadBackgroundTerminalsTerminateParams { + thread_id: "thr_123".to_string(), + process_id: "42".to_string(), + }, + }; + assert_eq!( + json!({ + "method": "thread/backgroundTerminals/terminate", + "id": 8, + "params": { + "threadId": "thr_123", + "processId": "42" + } + }), + serde_json::to_value(&request)?, + ); + Ok(()) + } + #[test] fn serialize_thread_realtime_start() -> Result<()> { let request = ClientRequest::ThreadRealtimeStart { diff --git a/codex-rs/app-server-protocol/src/protocol/v2/apps.rs b/codex-rs/app-server-protocol/src/protocol/v2/apps.rs index 9f46525e6c..94545eb788 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/apps.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/apps.rs @@ -102,6 +102,33 @@ pub struct AppInfo { pub plugin_display_names: Vec, } +impl AppInfo { + pub fn category(&self) -> Option { + self.branding + .as_ref() + .and_then(|branding| non_empty_category(branding.category.as_deref())) + .or_else(|| { + self.app_metadata + .as_ref() + .and_then(|metadata| metadata.categories.as_ref()) + .and_then(|categories| { + categories + .iter() + .find_map(|category| non_empty_category(Some(category.as_str()))) + }) + }) + } +} + +fn non_empty_category(category: Option<&str>) -> Option { + let category = category?.trim(); + if category.is_empty() { + None + } else { + Some(category.to_string()) + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -111,17 +138,18 @@ pub struct AppSummary { pub name: String, pub description: Option, pub install_url: Option, - pub needs_auth: bool, + pub category: Option, } impl From for AppSummary { fn from(value: AppInfo) -> Self { + let category = value.category(); Self { id: value.id, name: value.name, description: value.description, install_url: value.install_url, - needs_auth: false, + category, } } } diff --git a/codex-rs/app-server-protocol/src/protocol/v2/plugin.rs b/codex-rs/app-server-protocol/src/protocol/v2/plugin.rs index 6222eb6e73..c196fba917 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/plugin.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/plugin.rs @@ -661,6 +661,7 @@ pub struct AppTemplateSummary { pub template_id: String, pub name: String, pub description: Option, + pub category: Option, pub canonical_connector_id: Option, pub logo_url: Option, pub logo_url_dark: Option, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs index 0c7c362b52..5b412c2edd 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs @@ -594,6 +594,18 @@ pub struct ThreadArchiveParams { #[ts(export_to = "v2/")] pub struct ThreadArchiveResponse {} +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadDeleteParams { + pub thread_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadDeleteResponse {} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -935,6 +947,57 @@ pub struct ThreadBackgroundTerminalsCleanParams { #[ts(export_to = "v2/")] pub struct ThreadBackgroundTerminalsCleanResponse {} +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminalsListParams { + pub thread_id: String, + /// Opaque pagination cursor returned by a previous call. + #[ts(optional = nullable)] + pub cursor: Option, + /// Optional page size. + #[ts(optional = nullable)] + pub limit: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminal { + pub item_id: String, + pub process_id: String, + pub command: String, + pub cwd: AbsolutePathBuf, + pub os_pid: Option, + pub cpu_percent: Option, + pub rss_kb: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminalsListResponse { + pub data: Vec, + /// Opaque cursor to pass to the next call to continue after the last item. + /// If None, there are no more items to return. + pub next_cursor: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminalsTerminateParams { + pub thread_id: String, + pub process_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminalsTerminateResponse { + pub terminated: bool, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -1330,6 +1393,13 @@ pub struct ThreadArchivedNotification { pub thread_id: String, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadDeletedNotification { + pub thread_id: String, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs index 84567fb4c2..836c6d4e96 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs @@ -68,7 +68,13 @@ pub struct TurnStartParams { #[ts(optional = nullable)] pub client_user_message_id: Option, pub input: Vec, - /// Optional turn-scoped Responses API client metadata. + /// Optional metadata to enrich Codex's ResponsesAPI turn metadata. + /// + /// Entries are flattened into the JSON string sent as + /// `client_metadata["x-codex-turn-metadata"]` on ResponsesAPI HTTP and websocket requests. + /// + /// They are not sent as top-level ResponsesAPI `client_metadata` keys, and reserved keys + /// such as `session_id`, `thread_id`, `turn_id`, and `window_id` cannot be overridden. #[experimental("turn/start.responsesapiClientMetadata")] #[ts(optional = nullable)] pub responsesapi_client_metadata: Option>, @@ -161,7 +167,13 @@ pub struct TurnSteerParams { #[ts(optional = nullable)] pub client_user_message_id: Option, pub input: Vec, - /// Optional turn-scoped Responses API client metadata. + /// Optional metadata to enrich Codex's ResponsesAPI turn metadata. + /// + /// Entries are flattened into the JSON string sent as + /// `client_metadata["x-codex-turn-metadata"]` on ResponsesAPI HTTP and websocket requests. + /// + /// They are not sent as top-level ResponsesAPI `client_metadata` keys, and reserved keys + /// such as `session_id`, `thread_id`, `turn_id`, and `window_id` cannot be overridden. #[experimental("turn/steer.responsesapiClientMetadata")] #[ts(optional = nullable)] pub responsesapi_client_metadata: Option>, diff --git a/codex-rs/app-server-transport/src/transport/remote_control/tests.rs b/codex-rs/app-server-transport/src/transport/remote_control/tests.rs index 1e653c2969..1d46404855 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/tests.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/tests.rs @@ -116,6 +116,7 @@ fn remote_control_auth_dot_json(account_id: Option<&str>) -> AuthDotJson { last_refresh: Some(chrono::Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, } } diff --git a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs index e0208b1f1c..ef74527e06 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs @@ -1842,6 +1842,7 @@ mod tests { last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, } } diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 55ea78e617..756a394841 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -36,6 +36,7 @@ codex-cloud-config = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } codex-core-plugins = { workspace = true } +codex-home = { workspace = true } codex-exec-server = { workspace = true } codex-extension-api = { workspace = true } codex-external-agent-migration = { workspace = true } @@ -76,6 +77,7 @@ codex-thread-store = { workspace = true } codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-json-to-toml = { workspace = true } +codex-utils-path-uri = { workspace = true } chrono = { workspace = true } clap = { workspace = true, features = ["derive"] } futures = { workspace = true } diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 999cc8fa37..42ac510a8a 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -151,12 +151,15 @@ Example with notification opt-out: - `thread/settings/updated` — experimental notification emitted to subscribed clients when a loaded thread’s effective next-turn settings change; includes `threadId` and the full `threadSettings`. - `thread/status/changed` — notification emitted when a loaded thread’s status changes (`threadId` + new `status`). - `thread/archive` — move a thread’s rollout file into the archived directory and attempt to move any spawned descendant thread rollout files; returns `{}` on success and emits `thread/archived` for each archived thread. +- `thread/delete` — hard-delete an active or archived thread and any spawned descendant threads; returns `{}` on success and emits `thread/deleted` for each deleted thread. - `thread/unsubscribe` — unsubscribe this connection from thread turn/item events. If this was the last subscriber, the server keeps the thread loaded and unloads it only after it has had no subscribers and no thread activity for 30 minutes, then emits `thread/closed`. - `thread/name/set` — set or update a thread’s user-facing name for either a loaded thread or a persisted rollout; returns `{}` on success and emits `thread/name/updated` to initialized, opted-in clients. Thread names are not required to be unique; name lookups resolve to the most recently updated thread. - `thread/unarchive` — move an archived rollout file back into the sessions directory; returns the restored `thread` on success and emits `thread/unarchived`. - `thread/compact/start` — trigger conversation history compaction for a thread; returns `{}` immediately while progress streams through standard turn/item notifications. - `thread/shellCommand` — run a user-initiated `!` shell command against a thread; this runs unsandboxed with full access rather than inheriting the thread sandbox policy. Returns `{}` immediately while progress streams through standard turn/item notifications and any active turn receives the formatted output in its message stream. - `thread/backgroundTerminals/clean` — terminate all running background terminals for a thread (experimental; requires `capabilities.experimentalApi`); returns `{}` when the cleanup request is accepted. +- `thread/backgroundTerminals/list` — list running background terminals for a loaded thread (experimental; requires `capabilities.experimentalApi`); returns `data` with the running terminal ids. +- `thread/backgroundTerminals/terminate` — terminate one running background terminal by app-server `processId` (experimental; requires `capabilities.experimentalApi`); returns whether a process was terminated. - `thread/rollback` — drop the last N turns from the agent’s in-memory context and persist a rollback marker in the rollout so future resumes see the pruned history; returns the updated `thread` (with `turns` populated) on success. - `turn/start` — add user input to a thread and begin Codex generation; responds with the initial `turn` object and streams `turn/started`, `item/*`, and `turn/completed` notifications. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Experimental `runtimeWorkspaceRoots` replaces the thread-scoped runtime workspace roots used to materialize `:workspace_roots`; paths must be absolute. Prefer experimental `permissions` profile selection by id for permission overrides; the legacy `sandboxPolicy` field is still accepted but cannot be combined with `permissions`. For `collaborationMode`, `settings.developer_instructions: null` means "use built-in instructions for the selected mode". - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. @@ -203,7 +206,7 @@ Example with notification opt-out: - `marketplace/upgrade` — upgrade all configured Git plugin marketplaces, or one named marketplace when `marketplaceName` is provided. Returns selected marketplace names, upgraded roots, and per-marketplace errors. - `plugin/list` — list discovered plugin marketplaces and plugin state, including effective marketplace install/auth policy metadata, plugin `availability` (`AVAILABLE` by default or `DISABLED_BY_ADMIN` for remote plugins blocked upstream), fail-open `marketplaceLoadErrors` entries for marketplace files that could not be parsed or loaded, and best-effort `featuredPluginIds` for the official curated marketplace. `interface.category` uses the marketplace category when present; otherwise it falls back to the plugin manifest category (**under development; do not call from production clients yet**). - `plugin/installed` — list installed plugin rows plus any explicitly requested local install-suggestion plugin names, without fetching the broader remote catalog. Mention surfaces can use this narrower view when they need plugin mention payloads rather than plugin-page discovery data (**under development; do not call from production clients yet**). -- `plugin/read` — read one plugin by `marketplacePath` plus `pluginName`, returning marketplace info, a list-style `summary`, manifest descriptions/interface metadata, and bundled skills/hooks/apps/MCP server names. Returned plugin skills include their current `enabled` state after local config filtering; bundled hooks are returned as lightweight declaration summaries keyed for correlation with `hooks/list`. Plugin app summaries also include `needsAuth` when the server can determine connector accessibility (**under development; do not call from production clients yet**). +- `plugin/read` — read one plugin by `marketplacePath` plus `pluginName`, returning marketplace info, a list-style `summary`, manifest descriptions/interface metadata, and bundled skills/hooks/apps/MCP server names. Returned plugin skills include their current `enabled` state after local config filtering; bundled hooks are returned as lightweight declaration summaries keyed for correlation with `hooks/list`. Use `plugin/install`'s `appsNeedingAuth` to drive post-install authentication and `app/list`'s `isAccessible` to determine current connector accessibility (**under development; do not call from production clients yet**). - `plugin/skill/read` — read remote plugin skill markdown on demand by `remoteMarketplaceName`, `remotePluginId`, and `skillName`. This lets clients preview uninstalled remote plugin skills without downloading the plugin bundle. - `skills/changed` — notification emitted when watched local skill files change. - `app/list` — list available apps. @@ -610,6 +613,16 @@ Use `thread/archive` to move the persisted rollout (stored as a JSONL file on di An archived thread will not appear in `thread/list` unless `archived` is set to `true`. +### Example: Delete a thread + +Use `thread/delete` to hard-delete a thread and its spawned descendant threads. Existing rollout files and associated metadata must be removed before the request succeeds; missing rollout files are treated as already deleted. + +```json +{ "method": "thread/delete", "id": 23, "params": { "threadId": "thr_b" } } +{ "id": 23, "result": {} } +{ "method": "thread/deleted", "params": { "threadId": "thr_b" } } +``` + ### Example: Unarchive a thread Use `thread/unarchive` to move an archived rollout back into the sessions directory. @@ -872,6 +885,32 @@ Use `thread/backgroundTerminals/clean` to terminate all running background termi { "id": 35, "result": {} } ``` +### Example: List and terminate background terminals + +Use `thread/backgroundTerminals/list` to inspect running background terminals associated with a loaded thread. The `backgroundTerminals` segment intentionally follows the existing `thread/backgroundTerminals/clean` method. The returned `processId` is the app-server process id; host OS metadata is nullable. The request accepts the standard `cursor` and `limit` pagination fields. When `nextCursor` is non-null, pass it as `cursor` to fetch the next page. + +```json +{ "method": "thread/backgroundTerminals/list", "id": 36, "params": { "threadId": "thr_123" } } +{ "id": 36, "result": { "data": [ + { + "itemId": "item_456", + "processId": "42", + "command": "python3 -m http.server", + "cwd": "/workspace", + "osPid": null, + "cpuPercent": null, + "rssKb": null + } +], "nextCursor": null } } +``` + +Use `thread/backgroundTerminals/terminate` to terminate one running background terminal by that `processId`. + +```json +{ "method": "thread/backgroundTerminals/terminate", "id": 37, "params": { "threadId": "thr_123", "processId": "42" } } +{ "id": 37, "result": { "terminated": true } } +``` + ### Example: Steer an active turn Use `turn/steer` to append additional user input to the currently active regular turn. This does diff --git a/codex-rs/app-server/src/config/external_agent_config.rs b/codex-rs/app-server/src/config/external_agent_config.rs index eca927fc23..665588213f 100644 --- a/codex-rs/app-server/src/config/external_agent_config.rs +++ b/codex-rs/app-server/src/config/external_agent_config.rs @@ -1147,7 +1147,7 @@ fn configured_marketplace_plugins( ) -> io::Result>> { let plugins_input = config.plugins_config_input(); let marketplaces = plugins_manager - .list_marketplaces_for_config(&plugins_input, &[]) + .list_marketplaces_for_config(&plugins_input, &[], /*include_openai_curated*/ true) .map_err(|err| { invalid_data_error(format!("failed to list configured marketplaces: {err}")) })?; diff --git a/codex-rs/app-server/src/extensions.rs b/codex-rs/app-server/src/extensions.rs index 4916132c58..c979baacfa 100644 --- a/codex-rs/app-server/src/extensions.rs +++ b/codex-rs/app-server/src/extensions.rs @@ -21,11 +21,15 @@ use codex_protocol::error::CodexErr; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_rollout::state_db::StateDbHandle; +use codex_thread_store::ThreadStore; use crate::outgoing_message::OutgoingMessageSender; use crate::thread_state::ThreadListenerCommand; use crate::thread_state::ThreadStateManager; +// TODO(jif): Enable once /ps/mcp serves complete hosted skill packages. +const ORCHESTRATOR_SKILLS_ENABLED: bool = false; + pub(crate) struct ThreadExtensionDependencies { pub(crate) event_sink: Arc, pub(crate) auth_manager: Arc, @@ -34,6 +38,8 @@ pub(crate) struct ThreadExtensionDependencies { pub(crate) thread_manager: Weak, pub(crate) goal_service: Arc, pub(crate) executor_skill_provider: Arc, + /// Process-scoped persistence backend for extensions that need stored thread history. + pub(crate) thread_store: Arc, } pub(crate) fn thread_extensions( @@ -51,6 +57,7 @@ where thread_manager, goal_service, executor_skill_provider, + thread_store: _thread_store, } = dependencies; let mut builder = ExtensionRegistryBuilder::::with_event_sink(event_sink); if let Some(state_db) = state_db { @@ -69,10 +76,20 @@ where codex_mcp_extension::install(&mut builder); codex_web_search_extension::install(&mut builder, auth_manager.clone()); codex_image_generation_extension::install(&mut builder, auth_manager); + let mut skill_providers = codex_skills_extension::SkillProviders::new() + .with_executor_provider(executor_skill_provider); + if ORCHESTRATOR_SKILLS_ENABLED { + skill_providers = skill_providers.with_orchestrator_provider(Arc::new( + codex_skills_extension::OrchestratorSkillProvider::new(), + )); + } codex_skills_extension::install_with_providers( &mut builder, - codex_skills_extension::SkillProviders::new() - .with_executor_provider(executor_skill_provider), + skill_providers, + |config: &Config| codex_skills_extension::SkillsExtensionConfig { + include_instructions: config.include_skill_instructions, + bundled_skills_enabled: config.bundled_skills_enabled(), + }, ); Arc::new(builder.build()) } diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 448a389851..27191e7215 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -104,7 +104,9 @@ type PendingClientRequestResponse = std::result::Result bool { matches!( notification, - ServerNotification::TurnCompleted(_) | ServerNotification::ThreadSettingsUpdated(_) + ServerNotification::TurnCompleted(_) + | ServerNotification::ThreadSettingsUpdated(_) + | ServerNotification::ExternalAgentConfigImportCompleted(_) ) } @@ -729,6 +731,7 @@ mod tests { use super::*; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ConfigRequirementsReadResponse; + use codex_app_server_protocol::ExternalAgentConfigImportCompletedNotification; use codex_app_server_protocol::SessionSource as ApiSessionSource; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; @@ -893,5 +896,10 @@ mod tests { }, }) )); + assert!(server_notification_requires_delivery( + &ServerNotification::ExternalAgentConfigImportCompleted( + ExternalAgentConfigImportCompletedNotification {}, + ) + )); } } diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index b6f0b8c859..d9c75a54e1 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -1,3 +1,4 @@ +#![recursion_limit = "256"] #![deny(clippy::print_stdout, clippy::print_stderr)] use codex_arg0::Arg0DispatchPaths; @@ -74,6 +75,8 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::registry::Registry; use tracing_subscriber::util::SubscriberInitExt; +const SQLITE_RECOVERY_CONFIG_WARNING_SUMMARY: &str = "Codex rebuilt its local database."; + mod analytics_utils; mod app_server_tracing; mod attestation; @@ -533,8 +536,8 @@ pub async fn run_main_with_transport_options( } _ => None, }; - let state_db = match rollout_state_db::try_init(&config).await { - Ok(state_db) => Some(state_db), + let state_db_init = match init_sqlite_state_db_with_fresh_start_on_corruption(&config).await { + Ok(state_db_init) => state_db_init, Err(err) => { return Err(std::io::Error::other(format!( "failed to initialize sqlite state runtime under {}: {err}", @@ -542,6 +545,15 @@ pub async fn run_main_with_transport_options( ))); } }; + let state_db = state_db_init.state_db; + if let Some(recovery_notice) = state_db_init.recovery_notice { + config_warnings.push(ConfigWarningNotification { + summary: SQLITE_RECOVERY_CONFIG_WARNING_SUMMARY.to_string(), + details: Some(recovery_notice.details), + path: None, + range: None, + }); + } if should_run_personality_migration { let effective_toml = config.config_layer_stack.effective_config(); @@ -1099,6 +1111,121 @@ pub async fn run_main_with_transport_options( Ok(()) } +struct SqliteRecoveryNotice { + details: String, +} + +struct RecoveredSqliteDatabase { + database_path: String, + backup_folder: String, +} + +struct StateDbInitResult { + state_db: Option, + recovery_notice: Option, +} + +async fn init_sqlite_state_db_with_fresh_start_on_corruption( + config: &Config, +) -> anyhow::Result { + let mut attempted_backups = HashSet::new(); + let mut recovered_databases = Vec::new(); + loop { + let err = match rollout_state_db::try_init(config).await { + Ok(state_db) => { + let recovery_notice = sqlite_recovery_notice(&recovered_databases); + if recovery_notice.is_some() { + emit_state_db_backup_warning(SQLITE_RECOVERY_CONFIG_WARNING_SUMMARY); + for recovered_database in &recovered_databases { + emit_state_db_backup_warning(&format!( + "Database path: {}", + recovered_database.database_path + )); + emit_state_db_backup_warning(&format!( + "Backup folder: {}", + recovered_database.backup_folder + )); + } + } + return Ok(StateDbInitResult { + state_db: Some(state_db), + recovery_notice, + }); + } + Err(err) => err, + }; + if !codex_state::is_sqlite_corruption_error(&err) { + return Err(err); + } + + let database_path = codex_state::runtime_db_path_for_corruption_error(&err) + .unwrap_or_else(|| codex_state::state_db_path(config.sqlite_home.as_path())); + if !attempted_backups.insert(database_path.clone()) { + return Err(anyhow::anyhow!( + "failed to initialize sqlite state runtime after moving damaged database file into a backup folder: {err}" + )); + } + + let original_error = err.to_string(); + emit_state_db_backup_warning(&format!( + "Codex local database at {} appears damaged. Moving it into a backup folder so the app server can rebuild it from saved data.", + database_path.display() + )); + let backups = codex_state::backup_runtime_db_for_fresh_start(database_path.as_path()) + .await + .map_err(|backup_err| { + anyhow::anyhow!( + "failed to move damaged sqlite state database files into a backup folder: {backup_err}; original error: {original_error}" + ) + })?; + for backup in &backups { + emit_state_db_backup_warning(&format!( + "Moved damaged Codex local database file {} to {}", + backup.original_path.display(), + backup.backup_path.display() + )); + } + if let Some(first_backup) = backups.first() + && let Some(backup_folder) = first_backup.backup_path.parent() + { + recovered_databases.push(RecoveredSqliteDatabase { + database_path: first_backup.original_path.display().to_string(), + backup_folder: backup_folder.display().to_string(), + }); + } + } +} + +fn sqlite_recovery_notice( + recovered_databases: &[RecoveredSqliteDatabase], +) -> Option { + if recovered_databases.is_empty() { + return None; + } + + let details = recovered_databases + .iter() + .map(|recovered_database| { + format!( + "Database path: {}\nBackup folder: {}", + recovered_database.database_path, recovered_database.backup_folder + ) + }) + .collect::>() + .join("\n\n"); + Some(SqliteRecoveryNotice { details }) +} + +fn emit_state_db_backup_warning(message: &str) { + warn!("{message}"); + if !tracing::dispatcher::has_been_set() { + #[allow(clippy::print_stderr)] + { + eprintln!("{message}"); + } + } +} + fn analytics_rpc_transport(transport: &AppServerTransport) -> AppServerRpcTransport { match transport { AppServerTransport::Stdio => AppServerRpcTransport::Stdio, diff --git a/codex-rs/app-server/src/mcp_refresh.rs b/codex-rs/app-server/src/mcp_refresh.rs index b5431228d4..0fd63a5f02 100644 --- a/codex-rs/app-server/src/mcp_refresh.rs +++ b/codex-rs/app-server/src/mcp_refresh.rs @@ -113,6 +113,7 @@ mod tests { use codex_core::thread_store_from_config; use codex_exec_server::EnvironmentManager; use codex_extension_api::NoopExtensionEventSink; + use codex_home::CodexHomeUserInstructionsProvider; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_protocol::protocol::SessionSource; @@ -202,8 +203,12 @@ mod tests { thread_manager: thread_manager.clone(), goal_service: Arc::new(codex_goal_extension::GoalService::new()), executor_skill_provider: Arc::clone(&executor_skill_provider), + thread_store: Arc::clone(&thread_store), }, ), + Arc::new(CodexHomeUserInstructionsProvider::new( + good_config.codex_home.clone(), + )), /*analytics_events_client*/ None, Arc::clone(&thread_store), Some(state_db.clone()), diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 6bb5121dc9..10bbccd9e4 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -72,6 +72,7 @@ use codex_core::config::Config; use codex_exec_server::EnvironmentManager; use codex_feedback::CodexFeedback; use codex_goal_extension::GoalService; +use codex_home::CodexHomeUserInstructionsProvider; use codex_login::AuthManager; use codex_login::auth::ExternalAuth; use codex_login::auth::ExternalAuthRefreshContext; @@ -336,8 +337,12 @@ impl MessageProcessor { thread_manager: thread_manager.clone(), goal_service: Arc::clone(&goal_service), executor_skill_provider: Arc::clone(&executor_skill_provider), + thread_store: Arc::clone(&thread_store), }, ), + Arc::new(CodexHomeUserInstructionsProvider::new( + config.codex_home.clone(), + )), Some(analytics_events_client.clone()), Arc::clone(&thread_store), state_db.clone(), @@ -400,7 +405,7 @@ impl MessageProcessor { Arc::clone(&thread_manager), Arc::clone(&config), feedback, - log_db, + log_db.clone(), state_db.clone(), ); let git_processor = GitRequestProcessor::new(); @@ -454,6 +459,7 @@ impl MessageProcessor { Arc::clone(&thread_list_state_permit), thread_goal_processor.clone(), state_db, + log_db, Arc::clone(&skills_watcher), ); let turn_processor = TurnRequestProcessor::new( @@ -1071,6 +1077,11 @@ impl MessageProcessor { .thread_archive(request_id.clone(), params) .await } + ClientRequest::ThreadDelete { params, .. } => { + self.thread_processor + .thread_delete(request_id.clone(), params) + .await + } ClientRequest::ThreadIncrementElicitation { params, .. } => { self.thread_processor .thread_increment_elicitation(params) @@ -1126,6 +1137,16 @@ impl MessageProcessor { .thread_background_terminals_clean(&request_id, params) .await } + ClientRequest::ThreadBackgroundTerminalsList { params, .. } => { + self.thread_processor + .thread_background_terminals_list(params) + .await + } + ClientRequest::ThreadBackgroundTerminalsTerminate { params, .. } => { + self.thread_processor + .thread_background_terminals_terminate(params) + .await + } ClientRequest::ThreadRollback { params, .. } => { self.thread_processor .thread_rollback(&request_id, params) diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index 00f17937ba..83e704cdfd 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -171,13 +171,21 @@ use codex_app_server_protocol::ThreadApproveGuardianDeniedActionResponse; use codex_app_server_protocol::ThreadArchiveParams; use codex_app_server_protocol::ThreadArchiveResponse; use codex_app_server_protocol::ThreadArchivedNotification; +use codex_app_server_protocol::ThreadBackgroundTerminal; use codex_app_server_protocol::ThreadBackgroundTerminalsCleanParams; use codex_app_server_protocol::ThreadBackgroundTerminalsCleanResponse; +use codex_app_server_protocol::ThreadBackgroundTerminalsListParams; +use codex_app_server_protocol::ThreadBackgroundTerminalsListResponse; +use codex_app_server_protocol::ThreadBackgroundTerminalsTerminateParams; +use codex_app_server_protocol::ThreadBackgroundTerminalsTerminateResponse; use codex_app_server_protocol::ThreadClosedNotification; use codex_app_server_protocol::ThreadCompactStartParams; use codex_app_server_protocol::ThreadCompactStartResponse; use codex_app_server_protocol::ThreadDecrementElicitationParams; use codex_app_server_protocol::ThreadDecrementElicitationResponse; +use codex_app_server_protocol::ThreadDeleteParams; +use codex_app_server_protocol::ThreadDeleteResponse; +use codex_app_server_protocol::ThreadDeletedNotification; use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadForkResponse; use codex_app_server_protocol::ThreadGoal; @@ -414,6 +422,7 @@ use codex_rollout::state_db::reconcile_rollout; use codex_state::ThreadMetadata; use codex_state::log_db::LogDbLayer; use codex_thread_store::ArchiveThreadParams as StoreArchiveThreadParams; +use codex_thread_store::DeleteThreadParams as StoreDeleteThreadParams; use codex_thread_store::GitInfoPatch as StoreGitInfoPatch; use codex_thread_store::ListThreadsParams as StoreListThreadsParams; use codex_thread_store::LocalThreadStore; @@ -534,6 +543,7 @@ fn resolve_runtime_workspace_roots(workspace_roots: Vec) -> Vec mod config_errors; mod request_errors; +mod thread_delete; mod thread_goal_processor; mod thread_lifecycle; mod thread_resume_redaction; diff --git a/codex-rs/app-server/src/request_processors/account_processor.rs b/codex-rs/app-server/src/request_processors/account_processor.rs index a99ec93c2e..f03a3f2b37 100644 --- a/codex-rs/app-server/src/request_processors/account_processor.rs +++ b/codex-rs/app-server/src/request_processors/account_processor.rs @@ -158,6 +158,9 @@ impl AccountRequestProcessor { pub(crate) fn clear_external_auth(&self) { self.auth_manager.clear_external_auth(); + self.thread_manager + .plugins_manager() + .set_auth_mode(self.auth_manager.get_api_auth_mode()); } fn current_account_updated_notification(&self) -> AccountUpdatedNotification { @@ -173,6 +176,10 @@ impl AccountRequestProcessor { thread_manager: &Arc, auth: Option, ) { + thread_manager + .plugins_manager() + .set_auth_mode(auth.as_ref().map(CodexAuth::api_auth_mode)); + match config_manager .load_latest_config(/*fallback_cwd*/ None) .await diff --git a/codex-rs/app-server/src/request_processors/fs_processor.rs b/codex-rs/app-server/src/request_processors/fs_processor.rs index 99a8620b4f..c0d93bb158 100644 --- a/codex-rs/app-server/src/request_processors/fs_processor.rs +++ b/codex-rs/app-server/src/request_processors/fs_processor.rs @@ -29,6 +29,7 @@ use codex_exec_server::CreateDirectoryOptions; use codex_exec_server::EnvironmentManager; use codex_exec_server::ExecutorFileSystem; use codex_exec_server::RemoveOptions; +use codex_utils_path_uri::PathUri; use std::io; use std::sync::Arc; @@ -64,9 +65,10 @@ impl FsRequestProcessor { &self, params: FsReadFileParams, ) -> Result { + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let bytes = self .file_system()? - .read_file(¶ms.path, /*sandbox*/ None) + .read_file(&path, /*sandbox*/ None) .await .map_err(map_fs_error)?; Ok(FsReadFileResponse { @@ -83,8 +85,9 @@ impl FsRequestProcessor { "fs/writeFile requires valid base64 dataBase64: {err}" )) })?; + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; self.file_system()? - .write_file(¶ms.path, bytes, /*sandbox*/ None) + .write_file(&path, bytes, /*sandbox*/ None) .await .map_err(map_fs_error)?; Ok(FsWriteFileResponse {}) @@ -94,9 +97,10 @@ impl FsRequestProcessor { &self, params: FsCreateDirectoryParams, ) -> Result { + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; self.file_system()? .create_directory( - ¶ms.path, + &path, CreateDirectoryOptions { recursive: params.recursive.unwrap_or(true), }, @@ -111,9 +115,10 @@ impl FsRequestProcessor { &self, params: FsGetMetadataParams, ) -> Result { + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let metadata = self .file_system()? - .get_metadata(¶ms.path, /*sandbox*/ None) + .get_metadata(&path, /*sandbox*/ None) .await .map_err(map_fs_error)?; Ok(FsGetMetadataResponse { @@ -129,9 +134,10 @@ impl FsRequestProcessor { &self, params: FsReadDirectoryParams, ) -> Result { + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let entries = self .file_system()? - .read_directory(¶ms.path, /*sandbox*/ None) + .read_directory(&path, /*sandbox*/ None) .await .map_err(map_fs_error)?; Ok(FsReadDirectoryResponse { @@ -150,9 +156,10 @@ impl FsRequestProcessor { &self, params: FsRemoveParams, ) -> Result { + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; self.file_system()? .remove( - ¶ms.path, + &path, RemoveOptions { recursive: params.recursive.unwrap_or(true), force: params.force.unwrap_or(true), @@ -168,10 +175,13 @@ impl FsRequestProcessor { &self, params: FsCopyParams, ) -> Result { + let source_path = PathUri::from_abs_path(¶ms.source_path).map_err(map_fs_error)?; + let destination_path = + PathUri::from_abs_path(¶ms.destination_path).map_err(map_fs_error)?; self.file_system()? .copy( - ¶ms.source_path, - ¶ms.destination_path, + &source_path, + &destination_path, CopyOptions { recursive: params.recursive, }, diff --git a/codex-rs/app-server/src/request_processors/plugins.rs b/codex-rs/app-server/src/request_processors/plugins.rs index 54a5efbe7a..e4971b36fa 100644 --- a/codex-rs/app-server/src/request_processors/plugins.rs +++ b/codex-rs/app-server/src/request_processors/plugins.rs @@ -573,6 +573,7 @@ impl PluginRequestProcessor { .list_marketplaces_for_config( &config_for_marketplace_listing, &roots_for_marketplace_listing, + /*include_openai_curated*/ true, )?; Ok::< ( @@ -837,8 +838,11 @@ impl PluginRequestProcessor { let config_for_marketplace_listing = plugins_input.clone(); let shared_plugin_ids_by_local_path = load_shared_plugin_ids_by_local_path(config)?; match tokio::task::spawn_blocking(move || { - let outcome = plugins_manager - .list_marketplaces_for_config(&config_for_marketplace_listing, &roots)?; + let outcome = plugins_manager.list_marketplaces_for_config( + &config_for_marketplace_listing, + &roots, + /*include_openai_curated*/ true, + )?; Ok::< ( Vec, @@ -1031,12 +1035,10 @@ impl PluginRequestProcessor { } None => None, }; - let environment_manager = self.thread_manager.environment_manager(); let app_summaries = load_plugin_app_summaries( &config, &outcome.plugin.apps, - Arc::clone(&environment_manager), - self.thread_manager.mcp_manager(), + &outcome.plugin.app_category_by_id, ) .await; let visible_skills = outcome @@ -1114,14 +1116,13 @@ impl PluginRequestProcessor { .cloned() .map(codex_plugin::AppConnectorId) .collect::>(); - let environment_manager = self.thread_manager.environment_manager(); - let app_summaries = load_plugin_app_summaries( - &config, - &plugin_apps, - Arc::clone(&environment_manager), - self.thread_manager.mcp_manager(), - ) - .await; + let app_category_by_id = remote_detail + .app_manifest + .as_ref() + .map(plugin_app_category_by_id_from_value) + .unwrap_or_default(); + let app_summaries = + load_plugin_app_summaries(&config, &plugin_apps, &app_category_by_id).await; remote_plugin_detail_to_info(remote_detail, app_summaries) } }; @@ -1568,17 +1569,28 @@ impl PluginRequestProcessor { .into_iter() .map(codex_plugin::AppConnectorId) .collect::>(); + let app_category_by_id = remote_detail + .app_manifest + .as_ref() + .map(plugin_app_category_by_id_from_value) + .unwrap_or_default(); let all_connectors = connectors::list_cached_all_connectors(&config) .await .unwrap_or_default(); connectors::connectors_for_plugin_apps(all_connectors, &plugin_apps) .into_iter() - .map(|connector| AppSummary { - id: connector.id, - name: connector.name, - description: connector.description, - install_url: connector.install_url, - needs_auth: true, + .map(|connector| { + let category = app_category_by_id + .get(&connector.id) + .cloned() + .or_else(|| connector.category()); + AppSummary { + category, + id: connector.id, + name: connector.name, + description: connector.description, + install_url: connector.install_url, + } }) .collect() } @@ -1883,8 +1895,7 @@ impl PluginRequestProcessor { async fn load_plugin_app_summaries( config: &Config, plugin_apps: &[codex_plugin::AppConnectorId], - environment_manager: Arc, - mcp_manager: Arc, + app_category_by_id: &HashMap, ) -> Vec { if plugin_apps.is_empty() { return Vec::new(); @@ -1903,51 +1914,31 @@ async fn load_plugin_app_summaries( let plugin_connectors = connectors::connectors_for_plugin_apps(connectors, plugin_apps); - let accessible_connectors = - match connectors::list_accessible_connectors_from_mcp_tools_with_mcp_manager( - config, - /*force_refetch*/ false, - environment_manager, - mcp_manager, - ) - .await - { - Ok(status) if status.codex_apps_ready => status.connectors, - Ok(_) => { - return plugin_connectors - .into_iter() - .map(AppSummary::from) - .collect(); - } - Err(err) => { - warn!("failed to load app auth state for plugin/read: {err:#}"); - return plugin_connectors - .into_iter() - .map(AppSummary::from) - .collect(); - } - }; - - let accessible_ids = accessible_connectors - .iter() - .map(|connector| connector.id.as_str()) - .collect::>(); - plugin_connectors .into_iter() .map(|connector| { - let needs_auth = !accessible_ids.contains(connector.id.as_str()); + let category = app_category_by_id + .get(&connector.id) + .cloned() + .or_else(|| connector.category()); AppSummary { id: connector.id, name: connector.name, description: connector.description, install_url: connector.install_url, - needs_auth, + category, } }) .collect() } +fn plugin_app_category_by_id_from_value(value: &serde_json::Value) -> HashMap { + codex_core_plugins::loader::plugin_app_metadata_from_value(value) + .into_iter() + .filter_map(|app| app.category.map(|category| (app.id.0, category))) + .collect() +} + fn plugin_apps_needing_auth( all_connectors: &[AppInfo], accessible_connectors: &[AppInfo], @@ -1974,12 +1965,15 @@ fn plugin_apps_needing_auth( && !accessible_ids.contains(connector.id.as_str()) }) .cloned() - .map(|connector| AppSummary { - id: connector.id, - name: connector.name, - description: connector.description, - install_url: connector.install_url, - needs_auth: true, + .map(|connector| { + let category = connector.category(); + AppSummary { + category, + id: connector.id, + name: connector.name, + description: connector.description, + install_url: connector.install_url, + } }) .collect() } @@ -2067,6 +2061,7 @@ fn remote_plugin_detail_to_info( template_id: template.template_id, name: template.name, description: template.description, + category: template.category, canonical_connector_id: template.canonical_connector_id, logo_url: template.logo_url, logo_url_dark: template.logo_url_dark, diff --git a/codex-rs/app-server/src/request_processors/thread_delete.rs b/codex-rs/app-server/src/request_processors/thread_delete.rs new file mode 100644 index 0000000000..fc7e58573c --- /dev/null +++ b/codex-rs/app-server/src/request_processors/thread_delete.rs @@ -0,0 +1,188 @@ +//! `thread/delete` request handling. + +use super::thread_processor::core_thread_write_error; +use super::thread_processor::unsupported_thread_store_operation; +use super::*; + +impl ThreadRequestProcessor { + pub(crate) async fn thread_delete( + &self, + request_id: ConnectionRequestId, + params: ThreadDeleteParams, + ) -> Result, JSONRPCErrorError> { + let mut deleted_thread_ids = Vec::new(); + let result = { + let _thread_list_state_permit = self.acquire_thread_list_state_permit().await?; + self.thread_delete_response(params, &mut deleted_thread_ids) + .await + }; + match result { + Ok(response) => { + self.outgoing + .send_response(request_id.clone(), response) + .await; + self.send_thread_deleted_notifications(deleted_thread_ids) + .await; + Ok(None) + } + Err(error) => Err(error), + } + } + + async fn thread_delete_response( + &self, + params: ThreadDeleteParams, + deleted_thread_ids: &mut Vec, + ) -> Result { + let thread_id = ThreadId::from_string(¶ms.thread_id) + .map_err(|err| invalid_request(format!("invalid thread id: {err}")))?; + + let mut thread_ids = self.state_db_spawn_subtree_thread_ids(thread_id).await?; + let mut seen = thread_ids.iter().copied().collect::>(); + + match self + .thread_manager + .list_agent_subtree_thread_ids(thread_id) + .await + { + Ok(live_thread_ids) => { + for live_thread_id in live_thread_ids { + if seen.insert(live_thread_id) { + thread_ids.push(live_thread_id); + } + } + } + Err(err) => return Err(core_thread_write_error("delete thread", err)), + } + + self.validate_root_thread_delete(thread_id, thread_ids.len() > 1) + .await?; + for thread_id_to_delete in thread_ids.iter().copied() { + self.prepare_thread_for_delete(thread_id_to_delete).await; + } + + let mut delete_order: Vec<_> = thread_ids.iter().skip(1).rev().copied().collect(); + delete_order.push(thread_id); + + for thread_id_to_delete in delete_order.iter().copied() { + match self + .thread_store + .delete_thread(StoreDeleteThreadParams { + thread_id: thread_id_to_delete, + }) + .await + { + Ok(()) => {} + Err(ThreadStoreError::ThreadNotFound { .. }) => { + warn!( + "thread {thread_id_to_delete} was already missing while deleting {thread_id}" + ); + } + Err(err) => { + return Err(thread_store_delete_error(err)); + } + } + } + + if let Some(state_db) = self.state_db.as_ref() { + state_db + .delete_threads_strict(thread_ids.as_slice()) + .await + .map_err(|err| { + internal_error(format!( + "failed to delete app-server state for {thread_id}: {err}" + )) + })?; + } + + deleted_thread_ids.extend( + delete_order + .into_iter() + .map(|thread_id| thread_id.to_string()), + ); + Ok(ThreadDeleteResponse {}) + } + + async fn send_thread_deleted_notifications(&self, deleted_thread_ids: Vec) { + for thread_id in deleted_thread_ids { + self.outgoing + .send_server_notification(ServerNotification::ThreadDeleted( + ThreadDeletedNotification { thread_id }, + )) + .await; + } + } + + async fn validate_root_thread_delete( + &self, + thread_id: ThreadId, + has_descendants: bool, + ) -> Result<(), JSONRPCErrorError> { + if let Ok(thread) = self.thread_manager.get_thread(thread_id).await { + if !thread.config_snapshot().await.ephemeral { + return Ok(()); + } + return Err(invalid_request(format!( + "thread is not persisted and cannot be deleted: {thread_id}" + ))); + } + match self + .thread_store + .read_thread(StoreReadThreadParams { + thread_id, + include_archived: true, + include_history: false, + }) + .await + { + Ok(_) => Ok(()), + Err(ThreadStoreError::ThreadNotFound { .. }) => { + if has_descendants { + return Ok(()); + } + let Some(state_db) = self.state_db.as_ref() else { + return Err(thread_store_delete_error( + ThreadStoreError::ThreadNotFound { thread_id }, + )); + }; + if state_db + .get_thread(thread_id) + .await + .map_err(|err| { + internal_error(format!( + "failed to read app-server state for {thread_id}: {err}" + )) + })? + .is_some() + { + Ok(()) + } else { + Err(thread_store_delete_error( + ThreadStoreError::ThreadNotFound { thread_id }, + )) + } + } + Err(err) => Err(thread_store_delete_error(err)), + } + } + + async fn prepare_thread_for_delete(&self, thread_id: ThreadId) { + self.prepare_thread_for_removal(thread_id, "delete").await; + if let Some(log_db) = self.log_db.as_ref() { + log_db.flush().await; + } + } +} + +fn thread_store_delete_error(err: ThreadStoreError) -> JSONRPCErrorError { + match err { + ThreadStoreError::ThreadNotFound { thread_id } => { + invalid_request(format!("thread not found: {thread_id}")) + } + ThreadStoreError::InvalidRequest { message } => invalid_request(message), + ThreadStoreError::Unsupported { operation } => { + unsupported_thread_store_operation(operation) + } + err => internal_error(format!("failed to delete thread: {err}")), + } +} diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index fd6432531d..58719d55ba 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -325,6 +325,7 @@ pub(crate) struct ThreadRequestProcessor { pub(super) thread_list_state_permit: Arc, pub(super) thread_goal_processor: ThreadGoalRequestProcessor, pub(super) state_db: Option, + pub(super) log_db: Option, pub(super) background_tasks: TaskTracker, pub(super) skills_watcher: Arc, } @@ -356,6 +357,7 @@ impl ThreadRequestProcessor { thread_list_state_permit: Arc, thread_goal_processor: ThreadGoalRequestProcessor, state_db: Option, + log_db: Option, skills_watcher: Arc, ) -> Self { Self { @@ -372,6 +374,7 @@ impl ThreadRequestProcessor { thread_list_state_permit, thread_goal_processor, state_db, + log_db, background_tasks: TaskTracker::new(), skills_watcher, } @@ -569,6 +572,24 @@ impl ThreadRequestProcessor { .map(|response| Some(response.into())) } + pub(crate) async fn thread_background_terminals_list( + &self, + params: ThreadBackgroundTerminalsListParams, + ) -> Result, JSONRPCErrorError> { + self.thread_background_terminals_list_inner(params) + .await + .map(|response| Some(response.into())) + } + + pub(crate) async fn thread_background_terminals_terminate( + &self, + params: ThreadBackgroundTerminalsTerminateParams, + ) -> Result, JSONRPCErrorError> { + self.thread_background_terminals_terminate_inner(params) + .await + .map(|response| Some(response.into())) + } + pub(crate) async fn thread_rollback( &self, request_id: &ConnectionRequestId, @@ -678,7 +699,7 @@ impl ThreadRequestProcessor { Ok((thread_id, thread)) } - async fn acquire_thread_list_state_permit( + pub(super) async fn acquire_thread_list_state_permit( &self, ) -> Result, JSONRPCErrorError> { self.thread_list_state_permit @@ -750,6 +771,10 @@ impl ThreadRequestProcessor { } async fn prepare_thread_for_archive(&self, thread_id: ThreadId) { + self.prepare_thread_for_removal(thread_id, "archive").await; + } + + pub(super) async fn prepare_thread_for_removal(&self, thread_id: ThreadId, operation: &str) { let removed_conversation = self.thread_manager.remove_thread(&thread_id).await; if let Some(conversation) = removed_conversation { info!("thread {thread_id} was active; shutting down"); @@ -757,11 +782,11 @@ impl ThreadRequestProcessor { ThreadShutdownResult::Complete => {} ThreadShutdownResult::SubmitFailed => { error!( - "failed to submit Shutdown to thread {thread_id}; proceeding with archive" + "failed to submit Shutdown to thread {thread_id}; proceeding with {operation}" ); } ThreadShutdownResult::TimedOut => { - warn!("thread {thread_id} shutdown timed out; proceeding with archive"); + warn!("thread {thread_id} shutdown timed out; proceeding with {operation}"); } } } @@ -1294,23 +1319,7 @@ impl ThreadRequestProcessor { let thread_id = ThreadId::from_string(¶ms.thread_id) .map_err(|err| invalid_request(format!("invalid session id: {err}")))?; - let mut thread_ids = vec![thread_id]; - if let Some(state_db_ctx) = self.state_db.as_ref() { - let descendants = state_db_ctx - .list_thread_spawn_descendants(thread_id) - .await - .map_err(|err| { - internal_error(format!( - "failed to list spawned descendants for session {thread_id}: {err}" - )) - })?; - let mut seen = HashSet::from([thread_id]); - for descendant_id in descendants { - if seen.insert(descendant_id) { - thread_ids.push(descendant_id); - } - } - } + let thread_ids = self.state_db_spawn_subtree_thread_ids(thread_id).await?; let mut archive_thread_ids = Vec::new(); match self @@ -1395,6 +1404,31 @@ impl ThreadRequestProcessor { Ok((ThreadArchiveResponse {}, archived_thread_ids)) } + pub(super) async fn state_db_spawn_subtree_thread_ids( + &self, + thread_id: ThreadId, + ) -> Result, JSONRPCErrorError> { + let mut thread_ids = vec![thread_id]; + let Some(state_db_ctx) = self.state_db.as_ref() else { + return Ok(thread_ids); + }; + let mut seen = HashSet::from([thread_id]); + let descendants = state_db_ctx + .list_thread_spawn_descendants(thread_id) + .await + .map_err(|err| { + internal_error(format!( + "failed to list spawned descendants for thread id {thread_id}: {err}" + )) + })?; + for descendant_id in descendants { + if seen.insert(descendant_id) { + thread_ids.push(descendant_id); + } + } + Ok(thread_ids) + } + async fn thread_increment_elicitation_inner( &self, params: ThreadIncrementElicitationParams, @@ -1725,6 +1759,54 @@ impl ThreadRequestProcessor { Ok(ThreadBackgroundTerminalsCleanResponse {}) } + async fn thread_background_terminals_list_inner( + &self, + params: ThreadBackgroundTerminalsListParams, + ) -> Result { + let ThreadBackgroundTerminalsListParams { + thread_id, + cursor, + limit, + } = params; + + let (_, thread) = self.load_thread(&thread_id).await?; + let terminals = thread + .list_background_terminals() + .await + .into_iter() + .map(|terminal| ThreadBackgroundTerminal { + item_id: terminal.item_id, + process_id: terminal.process_id, + command: terminal.command, + cwd: terminal.cwd, + os_pid: None, + cpu_percent: None, + rss_kb: None, + }) + .collect::>(); + + let (data, next_cursor) = paginate_background_terminals(&terminals, cursor, limit)?; + + Ok(ThreadBackgroundTerminalsListResponse { data, next_cursor }) + } + + async fn thread_background_terminals_terminate_inner( + &self, + params: ThreadBackgroundTerminalsTerminateParams, + ) -> Result { + let ThreadBackgroundTerminalsTerminateParams { + thread_id, + process_id, + } = params; + let process_id = process_id.parse::().map_err(|err| { + invalid_request(format!("invalid background terminal process id: {err}")) + })?; + + let (_, thread) = self.load_thread(&thread_id).await?; + let terminated = thread.terminate_background_terminal(process_id).await; + Ok(ThreadBackgroundTerminalsTerminateResponse { terminated }) + } + async fn thread_shell_command_inner( &self, request_id: &ConnectionRequestId, @@ -3843,7 +3925,7 @@ fn thread_read_view_error(err: ThreadReadViewError) -> JSONRPCErrorError { } } -fn unsupported_thread_store_operation(operation: &'static str) -> JSONRPCErrorError { +pub(super) fn unsupported_thread_store_operation(operation: &'static str) -> JSONRPCErrorError { method_not_found(format!("{operation} is not supported yet")) } @@ -3964,7 +4046,7 @@ fn conversation_summary_rollout_path_read_error( } } -fn core_thread_write_error(operation: &str, err: CodexErr) -> JSONRPCErrorError { +pub(super) fn core_thread_write_error(operation: &str, err: CodexErr) -> JSONRPCErrorError { match err { CodexErr::ThreadNotFound(thread_id) => { invalid_request(format!("thread not found: {thread_id}")) @@ -4250,6 +4332,34 @@ fn build_thread_from_snapshot( } } +fn paginate_background_terminals( + terminals: &[ThreadBackgroundTerminal], + cursor: Option, + limit: Option, +) -> Result<(Vec, Option), JSONRPCErrorError> { + let start = match cursor { + Some(cursor) => { + let cursor = cursor + .parse::() + .map_err(|err| invalid_request(format!("invalid cursor: {err}")))?; + terminals + .iter() + .position(|terminal| { + terminal + .process_id + .parse::() + .is_ok_and(|process_id| process_id > cursor) + }) + .unwrap_or(terminals.len()) + } + None => 0, + }; + let effective_limit = limit.unwrap_or(terminals.len() as u32).max(1) as usize; + let end = start.saturating_add(effective_limit).min(terminals.len()); + let next_cursor = (end < terminals.len()).then(|| terminals[end - 1].process_id.clone()); + Ok((terminals[start..end].to_vec(), next_cursor)) +} + fn build_thread_from_loaded_snapshot( thread_id: ThreadId, config_snapshot: &ThreadConfigSnapshot, diff --git a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs index 54c0f0343f..7952517c92 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs @@ -36,6 +36,65 @@ mod thread_list_cwd_filter_tests { } } +mod background_terminal_pagination_tests { + use super::super::paginate_background_terminals; + use codex_app_server_protocol::ThreadBackgroundTerminal; + use codex_utils_absolute_path::AbsolutePathBuf; + use pretty_assertions::assert_eq; + + fn terminal(process_id: &str) -> ThreadBackgroundTerminal { + let cwd = if cfg!(windows) { r"C:\tmp" } else { "/tmp" }; + + ThreadBackgroundTerminal { + item_id: format!("item-{process_id}"), + process_id: process_id.to_string(), + command: format!("command-{process_id}"), + cwd: AbsolutePathBuf::from_absolute_path(cwd).expect("absolute cwd"), + os_pid: None, + cpu_percent: None, + rss_kb: None, + } + } + + #[test] + fn paginates_with_process_id_cursor() { + let terminals = vec![ + terminal("1"), + terminal("2"), + terminal("3"), + terminal("4"), + terminal("5"), + ]; + + let (data, next_cursor) = + paginate_background_terminals(&terminals, /*cursor*/ None, Some(2)) + .expect("valid page"); + + assert_eq!(data, vec![terminal("1"), terminal("2")]); + assert_eq!(next_cursor, Some("2".to_string())); + let first_cursor = next_cursor; + + let terminals_without_anchor = vec![terminal("1"), terminal("3"), terminal("4")]; + let (data, next_cursor) = + paginate_background_terminals(&terminals_without_anchor, first_cursor.clone(), Some(2)) + .expect("valid page"); + + assert_eq!(data, vec![terminal("3"), terminal("4")]); + assert_eq!(next_cursor, None); + + let (data, next_cursor) = + paginate_background_terminals(&terminals, first_cursor, Some(2)).expect("valid page"); + + assert_eq!(data, vec![terminal("3"), terminal("4")]); + assert_eq!(next_cursor, Some("4".to_string())); + + assert!( + paginate_background_terminals(&terminals, Some("missing".to_string()), Some(1)) + .is_err() + ); + } +} + mod thread_processor_behavior_tests { async fn forked_from_id_from_rollout(path: &Path) -> Option { codex_core::read_session_meta_line(path) diff --git a/codex-rs/app-server/tests/common/auth_fixtures.rs b/codex-rs/app-server/tests/common/auth_fixtures.rs index cf78e788de..18cfd7ed68 100644 --- a/codex-rs/app-server/tests/common/auth_fixtures.rs +++ b/codex-rs/app-server/tests/common/auth_fixtures.rs @@ -165,6 +165,7 @@ pub fn write_chatgpt_auth( last_refresh, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth(codex_home, &auth, cli_auth_credentials_store_mode).context("write auth.json") diff --git a/codex-rs/app-server/tests/common/models_cache.rs b/codex-rs/app-server/tests/common/models_cache.rs index 8233b1b296..2a0c10a27c 100644 --- a/codex-rs/app-server/tests/common/models_cache.rs +++ b/codex-rs/app-server/tests/common/models_cache.rs @@ -47,6 +47,7 @@ fn preset_to_info(preset: &ModelPreset, priority: i32) -> ModelInfo { context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs index 646de6ad21..91c42e2fe4 100644 --- a/codex-rs/app-server/tests/common/test_app_server.rs +++ b/codex-rs/app-server/tests/common/test_app_server.rs @@ -79,6 +79,7 @@ use codex_app_server_protocol::SkillsExtraRootsSetParams; use codex_app_server_protocol::SkillsListParams; use codex_app_server_protocol::ThreadArchiveParams; use codex_app_server_protocol::ThreadCompactStartParams; +use codex_app_server_protocol::ThreadDeleteParams; use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadInjectItemsParams; use codex_app_server_protocol::ThreadListParams; @@ -456,6 +457,15 @@ impl TestAppServer { self.send_request("thread/archive", params).await } + /// Send a `thread/delete` JSON-RPC request. + pub async fn send_thread_delete_request( + &mut self, + params: ThreadDeleteParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/delete", params).await + } + /// Send a `thread/name/set` JSON-RPC request. pub async fn send_thread_set_name_request( &mut self, diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index ace43d2847..fa4c2042af 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -119,6 +119,7 @@ async fn list_apps_returns_empty_with_api_key_auth() -> Result<()> { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }, AuthCredentialsStoreMode::File, )?; diff --git a/codex-rs/app-server/tests/suite/v2/client_metadata.rs b/codex-rs/app-server/tests/suite/v2/client_metadata.rs index 873a1cc044..681e11ae34 100644 --- a/codex-rs/app-server/tests/suite/v2/client_metadata.rs +++ b/codex-rs/app-server/tests/suite/v2/client_metadata.rs @@ -112,6 +112,7 @@ async fn turn_start_forwards_client_metadata_to_responses_request_v2() -> Result assert_eq!(metadata["origin"].as_str(), Some("gaas")); assert_eq!(metadata["thread_source"].as_str(), Some("client-supplied")); assert_eq!(metadata["turn_id"].as_str(), Some(turn.id.as_str())); + assert!(metadata.get("installation_id").is_some()); assert!(metadata.get("session_id").is_some()); assert_eq!( metadata["window_id"].as_str(), diff --git a/codex-rs/app-server/tests/suite/v2/compaction.rs b/codex-rs/app-server/tests/suite/v2/compaction.rs index 690c6d8bff..08ddfa4c61 100644 --- a/codex-rs/app-server/tests/suite/v2/compaction.rs +++ b/codex-rs/app-server/tests/suite/v2/compaction.rs @@ -29,6 +29,7 @@ use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::UserInput as V2UserInput; use codex_config::types::AuthCredentialsStoreMode; +use codex_features::Feature; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use core_test_support::responses; @@ -150,7 +151,7 @@ async fn auto_compaction_remote_emits_started_and_completed_items() -> Result<() write_mock_responses_config_toml( codex_home.path(), &server.uri(), - &BTreeMap::default(), + &BTreeMap::from([(Feature::RemoteCompactionV2, false)]), REMOTE_AUTO_COMPACT_LIMIT, Some(true), "mock_provider", diff --git a/codex-rs/app-server/tests/suite/v2/mcp_resource.rs b/codex-rs/app-server/tests/suite/v2/mcp_resource.rs index 2e3ffcb066..9e91e3e872 100644 --- a/codex-rs/app-server/tests/suite/v2/mcp_resource.rs +++ b/codex-rs/app-server/tests/suite/v2/mcp_resource.rs @@ -19,6 +19,8 @@ use codex_app_server_protocol::McpResourceReadResponse; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::UserInput; use codex_arg0::Arg0DispatchPaths; use codex_config::CloudConfigBundleLoader; use codex_config::LoaderOverrides; @@ -30,9 +32,14 @@ use codex_protocol::protocol::SessionSource; use core_test_support::responses; use pretty_assertions::assert_eq; use rmcp::handler::server::ServerHandler; +use rmcp::model::ListResourcesResult; +use rmcp::model::Meta; +use rmcp::model::PaginatedRequestParams; use rmcp::model::ProtocolVersion; +use rmcp::model::RawResource; use rmcp::model::ReadResourceRequestParams; use rmcp::model::ReadResourceResult; +use rmcp::model::Resource; use rmcp::model::ResourceContents; use rmcp::model::ServerCapabilities; use rmcp::model::ServerInfo; @@ -41,6 +48,7 @@ use rmcp::service::RoleServer; use rmcp::transport::StreamableHttpServerConfig; use rmcp::transport::StreamableHttpService; use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; +use serde_json::json; use tempfile::TempDir; use tokio::net::TcpListener; use tokio::task::JoinHandle; @@ -51,49 +59,34 @@ const TEST_RESOURCE_URI: &str = "test://codex/resource"; const TEST_BLOB_RESOURCE_URI: &str = "test://codex/resource.bin"; const TEST_RESOURCE_BLOB: &str = "YmluYXJ5LXJlc291cmNl"; const TEST_RESOURCE_TEXT: &str = "Resource body from the MCP server."; +const SKILL_NAME: &str = "demo-plugin:deploy"; +const RAW_SKILL_DESCRIPTION: &str = "Deploy\nthrough the orchestrator."; +const SKILL_DESCRIPTION: &str = "Deploy through the <hosted> orchestrator."; +const SKILL_RESOURCE_URI: &str = "skill://plugin_demo/deploy"; +const SKILL_MAIN_PROMPT_URI: &str = "skill://plugin_demo/deploy/SKILL.md"; +const SKILL_REFERENCE_URI: &str = "skill://plugin_demo/deploy/references/deploy.md"; +const SKILL_MARKER: &str = "ORCHESTRATOR_SKILL_BODY_MARKER"; +const SKILL_CONTENTS: &str = concat!( + "---\n", + "name: deploy\n", + "description: Deploy through the orchestrator.\n", + "---\n\n", + "# Deploy\n\n", + "ORCHESTRATOR_SKILL_BODY_MARKER\n\n", + "Read the [deployment reference](skill://plugin_demo/deploy/references/deploy.md).\n", +); +const SKILL_REFERENCE_CONTENTS: &str = + "# Deploy reference\n\nUse the orchestrator deployment API.\n"; +const SKILLS_LIST_CALL_ID: &str = "skills-list"; +const SKILLS_READ_CALL_ID: &str = "skills-read"; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn mcp_resource_read_returns_resource_contents() -> Result<()> { let responses_server = responses::start_mock_server().await; let (apps_server_url, apps_server_handle) = start_resource_apps_mcp_server().await?; - - let codex_home = TempDir::new()?; let responses_server_uri = responses_server.uri(); - std::fs::write( - codex_home.path().join("config.toml"), - format!( - r#" -model = "mock-model" -approval_policy = "untrusted" -sandbox_mode = "read-only" - -model_provider = "mock_provider" -chatgpt_base_url = "{apps_server_url}" -mcp_oauth_credentials_store = "file" - -[features] -apps = true - -[model_providers.mock_provider] -name = "Mock provider for test" -base_url = "{responses_server_uri}/v1" -wire_api = "responses" -request_max_retries = 0 -stream_max_retries = 0 -"# - ), - )?; - write_chatgpt_auth( - codex_home.path(), - ChatGptAuthFixture::new("chatgpt-token") - .account_id("account-123") - .chatgpt_user_id("user-123") - .chatgpt_account_id("account-123"), - AuthCredentialsStoreMode::File, - )?; - - let mut mcp = TestAppServer::new(codex_home.path()).await?; - timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + let (_codex_home, mut mcp) = + start_resource_test_app_server(&apps_server_url, &responses_server_uri).await?; let thread_start_id = mcp .send_thread_start_request(ThreadStartParams { @@ -120,7 +113,6 @@ stream_max_retries = 0 mcp.read_stream_until_response_message(RequestId::Integer(read_request_id)), ) .await??; - assert_eq!( to_response::(read_response)?, expected_resource_read_response() @@ -131,6 +123,165 @@ stream_max_retries = 0 Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "orchestrator skills are disabled until /ps/mcp serves complete skill packages"] +async fn orchestrator_skill_can_read_referenced_resource_without_an_executor() -> Result<()> { + let responses_server = responses::start_mock_server().await; + let (apps_server_url, apps_server_handle) = start_resource_apps_mcp_server().await?; + let responses_server_uri = responses_server.uri(); + let (_codex_home, mut mcp) = + start_resource_test_app_server(&apps_server_url, &responses_server_uri).await?; + + let thread_start_id = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + environments: Some(Vec::new()), + ..Default::default() + }) + .await?; + let thread_start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response(thread_start_resp)?; + + let response_mock = responses::mount_sse_sequence( + &responses_server, + vec![ + responses::sse(vec![ + responses::ev_response_created("resp-skills-list"), + responses::ev_function_call_with_namespace( + SKILLS_LIST_CALL_ID, + "skills", + "list", + &json!({ + "authority": { + "kind": "orchestrator", + }, + }) + .to_string(), + ), + responses::ev_completed("resp-skills-list"), + ]), + responses::sse(vec![ + responses::ev_response_created("resp-skills-read"), + responses::ev_function_call_with_namespace( + SKILLS_READ_CALL_ID, + "skills", + "read", + &json!({ + "authority": { + "kind": "orchestrator", + }, + "package": SKILL_RESOURCE_URI, + "resource": SKILL_REFERENCE_URI, + }) + .to_string(), + ), + responses::ev_completed("resp-skills-read"), + ]), + responses::sse(vec![ + responses::ev_response_created("resp-orchestrator-skill"), + responses::ev_assistant_message("msg-orchestrator-skill", "Done"), + responses::ev_completed("resp-orchestrator-skill"), + ]), + ], + ) + .await; + let turn_start_id = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![UserInput::Text { + text: format!("Use ${SKILL_NAME}"), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_start_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let requests = response_mock.requests(); + assert_eq!(requests.len(), 3); + let first_request = &requests[0]; + assert!(first_request.tool_by_name("skills", "list").is_some()); + assert!(first_request.tool_by_name("skills", "read").is_some()); + assert!(first_request.tool_by_name("skills", "search").is_none()); + + let developer_messages = first_request.message_input_texts("developer"); + let catalog_line = format!( + "- {SKILL_NAME}: {SKILL_DESCRIPTION} (orchestrator resource: {SKILL_RESOURCE_URI})" + ); + assert_eq!( + 1, + developer_messages + .iter() + .filter(|text| text.contains(&catalog_line)) + .count() + ); + assert!( + developer_messages + .iter() + .all(|text| !text.contains("ignored-plugin:ignored")) + ); + assert!( + developer_messages + .iter() + .any(|text| text.contains("do not treat `skill://` identifiers as filesystem paths")) + ); + let skill_fragments = first_request + .message_input_texts("user") + .into_iter() + .filter(|text| text.starts_with("")) + .collect::>(); + assert_eq!(1, skill_fragments.len()); + assert!(skill_fragments[0].contains(&format!("{SKILL_NAME}"))); + assert!(skill_fragments[0].contains(SKILL_MARKER)); + assert!(skill_fragments[0].contains(SKILL_REFERENCE_URI)); + + let list_output = requests[1] + .function_call_output_text(SKILLS_LIST_CALL_ID) + .ok_or_else(|| anyhow::anyhow!("skills.list output should be sent to the model"))?; + assert_eq!( + serde_json::from_str::(&list_output)?, + json!({ + "skills": [{ + "authority": { + "kind": "orchestrator", + }, + "package": SKILL_RESOURCE_URI, + "name": SKILL_NAME, + "description": SKILL_DESCRIPTION, + "main_resource": SKILL_MAIN_PROMPT_URI, + }], + "warnings": ["Orchestrator skill discovery stopped after 2 resource pages: failed to list orchestrator skill resources: resources/list failed for `codex_apps`: Mcp error: -32603: simulated later-page failure"], + }) + ); + + let read_output = requests[2] + .function_call_output_text(SKILLS_READ_CALL_ID) + .ok_or_else(|| anyhow::anyhow!("skills.read output should be sent to the model"))?; + assert_eq!( + serde_json::from_str::(&read_output)?, + json!({ + "resource": SKILL_REFERENCE_URI, + "contents": SKILL_REFERENCE_CONTENTS, + }) + ); + apps_server_handle.abort(); + let _ = apps_server_handle.await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn mcp_resource_read_returns_resource_contents_without_thread() -> Result<()> { let (apps_server_url, apps_server_handle) = start_resource_apps_mcp_server().await?; @@ -246,6 +397,52 @@ async fn mcp_resource_read_returns_error_for_unknown_thread() -> Result<()> { Ok(()) } +async fn start_resource_test_app_server( + apps_server_url: &str, + responses_server_uri: &str, +) -> Result<(TempDir, TestAppServer)> { + let codex_home = TempDir::new()?; + std::fs::write( + codex_home.path().join("config.toml"), + format!( + r#" +model = "mock-model" +approval_policy = "untrusted" +sandbox_mode = "read-only" + +model_provider = "mock_provider" +chatgpt_base_url = "{apps_server_url}" +mcp_oauth_credentials_store = "file" + +[features] +apps = true + +[skills] +include_instructions = true + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "{responses_server_uri}/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +"# + ), + )?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("chatgpt-token") + .account_id("account-123") + .chatgpt_user_id("user-123") + .chatgpt_account_id("account-123"), + AuthCredentialsStoreMode::File, + )?; + + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + Ok((codex_home, mcp)) +} + async fn start_resource_apps_mcp_server() -> Result<(String, JoinHandle<()>)> { let listener = TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; @@ -292,12 +489,79 @@ impl ServerHandler for ResourceAppsMcpServer { .with_protocol_version(ProtocolVersion::V_2025_06_18) } + async fn list_resources( + &self, + request: Option, + _context: RequestContext, + ) -> Result { + let cursor = request.and_then(|request| request.cursor); + if cursor.is_none() { + return Ok(ListResourcesResult { + resources: vec![skill_resource( + "skill://plugin_ignored/ignored", + "plugin_ignored/ignored", + "Not an MCP skill resource.", + "text/plain", + "ignored-plugin", + "ignored", + )], + next_cursor: Some("skills-page".to_string()), + meta: None, + }); + } + if cursor.as_deref() == Some("failing-page") { + return Err(rmcp::ErrorData::internal_error( + "simulated later-page failure", + /*data*/ None, + )); + } + if cursor.as_deref() != Some("skills-page") { + return Err(rmcp::ErrorData::invalid_params( + "unexpected resources/list cursor", + /*data*/ None, + )); + } + + Ok(ListResourcesResult { + resources: vec![skill_resource( + SKILL_RESOURCE_URI, + "plugin_demo/deploy", + RAW_SKILL_DESCRIPTION, + "mcp/skill", + "demo-plugin", + "deploy", + )], + next_cursor: Some("failing-page".to_string()), + meta: None, + }) + } + async fn read_resource( &self, request: ReadResourceRequestParams, _context: RequestContext, ) -> Result { let uri = request.uri; + if uri == SKILL_MAIN_PROMPT_URI { + return Ok(ReadResourceResult::new(vec![ + ResourceContents::TextResourceContents { + uri: SKILL_MAIN_PROMPT_URI.to_string(), + mime_type: Some("text/markdown".to_string()), + text: SKILL_CONTENTS.to_string(), + meta: None, + }, + ])); + } + if uri == SKILL_REFERENCE_URI { + return Ok(ReadResourceResult::new(vec![ + ResourceContents::TextResourceContents { + uri: SKILL_REFERENCE_URI.to_string(), + mime_type: Some("text/markdown".to_string()), + text: SKILL_REFERENCE_CONTENTS.to_string(), + meta: None, + }, + ])); + } if uri != TEST_RESOURCE_URI { return Err(rmcp::ErrorData::resource_not_found( format!("resource not found: {uri}"), @@ -321,3 +585,27 @@ impl ServerHandler for ResourceAppsMcpServer { ])) } } + +fn skill_resource( + uri: &str, + name: &str, + description: &str, + mime_type: &str, + plugin_name: &str, + skill_name: &str, +) -> Resource { + Resource::new( + RawResource::new(uri, name) + .with_description(description) + .with_mime_type(mime_type) + .with_meta(skill_resource_meta(plugin_name, skill_name)), + /*annotations*/ None, + ) +} + +fn skill_resource_meta(plugin_name: &str, skill_name: &str) -> Meta { + Meta(serde_json::Map::from_iter([ + ("plugin_name".to_string(), json!(plugin_name)), + ("skill_name".to_string(), json!(skill_name)), + ])) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 77d4b37589..e038a131e4 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -50,6 +50,7 @@ mod review; mod safety_check_downgrade; mod skills_list; mod thread_archive; +mod thread_delete; mod thread_fork; mod thread_inject_items; mod thread_list; diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs index ca490ad5e8..1be64b8fac 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs @@ -292,6 +292,14 @@ async fn plugin_install_writes_remote_plugin_to_cloud_and_cache() -> Result<()> async fn plugin_install_uses_remote_apps_needing_auth_response() -> Result<()> { let codex_home = TempDir::new()?; let server = MockServer::start().await; + let remote_app_manifest = json!({ + "apps": { + "alpha": { + "id": "alpha", + "category": "Developer Tools" + } + } + }); let bundle_url = mount_remote_plugin_bundle( &server, /*status_code*/ 200, @@ -299,7 +307,14 @@ async fn plugin_install_uses_remote_apps_needing_auth_response() -> Result<()> { ) .await; configure_remote_plugin_with_apps_test(codex_home.path(), &server)?; - mount_remote_plugin_detail(&server, REMOTE_PLUGIN_ID, "1.2.3", Some(&bundle_url)).await; + mount_remote_plugin_detail_with_app_manifest( + &server, + REMOTE_PLUGIN_ID, + "1.2.3", + Some(&bundle_url), + remote_app_manifest, + ) + .await; mount_empty_remote_installed_plugins(&server).await; mount_remote_plugin_install_with_apps_needing_auth(&server, REMOTE_PLUGIN_ID, &["alpha"]).await; @@ -327,7 +342,7 @@ async fn plugin_install_uses_remote_apps_needing_auth_response() -> Result<()> { name: "alpha".to_string(), description: None, install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), - needs_auth: true, + category: Some("Developer Tools".to_string()), }], } ); @@ -1001,7 +1016,7 @@ async fn plugin_install_returns_apps_needing_auth() -> Result<()> { name: "Alpha".to_string(), description: Some("Alpha connector".to_string()), install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), - needs_auth: true, + category: None, }], } ); @@ -1089,7 +1104,7 @@ async fn plugin_install_filters_disallowed_apps_needing_auth() -> Result<()> { name: "Alpha".to_string(), description: Some("Alpha connector".to_string()), install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), - needs_auth: true, + category: None, }], } ); diff --git a/codex-rs/app-server/tests/suite/v2/plugin_read.rs b/codex-rs/app-server/tests/suite/v2/plugin_read.rs index 4d8291a328..51bbd75bfb 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_read.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_read.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::sync::Arc; use std::sync::Mutex as StdMutex; use std::time::Duration; @@ -17,6 +16,7 @@ use axum::http::Uri; use axum::http::header::AUTHORIZATION; use axum::routing::get; use codex_app_server_protocol::AppInfo; +use codex_app_server_protocol::AppMetadata; use codex_app_server_protocol::AppTemplateSummary; use codex_app_server_protocol::AppTemplateUnavailableReason; use codex_app_server_protocol::HookEventName; @@ -37,17 +37,6 @@ use codex_app_server_protocol::RequestId; use codex_config::types::AuthCredentialsStoreMode; use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; -use rmcp::handler::server::ServerHandler; -use rmcp::model::JsonObject; -use rmcp::model::ListToolsResult; -use rmcp::model::Meta; -use rmcp::model::ServerCapabilities; -use rmcp::model::ServerInfo; -use rmcp::model::Tool; -use rmcp::model::ToolAnnotations; -use rmcp::transport::StreamableHttpServerConfig; -use rmcp::transport::StreamableHttpService; -use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; use serde_json::json; use tempfile::TempDir; use tokio::net::TcpListener; @@ -434,6 +423,7 @@ async fn plugin_read_reads_remote_plugin_details_when_remote_plugin_enabled() -> "template_id": "templated_apps_GitHubEnterprise", "name": "GitHub Enterprise", "description": "Connect GitHub Enterprise", + "category": "Developer Tools", "canonical_connector_id": "github_enterprise", "logo_url": "https://example.com/ghe-light.png", "logo_url_dark": "https://example.com/ghe-dark.png", @@ -595,6 +585,7 @@ async fn plugin_read_reads_remote_plugin_details_when_remote_plugin_enabled() -> template_id: "templated_apps_GitHubEnterprise".to_string(), name: "GitHub Enterprise".to_string(), description: Some("Connect GitHub Enterprise".to_string()), + category: Some("Developer Tools".to_string()), canonical_connector_id: Some("github_enterprise".to_string()), logo_url: Some("https://example.com/ghe-light.png".to_string()), logo_url_dark: Some("https://example.com/ghe-dark.png".to_string()), @@ -605,6 +596,7 @@ async fn plugin_read_reads_remote_plugin_details_when_remote_plugin_enabled() -> template_id: "templated_apps_Databricks".to_string(), name: "Databricks".to_string(), description: None, + category: None, canonical_connector_id: None, logo_url: None, logo_url_dark: None, @@ -1313,7 +1305,8 @@ description: Visible only for ChatGPT r#"{ "apps": { "gmail": { - "id": "gmail" + "id": "gmail", + "category": "Communication" } } }"#, @@ -1483,14 +1476,17 @@ enabled = false response.plugin.apps[0].install_url.as_deref(), Some("https://chatgpt.com/apps/gmail/gmail") ); - assert_eq!(response.plugin.apps[0].needs_auth, true); + assert_eq!( + response.plugin.apps[0].category.as_deref(), + Some("Communication") + ); assert_eq!(response.plugin.mcp_servers.len(), 1); assert_eq!(response.plugin.mcp_servers[0], "demo"); Ok(()) } #[tokio::test] -async fn plugin_read_returns_app_needs_auth() -> Result<()> { +async fn plugin_read_returns_app_metadata_category() -> Result<()> { let connectors = vec![ AppInfo { id: "alpha".to_string(), @@ -1500,7 +1496,20 @@ async fn plugin_read_returns_app_needs_auth() -> Result<()> { logo_url_dark: None, distribution_channel: Some("featured".to_string()), branding: None, - app_metadata: None, + app_metadata: Some(AppMetadata { + review: None, + categories: Some(vec!["Productivity".to_string()]), + sub_categories: None, + seo_description: None, + screenshots: None, + developer: None, + version: None, + version_id: None, + version_notes: None, + first_party_type: None, + first_party_requires_install: None, + show_in_composer_when_unlinked: None, + }), labels: None, install_url: None, is_accessible: false, @@ -1523,8 +1532,7 @@ async fn plugin_read_returns_app_needs_auth() -> Result<()> { plugin_display_names: Vec::new(), }, ]; - let tools = vec![connector_tool("beta", "Beta App")?]; - let (server_url, server_handle) = start_apps_server(connectors, tools).await?; + let (server_url, server_handle) = start_apps_server(connectors).await?; let codex_home = TempDir::new()?; write_connectors_config(codex_home.path(), &server_url)?; @@ -1571,9 +1579,9 @@ async fn plugin_read_returns_app_needs_auth() -> Result<()> { .plugin .apps .iter() - .map(|app| (app.id.as_str(), app.needs_auth)) + .map(|app| (app.id.as_str(), app.category.as_deref())) .collect::>(), - vec![("alpha", true), ("beta", false)] + vec![("alpha", Some("Productivity")), ("beta", None)] ); server_handle.abort(); @@ -1851,70 +1859,22 @@ struct AppsServerState { response: Arc>, } -#[derive(Clone)] -struct PluginReadMcpServer { - tools: Arc>>, -} - -impl ServerHandler for PluginReadMcpServer { - fn get_info(&self) -> ServerInfo { - ServerInfo::new(ServerCapabilities::builder().enable_tools().build()) - } - - fn list_tools( - &self, - _request: Option, - _context: rmcp::service::RequestContext, - ) -> impl std::future::Future> + Send + '_ - { - let tools = self.tools.clone(); - async move { - let tools = tools - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clone(); - Ok(ListToolsResult { - tools, - next_cursor: None, - meta: None, - }) - } - } -} - -async fn start_apps_server( - connectors: Vec, - tools: Vec, -) -> Result<(String, JoinHandle<()>)> { +async fn start_apps_server(connectors: Vec) -> Result<(String, JoinHandle<()>)> { let state = Arc::new(AppsServerState { response: Arc::new(StdMutex::new( json!({ "apps": connectors, "next_token": null }), )), }); - let tools = Arc::new(StdMutex::new(tools)); let listener = TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let mcp_service = StreamableHttpService::new( - { - let tools = tools.clone(); - move || { - Ok(PluginReadMcpServer { - tools: tools.clone(), - }) - } - }, - Arc::new(LocalSessionManager::default()), - StreamableHttpServerConfig::default(), - ); let router = Router::new() .route("/connectors/directory/list", get(list_directory_connectors)) .route( "/connectors/directory/list_workspace", get(list_directory_connectors), ) - .with_state(state) - .nest_service("/api/codex/ps/mcp", mcp_service); + .with_state(state); let handle = tokio::spawn(async move { let _ = axum::serve(listener, router).await; @@ -1954,27 +1914,6 @@ async fn list_directory_connectors( } } -fn connector_tool(connector_id: &str, connector_name: &str) -> Result { - let schema: JsonObject = serde_json::from_value(json!({ - "type": "object", - "additionalProperties": false - }))?; - let mut tool = Tool::new( - Cow::Owned(format!("connector_{connector_id}")), - Cow::Borrowed("Connector test tool"), - Arc::new(schema), - ); - tool.annotations = Some(ToolAnnotations::new().read_only(true)); - - let mut meta = Meta::new(); - meta.0 - .insert("connector_id".to_string(), json!(connector_id)); - meta.0 - .insert("connector_name".to_string(), json!(connector_name)); - tool.meta = Some(meta); - Ok(tool) -} - fn write_connectors_config(codex_home: &std::path::Path, base_url: &str) -> std::io::Result<()> { std::fs::write( codex_home.join("config.toml"), diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index bd03af8e02..ddc0eae404 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -36,7 +36,10 @@ use codex_app_server_protocol::ThreadRealtimeTranscriptDoneNotification; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnStartedNotification; +use codex_app_server_protocol::UserInput as V2UserInput; use codex_features::FEATURES; use codex_features::Feature; use codex_protocol::protocol::RealtimeConversationVersion; @@ -1342,6 +1345,136 @@ async fn webrtc_v1_handoff_request_delegates_and_appends_result() -> Result<()> Ok(()) } +#[tokio::test] +async fn webrtc_assistant_output_without_handoff_reaches_realtime() -> Result<()> { + skip_if_no_network!(Ok(())); + + let final_answer = "long output ".repeat(1_000); + for (version, expected_version, preamble) in [ + ( + RealtimeTestVersion::V1, + RealtimeConversationVersion::V1, + "direct preamble from v1", + ), + ( + RealtimeTestVersion::V2, + RealtimeConversationVersion::V2, + "direct preamble from v2", + ), + ] { + let mut harness = RealtimeE2eHarness::new( + version, + main_loop_responses(vec![responses::sse(vec![ + responses::ev_response_created("resp-1"), + json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "id": "msg-preamble", + "phase": "commentary", + "content": [{"type": "output_text", "text": preamble}] + } + }), + responses::ev_assistant_message("msg-final", &final_answer), + responses::ev_completed("resp-1"), + ])]), + realtime_sideband(vec![realtime_sideband_connection(vec![ + vec![session_updated("sess_standalone_output")], + vec![], + match version { + RealtimeTestVersion::V1 => vec![], + RealtimeTestVersion::V2 => vec![ + json!({ + "type": "response.created", + "response": { "id": "resp_preamble" } + }), + json!({ + "type": "response.done", + "response": { "id": "resp_preamble" } + }), + ], + }, + vec![], + vec![], + ])]), + ) + .await?; + + let started = harness.start_webrtc_realtime("v=offer\r\n").await?; + assert_eq!(started.started.version, expected_version); + + let request_id = harness + .mcp + .send_turn_start_request(TurnStartParams { + thread_id: harness.thread_id.clone(), + input: vec![V2UserInput::Text { + text: "direct text turn".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + harness + .mcp + .read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let _: TurnStartResponse = to_response(response)?; + let _ = harness + .read_notification::("turn/completed") + .await?; + + let preamble_request = harness.sideband_outbound_request(/*request_index*/ 1).await; + let output_text = match version { + RealtimeTestVersion::V1 => { + let final_request = harness.sideband_outbound_request(/*request_index*/ 2).await; + assert_eq!( + preamble_request, + json!({ + "type": "conversation.handoff.append", + "handoff_id": "codex", + "output_text": preamble, + }) + ); + assert_eq!(final_request["type"], "conversation.handoff.append"); + assert_eq!(final_request["handoff_id"], "codex"); + final_request["output_text"] + .as_str() + .expect("output text") + .to_string() + } + RealtimeTestVersion::V2 => { + assert_v2_progress_update(&preamble_request, preamble); + assert_v2_response_create( + &harness.sideband_outbound_request(/*request_index*/ 2).await, + ); + let final_request = harness.sideband_outbound_request(/*request_index*/ 3).await; + assert_eq!(final_request["type"], "conversation.item.create"); + assert_eq!(final_request["item"]["type"], "message"); + assert_eq!(final_request["item"]["role"], "user"); + assert_eq!(final_request["item"]["content"][0]["type"], "input_text"); + let output_text = final_request["item"]["content"][0]["text"] + .as_str() + .expect("output text"); + assert!(output_text.starts_with("[BACKEND] ")); + assert_v2_response_create( + &harness.sideband_outbound_request(/*request_index*/ 4).await, + ); + output_text.to_string() + } + }; + assert!(output_text.contains("tokens truncated")); + assert!(output_text.len() <= 4_000); + + harness.shutdown().await; + } + + Ok(()) +} + #[tokio::test] async fn webrtc_v2_forwards_audio_and_text_between_client_and_sideband() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs b/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs index ca2121dd4c..18d1dcd43f 100644 --- a/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs +++ b/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use anyhow::Result; use app_test_support::create_mock_responses_server_repeating_assistant; use codex_app_server::in_process; +use codex_app_server::in_process::InProcessClientHandle; use codex_app_server::in_process::InProcessServerEvent; use codex_app_server::in_process::InProcessStartArgs; use codex_app_server_protocol::ClientInfo; @@ -27,6 +28,8 @@ use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::ThreadDeleteParams; +use codex_app_server_protocol::ThreadDeleteResponse; use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadListResponse; use codex_app_server_protocol::ThreadResumeParams; @@ -42,8 +45,14 @@ use codex_core::config::Config; use codex_core::config::ConfigBuilder; use codex_exec_server::EnvironmentManager; use codex_feedback::CodexFeedback; +use codex_protocol::ThreadId; +use codex_protocol::models::BaseInstructions; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadMemoryMode; +use codex_thread_store::CreateThreadParams as StoreCreateThreadParams; use codex_thread_store::InMemoryThreadStore; +use codex_thread_store::ThreadPersistenceMetadata; +use codex_thread_store::ThreadStore; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::timeout; @@ -52,7 +61,7 @@ use uuid::Uuid; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] -async fn thread_start_with_non_local_thread_store_does_not_create_local_persistence() -> Result<()> +async fn thread_delete_with_non_local_thread_store_does_not_create_local_persistence() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; let codex_home = TempDir::new()?; @@ -61,43 +70,10 @@ async fn thread_start_with_non_local_thread_store_does_not_create_local_persiste // here so this regression stays focused on thread persistence artifacts. create_config_toml_with_thread_store(codex_home.path(), &server.uri(), &store_id)?; - let loader_overrides = LoaderOverrides::without_managed_config_for_tests(); - let config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .fallback_cwd(Some(codex_home.path().to_path_buf())) - .loader_overrides(loader_overrides.clone()) - .build() - .await?; - let thread_store = InMemoryThreadStore::for_id(store_id.clone()); let _in_memory_store = InMemoryThreadStoreId { store_id }; - let mut client = in_process::start(InProcessStartArgs { - arg0_paths: Arg0DispatchPaths::default(), - config: Arc::new(config), - cli_overrides: Vec::new(), - loader_overrides, - strict_config: false, - cloud_config_bundle: CloudConfigBundleLoader::default(), - thread_config_loader: Arc::new(NoopThreadConfigLoader), - feedback: CodexFeedback::new(), - log_db: None, - state_db: None, - environment_manager: Arc::new(EnvironmentManager::default_for_tests()), - config_warnings: Vec::new(), - session_source: SessionSource::Cli, - enable_codex_api_key_env: false, - initialize: InitializeParams { - client_info: ClientInfo { - name: "codex-app-server-tests".to_string(), - title: None, - version: "0.1.0".to_string(), - }, - capabilities: None, - }, - channel_capacity: in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY, - }) - .await?; + let mut client = start_in_process_server(codex_home.path()).await?; let response = client .request(ClientRequest::ThreadStart { @@ -166,11 +142,39 @@ async fn thread_start_with_non_local_thread_store_does_not_create_local_persiste assert_eq!(data[0].id, thread.id); assert_eq!(data[0].path, None); + delete_thread(&client, /*request_id*/ 4, thread.id.clone()).await?; + let unloaded_thread_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; + thread_store + .create_thread(StoreCreateThreadParams { + thread_id: unloaded_thread_id, + extra_config: None, + forked_from_id: None, + parent_thread_id: None, + source: SessionSource::Cli, + thread_source: None, + base_instructions: BaseInstructions::default(), + dynamic_tools: Vec::new(), + multi_agent_version: None, + metadata: ThreadPersistenceMetadata { + cwd: Some(codex_home.path().to_path_buf()), + model_provider: "mock_provider".to_string(), + memory_mode: ThreadMemoryMode::Enabled, + }, + }) + .await?; + delete_thread( + &client, + /*request_id*/ 5, + unloaded_thread_id.to_string(), + ) + .await?; + client.shutdown().await?; let calls = thread_store.calls().await; - assert_eq!(calls.create_thread, 1); + assert_eq!(calls.create_thread, 2); assert_eq!(calls.list_threads, 1); + assert_eq!(calls.delete_thread, 2); assert!( calls.append_items > 0, "turn/start should append rollout items through the injected store" @@ -269,10 +273,24 @@ async fn cold_thread_resume_reuses_non_local_history_probe() -> Result<()> { Ok(()) } +async fn start_in_process_server(codex_home: &Path) -> Result { + let loader_overrides = LoaderOverrides::without_managed_config_for_tests(); + let config = Arc::new( + ConfigBuilder::default() + .codex_home(codex_home.to_path_buf()) + .fallback_cwd(Some(codex_home.to_path_buf())) + .loader_overrides(loader_overrides.clone()) + .build() + .await?, + ); + + Ok(start_in_process_client(config, loader_overrides).await?) +} + async fn start_in_process_client( config: Arc, loader_overrides: LoaderOverrides, -) -> std::io::Result { +) -> std::io::Result { in_process::start(InProcessStartArgs { arg0_paths: Arg0DispatchPaths::default(), config, @@ -301,6 +319,22 @@ async fn start_in_process_client( .await } +async fn delete_thread( + client: &InProcessClientHandle, + request_id: i64, + thread_id: String, +) -> Result<()> { + let response = client + .request(ClientRequest::ThreadDelete { + request_id: RequestId::Integer(request_id), + params: ThreadDeleteParams { thread_id }, + }) + .await? + .map_err(|error| anyhow::anyhow!("thread/delete failed: {}", error.message))?; + let _: ThreadDeleteResponse = serde_json::from_value(response)?; + Ok(()) +} + fn assert_no_local_persistence_artifacts(codex_home: &Path) -> Result<()> { // These are the observable tripwires for accidental local persistence. If a // future code path constructs a local rollout/session store or opens the diff --git a/codex-rs/app-server/tests/suite/v2/thread_delete.rs b/codex-rs/app-server/tests/suite/v2/thread_delete.rs new file mode 100644 index 0000000000..bef155f9ad --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/thread_delete.rs @@ -0,0 +1,200 @@ +use anyhow::Result; +use app_test_support::TestAppServer; +use app_test_support::create_fake_rollout; +use app_test_support::to_response; +use codex_app_server_protocol::JSONRPCError; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ThreadDeleteParams; +use codex_app_server_protocol::ThreadDeleteResponse; +use codex_app_server_protocol::ThreadDeletedNotification; +use codex_app_server_protocol::ThreadLoadedListParams; +use codex_app_server_protocol::ThreadLoadedListResponse; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_core::find_thread_path_by_id_str; +use codex_protocol::ThreadId; +use codex_state::DirectionalThreadSpawnEdgeStatus; +use codex_state::StateRuntime; +use pretty_assertions::assert_eq; +use std::path::Path; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +#[tokio::test] +async fn thread_delete_deletes_spawned_descendants() -> Result<()> { + let codex_home = TempDir::new()?; + + let parent_id = create_delete_test_rollout(codex_home.path(), /*minute*/ 0, "parent")?; + let child_id = create_delete_test_rollout(codex_home.path(), /*minute*/ 1, "child")?; + let grandchild_id = + create_delete_test_rollout(codex_home.path(), /*minute*/ 2, "grandchild")?; + + let state_db = + StateRuntime::init(codex_home.path().to_path_buf(), "mock_provider".into()).await?; + let parent_thread_id = ThreadId::from_string(&parent_id)?; + let child_thread_id = ThreadId::from_string(&child_id)?; + let grandchild_thread_id = ThreadId::from_string(&grandchild_id)?; + + for (parent, child, status) in [ + ( + parent_thread_id, + child_thread_id, + DirectionalThreadSpawnEdgeStatus::Closed, + ), + ( + child_thread_id, + grandchild_thread_id, + DirectionalThreadSpawnEdgeStatus::Open, + ), + ] { + state_db + .upsert_thread_spawn_edge(parent, child, status) + .await?; + } + + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let delete_id = mcp + .send_thread_delete_request(ThreadDeleteParams { + thread_id: parent_id.clone(), + }) + .await?; + let delete_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(delete_id)), + ) + .await??; + let _: ThreadDeleteResponse = to_response::(delete_resp)?; + + let mut deleted_ids = Vec::new(); + for _ in 0..3 { + let notification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("thread/deleted"), + ) + .await??; + let deleted_notification: ThreadDeletedNotification = serde_json::from_value( + notification + .params + .expect("thread/deleted notification params"), + )?; + deleted_ids.push(deleted_notification.thread_id); + } + assert_eq!(deleted_ids, vec![grandchild_id, child_id, parent_id]); + + for thread_id in [parent_thread_id, child_thread_id, grandchild_thread_id] { + let rollout_path = find_thread_path_by_id_str( + codex_home.path(), + &thread_id.to_string(), + /*state_db_ctx*/ None, + ) + .await?; + assert!( + rollout_path.is_none(), + "expected active rollout for {thread_id} to be deleted" + ); + } + assert_eq!( + state_db + .list_thread_spawn_descendants(parent_thread_id) + .await?, + Vec::::new() + ); + Ok(()) +} + +fn create_delete_test_rollout(codex_home: &Path, minute: u8, preview: &str) -> Result { + create_fake_rollout( + codex_home, + &format!("2025-01-01T00-{minute:02}-00"), + &format!("2025-01-01T00:{minute:02}:00Z"), + preview, + Some("mock_provider"), + /*git_info*/ None, + ) +} + +#[tokio::test] +async fn thread_delete_handles_live_threads_before_rollout_exists() -> Result<()> { + let codex_home = TempDir::new()?; + + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let start_id = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let persisted_thread = to_response::(start_resp)?.thread; + let rollout_path = find_thread_path_by_id_str( + codex_home.path(), + &persisted_thread.id, + /*state_db_ctx*/ None, + ) + .await?; + assert_eq!(rollout_path, None); + + let delete_id = mcp + .send_thread_delete_request(ThreadDeleteParams { + thread_id: persisted_thread.id, + }) + .await?; + let delete_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(delete_id)), + ) + .await??; + let _: ThreadDeleteResponse = to_response::(delete_resp)?; + + let start_id = mcp + .send_thread_start_request(ThreadStartParams { + ephemeral: Some(true), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let delete_id = mcp + .send_thread_delete_request(ThreadDeleteParams { + thread_id: thread.id.clone(), + }) + .await?; + let delete_err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(delete_id)), + ) + .await??; + let expected_message = format!( + "thread is not persisted and cannot be deleted: {}", + thread.id + ); + assert_eq!(delete_err.error.message, expected_message); + + let list_id = mcp + .send_thread_loaded_list_request(ThreadLoadedListParams::default()) + .await?; + let list_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(list_id)), + ) + .await??; + let ThreadLoadedListResponse { mut data, .. } = + to_response::(list_resp)?; + data.sort(); + assert_eq!(data, vec![thread.id]); + + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/thread_start.rs b/codex-rs/app-server/tests/suite/v2/thread_start.rs index 6366c6fbca..82a2b6b2c0 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_start.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_start.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use anyhow::Result; use app_test_support::ChatGptAuthFixture; use app_test_support::PathBufExt; @@ -21,6 +22,8 @@ use codex_app_server_protocol::ThreadStartedNotification; use codex_app_server_protocol::ThreadStatus; use codex_app_server_protocol::ThreadStatusChangedNotification; use codex_app_server_protocol::TurnEnvironmentParams; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::UserInput as V2UserInput; use codex_config::loader::project_trust_key; use codex_config::types::AuthCredentialsStoreMode; use codex_core::config::set_project_trust_level; @@ -400,11 +403,13 @@ async fn thread_start_response_excludes_empty_project_instruction_source() -> Re } #[tokio::test] -async fn thread_start_without_selected_environment_excludes_instruction_sources() -> Result<()> { +async fn thread_start_without_selected_environment_includes_only_global_instruction_source() +-> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; let codex_home = TempDir::new()?; create_config_toml_without_approval_policy(codex_home.path(), &server.uri())?; - std::fs::write(codex_home.path().join("AGENTS.md"), "global instructions")?; + let global_agents_path = codex_home.path().join("AGENTS.md"); + std::fs::write(&global_agents_path, "global instructions")?; let workspace = TempDir::new()?; std::fs::write(workspace.path().join("AGENTS.md"), "project instructions")?; @@ -424,11 +429,56 @@ async fn thread_start_without_selected_environment_excludes_instruction_sources( ) .await??; let ThreadStartResponse { + thread, instruction_sources, .. } = to_response::(response)?; - assert!(instruction_sources.is_empty()); + assert_eq!( + instruction_sources + .into_iter() + .map(normalize_path_for_comparison) + .collect::>(), + vec![normalize_path_for_comparison(std::fs::canonicalize( + global_agents_path, + )?)] + ); + + let turn_request_id = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "inspect instructions".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_request_id)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let requests = server + .received_requests() + .await + .context("failed to fetch received requests")?; + let model_request = requests + .iter() + .find(|request| request.url.path().ends_with("/responses")) + .context("expected model request")?; + let model_request_body = model_request + .body_json::() + .context("model request body should be JSON")? + .to_string(); + assert!(model_request_body.contains("global instructions")); + assert!(!model_request_body.contains("project instructions")); Ok(()) } diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 823851b715..7fc0d8c71d 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -43,7 +43,12 @@ use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ServerRequestResolvedNotification; use codex_app_server_protocol::SubAgentActivityKind; use codex_app_server_protocol::TextElement; +use codex_app_server_protocol::ThreadDeleteParams; +use codex_app_server_protocol::ThreadDeleteResponse; +use codex_app_server_protocol::ThreadDeletedNotification; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadLoadedListParams; +use codex_app_server_protocol::ThreadLoadedListResponse; use codex_app_server_protocol::ThreadSource; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; @@ -3370,6 +3375,49 @@ async fn turn_start_emits_spawn_agent_item_with_model_metadata_v2() -> Result<() assert_eq!(turn_completed.thread_id, thread.id); assert_eq!(turn_completed.turn.id, turn.turn.id); + // Reuse this live spawn setup to cover thread/delete's ThreadManager descendant path. + let delete_req = mcp + .send_thread_delete_request(ThreadDeleteParams { + thread_id: thread.id.clone(), + }) + .await?; + let delete_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(delete_req)), + ) + .await??; + let _: ThreadDeleteResponse = to_response::(delete_resp)?; + + let mut deleted_thread_ids = Vec::new(); + for _ in 0..2 { + let deleted_notif = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("thread/deleted"), + ) + .await??; + let deleted: ThreadDeletedNotification = serde_json::from_value( + deleted_notif + .params + .expect("thread/deleted notification params"), + )?; + deleted_thread_ids.push(deleted.thread_id); + } + assert_eq!( + deleted_thread_ids, + vec![receiver_thread_id, thread.id.clone()] + ); + + let list_req = mcp + .send_thread_loaded_list_request(ThreadLoadedListParams::default()) + .await?; + let list_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(list_req)), + ) + .await??; + let ThreadLoadedListResponse { data, .. } = to_response::(list_resp)?; + assert_eq!(data, Vec::::new()); + Ok(()) } diff --git a/codex-rs/apply-patch/Cargo.toml b/codex-rs/apply-patch/Cargo.toml index 2584338618..8e1d7f651f 100644 --- a/codex-rs/apply-patch/Cargo.toml +++ b/codex-rs/apply-patch/Cargo.toml @@ -20,6 +20,7 @@ workspace = true anyhow = { workspace = true } codex-exec-server = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-path-uri = { workspace = true } similar = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/codex-rs/apply-patch/src/invocation.rs b/codex-rs/apply-patch/src/invocation.rs index 1aa429e0d3..b66ee2291d 100644 --- a/codex-rs/apply-patch/src/invocation.rs +++ b/codex-rs/apply-patch/src/invocation.rs @@ -21,6 +21,7 @@ use crate::parser::Hunk; use crate::parser::ParseError; use crate::parser::parse_patch; use crate::unified_diff_from_chunks; +use codex_utils_path_uri::PathUri; use std::str::Utf8Error; use tree_sitter::LanguageError; @@ -185,7 +186,18 @@ pub async fn verify_apply_patch_args( ); } Hunk::DeleteFile { .. } => { - let content = match fs.read_file_text(&path, sandbox).await { + let path_uri = match PathUri::from_abs_path(&path) { + Ok(path_uri) => path_uri, + Err(e) => { + return MaybeApplyPatchVerified::CorrectnessError( + ApplyPatchError::IoError(IoError { + context: format!("Failed to read {}", path.display()), + source: e, + }), + ); + } + }; + let content = match fs.read_file_text(&path_uri, sandbox).await { Ok(content) => content, Err(e) => { return MaybeApplyPatchVerified::CorrectnessError( diff --git a/codex-rs/apply-patch/src/lib.rs b/codex-rs/apply-patch/src/lib.rs index 29d42e1c07..0800ed04b6 100644 --- a/codex-rs/apply-patch/src/lib.rs +++ b/codex-rs/apply-patch/src/lib.rs @@ -16,6 +16,7 @@ use codex_exec_server::ExecutorFileSystem; use codex_exec_server::FileSystemSandboxContext; use codex_exec_server::RemoveOptions; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; pub use parser::Hunk; pub use parser::ParseError; use parser::ParseError::*; @@ -390,6 +391,7 @@ async fn apply_hunks_to_files( for hunk in hunks { let affected_path = hunk.path().to_path_buf(); let path_abs = hunk.resolve_path(cwd); + let path_uri = PathUri::from_abs_path(&path_abs)?; match hunk { Hunk::AddFile { contents, .. } => { let overwritten_content = @@ -415,7 +417,7 @@ async fn apply_hunks_to_files( } Hunk::DeleteFile { .. } => { note_existing_path_delta_support(&path_abs, fs, sandbox, &mut delta.exact).await; - let deleted_content = fs.read_file_text(&path_abs, sandbox).await.ok(); + let deleted_content = fs.read_file_text(&path_uri, sandbox).await.ok(); if deleted_content.is_none() { delta.exact = false; } @@ -424,7 +426,7 @@ async fn apply_hunks_to_files( .with_context(|| format!("Failed to delete file {}", path_abs.display()))?; if let Err(error) = fs .remove( - &path_abs, + &path_uri, RemoveOptions { recursive: false, force: false, @@ -488,7 +490,7 @@ async fn apply_hunks_to_files( })?; if let Err(error) = fs .remove( - &path_abs, + &path_uri, RemoveOptions { recursive: false, force: false, @@ -521,7 +523,7 @@ async fn apply_hunks_to_files( modified.push(affected_path); } else { try_write!( - fs.write_file(&path_abs, new_contents.clone().into_bytes(), sandbox) + fs.write_file(&path_uri, new_contents.clone().into_bytes(), sandbox) .await .with_context(|| format!( "Failed to write file {}", @@ -554,7 +556,8 @@ async fn ensure_not_directory( fs: &dyn ExecutorFileSystem, sandbox: Option<&FileSystemSandboxContext>, ) -> io::Result<()> { - let metadata = fs.get_metadata(path, sandbox).await?; + let path_uri = PathUri::from_abs_path(path)?; + let metadata = fs.get_metadata(&path_uri, sandbox).await?; if metadata.is_directory { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -570,9 +573,12 @@ async fn remove_failure_was_side_effect_free( fs: &dyn ExecutorFileSystem, sandbox: Option<&FileSystemSandboxContext>, ) -> bool { + let Ok(path_uri) = PathUri::from_abs_path(path) else { + return false; + }; match expected_content { Some(expected_content) => fs - .read_file_text(path, sandbox) + .read_file_text(&path_uri, sandbox) .await .is_ok_and(|content| content == expected_content), None => false, @@ -586,7 +592,14 @@ async fn read_optional_file_text_for_delta( exact: &mut bool, ) -> Option { note_existing_path_delta_support(path, fs, sandbox, exact).await; - match fs.read_file_text(path, sandbox).await { + let path_uri = match PathUri::from_abs_path(path) { + Ok(path_uri) => path_uri, + Err(_) => { + *exact = false; + return None; + } + }; + match fs.read_file_text(&path_uri, sandbox).await { Ok(content) => Some(content), Err(source) if source.kind() == io::ErrorKind::NotFound => None, Err(_) => { @@ -602,7 +615,11 @@ async fn note_existing_path_delta_support( sandbox: Option<&FileSystemSandboxContext>, exact: &mut bool, ) { - match fs.get_metadata(path, sandbox).await { + let Ok(path_uri) = PathUri::from_abs_path(path) else { + *exact = false; + return; + }; + match fs.get_metadata(&path_uri, sandbox).await { Ok(metadata) if metadata.is_file && !metadata.is_symlink => {} Ok(_) => *exact = false, Err(source) if source.kind() == io::ErrorKind::NotFound => {} @@ -616,12 +633,14 @@ async fn write_file_with_missing_parent_retry( contents: Vec, sandbox: Option<&FileSystemSandboxContext>, ) -> anyhow::Result<()> { - match fs.write_file(path_abs, contents.clone(), sandbox).await { + let path_uri = PathUri::from_abs_path(path_abs)?; + match fs.write_file(&path_uri, contents.clone(), sandbox).await { Ok(()) => Ok(()), Err(err) if err.kind() == io::ErrorKind::NotFound => { if let Some(parent_abs) = path_abs.parent() { + let parent_uri = PathUri::from_abs_path(&parent_abs)?; fs.create_directory( - &parent_abs, + &parent_uri, CreateDirectoryOptions { recursive: true }, sandbox, ) @@ -633,7 +652,7 @@ async fn write_file_with_missing_parent_retry( ) })?; } - fs.write_file(path_abs, contents, sandbox) + fs.write_file(&path_uri, contents, sandbox) .await .with_context(|| format!("Failed to write file {}", path_abs.display()))?; Ok(()) @@ -657,7 +676,8 @@ async fn derive_new_contents_from_chunks( fs: &dyn ExecutorFileSystem, sandbox: Option<&FileSystemSandboxContext>, ) -> std::result::Result { - let original_contents = fs.read_file_text(path_abs, sandbox).await.map_err(|err| { + let path_uri = PathUri::from_abs_path(path_abs)?; + let original_contents = fs.read_file_text(&path_uri, sandbox).await.map_err(|err| { ApplyPatchError::IoError(IoError { context: format!("Failed to read file to update {}", path_abs.display()), source: err, diff --git a/codex-rs/cli/BUILD.bazel b/codex-rs/cli/BUILD.bazel index faa8627477..97c868845b 100644 --- a/codex-rs/cli/BUILD.bazel +++ b/codex-rs/cli/BUILD.bazel @@ -1,4 +1,5 @@ -load("//:defs.bzl", "MACOS_WEBRTC_RUSTC_LINK_FLAGS", "codex_rust_crate", "multiplatform_binaries") +load("//:defs.bzl", "MACOS_WEBRTC_RUSTC_LINK_FLAGS", "codex_rust_crate") +load("//bazel/platforms:release_binaries.bzl", "multiplatform_binaries") codex_rust_crate( name = "cli", diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index 6a4d78fbc7..727f367502 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -33,6 +33,7 @@ codex-utils-cli = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } codex-core-plugins = { workspace = true } +codex-home = { workspace = true } codex-exec = { workspace = true } codex-exec-server = { workspace = true } codex-execpolicy = { workspace = true } diff --git a/codex-rs/cli/src/doctor.rs b/codex-rs/cli/src/doctor.rs index d817f1a192..0b7e7cfe7b 100644 --- a/codex-rs/cli/src/doctor.rs +++ b/codex-rs/cli/src/doctor.rs @@ -1324,6 +1324,7 @@ fn stored_auth_mode(auth: &codex_login::AuthDotJson) -> &'static str { codex_app_server_protocol::AuthMode::ChatgptAuthTokens => "chatgpt_auth_tokens", codex_app_server_protocol::AuthMode::AgentIdentity => "agent_identity", codex_app_server_protocol::AuthMode::PersonalAccessToken => "personal_access_token", + codex_app_server_protocol::AuthMode::BedrockApiKey => "bedrock_api_key", } } @@ -1331,10 +1332,12 @@ fn stored_auth_mode_value(auth: &AuthDotJson) -> codex_app_server_protocol::Auth if let Some(mode) = auth.auth_mode { return mode; } - if auth.openai_api_key.is_some() { - codex_app_server_protocol::AuthMode::ApiKey - } else if auth.personal_access_token.is_some() { + if auth.personal_access_token.is_some() { codex_app_server_protocol::AuthMode::PersonalAccessToken + } else if auth.bedrock_api_key.is_some() { + codex_app_server_protocol::AuthMode::BedrockApiKey + } else if auth.openai_api_key.is_some() { + codex_app_server_protocol::AuthMode::ApiKey } else { codex_app_server_protocol::AuthMode::Chatgpt } @@ -1407,6 +1410,11 @@ fn stored_auth_issues( issues.push("personal access token auth is missing a personal access token"); } } + codex_app_server_protocol::AuthMode::BedrockApiKey => { + if auth.bedrock_api_key.is_none() { + issues.push("Bedrock API key auth is missing a Bedrock API key"); + } + } } issues } @@ -2141,8 +2149,9 @@ async fn state_check(config: &Config) -> DoctorCheck { }; let mut check = DoctorCheck::new("state.paths", "state", status, summary).details(details); if status == CheckStatus::Fail { - check = check - .remediation("Back up CODEX_HOME, then remove or repair the affected SQLite database."); + check = check.remediation( + "Move the damaged SQLite database aside, then restart the interactive CLI or app server so it can rebuild that runtime database from saved data. Other entry points may not rebuild automatically.", + ); } check } @@ -2436,6 +2445,7 @@ fn auth_mode_name(auth: &CodexAuth) -> &'static str { codex_app_server_protocol::AuthMode::ChatgptAuthTokens => "chatgpt_auth_tokens", codex_app_server_protocol::AuthMode::AgentIdentity => "agent_identity", codex_app_server_protocol::AuthMode::PersonalAccessToken => "personal_access_token", + codex_app_server_protocol::AuthMode::BedrockApiKey => "bedrock_api_key", } } @@ -2565,7 +2575,10 @@ fn provider_auth_reachability_mode_from_auth( return ProviderAuthReachabilityMode::Chatgpt; } match stored_auth.map(stored_auth_mode_value) { - Some(codex_app_server_protocol::AuthMode::ApiKey) => ProviderAuthReachabilityMode::ApiKey, + Some( + codex_app_server_protocol::AuthMode::ApiKey + | codex_app_server_protocol::AuthMode::BedrockApiKey, + ) => ProviderAuthReachabilityMode::ApiKey, Some( codex_app_server_protocol::AuthMode::Chatgpt | codex_app_server_protocol::AuthMode::ChatgptAuthTokens @@ -3470,6 +3483,7 @@ mod tests { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; assert_eq!( @@ -3488,6 +3502,7 @@ mod tests { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; assert_eq!( @@ -3508,6 +3523,7 @@ mod tests { last_refresh: None, agent_identity: None, personal_access_token: Some("at-test".to_string()), + bedrock_api_key: None, }; assert_eq!(stored_auth_mode(&auth), "personal_access_token"); @@ -3530,6 +3546,7 @@ mod tests { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; assert_eq!( diff --git a/codex-rs/cli/src/login.rs b/codex-rs/cli/src/login.rs index a114825fec..8378ee1277 100644 --- a/codex-rs/cli/src/login.rs +++ b/codex-rs/cli/src/login.rs @@ -430,6 +430,10 @@ pub async fn run_login_status(cli_config_overrides: CliConfigOverrides) -> ! { eprintln!("Logged in using personal access token"); std::process::exit(0); } + AuthMode::BedrockApiKey => { + eprintln!("Logged in using Amazon Bedrock API key"); + std::process::exit(0); + } }, Ok(None) => { eprintln!("Not logged in"); diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index b00f14d1b4..c6f4c069f8 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -37,10 +37,12 @@ use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_cli::CliConfigOverrides; use codex_utils_cli::ProfileV2Name; use codex_utils_cli::SharedCliOptions; -use codex_utils_cli::resume_hint; use owo_colors::OwoColorize; +use std::collections::HashSet; use std::io::IsTerminal; +use std::io::Write; use std::path::PathBuf; +use std::sync::Arc; use supports_color::Stream; #[cfg(any(target_os = "macos", target_os = "windows"))] @@ -76,6 +78,7 @@ use codex_features::FEATURES; use codex_features::Feature; use codex_features::Stage; use codex_features::is_known_feature_key; +use codex_home::CodexHomeUserInstructionsProvider; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_login::read_codex_access_token_from_env; @@ -181,6 +184,9 @@ enum Subcommand { /// Archive a saved session by id or session name. Archive(SessionArchiveCommand), + /// Permanently delete a saved session by id or session name. + Delete(DeleteCommand), + /// Unarchive a saved session by id or session name. Unarchive(SessionArchiveCommand), @@ -355,6 +361,16 @@ struct SessionArchiveConfigOverrides { config_overrides: CliConfigOverrides, } +#[derive(Debug, Args)] +struct DeleteCommand { + #[clap(flatten)] + session: SessionArchiveCommand, + + /// Delete without prompting. SESSION must be a UUID. + #[arg(long, default_value_t = false)] + force: bool, +} + #[derive(Debug, Parser)] struct ForkCommand { /// Conversation/session id (UUID). When provided, forks this session. @@ -686,10 +702,11 @@ fn parse_socket_path(raw: &str) -> Result { } fn format_exit_messages(exit_info: AppExitInfo, color_enabled: bool) -> Vec { + let is_fatal = matches!(&exit_info.exit_reason, ExitReason::Fatal(_)); let AppExitInfo { token_usage, thread_id: conversation_id, - thread_name, + resume_hint, .. } = exit_info; @@ -698,13 +715,15 @@ fn format_exit_messages(exit_info: AppExitInfo, color_enabled: bool) -> Vec Vec anyhow::Result<()> { - match exit_info.exit_reason { + let is_fatal = match &exit_info.exit_reason { ExitReason::Fatal(message) => { eprintln!("ERROR: {message}"); - std::process::exit(1); + true } - ExitReason::UserRequested => { /* normal exit */ } - } + ExitReason::UserRequested => false, + }; let update_action = exit_info.update_action; let color_enabled = supports_color::on(Stream::Stdout).is_some(); for line in format_exit_messages(exit_info, color_enabled) { println!("{line}"); } + if is_fatal { + std::io::stdout().flush()?; + std::process::exit(1); + } if let Some(action) = update_action { run_update_action(action)?; } @@ -830,6 +853,17 @@ async fn run_session_archive_cli_command( .map_err(|err| anyhow::anyhow!("{err}")) } +fn delete_action(target: &str, force: bool) -> anyhow::Result { + if force && codex_protocol::ThreadId::from_string(target).is_err() { + anyhow::bail!("--force requires a session UUID; names must be confirmed interactively"); + } + let confirmation = match force { + true => codex_tui::DeleteConfirmation::Skip, + false => codex_tui::DeleteConfirmation::Prompt, + }; + Ok(codex_tui::SessionArchiveAction::Delete(confirmation)) +} + async fn run_debug_app_server_command(cmd: DebugAppServerCommand) -> anyhow::Result<()> { match cmd.subcommand { DebugAppServerSubcommand::SendMessageV2(cmd) => { @@ -1244,6 +1278,20 @@ async fn cli_main(arg0_paths: Arg0DispatchPaths) -> anyhow::Result<()> { .await?; println!("{output}"); } + Some(Subcommand::Delete(DeleteCommand { session, force })) => { + let action = delete_action(&session.target, force)?; + let output = run_session_archive_cli_command( + action, + session, + interactive, + root_config_overrides.clone(), + root_remote.clone(), + root_remote_auth_token_env.clone(), + arg0_paths.clone(), + ) + .await?; + println!("{output}"); + } Some(Subcommand::Unarchive(cmd)) => { let output = run_session_archive_cli_command( codex_tui::SessionArchiveAction::Unarchive, @@ -1608,6 +1656,7 @@ fn profile_v2_for_subcommand<'a>( | Subcommand::Review(_) | Subcommand::Resume(_) | Subcommand::Archive(_) + | Subcommand::Delete(_) | Subcommand::Unarchive(_) | Subcommand::Fork(_) | Subcommand::Mcp(_) @@ -1616,7 +1665,7 @@ fn profile_v2_for_subcommand<'a>( subcommand: DebugSubcommand::PromptInput(_), }) => Ok(Some(profile_v2)), _ => anyhow::bail!( - "--profile only applies to runtime commands and `codex mcp`: `codex`, `codex exec`, `codex review`, `codex resume`, `codex archive`, `codex unarchive`, `codex fork`, `codex mcp`, `codex sandbox`, and `codex debug prompt-input`." + "--profile only applies to runtime commands and `codex mcp`: `codex`, `codex exec`, `codex review`, `codex resume`, `codex archive`, `codex delete`, `codex unarchive`, `codex fork`, `codex mcp`, `codex sandbox`, and `codex debug prompt-input`." ), } } @@ -1910,7 +1959,16 @@ async fn run_debug_prompt_input_command( }); } - let prompt_input = codex_core::build_prompt_input(config, input, /*state_db*/ None).await?; + let user_instructions_provider = Arc::new(CodexHomeUserInstructionsProvider::new( + config.codex_home.clone(), + )); + let prompt_input = codex_core::build_prompt_input( + config, + input, + /*state_db*/ None, + user_instructions_provider, + ) + .await?; println!("{}", serde_json::to_string_pretty(&prompt_input)?); Ok(()) @@ -2041,6 +2099,7 @@ fn unsupported_subcommand_name_for_strict_config( | Some(Subcommand::ExecServer(_)) | Some(Subcommand::Resume(_)) | Some(Subcommand::Archive(_)) + | Some(Subcommand::Delete(_)) | Some(Subcommand::Unarchive(_)) | Some(Subcommand::Fork(_)) | Some(Subcommand::Doctor(_)) => None, @@ -2204,7 +2263,7 @@ async fn run_interactive_tui( remote_endpoint.clone(), ) }; - let mut attempted_repair = false; + let mut attempted_backups = HashSet::new(); loop { let err = match start_tui().await { Ok(exit_info) => return Ok(exit_info), @@ -2217,25 +2276,25 @@ async fn run_interactive_tui( local_state_db::print_locked_guidance(startup_error); return Ok(AppExitInfo::fatal(startup_error.to_string())); } - if attempted_repair { + if !local_state_db::is_auto_backup_recoverable(startup_error) { local_state_db::print_diagnostic_guidance(startup_error); return Ok(AppExitInfo::fatal(startup_error.to_string())); } - if !local_state_db::confirm_repair(startup_error)? { + if !attempted_backups.insert(startup_error.database_path().to_path_buf()) { local_state_db::print_diagnostic_guidance(startup_error); return Ok(AppExitInfo::fatal(startup_error.to_string())); } - match local_state_db::repair_files(startup_error).await { - Ok(backups) => local_state_db::print_repair_backups(&backups), - Err(repair_err) => { + local_state_db::print_auto_backup_start(startup_error); + match local_state_db::backup_files_for_fresh_start(startup_error).await { + Ok(backups) => local_state_db::confirm_fresh_start_rebuild(startup_error, &backups)?, + Err(backup_err) => { local_state_db::print_diagnostic_guidance(startup_error); return Ok(AppExitInfo::fatal(format!( - "failed to repair Codex local data automatically: {repair_err}" + "failed to move damaged Codex local database files into a backup folder automatically: {backup_err}" ))); } } - attempted_repair = true; } } @@ -2615,6 +2674,14 @@ mod tests { ); } + #[test] + fn import_remains_an_interactive_prompt() { + let cli = MultitoolCli::try_parse_from(["codex", "import"]).expect("parse"); + + assert!(cli.subcommand.is_none()); + assert_eq!(cli.interactive.prompt.as_deref(), Some("import")); + } + #[test] fn profile_v2_rejects_non_plain_names_at_parse_time() { assert!( @@ -2875,6 +2942,17 @@ mod tests { assert!(interactive.bypass_hook_trust); } + #[test] + fn delete_force_requires_uuid() { + assert!(delete_action("123e4567-e89b-12d3-a456-426614174000", true).is_ok()); + + let err = delete_action("my-thread", true).expect_err("name should require prompt"); + assert_eq!( + err.to_string(), + "--force requires a session UUID; names must be confirmed interactively" + ); + } + #[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))] #[test] fn sandbox_parses_permissions_profile() { @@ -2993,12 +3071,13 @@ mod tests { total_tokens: 2, ..Default::default() }; + let thread_id = conversation_id + .map(ThreadId::from_string) + .map(Result::unwrap); AppExitInfo { token_usage, - thread_id: conversation_id - .map(ThreadId::from_string) - .map(Result::unwrap), - thread_name: thread_name.map(str::to_string), + thread_id, + resume_hint: codex_utils_cli::resume_hint(thread_name, thread_id), update_action: None, exit_reason: ExitReason::UserRequested, } @@ -3009,7 +3088,7 @@ mod tests { let exit_info = AppExitInfo { token_usage: TokenUsage::default(), thread_id: None, - thread_name: None, + resume_hint: None, update_action: None, exit_reason: ExitReason::UserRequested, }; @@ -3017,6 +3096,40 @@ mod tests { assert!(lines.is_empty()); } + #[test] + fn format_exit_messages_includes_session_id_for_fatal_exit_without_resume_hint() { + let exit_info = AppExitInfo { + token_usage: TokenUsage::default(), + thread_id: Some(ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap()), + resume_hint: None, + update_action: None, + exit_reason: ExitReason::Fatal("boom".to_string()), + }; + let lines = format_exit_messages(exit_info, /*color_enabled*/ false); + assert_eq!( + lines, + vec!["Session ID: 123e4567-e89b-12d3-a456-426614174000".to_string()] + ); + } + + #[test] + fn format_exit_messages_includes_resume_hint_for_fatal_exit() { + let mut exit_info = sample_exit_info( + Some("123e4567-e89b-12d3-a456-426614174000"), + /*thread_name*/ None, + ); + exit_info.exit_reason = ExitReason::Fatal("boom".to_string()); + let lines = format_exit_messages(exit_info, /*color_enabled*/ false); + assert_eq!( + lines, + vec![ + "Token usage: total=2 input=0 output=2".to_string(), + "To continue this session, run codex resume 123e4567-e89b-12d3-a456-426614174000" + .to_string(), + ] + ); + } + #[test] fn format_exit_messages_includes_resume_hint_without_color() { let exit_info = sample_exit_info( diff --git a/codex-rs/cli/src/plugin_cmd.rs b/codex-rs/cli/src/plugin_cmd.rs index 94c388066d..d6e28b5da0 100644 --- a/codex-rs/cli/src/plugin_cmd.rs +++ b/codex-rs/cli/src/plugin_cmd.rs @@ -204,7 +204,7 @@ pub async fn run_plugin_list( .. } = load_plugin_command_context(overrides).await?; let outcome = manager - .list_marketplaces_for_config(&plugins_input, &[]) + .list_marketplaces_for_config(&plugins_input, &[], /*include_openai_curated*/ true) .context("failed to list marketplace plugins")?; ensure_configured_marketplace_snapshots_loaded( codex_home.as_path(), @@ -609,7 +609,7 @@ fn find_marketplace_for_plugin( plugin_name: &str, ) -> Result { let outcome = manager - .list_marketplaces_for_config(plugins_input, &[]) + .list_marketplaces_for_config(plugins_input, &[], /*include_openai_curated*/ true) .context("failed to list marketplace plugins")?; ensure_configured_marketplace_snapshots_loaded( codex_home, diff --git a/codex-rs/cli/src/state_db_recovery.rs b/codex-rs/cli/src/state_db_recovery.rs index 7aeffaca3a..89bb16643c 100644 --- a/codex-rs/cli/src/state_db_recovery.rs +++ b/codex-rs/cli/src/state_db_recovery.rs @@ -1,10 +1,12 @@ -//! CLI recovery for local state database startup failures. +//! CLI handling for local state database startup failures. //! -//! This keeps user-facing repair and lock-contention handling out of the main +//! This keeps user-facing backup and lock-contention handling out of the main //! CLI dispatch path while preserving the TUI startup error as the boundary type. +use codex_state::RuntimeDbBackup; use codex_tui::LocalStateDbStartupError; -use std::path::PathBuf; +use std::io::IsTerminal; +use std::path::Path; pub(crate) fn startup_error(err: &std::io::Error) -> Option<&LocalStateDbStartupError> { err.get_ref() @@ -12,66 +14,60 @@ pub(crate) fn startup_error(err: &std::io::Error) -> Option<&LocalStateDbStartup } pub(crate) fn is_locked(detail: &str) -> bool { - let detail = detail.to_ascii_lowercase(); - detail.contains("database is locked") || detail.contains("database is busy") + codex_state::sqlite_error_detail_is_lock(detail) } -pub(crate) fn confirm_repair(startup_error: &LocalStateDbStartupError) -> std::io::Result { +pub(crate) fn is_corruption(detail: &str) -> bool { + codex_state::sqlite_error_detail_is_corruption(detail) +} + +pub(crate) fn is_auto_backup_recoverable(startup_error: &LocalStateDbStartupError) -> bool { + is_corruption(startup_error.detail()) || sqlite_home_is_blocking_file(startup_error) +} + +fn sqlite_home_is_blocking_file(startup_error: &LocalStateDbStartupError) -> bool { + startup_error + .database_path() + .parent() + .and_then(|path| std::fs::metadata(path).ok()) + .is_some_and(|metadata| metadata.is_file()) +} + +pub(crate) fn print_auto_backup_start(startup_error: &LocalStateDbStartupError) { eprintln!("Codex couldn't start because its local database appears to be damaged."); - eprintln!("Codex can try a safe repair by backing up those files and rebuilding them."); + eprintln!("Moving the damaged local database aside so Codex can rebuild it from saved data."); print_technical_details(startup_error); - crate::confirm("Repair Codex local data now? [y/N]: ") } -pub(crate) async fn repair_files( +pub(crate) async fn backup_files_for_fresh_start( startup_error: &LocalStateDbStartupError, -) -> std::io::Result> { - let state_db_path = startup_error.state_db_path(); - let sqlite_home = state_db_path.parent().ok_or_else(|| { - std::io::Error::other("state database path does not have a parent directory") - })?; - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |duration| duration.as_secs()); - let repair_suffix = format!("codex-repair-{timestamp}"); - let mut backups = Vec::new(); - - match tokio::fs::metadata(sqlite_home).await { - Ok(metadata) if metadata.is_dir() => {} - Ok(_) => { - backups.push(backup_path(sqlite_home, &repair_suffix).await?); - tokio::fs::create_dir_all(sqlite_home).await?; - } - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - tokio::fs::create_dir_all(sqlite_home).await?; - } - Err(err) => return Err(err), - } - - for path in codex_state::runtime_db_paths(sqlite_home) - .into_iter() - .flat_map(|db| sqlite_paths(db.path.as_path())) - { - if tokio::fs::try_exists(path.as_path()).await? { - backups.push(backup_path(path.as_path(), &repair_suffix).await?); - } - } - - if backups.is_empty() { - return Err(std::io::Error::other( - "no repairable Codex local data files were found", - )); - } - - Ok(backups) +) -> std::io::Result> { + codex_state::backup_runtime_db_for_fresh_start(startup_error.database_path()).await } -pub(crate) fn print_repair_backups(backups: &[PathBuf]) { - eprintln!("Backed up Codex local data before repair:"); - for backup in backups { - eprintln!(" {}", backup.display()); +pub(crate) fn confirm_fresh_start_rebuild( + startup_error: &LocalStateDbStartupError, + backups: &[RuntimeDbBackup], +) -> std::io::Result<()> { + eprintln!("Codex rebuilt its local database."); + eprintln!( + "Codex detected a damaged local database, moved it into a backup folder, and will continue startup with a fresh database." + ); + eprintln!("Database path: {}", startup_error.database_path().display()); + if let Some(backup_folder) = backup_folder(backups) { + eprintln!("Backup folder: {}", backup_folder.display()); + } else { + eprintln!("Backup folder: unavailable"); } - eprintln!("Retrying startup with rebuilt local data..."); + + if std::io::stdin().is_terminal() && std::io::stderr().is_terminal() { + eprintln!("Press Enter to continue."); + let mut input = String::new(); + std::io::stdin().read_line(&mut input)?; + } else { + eprintln!("Continuing startup with a fresh local database..."); + } + Ok(()) } pub(crate) fn print_diagnostic_guidance(startup_error: &LocalStateDbStartupError) { @@ -87,79 +83,50 @@ pub(crate) fn print_locked_guidance(startup_error: &LocalStateDbStartupError) { print_technical_details(startup_error); } -fn sqlite_paths(db_path: &std::path::Path) -> Vec { - let mut wal_path = db_path.as_os_str().to_os_string(); - wal_path.push("-wal"); - let mut shm_path = db_path.as_os_str().to_os_string(); - shm_path.push("-shm"); - vec![ - db_path.to_path_buf(), - PathBuf::from(wal_path), - PathBuf::from(shm_path), - ] -} - -async fn backup_path(path: &std::path::Path, repair_suffix: &str) -> std::io::Result { - let file_name = path.file_name().ok_or_else(|| { - std::io::Error::other(format!( - "cannot create a repair backup name for {}", - path.display() - )) - })?; - let mut sequence = 0; - loop { - let mut backup_name = file_name.to_os_string(); - backup_name.push(format!(".{repair_suffix}.{sequence}.bak")); - let backup_path = path.with_file_name(backup_name); - if !tokio::fs::try_exists(backup_path.as_path()).await? { - tokio::fs::rename(path, backup_path.as_path()).await?; - return Ok(backup_path); - } - sequence += 1; - } -} - fn print_technical_details(startup_error: &LocalStateDbStartupError) { eprintln!("Technical details:"); - eprintln!(" Location: {}", startup_error.state_db_path().display()); + eprintln!(" Location: {}", startup_error.database_path().display()); eprintln!(" Cause: {}", startup_error.detail()); } +fn backup_folder(backups: &[RuntimeDbBackup]) -> Option<&Path> { + backups.first()?.backup_path.parent() +} + #[cfg(test)] mod tests { use super::*; use pretty_assertions::assert_eq; + use std::path::PathBuf; use tempfile::TempDir; #[tokio::test] - async fn repair_backs_up_owned_database_files() -> std::io::Result<()> { + async fn backup_backs_up_only_failed_database_file() -> std::io::Result<()> { let temp_dir = TempDir::new()?; let state_path = codex_state::state_db_path(temp_dir.path()); - let logs_path = codex_state::logs_db_path(temp_dir.path()); - let goals_path = codex_state::goals_db_path(temp_dir.path()); - let state_sidecars = sqlite_paths(state_path.as_path()); + let failed_db_path = codex_state::logs_db_path(temp_dir.path()); tokio::fs::write(state_path.as_path(), b"state").await?; - tokio::fs::write(state_sidecars[1].as_path(), b"state-wal").await?; - tokio::fs::write(logs_path.as_path(), b"logs").await?; - tokio::fs::write(goals_path.as_path(), b"goals").await?; + tokio::fs::write(failed_db_path.as_path(), b"logs").await?; let startup_error = - LocalStateDbStartupError::new(state_path.clone(), "corrupt".to_string()); - let backups = repair_files(&startup_error).await?; + LocalStateDbStartupError::new(failed_db_path.clone(), "corrupt".to_string()); + let backups = backup_files_for_fresh_start(&startup_error).await?; - assert_eq!(backups.len(), 4); - assert!(!tokio::fs::try_exists(state_path.as_path()).await?); - assert!(!tokio::fs::try_exists(state_sidecars[1].as_path()).await?); - assert!(!tokio::fs::try_exists(logs_path.as_path()).await?); - assert!(!tokio::fs::try_exists(goals_path.as_path()).await?); - for backup in backups { - assert!(tokio::fs::try_exists(backup.as_path()).await?); - } + assert_eq!( + backups + .iter() + .map(|backup| &backup.original_path) + .collect::>(), + vec![&failed_db_path] + ); + assert!(!tokio::fs::try_exists(failed_db_path.as_path()).await?); + assert!(tokio::fs::try_exists(state_path.as_path()).await?); + assert!(tokio::fs::try_exists(backups[0].backup_path.as_path()).await?); Ok(()) } #[tokio::test] - async fn repair_replaces_blocking_sqlite_home_file() -> std::io::Result<()> { + async fn backup_replaces_blocking_sqlite_home_file() -> std::io::Result<()> { let temp_dir = TempDir::new()?; let sqlite_home = temp_dir.path().join("sqlite-home"); tokio::fs::write(sqlite_home.as_path(), b"not-a-directory").await?; @@ -168,18 +135,25 @@ mod tests { "File exists".to_string(), ); - let backups = repair_files(&startup_error).await?; + assert!(is_auto_backup_recoverable(&startup_error)); + let backups = backup_files_for_fresh_start(&startup_error).await?; assert_eq!(backups.len(), 1); assert!(tokio::fs::metadata(sqlite_home.as_path()).await?.is_dir()); - assert!(tokio::fs::try_exists(backups[0].as_path()).await?); + assert!(tokio::fs::try_exists(backups[0].backup_path.as_path()).await?); Ok(()) } #[test] - fn lock_failures_skip_repair() { - assert!(is_locked("database is locked")); - assert!(is_locked("database is busy")); - assert!(!is_locked("database disk image is malformed")); + fn backup_folder_uses_parent_of_first_backup_path() { + let backups = vec![RuntimeDbBackup { + original_path: PathBuf::from("/tmp/state_5.sqlite"), + backup_path: PathBuf::from("/tmp/db-backups/sqlite-1-0/state_5.sqlite"), + }]; + + assert_eq!( + backup_folder(&backups), + Some(Path::new("/tmp/db-backups/sqlite-1-0")) + ); } } diff --git a/codex-rs/cli/tests/delete.rs b/codex-rs/cli/tests/delete.rs new file mode 100644 index 0000000000..854b87b56e --- /dev/null +++ b/codex-rs/cli/tests/delete.rs @@ -0,0 +1,17 @@ +use predicates::prelude::*; + +#[test] +fn missing_session_fails_before_delete_confirmation() -> anyhow::Result<()> { + let codex_home = tempfile::tempdir()?; + let mut cmd = assert_cmd::Command::new(codex_utils_cargo_bin::cargo_bin("codex")?); + cmd.env("CODEX_HOME", codex_home.path()) + .args(["delete", "123e4567-e89b-12d3-a456-426614174000"]); + + cmd.assert() + .failure() + .stderr(predicate::str::contains( + "No active or archived session found matching", + )) + .stderr(predicate::str::contains("cannot confirm").not()); + Ok(()) +} diff --git a/codex-rs/cloud-config/src/cache.rs b/codex-rs/cloud-config/src/cache.rs index 1e68646159..e5d1d4dc84 100644 --- a/codex-rs/cloud-config/src/cache.rs +++ b/codex-rs/cloud-config/src/cache.rs @@ -22,7 +22,7 @@ use tokio::fs; const CLOUD_CONFIG_BUNDLE_CACHE_VERSION: u32 = 1; pub(super) const CLOUD_CONFIG_BUNDLE_CACHE_FILENAME: &str = "cloud-config-bundle-cache.json"; -const CLOUD_CONFIG_BUNDLE_CACHE_TTL: Duration = Duration::from_secs(30 * 60); +const CLOUD_CONFIG_BUNDLE_CACHE_TTL: Duration = Duration::from_secs(60 * 60); const CLOUD_CONFIG_BUNDLE_CACHE_WRITE_HMAC_KEY: &[u8] = b"codex-cloud-config-bundle-cache-v1-6160ae70-bcfd-4ca8-a99b-40f73b3b072e"; const CLOUD_CONFIG_BUNDLE_CACHE_READ_HMAC_KEYS: &[&[u8]] = diff --git a/codex-rs/cloud-config/src/cache_tests.rs b/codex-rs/cloud-config/src/cache_tests.rs index 0416ed7e77..28899f930d 100644 --- a/codex-rs/cloud-config/src/cache_tests.rs +++ b/codex-rs/cloud-config/src/cache_tests.rs @@ -81,7 +81,7 @@ async fn save_writes_signed_payload_and_loads_for_matching_identity() { .expect("parse cache"); assert!( cache_file.signed_payload.expires_at - <= cache_file.signed_payload.cached_at + ChronoDuration::minutes(30) + <= cache_file.signed_payload.cached_at + ChronoDuration::minutes(60) ); assert!(cache_file.signed_payload.expires_at > cache_file.signed_payload.cached_at); assert_eq!( diff --git a/codex-rs/cloud-config/src/service.rs b/codex-rs/cloud-config/src/service.rs index eed94c0a10..1a7104bf52 100644 --- a/codex-rs/cloud-config/src/service.rs +++ b/codex-rs/cloud-config/src/service.rs @@ -32,7 +32,7 @@ use tokio::time::timeout; pub(crate) const CLOUD_CONFIG_BUNDLE_TIMEOUT: Duration = Duration::from_secs(15); const CLOUD_CONFIG_BUNDLE_MAX_ATTEMPTS: usize = 5; -const CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(5 * 60); +const CLOUD_CONFIG_BUNDLE_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60); const CLOUD_CONFIG_BUNDLE_LOAD_FAILED_MESSAGE: &str = "Failed to load cloud config bundle (workspace-managed policies)."; const CLOUD_CONFIG_BUNDLE_AUTH_RECOVERY_FAILED_MESSAGE: &str = concat!( diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index ad26549a26..d9a67c7b97 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -291,6 +291,18 @@ impl RealtimeWebsocketWriter { .await } + pub async fn send_conversation_handoff_append( + &self, + handoff_id: String, + output_text: String, + ) -> Result<(), ApiError> { + self.send_json(&RealtimeOutboundMessage::ConversationHandoffAppend { + handoff_id, + output_text, + }) + .await + } + pub async fn send_conversation_function_call_output( &self, call_id: String, diff --git a/codex-rs/codex-api/tests/models_integration.rs b/codex-rs/codex-api/tests/models_integration.rs index 4ee6069898..5f421dc1ad 100644 --- a/codex-rs/codex-api/tests/models_integration.rs +++ b/codex-rs/codex-api/tests/models_integration.rs @@ -93,6 +93,7 @@ async fn models_client_hits_models_endpoint() { context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), diff --git a/codex-rs/codex-home/BUILD.bazel b/codex-rs/codex-home/BUILD.bazel new file mode 100644 index 0000000000..a5a01e4e34 --- /dev/null +++ b/codex-rs/codex-home/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "codex-home", + crate_name = "codex_home", +) diff --git a/codex-rs/codex-home/Cargo.toml b/codex-rs/codex-home/Cargo.toml new file mode 100644 index 0000000000..a8fa625dcf --- /dev/null +++ b/codex-rs/codex-home/Cargo.toml @@ -0,0 +1,21 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-home" +version.workspace = true + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +codex-extension-api = { workspace = true } +codex-utils-absolute-path = { workspace = true } +tokio = { workspace = true, features = ["fs"] } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/codex-rs/codex-home/src/instructions/mod.rs b/codex-rs/codex-home/src/instructions/mod.rs new file mode 100644 index 0000000000..0840263a13 --- /dev/null +++ b/codex-rs/codex-home/src/instructions/mod.rs @@ -0,0 +1,83 @@ +use std::io; + +use codex_extension_api::LoadUserInstructionsFuture; +use codex_extension_api::LoadedUserInstructions; +use codex_extension_api::UserInstructions; +use codex_extension_api::UserInstructionsProvider; +use codex_utils_absolute_path::AbsolutePathBuf; + +const DEFAULT_AGENTS_MD_FILENAME: &str = "AGENTS.md"; +const LOCAL_AGENTS_MD_FILENAME: &str = "AGENTS.override.md"; + +/// Loads user instructions from a Codex home directory. +#[derive(Clone, Debug)] +pub struct CodexHomeUserInstructionsProvider { + codex_home: AbsolutePathBuf, +} + +impl CodexHomeUserInstructionsProvider { + /// Creates a provider rooted at the supplied absolute Codex home directory. + pub fn new(codex_home: AbsolutePathBuf) -> Self { + Self { codex_home } + } + + async fn load_from_codex_home(&self) -> LoadedUserInstructions { + let mut warnings = Vec::new(); + for candidate in [LOCAL_AGENTS_MD_FILENAME, DEFAULT_AGENTS_MD_FILENAME] { + let path = self.codex_home.join(candidate); + match tokio::fs::metadata(path.as_path()).await { + Ok(metadata) if !metadata.is_file() => continue, + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => { + warnings.push(format!( + "Failed to read global AGENTS.md instructions from `{}`: {err}", + path.display() + )); + continue; + } + } + let data = match tokio::fs::read(path.as_path()).await { + Ok(data) => data, + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => { + warnings.push(format!( + "Failed to read global AGENTS.md instructions from `{}`: {err}", + path.display() + )); + continue; + } + }; + if let Err(err) = std::str::from_utf8(&data) { + warnings.push(format!( + "Global AGENTS.md instructions from `{}` contain invalid UTF-8: {err}. Invalid byte sequences were replaced.", + path.display() + )); + } + let contents = String::from_utf8_lossy(&data); + let trimmed = contents.trim(); + if !trimmed.is_empty() { + return LoadedUserInstructions { + instructions: Some(UserInstructions { + text: trimmed.to_string(), + source: path, + }), + warnings, + }; + } + } + LoadedUserInstructions { + instructions: None, + warnings, + } + } +} + +impl UserInstructionsProvider for CodexHomeUserInstructionsProvider { + fn load_user_instructions(&self) -> LoadUserInstructionsFuture<'_> { + Box::pin(self.load_from_codex_home()) + } +} + +#[cfg(test)] +mod tests; diff --git a/codex-rs/codex-home/src/instructions/tests.rs b/codex-rs/codex-home/src/instructions/tests.rs new file mode 100644 index 0000000000..e884210403 --- /dev/null +++ b/codex-rs/codex-home/src/instructions/tests.rs @@ -0,0 +1,152 @@ +use std::fs; +use std::path::Path; + +use codex_extension_api::LoadedUserInstructions; +use codex_extension_api::UserInstructions; +use codex_extension_api::UserInstructionsProvider; +use codex_utils_absolute_path::AbsolutePathBuf; +use pretty_assertions::assert_eq; +use tempfile::TempDir; + +use super::CodexHomeUserInstructionsProvider; +use super::DEFAULT_AGENTS_MD_FILENAME; +use super::LOCAL_AGENTS_MD_FILENAME; + +fn provider(home: &TempDir) -> CodexHomeUserInstructionsProvider { + CodexHomeUserInstructionsProvider::new( + AbsolutePathBuf::try_from(home.path().to_path_buf()).expect("absolute temp dir"), + ) +} + +fn expected( + home: &TempDir, + filename: &str, + text: &str, + warnings: Vec, +) -> LoadedUserInstructions { + LoadedUserInstructions { + instructions: Some(UserInstructions { + text: text.to_string(), + source: AbsolutePathBuf::try_from(home.path().join(filename)) + .expect("absolute source path"), + }), + warnings, + } +} + +#[cfg(unix)] +fn create_symlink_loop(path: &Path) { + std::os::unix::fs::symlink( + path.file_name().expect("override path should have a name"), + path, + ) + .expect("create symlink loop"); +} + +#[cfg(windows)] +fn create_symlink_loop(path: &Path) { + std::os::windows::fs::symlink_file( + path.file_name().expect("override path should have a name"), + path, + ) + .expect("create symlink loop"); +} + +#[tokio::test] +async fn missing_files_return_no_instructions() { + let home = TempDir::new().expect("temp dir"); + + assert_eq!( + provider(&home).load_user_instructions().await, + LoadedUserInstructions::default() + ); +} + +#[tokio::test] +async fn override_takes_precedence_over_default() { + let home = TempDir::new().expect("temp dir"); + fs::write(home.path().join(DEFAULT_AGENTS_MD_FILENAME), "default").expect("write default"); + fs::write(home.path().join(LOCAL_AGENTS_MD_FILENAME), "override").expect("write override"); + + assert_eq!( + provider(&home).load_user_instructions().await, + expected(&home, LOCAL_AGENTS_MD_FILENAME, "override", Vec::new()) + ); +} + +#[tokio::test] +async fn empty_override_falls_back_to_trimmed_default() { + let home = TempDir::new().expect("temp dir"); + fs::write(home.path().join(LOCAL_AGENTS_MD_FILENAME), " \n\t").expect("write override"); + fs::write( + home.path().join(DEFAULT_AGENTS_MD_FILENAME), + "\n default instructions \n", + ) + .expect("write default"); + + assert_eq!( + provider(&home).load_user_instructions().await, + expected( + &home, + DEFAULT_AGENTS_MD_FILENAME, + "default instructions", + Vec::new() + ) + ); +} + +#[tokio::test] +async fn directory_override_falls_back_to_default() { + let home = TempDir::new().expect("temp dir"); + fs::create_dir(home.path().join(LOCAL_AGENTS_MD_FILENAME)).expect("create override directory"); + fs::write(home.path().join(DEFAULT_AGENTS_MD_FILENAME), "default").expect("write default"); + + assert_eq!( + provider(&home).load_user_instructions().await, + expected(&home, DEFAULT_AGENTS_MD_FILENAME, "default", Vec::new()) + ); +} + +#[tokio::test] +async fn recoverable_override_read_error_warns_and_falls_back_to_default() { + let home = TempDir::new().expect("temp dir"); + let override_path = home.path().join(LOCAL_AGENTS_MD_FILENAME); + create_symlink_loop(&override_path); + fs::write(home.path().join(DEFAULT_AGENTS_MD_FILENAME), "default").expect("write default"); + let read_error = fs::read(&override_path).expect_err("symlink loop should not be readable"); + let warning = format!( + "Failed to read global AGENTS.md instructions from `{}`: {read_error}", + override_path.display() + ); + + assert_eq!( + provider(&home).load_user_instructions().await, + expected(&home, DEFAULT_AGENTS_MD_FILENAME, "default", vec![warning]) + ); +} + +#[tokio::test] +async fn invalid_utf8_is_lossy_and_warned() { + let home = TempDir::new().expect("temp dir"); + let path = home.path().join(DEFAULT_AGENTS_MD_FILENAME); + let mut invalid_utf8 = b"global".to_vec(); + invalid_utf8.push(0xff); + invalid_utf8.extend_from_slice(b" doc"); + fs::write(&path, &invalid_utf8).expect("write invalid utf-8"); + + let outcome = provider(&home).load_user_instructions().await; + let utf8_error = std::str::from_utf8(&invalid_utf8).expect_err("invalid utf-8"); + let warning = format!( + "Global AGENTS.md instructions from `{}` contain invalid UTF-8: {utf8_error}. Invalid byte sequences were replaced.", + path.display(), + ); + assert_eq!( + outcome, + expected( + &home, + DEFAULT_AGENTS_MD_FILENAME, + "global\u{fffd} doc", + vec![warning] + ) + ); +} diff --git a/codex-rs/codex-home/src/lib.rs b/codex-rs/codex-home/src/lib.rs new file mode 100644 index 0000000000..7ca5e58072 --- /dev/null +++ b/codex-rs/codex-home/src/lib.rs @@ -0,0 +1,3 @@ +mod instructions; + +pub use instructions::CodexHomeUserInstructionsProvider; diff --git a/codex-rs/codex-mcp/Cargo.toml b/codex-rs/codex-mcp/Cargo.toml index d7634f0aae..433cbde8b8 100644 --- a/codex-rs/codex-mcp/Cargo.toml +++ b/codex-rs/codex-mcp/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [dependencies] anyhow = { workspace = true } +arc-swap = { workspace = true } async-channel = { workspace = true } codex-async-utils = { workspace = true } codex-api = { workspace = true } diff --git a/codex-rs/codex-mcp/src/catalog.rs b/codex-rs/codex-mcp/src/catalog.rs new file mode 100644 index 0000000000..68f0a1b5ad --- /dev/null +++ b/codex-rs/codex-mcp/src/catalog.rs @@ -0,0 +1,333 @@ +use std::cmp::Reverse; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::collections::HashMap; + +use codex_config::McpServerConfig; + +/// The component that declared an MCP server registration. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum McpServerSource { + Plugin { plugin_id: String }, + Config, + Compatibility { id: String }, + Extension { id: String }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum RegistrationPrecedence { + Plugin(Reverse), + Config, + Compatibility, + Extension(usize), +} + +impl RegistrationPrecedence { + fn tier(self) -> u8 { + match self { + Self::Plugin(_) => 0, + Self::Config => 1, + Self::Compatibility => 2, + Self::Extension(_) => 3, + } + } +} + +/// One named MCP server declaration before source resolution. +#[derive(Clone, Debug, PartialEq)] +pub struct McpServerRegistration { + name: String, + source: McpServerSource, + config: McpServerConfig, + precedence: RegistrationPrecedence, +} + +impl McpServerRegistration { + pub fn from_config(name: String, config: McpServerConfig) -> Self { + Self::new( + name, + McpServerSource::Config, + config, + RegistrationPrecedence::Config, + ) + } + + pub fn from_plugin( + name: String, + plugin_id: String, + plugin_order: usize, + config: McpServerConfig, + ) -> Self { + Self::new( + name, + McpServerSource::Plugin { plugin_id }, + config, + RegistrationPrecedence::Plugin(Reverse(plugin_order)), + ) + } + + pub fn from_compatibility( + name: String, + id: impl Into, + config: McpServerConfig, + ) -> Self { + Self::new( + name, + McpServerSource::Compatibility { id: id.into() }, + config, + RegistrationPrecedence::Compatibility, + ) + } + + pub fn from_extension( + name: String, + id: impl Into, + contribution_order: usize, + config: McpServerConfig, + ) -> Self { + Self::new( + name, + McpServerSource::Extension { id: id.into() }, + config, + RegistrationPrecedence::Extension(contribution_order), + ) + } + + fn new( + name: String, + source: McpServerSource, + config: McpServerConfig, + precedence: RegistrationPrecedence, + ) -> Self { + Self { + name, + source, + config, + precedence, + } + } +} + +/// One side of an MCP server conflict, including whether it registers or +/// removes the server. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum McpServerConflictAction { + Register(McpServerSource), + Remove(McpServerSource), +} + +/// A same-tier name collision and the final outcome after all precedence is applied. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct McpServerConflict { + pub name: String, + pub outcome: McpServerConflictAction, + pub contenders: Vec, +} + +#[derive(Clone, Debug)] +enum CatalogAction { + Register(Box), + Remove { + name: String, + source: McpServerSource, + precedence: RegistrationPrecedence, + }, +} + +impl CatalogAction { + fn name(&self) -> &str { + match self { + Self::Register(registration) => ®istration.name, + Self::Remove { name, .. } => name, + } + } + + fn precedence(&self) -> RegistrationPrecedence { + match self { + Self::Register(registration) => registration.precedence, + Self::Remove { precedence, .. } => *precedence, + } + } + + fn conflict_action(&self) -> McpServerConflictAction { + match self { + Self::Register(registration) => { + McpServerConflictAction::Register(registration.source.clone()) + } + Self::Remove { source, .. } => McpServerConflictAction::Remove(source.clone()), + } + } +} + +/// Mutable inputs used to produce an immutable resolved catalog. +#[derive(Clone, Debug, Default)] +pub struct McpCatalogBuilder { + actions: Vec, + disabled_server_names: BTreeSet, +} + +impl McpCatalogBuilder { + pub fn register(&mut self, registration: McpServerRegistration) { + self.actions + .push(CatalogAction::Register(Box::new(registration))); + } + + /// Applies the legacy name-scoped disabled veto after source resolution. + pub fn disable(&mut self, name: String) { + self.disabled_server_names.insert(name); + } + + pub fn remove_compatibility(&mut self, name: String, id: impl Into) { + self.actions.push(CatalogAction::Remove { + name, + source: McpServerSource::Compatibility { id: id.into() }, + precedence: RegistrationPrecedence::Compatibility, + }); + } + + pub fn remove_extension( + &mut self, + name: String, + id: impl Into, + contribution_order: usize, + ) { + self.actions.push(CatalogAction::Remove { + name, + source: McpServerSource::Extension { id: id.into() }, + precedence: RegistrationPrecedence::Extension(contribution_order), + }); + } + + pub fn build(mut self) -> ResolvedMcpCatalog { + // Stable sorting makes action order the tie-breaker when precedence is equal. + self.actions.sort_by_key(CatalogAction::precedence); + + let mut winners = BTreeMap::::new(); + let mut actions_by_name_and_tier = BTreeMap::<(String, u8), Vec<&CatalogAction>>::new(); + for action in &self.actions { + winners.insert(action.name().to_string(), action.clone()); + actions_by_name_and_tier + .entry((action.name().to_string(), action.precedence().tier())) + .or_default() + .push(action); + } + + let mut conflicts = Vec::new(); + for ((name, _), actions) in actions_by_name_and_tier { + if actions.len() < 2 { + continue; + } + let Some(outcome) = winners.get(&name).map(CatalogAction::conflict_action) else { + continue; + }; + conflicts.push(McpServerConflict { + name, + outcome, + contenders: actions + .into_iter() + .map(CatalogAction::conflict_action) + .collect(), + }); + } + + let mut disabled_server_names = self.disabled_server_names; + let servers = winners + .into_iter() + .filter_map(|(name, action)| match action { + CatalogAction::Register(registration) => { + let mut registration = *registration; + // Effective disabled winners remain name-scoped vetoes for later overlays. + if !registration.config.enabled || disabled_server_names.contains(&name) { + registration.config.enabled = false; + disabled_server_names.insert(name.clone()); + } + Some(( + name, + ResolvedMcpServer { + source: registration.source, + config: registration.config, + }, + )) + } + CatalogAction::Remove { .. } => None, + }) + .collect(); + + ResolvedMcpCatalog { + actions: self.actions, + disabled_server_names, + servers, + conflicts, + } + } +} + +/// A single winning MCP registration. +#[derive(Clone, Debug, PartialEq)] +pub struct ResolvedMcpServer { + source: McpServerSource, + config: McpServerConfig, +} + +impl ResolvedMcpServer { + pub fn source(&self) -> &McpServerSource { + &self.source + } + + pub fn config(&self) -> &McpServerConfig { + &self.config + } +} + +/// Immutable result of MCP registration resolution. +#[derive(Clone, Debug, Default)] +pub struct ResolvedMcpCatalog { + actions: Vec, + disabled_server_names: BTreeSet, + servers: BTreeMap, + conflicts: Vec, +} + +impl ResolvedMcpCatalog { + pub fn builder() -> McpCatalogBuilder { + McpCatalogBuilder::default() + } + + pub fn to_builder(&self) -> McpCatalogBuilder { + McpCatalogBuilder { + actions: self.actions.clone(), + disabled_server_names: self.disabled_server_names.clone(), + } + } + + pub fn server(&self, name: &str) -> Option<&ResolvedMcpServer> { + self.servers.get(name) + } + + pub fn configured_servers(&self) -> HashMap { + self.servers + .iter() + .map(|(name, server)| (name.clone(), server.config.clone())) + .collect() + } + + pub fn plugin_ids_by_server_name(&self) -> HashMap { + self.servers + .iter() + .filter_map(|(name, server)| match server.source() { + McpServerSource::Plugin { plugin_id } => Some((name.clone(), plugin_id.clone())), + McpServerSource::Config + | McpServerSource::Compatibility { .. } + | McpServerSource::Extension { .. } => None, + }) + .collect() + } + + pub fn conflicts(&self) -> &[McpServerConflict] { + &self.conflicts + } +} + +#[cfg(test)] +#[path = "catalog_tests.rs"] +mod tests; diff --git a/codex-rs/codex-mcp/src/catalog_tests.rs b/codex-rs/codex-mcp/src/catalog_tests.rs new file mode 100644 index 0000000000..6909959a54 --- /dev/null +++ b/codex-rs/codex-mcp/src/catalog_tests.rs @@ -0,0 +1,257 @@ +use std::collections::HashMap; +use std::time::Duration; + +use codex_config::AppToolApproval; +use codex_config::DEFAULT_MCP_SERVER_ENVIRONMENT_ID; +use codex_config::McpServerConfig; +use codex_config::McpServerToolConfig; +use codex_config::McpServerTransportConfig; +use pretty_assertions::assert_eq; + +use super::McpServerConflict; +use super::McpServerConflictAction; +use super::McpServerRegistration; +use super::McpServerSource; +use super::ResolvedMcpCatalog; + +fn server(url: &str) -> McpServerConfig { + McpServerConfig { + transport: McpServerTransportConfig::StreamableHttp { + url: url.to_string(), + bearer_token_env_var: None, + http_headers: None, + env_http_headers: None, + }, + environment_id: DEFAULT_MCP_SERVER_ENVIRONMENT_ID.to_string(), + enabled: true, + required: true, + supports_parallel_tool_calls: true, + disabled_reason: None, + startup_timeout_sec: Some(Duration::from_secs(7)), + tool_timeout_sec: Some(Duration::from_secs(11)), + default_tools_approval_mode: Some(AppToolApproval::Prompt), + enabled_tools: Some(vec!["read".to_string()]), + disabled_tools: Some(vec!["write".to_string()]), + scopes: None, + oauth: None, + oauth_resource: None, + tools: HashMap::from([( + "read".to_string(), + McpServerToolConfig { + approval_mode: Some(AppToolApproval::Approve), + }, + )]), + } +} + +fn plugin_source(plugin_id: &str) -> McpServerSource { + McpServerSource::Plugin { + plugin_id: plugin_id.to_string(), + } +} + +fn compatibility_source(id: &str) -> McpServerSource { + McpServerSource::Compatibility { id: id.to_string() } +} + +fn extension_source(id: &str) -> McpServerSource { + McpServerSource::Extension { id: id.to_string() } +} + +fn register(source: McpServerSource) -> McpServerConflictAction { + McpServerConflictAction::Register(source) +} + +fn remove(source: McpServerSource) -> McpServerConflictAction { + McpServerConflictAction::Remove(source) +} + +#[test] +fn source_precedence_preserves_the_winning_registration() { + let extension = server("https://extension.example/mcp"); + let mut plugin = server("https://plugin.example/mcp"); + plugin.enabled = false; + let mut builder = ResolvedMcpCatalog::builder(); + builder.register(McpServerRegistration::from_extension( + "docs".to_string(), + "hosted", + /*contribution_order*/ 0, + extension.clone(), + )); + builder.register(McpServerRegistration::from_plugin( + "docs".to_string(), + "plugin@test".to_string(), + /*plugin_order*/ 0, + plugin, + )); + builder.register(McpServerRegistration::from_plugin( + "docs".to_string(), + "other-plugin@test".to_string(), + /*plugin_order*/ 1, + server("https://other-plugin.example/mcp"), + )); + builder.register(McpServerRegistration::from_compatibility( + "docs".to_string(), + "legacy", + server("https://compatibility.example/mcp"), + )); + builder.register(McpServerRegistration::from_config( + "docs".to_string(), + server("https://config.example/mcp"), + )); + + let catalog = builder.build(); + let resolved = catalog.server("docs").expect("resolved server"); + + assert_eq!( + resolved.source(), + &McpServerSource::Extension { + id: "hosted".to_string(), + } + ); + assert_eq!(resolved.config(), &extension); + assert!(catalog.plugin_ids_by_server_name().is_empty()); + assert_eq!( + catalog.conflicts(), + &[McpServerConflict { + name: "docs".to_string(), + outcome: register(extension_source("hosted")), + contenders: vec![ + register(plugin_source("other-plugin@test")), + register(plugin_source("plugin@test")), + ], + }] + ); +} + +#[test] +fn disabled_veto_only_disables_the_winning_registration() { + let extension = server("https://extension.example/mcp"); + let mut expected = extension.clone(); + expected.enabled = false; + let mut builder = ResolvedMcpCatalog::builder(); + builder.register(McpServerRegistration::from_extension( + "docs".to_string(), + "hosted", + /*contribution_order*/ 0, + extension, + )); + builder.disable("docs".to_string()); + + let actual = builder + .build() + .server("docs") + .expect("resolved server") + .config() + .clone(); + + assert_eq!(actual, expected); +} + +#[test] +fn disabled_winner_remains_a_veto_when_the_catalog_is_extended() { + let mut disabled = server("https://config.example/mcp"); + disabled.enabled = false; + let mut expected = server("https://extension.example/mcp"); + expected.enabled = false; + let mut builder = ResolvedMcpCatalog::builder(); + builder.register(McpServerRegistration::from_config( + "docs".to_string(), + disabled, + )); + let mut builder = builder.build().to_builder(); + builder.register(McpServerRegistration::from_extension( + "docs".to_string(), + "hosted", + /*contribution_order*/ 0, + server("https://extension.example/mcp"), + )); + + let resolved = builder.build(); + + assert_eq!( + resolved.server("docs"), + Some(&super::ResolvedMcpServer { + source: extension_source("hosted"), + config: expected, + }) + ); +} + +#[test] +fn earlier_plugin_wins_with_an_explicit_conflict() { + let mut builder = ResolvedMcpCatalog::builder(); + builder.register(McpServerRegistration::from_plugin( + "docs".to_string(), + "alpha@test".to_string(), + /*plugin_order*/ 0, + server("https://alpha.example/mcp"), + )); + builder.register(McpServerRegistration::from_plugin( + "docs".to_string(), + "beta@test".to_string(), + /*plugin_order*/ 1, + server("https://beta.example/mcp"), + )); + + let catalog = builder.build(); + + assert_eq!( + catalog.plugin_ids_by_server_name(), + HashMap::from([("docs".to_string(), "alpha@test".to_string())]) + ); + assert_eq!( + catalog.conflicts(), + &[McpServerConflict { + name: "docs".to_string(), + outcome: register(plugin_source("alpha@test")), + contenders: vec![ + register(plugin_source("beta@test")), + register(plugin_source("alpha@test")), + ], + }] + ); +} + +#[test] +fn equal_precedence_uses_insertion_order_not_source_identity() { + let mut builder = ResolvedMcpCatalog::builder(); + builder.register(McpServerRegistration::from_compatibility( + "docs".to_string(), + "z-first", + server("https://first.example/mcp"), + )); + builder.register(McpServerRegistration::from_compatibility( + "docs".to_string(), + "a-second", + server("https://second.example/mcp"), + )); + + let catalog = builder.build(); + + assert_eq!( + catalog.server("docs"), + Some(&super::ResolvedMcpServer { + source: compatibility_source("a-second"), + config: server("https://second.example/mcp"), + }) + ); + let mut builder = catalog.to_builder(); + builder.remove_compatibility("docs".to_string(), "remove-last"); + + let catalog = builder.build(); + + assert_eq!(catalog.server("docs"), None); + assert_eq!( + catalog.conflicts(), + &[McpServerConflict { + name: "docs".to_string(), + outcome: remove(compatibility_source("remove-last")), + contenders: vec![ + register(compatibility_source("z-first")), + register(compatibility_source("a-second")), + remove(compatibility_source("remove-last")), + ], + }] + ); +} diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rs index 659d357189..9d1be43d42 100644 --- a/codex-rs/codex-mcp/src/connection_manager.rs +++ b/codex-rs/codex-mcp/src/connection_manager.rs @@ -348,6 +348,10 @@ impl McpConnectionManager { !self.clients.is_empty() } + pub(crate) fn contains_server(&self, server_name: &str) -> bool { + self.clients.contains_key(server_name) + } + /// Stop all MCP clients owned by this manager and terminate stdio server processes. pub async fn shutdown(&self) { self.startup_cancellation_token.cancel(); diff --git a/codex-rs/codex-mcp/src/lib.rs b/codex-rs/codex-mcp/src/lib.rs index f0c7c3c8ca..a270eb4e7e 100644 --- a/codex-rs/codex-mcp/src/lib.rs +++ b/codex-rs/codex-mcp/src/lib.rs @@ -3,11 +3,22 @@ pub use connection_manager::tool_is_model_visible; pub use elicitation::ElicitationReviewRequest; pub use elicitation::ElicitationReviewer; pub use elicitation::ElicitationReviewerHandle; +pub use resource_client::McpResourceClient; +pub use resource_client::McpResourcePage; +pub use resource_client::McpResourceReadResult; pub use rmcp_client::MCP_SANDBOX_STATE_META_CAPABILITY; pub use runtime::McpRuntimeContext; pub use runtime::SandboxState; pub use tools::ToolInfo; +pub use catalog::McpCatalogBuilder; +pub use catalog::McpServerConflict; +pub use catalog::McpServerConflictAction; +pub use catalog::McpServerRegistration; +pub use catalog::McpServerSource; +pub use catalog::ResolvedMcpCatalog; +pub use catalog::ResolvedMcpServer; + pub use mcp::CODEX_APPS_MCP_SERVER_NAME; pub use mcp::McpConfig; pub use mcp::ToolPluginProvenance; @@ -54,10 +65,12 @@ pub use mcp::qualified_mcp_tool_name_prefix; pub use tools::declared_openai_file_input_param_names; pub(crate) mod auth_elicitation; +mod catalog; pub(crate) mod codex_apps; pub(crate) mod connection_manager; pub(crate) mod elicitation; pub(crate) mod mcp; +mod resource_client; pub(crate) mod rmcp_client; pub(crate) mod runtime; pub(crate) mod server; diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index e6b17b14c3..eca8a28c57 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -37,6 +37,7 @@ use rmcp::model::ReadResourceResult; use serde_json::Value; use tokio_util::sync::CancellationToken; +use crate::ResolvedMcpCatalog; use crate::codex_apps::codex_apps_tools_cache_key; use crate::connection_manager::McpConnectionManager; use crate::runtime::McpRuntimeContext; @@ -135,13 +136,8 @@ pub struct McpConfig { pub prefix_mcp_tool_names: bool, /// Client-side elicitation capabilities advertised during MCP initialization. pub client_elicitation_capability: ElicitationCapability, - /// Materialized MCP servers keyed by server name. - /// - /// A host may add compatibility built-ins and extension overlays before - /// calling runtime entry points in this crate. - pub configured_mcp_servers: HashMap, - /// Winning plugin owner for plugin-provided MCP servers, keyed by server name. - pub plugin_ids_by_mcp_server_name: HashMap, + /// Resolved MCP registrations keyed by logical server name. + pub mcp_server_catalog: ResolvedMcpCatalog, /// Plugin metadata used to attribute MCP tools/connectors to plugin display names. pub plugin_capability_summaries: Vec, } @@ -176,6 +172,7 @@ impl ToolPluginProvenance { fn from_config(config: &McpConfig) -> Self { let mut tool_plugin_provenance = Self::default(); + let plugin_ids_by_mcp_server_name = config.mcp_server_catalog.plugin_ids_by_server_name(); for plugin in &config.plugin_capability_summaries { for connector_id in &plugin.app_connector_ids { tool_plugin_provenance @@ -185,7 +182,9 @@ impl ToolPluginProvenance { .push(plugin.display_name.clone()); } - for server_name in &plugin.mcp_server_names { + for server_name in plugin.mcp_server_names.iter().filter(|server_name| { + plugin_ids_by_mcp_server_name.get(*server_name) == Some(&plugin.config_name) + }) { tool_plugin_provenance .plugin_display_names_by_mcp_server_name .entry(server_name.clone()) @@ -206,8 +205,7 @@ impl ToolPluginProvenance { plugin_names.sort_unstable(); plugin_names.dedup(); } - tool_plugin_provenance.plugin_ids_by_mcp_server_name = - config.plugin_ids_by_mcp_server_name.clone(); + tool_plugin_provenance.plugin_ids_by_mcp_server_name = plugin_ids_by_mcp_server_name; tool_plugin_provenance } @@ -218,7 +216,7 @@ pub fn host_owned_codex_apps_enabled(config: &McpConfig, auth: Option<&CodexAuth } pub fn configured_mcp_servers(config: &McpConfig) -> HashMap { - config.configured_mcp_servers.clone() + config.mcp_server_catalog.configured_servers() } pub fn effective_mcp_servers( diff --git a/codex-rs/codex-mcp/src/mcp/mod_tests.rs b/codex-rs/codex-mcp/src/mcp/mod_tests.rs index 767f20611f..111c810a19 100644 --- a/codex-rs/codex-mcp/src/mcp/mod_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/mod_tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::McpServerRegistration; use codex_config::Constrained; use codex_config::types::AppToolApproval; use codex_login::CodexAuth; @@ -28,8 +29,7 @@ fn test_mcp_config(codex_home: PathBuf) -> McpConfig { apps_enabled: false, prefix_mcp_tool_names: true, client_elicitation_capability: ElicitationCapability::default(), - configured_mcp_servers: HashMap::new(), - plugin_ids_by_mcp_server_name: HashMap::new(), + mcp_server_catalog: ResolvedMcpCatalog::default(), plugin_capability_summaries: Vec::new(), } } @@ -122,16 +122,24 @@ fn mcp_prompt_auto_approval_rejects_auto_mode_in_default_permission_mode() { #[test] fn tool_plugin_provenance_collects_app_and_mcp_sources() { let mut config = test_mcp_config(PathBuf::new()); - config.plugin_ids_by_mcp_server_name = - HashMap::from([("alpha".to_string(), "alpha@test".to_string())]); + let mut catalog = ResolvedMcpCatalog::builder(); + catalog.register(McpServerRegistration::from_plugin( + "alpha".to_string(), + "alpha@test".to_string(), + /*plugin_order*/ 0, + codex_apps_mcp_server_config("https://alpha.example", /*apps_mcp_product_sku*/ None), + )); + config.mcp_server_catalog = catalog.build(); config.plugin_capability_summaries = vec![ PluginCapabilitySummary { + config_name: "alpha@test".to_string(), display_name: "alpha-plugin".to_string(), app_connector_ids: vec![AppConnectorId("connector_example".to_string())], mcp_server_names: vec!["alpha".to_string()], ..PluginCapabilitySummary::default() }, PluginCapabilitySummary { + config_name: "beta@test".to_string(), display_name: "beta-plugin".to_string(), app_connector_ids: vec![ AppConnectorId("connector_example".to_string()), @@ -156,10 +164,10 @@ fn tool_plugin_provenance_collects_app_and_mcp_sources() { vec!["beta-plugin".to_string()], ), ]), - plugin_display_names_by_mcp_server_name: HashMap::from([ - ("alpha".to_string(), vec!["alpha-plugin".to_string()]), - ("beta".to_string(), vec!["beta-plugin".to_string()]), - ]), + plugin_display_names_by_mcp_server_name: HashMap::from([( + "alpha".to_string(), + vec!["alpha-plugin".to_string()], + )]), plugin_ids_by_mcp_server_name: HashMap::from([( "alpha".to_string(), "alpha@test".to_string(), @@ -235,7 +243,8 @@ async fn effective_mcp_servers_preserve_runtime_servers() { config.apps_enabled = true; let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); - config.configured_mcp_servers.insert( + let mut catalog = ResolvedMcpCatalog::builder(); + catalog.register(McpServerRegistration::from_config( "sample".to_string(), McpServerConfig { transport: McpServerTransportConfig::StreamableHttp { @@ -259,8 +268,8 @@ async fn effective_mcp_servers_preserve_runtime_servers() { oauth_resource: None, tools: HashMap::new(), }, - ); - config.configured_mcp_servers.insert( + )); + catalog.register(McpServerRegistration::from_config( "docs".to_string(), McpServerConfig { transport: McpServerTransportConfig::StreamableHttp { @@ -284,14 +293,15 @@ async fn effective_mcp_servers_preserve_runtime_servers() { oauth_resource: None, tools: HashMap::new(), }, - ); - config.configured_mcp_servers.insert( + )); + catalog.register(McpServerRegistration::from_config( CODEX_APPS_MCP_SERVER_NAME.to_string(), codex_apps_mcp_server_config( &config.chatgpt_base_url, config.apps_mcp_product_sku.as_deref(), ), - ); + )); + config.mcp_server_catalog = catalog.build(); let effective = effective_mcp_servers(&config, Some(&auth)); diff --git a/codex-rs/codex-mcp/src/resource_client.rs b/codex-rs/codex-mcp/src/resource_client.rs new file mode 100644 index 0000000000..85dc974f20 --- /dev/null +++ b/codex-rs/codex-mcp/src/resource_client.rs @@ -0,0 +1,108 @@ +use std::sync::Arc; + +use anyhow::Context; +use anyhow::Result; +use arc_swap::ArcSwap; +use codex_protocol::mcp::Resource; +use codex_protocol::mcp::ResourceContent; +use rmcp::model::PaginatedRequestParams; +use rmcp::model::ReadResourceRequestParams; + +use crate::McpConnectionManager; + +/// One page of resources returned by an MCP server. +#[derive(Clone, Debug, PartialEq)] +pub struct McpResourcePage { + /// Resources advertised on this page. + pub resources: Vec, + /// Opaque cursor to supply when requesting the next page. + pub next_cursor: Option, +} + +/// Contents returned after reading one MCP resource. +#[derive(Clone, Debug, PartialEq)] +pub struct McpResourceReadResult { + /// Text or blob content returned for the requested resource. + pub contents: Vec, +} + +/// Session-scoped access to MCP resources through the currently installed manager. +/// +/// The client retains the manager's shared publication handle rather than a manager +/// snapshot, so calls automatically use replacements installed during startup and refresh. +#[derive(Clone)] +pub struct McpResourceClient { + manager: Arc>, +} + +impl std::fmt::Debug for McpResourceClient { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("McpResourceClient") + .finish_non_exhaustive() + } +} + +impl McpResourceClient { + /// Creates a resource client backed by the session's replaceable MCP manager. + pub fn new(manager: Arc>) -> Self { + Self { manager } + } + + /// Returns whether the current manager contains the named server. + /// + /// This does not wait for server startup or imply that startup succeeded. + pub async fn has_server(&self, server: &str) -> bool { + self.manager.load_full().contains_server(server) + } + + /// Lists one resource page from the named server. + pub async fn list_resources( + &self, + server: &str, + cursor: Option, + ) -> Result { + let params = + cursor.map(|cursor| PaginatedRequestParams::default().with_cursor(Some(cursor))); + let result = self + .manager + .load_full() + .list_resources(server, params) + .await?; + let resources = result + .resources + .into_iter() + .map(resource_from_rmcp) + .collect::>>()?; + Ok(McpResourcePage { + resources, + next_cursor: result.next_cursor, + }) + } + + /// Reads one resource from the named server. + pub async fn read_resource(&self, server: &str, uri: &str) -> Result { + let result = self + .manager + .load_full() + .read_resource(server, ReadResourceRequestParams::new(uri.to_string())) + .await?; + let contents = result + .contents + .into_iter() + .map(resource_content_from_rmcp) + .collect::>>()?; + Ok(McpResourceReadResult { contents }) + } +} + +fn resource_from_rmcp(resource: rmcp::model::Resource) -> Result { + let value = serde_json::to_value(resource).context("failed to serialize MCP resource")?; + Resource::from_mcp_value(value).context("failed to convert MCP resource") +} + +fn resource_content_from_rmcp(content: rmcp::model::ResourceContents) -> Result { + let value = + serde_json::to_value(content).context("failed to serialize MCP resource content")?; + serde_json::from_value(value).context("failed to convert MCP resource content") +} diff --git a/codex-rs/config/Cargo.toml b/codex-rs/config/Cargo.toml index f77a76b764..e2c650dccc 100644 --- a/codex-rs/config/Cargo.toml +++ b/codex-rs/config/Cargo.toml @@ -25,6 +25,7 @@ codex-network-proxy = { workspace = true } codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-path = { workspace = true } +codex-utils-path-uri = { workspace = true } dunce = { workspace = true } futures = { workspace = true, features = ["alloc", "std"] } gethostname = { workspace = true } diff --git a/codex-rs/config/src/loader/layer_io.rs b/codex-rs/config/src/loader/layer_io.rs index 415d82e405..7cdb4f5359 100644 --- a/codex-rs/config/src/loader/layer_io.rs +++ b/codex-rs/config/src/loader/layer_io.rs @@ -10,6 +10,7 @@ use crate::strict_config::config_error_from_ignored_toml_value_fields; use codex_file_system::ExecutorFileSystem; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::AbsolutePathBufGuard; +use codex_utils_path_uri::PathUri; use std::io; use std::path::Path; use std::path::PathBuf; @@ -106,7 +107,8 @@ pub(super) async fn read_config_from_path( log_missing_as_info: bool, strict_config: bool, ) -> io::Result> { - match fs.read_file_text(path, /*sandbox*/ None).await { + let path_uri = PathUri::from_abs_path(path)?; + match fs.read_file_text(&path_uri, /*sandbox*/ None).await { Ok(contents) => match toml::from_str::(&contents) { Ok(value) => { if strict_config { diff --git a/codex-rs/config/src/loader/mod.rs b/codex-rs/config/src/loader/mod.rs index ccd2835f6d..8ff39a7e9d 100644 --- a/codex-rs/config/src/loader/mod.rs +++ b/codex-rs/config/src/loader/mod.rs @@ -40,6 +40,7 @@ use codex_protocol::config_types::TrustLevel; use codex_protocol::protocol::AskForApproval; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::AbsolutePathBufGuard; +use codex_utils_path_uri::PathUri; use dunce::canonicalize as normalize_path; use serde::Deserialize; use std::io; @@ -471,7 +472,8 @@ async fn load_config_toml_for_required_layer( strict_config: bool, create_entry: impl FnOnce(TomlValue) -> ConfigLayerEntry, ) -> io::Result { - let toml_value = match fs.read_file_text(toml_file, /*sandbox*/ None).await { + let toml_file_uri = PathUri::from_abs_path(toml_file)?; + let toml_value = match fs.read_file_text(&toml_file_uri, /*sandbox*/ None).await { Ok(contents) => { let config_parent = toml_file.as_path().parent().ok_or_else(|| { io::Error::new( @@ -566,8 +568,9 @@ pub async fn load_requirements_toml( fs: &dyn ExecutorFileSystem, requirements_toml_file: &AbsolutePathBuf, ) -> io::Result> { + let requirements_toml_file_uri = PathUri::from_abs_path(requirements_toml_file)?; match fs - .read_file_text(requirements_toml_file, /*sandbox*/ None) + .read_file_text(&requirements_toml_file_uri, /*sandbox*/ None) .await { Ok(contents) => { @@ -1140,8 +1143,9 @@ async fn find_project_root( for ancestor in cwd.ancestors() { for marker in project_root_markers { let marker_path = ancestor.join(marker); + let marker_path_uri = PathUri::from_abs_path(&marker_path)?; if fs - .get_metadata(&marker_path, /*sandbox*/ None) + .get_metadata(&marker_path_uri, /*sandbox*/ None) .await .is_ok() { @@ -1156,14 +1160,20 @@ async fn find_git_checkout_root( fs: &dyn ExecutorFileSystem, cwd: &AbsolutePathBuf, ) -> Option { - let base = match fs.get_metadata(cwd, /*sandbox*/ None).await { + let cwd_uri = PathUri::from_abs_path(cwd).ok()?; + let base = match fs.get_metadata(&cwd_uri, /*sandbox*/ None).await { Ok(metadata) if metadata.is_directory => cwd.clone(), _ => cwd.parent()?, }; for dir in base.ancestors() { let dot_git = dir.join(".git"); - if fs.get_metadata(&dot_git, /*sandbox*/ None).await.is_ok() { + let dot_git_uri = PathUri::from_abs_path(&dot_git).ok()?; + if fs + .get_metadata(&dot_git_uri, /*sandbox*/ None) + .await + .is_ok() + { return Some(dir); } } @@ -1211,8 +1221,9 @@ async fn load_project_layers( let mut startup_warnings = Vec::new(); for dir in dirs { let dot_codex_abs = dir.join(".codex"); + let dot_codex_uri = PathUri::from_abs_path(&dot_codex_abs)?; if !fs - .get_metadata(&dot_codex_abs, /*sandbox*/ None) + .get_metadata(&dot_codex_uri, /*sandbox*/ None) .await .map(|metadata| metadata.is_directory) .unwrap_or(false) @@ -1229,7 +1240,8 @@ async fn load_project_layers( continue; } let config_file = dot_codex_abs.join(CONFIG_TOML_FILE); - match fs.read_file_text(&config_file, /*sandbox*/ None).await { + let config_file_uri = PathUri::from_abs_path(&config_file)?; + match fs.read_file_text(&config_file_uri, /*sandbox*/ None).await { Ok(contents) => { let config: TomlValue = match toml::from_str(&contents) { Ok(config) => config, @@ -1332,8 +1344,9 @@ async fn merge_root_checkout_project_hooks( return Ok(config); }; let hooks_config_file = hooks_config_folder.join(CONFIG_TOML_FILE); + let hooks_config_file_uri = PathUri::from_abs_path(&hooks_config_file)?; let root_config = match fs - .read_file_text(&hooks_config_file, /*sandbox*/ None) + .read_file_text(&hooks_config_file_uri, /*sandbox*/ None) .await { Ok(contents) => { diff --git a/codex-rs/config/src/loader/tests.rs b/codex-rs/config/src/loader/tests.rs index 6492f9a0c0..e349ba0c68 100644 --- a/codex-rs/config/src/loader/tests.rs +++ b/codex-rs/config/src/loader/tests.rs @@ -7,8 +7,8 @@ use codex_file_system::FileSystemResult; use codex_file_system::FileSystemSandboxContext; use codex_file_system::ReadDirectoryEntry; use codex_file_system::RemoveOptions; +use codex_utils_path_uri::PathUri; use pretty_assertions::assert_eq; -use std::path::Path; use tempfile::tempdir; struct TestFileSystem; @@ -17,35 +17,26 @@ struct TestFileSystem; impl ExecutorFileSystem for TestFileSystem { async fn canonicalize( &self, - path: &AbsolutePathBuf, + path: &PathUri, _sandbox: Option<&FileSystemSandboxContext>, - ) -> FileSystemResult { - path.canonicalize() - } - - async fn join( - &self, - base_path: &AbsolutePathBuf, - path: &Path, - ) -> FileSystemResult { - Ok(base_path.join(path)) - } - - async fn parent(&self, path: &AbsolutePathBuf) -> FileSystemResult> { - Ok(path.parent()) + ) -> FileSystemResult { + let path = path.to_abs_path()?; + let canonicalized = path.canonicalize()?; + PathUri::from_abs_path(&canonicalized) } async fn read_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { + let path = path.to_abs_path()?; tokio::fs::read(path.as_path()).await } async fn write_file( &self, - _path: &AbsolutePathBuf, + _path: &PathUri, _contents: Vec, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -54,7 +45,7 @@ impl ExecutorFileSystem for TestFileSystem { async fn create_directory( &self, - _path: &AbsolutePathBuf, + _path: &PathUri, _create_directory_options: CreateDirectoryOptions, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -63,7 +54,7 @@ impl ExecutorFileSystem for TestFileSystem { async fn get_metadata( &self, - _path: &AbsolutePathBuf, + _path: &PathUri, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult { unimplemented!("test filesystem only supports reads") @@ -71,7 +62,7 @@ impl ExecutorFileSystem for TestFileSystem { async fn read_directory( &self, - _path: &AbsolutePathBuf, + _path: &PathUri, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { unimplemented!("test filesystem only supports reads") @@ -79,7 +70,7 @@ impl ExecutorFileSystem for TestFileSystem { async fn remove( &self, - _path: &AbsolutePathBuf, + _path: &PathUri, _remove_options: RemoveOptions, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -88,8 +79,8 @@ impl ExecutorFileSystem for TestFileSystem { async fn copy( &self, - _source_path: &AbsolutePathBuf, - _destination_path: &AbsolutePathBuf, + _source_path: &PathUri, + _destination_path: &PathUri, _copy_options: CopyOptions, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { diff --git a/codex-rs/core-api/Cargo.toml b/codex-rs/core-api/Cargo.toml index 998da39ce7..f4c89e5640 100644 --- a/codex-rs/core-api/Cargo.toml +++ b/codex-rs/core-api/Cargo.toml @@ -20,6 +20,7 @@ codex-analytics = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } +codex-home = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } codex-login = { workspace = true } diff --git a/codex-rs/core-api/src/lib.rs b/codex-rs/core-api/src/lib.rs index 47452eb032..74fe9d5a0c 100644 --- a/codex-rs/core-api/src/lib.rs +++ b/codex-rs/core-api/src/lib.rs @@ -48,9 +48,14 @@ pub use codex_core::skills::SkillsManager; pub use codex_core::thread_store_from_config; pub use codex_exec_server::EnvironmentManager; pub use codex_exec_server::ExecServerRuntimePaths; +pub use codex_extension_api::LoadUserInstructionsFuture; +pub use codex_extension_api::LoadedUserInstructions; +pub use codex_extension_api::UserInstructions; +pub use codex_extension_api::UserInstructionsProvider; pub use codex_extension_api::empty_extension_registry; pub use codex_features::Feature; pub use codex_features::Features; +pub use codex_home::CodexHomeUserInstructionsProvider; pub use codex_login::AuthManager; pub use codex_login::default_client::set_default_originator; pub use codex_model_provider_info::OPENAI_PROVIDER_ID; diff --git a/codex-rs/core-plugins/src/discoverable.rs b/codex-rs/core-plugins/src/discoverable.rs index 28a90c1928..a630877f55 100644 --- a/codex-rs/core-plugins/src/discoverable.rs +++ b/codex-rs/core-plugins/src/discoverable.rs @@ -68,6 +68,7 @@ pub struct ToolSuggestPluginDiscoveryInput { #[derive(Clone, Debug, PartialEq, Eq)] pub struct ToolSuggestDiscoverablePlugin { pub id: String, + pub remote_plugin_id: Option, pub name: String, pub description: Option, pub has_skills: bool, @@ -86,7 +87,11 @@ impl PluginsManager { } let marketplaces = self - .list_marketplaces_for_config(&input.plugins, &[]) + .list_marketplaces_for_config( + &input.plugins, + &[], + /*include_openai_curated*/ !input.plugins.remote_plugin_enabled, + ) .context("failed to list plugin marketplaces for tool suggestions")? .marketplaces; let mut installed_app_connector_ids = self @@ -109,11 +114,6 @@ impl PluginsManager { let mut discoverable_plugins = Vec::::new(); for marketplace in marketplaces { let marketplace_name = marketplace.name; - if input.plugins.remote_plugin_enabled - && marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME - { - continue; - } let use_legacy_local_curated_filter = should_use_legacy_local_curated_discovery_filter( &marketplace_name, marketplace.path.as_path(), @@ -162,6 +162,7 @@ impl PluginsManager { discoverable_plugins.push(ToolSuggestDiscoverablePlugin { id: plugin.config_name, + remote_plugin_id: None, name: plugin.display_name, description: plugin.description, has_skills: plugin.has_skills, @@ -217,6 +218,7 @@ impl PluginsManager { discoverable_plugins.push(ToolSuggestDiscoverablePlugin { id: plugin.config_id, + remote_plugin_id: Some(plugin.remote_plugin_id), name: plugin.name, description: plugin.description, has_skills: plugin.has_skills, diff --git a/codex-rs/core-plugins/src/loader.rs b/codex-rs/core-plugins/src/loader.rs index 9dfa0bbc6f..1cc9430172 100644 --- a/codex-rs/core-plugins/src/loader.rs +++ b/codex-rs/core-plugins/src/loader.rs @@ -61,6 +61,12 @@ pub struct PluginHookLoadOutcome { pub hook_load_warnings: Vec, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PluginAppMetadata { + pub id: AppConnectorId, + pub category: Option, +} + enum PluginLoadScope<'a> { AllCapabilities { restriction_product: Option, @@ -123,6 +129,7 @@ struct PluginAppFile { #[derive(Debug, Default, Deserialize)] struct PluginAppConfig { id: String, + category: Option, } pub async fn load_plugins_from_layer_stack( @@ -848,6 +855,14 @@ fn default_mcp_config_paths(plugin_root: &Path) -> Vec { } pub async fn load_plugin_apps(plugin_root: &Path) -> Vec { + load_plugin_app_metadata(plugin_root) + .await + .into_iter() + .map(|app| app.id) + .collect() +} + +pub async fn load_plugin_app_metadata(plugin_root: &Path) -> Vec { if let Some(manifest) = load_plugin_manifest(plugin_root) { return load_apps_from_paths( plugin_root, @@ -858,6 +873,16 @@ pub async fn load_plugin_apps(plugin_root: &Path) -> Vec { load_apps_from_paths(plugin_root, default_app_config_paths(plugin_root)).await } +pub fn plugin_app_metadata_from_value(value: &JsonValue) -> Vec { + let Ok(parsed) = serde_json::from_value::(value.clone()) else { + return Vec::new(); + }; + let mut apps = plugin_app_metadata_from_file(parsed, /*plugin_root*/ None); + let mut seen_connector_ids = HashSet::new(); + apps.retain(|app| seen_connector_ids.insert(app.id.0.clone())); + apps +} + fn plugin_app_config_paths( plugin_root: &Path, manifest_paths: &PluginManifestPaths, @@ -994,8 +1019,8 @@ fn append_plugin_hook_file( async fn load_apps_from_paths( plugin_root: &Path, app_config_paths: Vec, -) -> Vec { - let mut connector_ids = Vec::new(); +) -> Vec { + let mut apps = Vec::new(); for app_config_path in app_config_paths { let Ok(contents) = tokio::fs::read_to_string(app_config_path.as_path()).await else { continue; @@ -1011,21 +1036,40 @@ async fn load_apps_from_paths( } }; - connector_ids.extend(parsed.apps.into_values().filter_map(|app| { - if app.id.trim().is_empty() { - warn!( - plugin = %plugin_root.display(), - "plugin app config is missing an app id" - ); - None - } else { - Some(AppConnectorId(app.id)) - } - })); + apps.extend(plugin_app_metadata_from_file(parsed, Some(plugin_root))); } let mut seen_connector_ids = HashSet::new(); - connector_ids.retain(|connector_id| seen_connector_ids.insert(connector_id.0.clone())); - connector_ids + apps.retain(|app| seen_connector_ids.insert(app.id.0.clone())); + apps +} + +fn plugin_app_metadata_from_file( + parsed: PluginAppFile, + plugin_root: Option<&Path>, +) -> Vec { + parsed + .apps + .into_values() + .filter_map(|app| { + if app.id.trim().is_empty() { + if let Some(plugin_root) = plugin_root { + warn!( + plugin = %plugin_root.display(), + "plugin app config is missing an app id" + ); + } + None + } else { + Some(PluginAppMetadata { + id: AppConnectorId(app.id), + category: app + .category + .map(|category| category.trim().to_string()) + .filter(|category| !category.is_empty()), + }) + } + }) + .collect() } pub async fn plugin_telemetry_metadata_from_root( @@ -1063,7 +1107,10 @@ pub async fn plugin_telemetry_metadata_from_root( plugin_root.as_path(), plugin_app_config_paths(plugin_root.as_path(), manifest_paths), ) - .await, + .await + .into_iter() + .map(|app| app.id) + .collect(), }), } } diff --git a/codex-rs/core-plugins/src/manager.rs b/codex-rs/core-plugins/src/manager.rs index 095c5fd69d..59bf365487 100644 --- a/codex-rs/core-plugins/src/manager.rs +++ b/codex-rs/core-plugins/src/manager.rs @@ -5,7 +5,7 @@ use crate::loader::PluginHookLoadOutcome; use crate::loader::configured_curated_plugin_ids_from_codex_home; use crate::loader::curated_plugin_cache_version; use crate::loader::installed_plugin_telemetry_metadata; -use crate::loader::load_plugin_apps; +use crate::loader::load_plugin_app_metadata; use crate::loader::load_plugin_hooks; use crate::loader::load_plugin_hooks_from_layer_stack; use crate::loader::load_plugin_mcp_servers; @@ -48,6 +48,7 @@ use crate::store::PluginInstallResult as StorePluginInstallResult; use crate::store::PluginStore; use crate::store::PluginStoreError; use codex_analytics::AnalyticsEventsClient; +use codex_app_server_protocol::AuthMode; use codex_config::ConfigLayerStack; use codex_config::clear_user_plugin; use codex_config::set_user_plugin_enabled; @@ -203,6 +204,13 @@ fn featured_plugin_ids_cache_key( } } +fn project_plugin_load_outcome_for_auth( + outcome: PluginLoadOutcome, + _auth_mode: Option, +) -> PluginLoadOutcome { + outcome +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct PluginInstallRequest { pub plugin_name: String, @@ -246,6 +254,7 @@ pub struct PluginDetail { pub disabled_skill_paths: HashSet, pub hooks: Vec, pub apps: Vec, + pub app_category_by_id: HashMap, pub mcp_server_names: Vec, pub details_unavailable_reason: Option, } @@ -319,6 +328,7 @@ pub struct PluginsManager { remote_installed_plugins_cache_refresh_state: RwLock, global_remote_catalog_cache_refresh_state: RwLock, restriction_product: Option, + auth_mode: RwLock>, analytics_events_client: RwLock>, } @@ -375,10 +385,30 @@ impl PluginsManager { GlobalRemoteCatalogCacheRefreshState::default(), ), restriction_product, + auth_mode: RwLock::new(None), analytics_events_client: RwLock::new(None), } } + pub fn set_auth_mode(&self, auth_mode: Option) -> bool { + let mut stored_auth_mode = match self.auth_mode.write() { + Ok(auth_mode_guard) => auth_mode_guard, + Err(err) => err.into_inner(), + }; + if *stored_auth_mode == auth_mode { + return false; + } + *stored_auth_mode = auth_mode; + true + } + + pub fn auth_mode(&self) -> Option { + match self.auth_mode.read() { + Ok(auth_mode_guard) => *auth_mode_guard, + Err(err) => *err.into_inner(), + } + } + pub fn set_analytics_events_client(&self, analytics_events_client: AnalyticsEventsClient) { let mut stored_client = match self.analytics_events_client.write() { Ok(client_guard) => client_guard, @@ -417,7 +447,7 @@ impl PluginsManager { remote_plugin_enabled: config.remote_plugin_enabled, }; if !force_reload && let Some(outcome) = self.cached_enabled_outcome(&cache_key) { - return outcome; + return self.project_plugins_for_auth(outcome); } let Ok(_load_permit) = self.enabled_outcome_load_semaphore.acquire().await else { @@ -425,7 +455,7 @@ impl PluginsManager { return PluginLoadOutcome::default(); }; if !force_reload && let Some(outcome) = self.cached_enabled_outcome(&cache_key) { - return outcome; + return self.project_plugins_for_auth(outcome); } let cache_generation = self.enabled_outcome_cache_generation(); let outcome = load_plugins_from_layer_stack( @@ -438,7 +468,11 @@ impl PluginsManager { .await; log_plugin_load_errors(&outcome); self.cache_enabled_outcome_if_current(cache_generation, cache_key, outcome.clone()); - outcome + self.project_plugins_for_auth(outcome) + } + + fn project_plugins_for_auth(&self, outcome: PluginLoadOutcome) -> PluginLoadOutcome { + project_plugin_load_outcome_for_auth(outcome, self.auth_mode()) } pub fn clear_cache(&self) { @@ -468,14 +502,15 @@ impl PluginsManager { if !config.plugins_enabled { return PluginLoadOutcome::default(); } - load_plugins_from_layer_stack( + let outcome = load_plugins_from_layer_stack( config_layer_stack, self.remote_installed_plugin_configs(), &self.store, self.restriction_product, config.remote_plugin_enabled, ) - .await + .await; + self.project_plugins_for_auth(outcome) } /// Resolve plugin hooks for a config layer stack without loading other plugin capabilities. @@ -992,14 +1027,19 @@ impl PluginsManager { &self, config: &PluginsConfigInput, additional_roots: &[AbsolutePathBuf], + include_openai_curated: bool, ) -> Result { if !config.plugins_enabled { return Ok(ConfiguredMarketplaceListOutcome::default()); } let (installed_plugins, enabled_plugins) = self.configured_plugin_states(config); - let marketplace_outcome = - self.discover_marketplaces_for_config(config, additional_roots)?; + let mut marketplace_roots = self.marketplace_roots(config, additional_roots); + if !include_openai_curated { + let curated_repo_root = curated_plugins_repo_path(self.codex_home.as_path()); + marketplace_roots.retain(|root| root.as_path() != curated_repo_root.as_path()); + } + let marketplace_outcome = list_marketplaces(&marketplace_roots)?; let mut seen_plugin_keys = HashSet::new(); let marketplaces = marketplace_outcome .marketplaces @@ -1185,6 +1225,7 @@ impl PluginsManager { disabled_skill_paths: HashSet::new(), hooks: Vec::new(), apps: Vec::new(), + app_category_by_id: HashMap::new(), mcp_server_names: Vec::new(), details_unavailable_reason: Some( PluginDetailsUnavailableReason::InstallRequiredForRemoteSource, @@ -1251,7 +1292,12 @@ impl PluginsManager { event_name: hook.event_name, }) .collect(); - let apps = load_plugin_apps(source_path.as_path()).await; + let app_metadata = load_plugin_app_metadata(source_path.as_path()).await; + let apps = app_metadata.iter().map(|app| app.id.clone()).collect(); + let app_category_by_id = app_metadata + .into_iter() + .filter_map(|app| app.category.map(|category| (app.id.0, category))) + .collect(); let mut mcp_server_names = load_plugin_mcp_servers(source_path.as_path()) .await .into_keys() @@ -1274,6 +1320,7 @@ impl PluginsManager { disabled_skill_paths: resolved_skills.disabled_skill_paths, hooks, apps, + app_category_by_id, mcp_server_names, details_unavailable_reason: None, }) diff --git a/codex-rs/core-plugins/src/manager_tests.rs b/codex-rs/core-plugins/src/manager_tests.rs index c3961796cf..1a268a4888 100644 --- a/codex-rs/core-plugins/src/manager_tests.rs +++ b/codex-rs/core-plugins/src/manager_tests.rs @@ -17,6 +17,7 @@ use crate::test_support::load_plugins_config as load_plugins_config_input; use crate::test_support::write_curated_plugin_sha_with as write_curated_plugin_sha; use crate::test_support::write_file; use crate::test_support::write_openai_curated_marketplace; +use codex_app_server_protocol::AuthMode; use codex_app_server_protocol::ConfigLayerSource; use codex_config::AppToolApproval; use codex_config::CONFIG_TOML_FILE; @@ -47,6 +48,21 @@ use wiremock::matchers::query_param; const MAX_CAPABILITY_SUMMARY_DESCRIPTION_LEN: usize = 1024; +#[test] +fn plugins_manager_tracks_auth_mode() { + let tmp = TempDir::new().unwrap(); + let manager = PluginsManager::new(tmp.path().to_path_buf()); + + assert_eq!(manager.auth_mode(), None); + assert!(manager.set_auth_mode(Some(AuthMode::ApiKey))); + assert_eq!(manager.auth_mode(), Some(AuthMode::ApiKey)); + assert!(!manager.set_auth_mode(Some(AuthMode::ApiKey))); + assert!(manager.set_auth_mode(Some(AuthMode::ChatgptAuthTokens))); + assert_eq!(manager.auth_mode(), Some(AuthMode::ChatgptAuthTokens)); + assert!(manager.set_auth_mode(/*auth_mode*/ None)); + assert_eq!(manager.auth_mode(), None); +} + fn write_plugin_with_version( root: &Path, dir_name: &str, @@ -1815,7 +1831,11 @@ enabled = false let config = load_config(tmp.path(), &repo_root).await; let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) - .list_marketplaces_for_config(&config, &[AbsolutePathBuf::try_from(repo_root).unwrap()]) + .list_marketplaces_for_config( + &config, + &[AbsolutePathBuf::try_from(repo_root).unwrap()], + /*include_openai_curated*/ true, + ) .unwrap() .marketplaces; @@ -1917,7 +1937,11 @@ enabled = true let config = load_config(tmp.path(), &repo_root).await; let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) - .list_marketplaces_for_config(&config, &[AbsolutePathBuf::try_from(repo_root).unwrap()]) + .list_marketplaces_for_config( + &config, + &[AbsolutePathBuf::try_from(repo_root).unwrap()], + /*include_openai_curated*/ true, + ) .unwrap() .marketplaces; @@ -1965,7 +1989,11 @@ plugins = true let config = load_config(tmp.path(), &repo_root).await; let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) - .list_marketplaces_for_config(&config, &[AbsolutePathBuf::try_from(repo_root).unwrap()]) + .list_marketplaces_for_config( + &config, + &[AbsolutePathBuf::try_from(repo_root).unwrap()], + /*include_openai_curated*/ true, + ) .unwrap() .marketplaces; @@ -2402,7 +2430,11 @@ enabled = true let config = load_config(tmp.path(), &repo_root).await; let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) - .list_marketplaces_for_config(&config, &[AbsolutePathBuf::try_from(repo_root).unwrap()]) + .list_marketplaces_for_config( + &config, + &[AbsolutePathBuf::try_from(repo_root).unwrap()], + /*include_openai_curated*/ true, + ) .unwrap() .marketplaces; @@ -2496,7 +2528,7 @@ plugins = true let config = load_config(tmp.path(), tmp.path()).await; let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) - .list_marketplaces_for_config(&config, &[]) + .list_marketplaces_for_config(&config, &[], /*include_openai_curated*/ true) .unwrap() .marketplaces; @@ -2534,6 +2566,37 @@ plugins = true ); } +#[tokio::test] +async fn list_marketplaces_can_skip_openai_curated_before_loading() { + let tmp = tempfile::tempdir().unwrap(); + let curated_root = curated_plugins_repo_path(tmp.path()); + + write_file( + &tmp.path().join(CONFIG_TOML_FILE), + r#"[features] +plugins = true +"#, + ); + write_file( + &curated_root.join(".agents/plugins/marketplace.json"), + "{not valid json", + ); + + let config = load_config(tmp.path(), tmp.path()).await; + let outcome = PluginsManager::new(tmp.path().to_path_buf()) + .list_marketplaces_for_config(&config, &[], /*include_openai_curated*/ false) + .unwrap(); + + assert_eq!(outcome.errors, Vec::new()); + assert_eq!( + outcome + .marketplaces + .iter() + .any(|marketplace| marketplace.name == OPENAI_CURATED_MARKETPLACE_NAME), + false + ); +} + #[tokio::test] async fn list_marketplaces_includes_installed_marketplace_roots() { let tmp = tempfile::tempdir().unwrap(); @@ -2576,7 +2639,7 @@ source = "/tmp/debug" .unwrap(); let config = load_config(tmp.path(), tmp.path()).await; let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) - .list_marketplaces_for_config(&config, &[]) + .list_marketplaces_for_config(&config, &[], /*include_openai_curated*/ true) .unwrap() .marketplaces; @@ -2652,7 +2715,7 @@ source = "/tmp/debug" let config = load_config(tmp.path(), tmp.path()).await; let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) - .list_marketplaces_for_config(&config, &[]) + .list_marketplaces_for_config(&config, &[], /*include_openai_curated*/ true) .unwrap() .marketplaces; @@ -2707,7 +2770,7 @@ plugins = true .unwrap(); let config = load_config(tmp.path(), tmp.path()).await; let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) - .list_marketplaces_for_config(&config, &[]) + .list_marketplaces_for_config(&config, &[], /*include_openai_curated*/ true) .unwrap() .marketplaces; @@ -2792,6 +2855,7 @@ enabled = false AbsolutePathBuf::try_from(repo_a_root).unwrap(), AbsolutePathBuf::try_from(repo_b_root).unwrap(), ], + /*include_openai_curated*/ true, ) .unwrap() .marketplaces; @@ -2902,7 +2966,11 @@ enabled = true let config = load_config(tmp.path(), &repo_root).await; let marketplaces = PluginsManager::new(tmp.path().to_path_buf()) - .list_marketplaces_for_config(&config, &[AbsolutePathBuf::try_from(repo_root).unwrap()]) + .list_marketplaces_for_config( + &config, + &[AbsolutePathBuf::try_from(repo_root).unwrap()], + /*include_openai_curated*/ true, + ) .unwrap() .marketplaces; diff --git a/codex-rs/core-plugins/src/remote.rs b/codex-rs/core-plugins/src/remote.rs index 2beeed2a1b..b948c51733 100644 --- a/codex-rs/core-plugins/src/remote.rs +++ b/codex-rs/core-plugins/src/remote.rs @@ -178,6 +178,7 @@ pub struct RemoteAppTemplate { pub template_id: String, pub name: String, pub description: Option, + pub category: Option, pub canonical_connector_id: Option, pub logo_url: Option, pub logo_url_dark: Option, @@ -458,6 +459,8 @@ struct RemoteAppTemplateResponse { #[serde(default)] description: Option, #[serde(default)] + category: Option, + #[serde(default)] canonical_connector_id: Option, #[serde(default)] logo_url: Option, @@ -1059,6 +1062,7 @@ async fn build_remote_plugin_detail( template_id: template.template_id, name: template.name, description: template.description, + category: template.category, canonical_connector_id: template.canonical_connector_id, logo_url: template.logo_url, logo_url_dark: template.logo_url_dark, diff --git a/codex-rs/core-skills/Cargo.toml b/codex-rs/core-skills/Cargo.toml index 3c18bee603..9473fa8e46 100644 --- a/codex-rs/core-skills/Cargo.toml +++ b/codex-rs/core-skills/Cargo.toml @@ -26,6 +26,7 @@ codex-protocol = { workspace = true } codex-skills = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-output-truncation = { workspace = true } +codex-utils-path-uri = { workspace = true } codex-utils-plugins = { workspace = true } dirs = { workspace = true } dunce = { workspace = true } diff --git a/codex-rs/core-skills/src/injection.rs b/codex-rs/core-skills/src/injection.rs index a358ddc637..03bb0a6ddd 100644 --- a/codex-rs/core-skills/src/injection.rs +++ b/codex-rs/core-skills/src/injection.rs @@ -13,6 +13,7 @@ use codex_exec_server::LOCAL_FS; use codex_otel::SessionTelemetry; use codex_protocol::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use codex_utils_plugins::mention_syntax::TOOL_MENTION_SIGIL; #[derive(Debug, Default)] @@ -75,10 +76,21 @@ pub async fn build_skill_injections( let fs = loaded_skills .and_then(|outcome| outcome.file_system_for_skill(skill)) .unwrap_or_else(|| Arc::clone(&LOCAL_FS)); - match fs - .read_file_text(&skill.path_to_skills_md, /*sandbox*/ None) - .await - { + // Skill metadata may point at a file that is absent, but a path the host + // cannot represent means the configured skill cannot be loaded correctly. + let path = match PathUri::from_abs_path(&skill.path_to_skills_md) { + Ok(path) => path, + Err(err) => { + emit_skill_injected_metric(otel, skill, "error"); + result.warnings.push(format!( + "Failed to load skill {name} at {path}: {err:#}", + name = skill.name, + path = skill.path_to_skills_md.display() + )); + continue; + } + }; + match fs.read_file_text(&path, /*sandbox*/ None).await { Ok(contents) => { emit_skill_injected_metric(otel, skill, "ok"); invocations.push(SkillInvocation { diff --git a/codex-rs/core-skills/src/loader.rs b/codex-rs/core-skills/src/loader.rs index e2c3896cba..813f13018d 100644 --- a/codex-rs/core-skills/src/loader.rs +++ b/codex-rs/core-skills/src/loader.rs @@ -19,6 +19,7 @@ use codex_protocol::protocol::Product; use codex_protocol::protocol::SkillScope; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::AbsolutePathBufGuard; +use codex_utils_path_uri::PathUri; use codex_utils_plugins::PluginSkillRoot; use codex_utils_plugins::plugin_namespace_for_skill_path; use dirs::home_dir; @@ -374,7 +375,17 @@ async fn repo_agents_skill_roots( let mut roots = Vec::new(); for dir in dirs { let agents_skills = dir.join(AGENTS_DIR_NAME).join(SKILLS_DIR_NAME); - match fs.get_metadata(&agents_skills, /*sandbox*/ None).await { + let agents_skills_uri = match PathUri::from_abs_path(&agents_skills) { + Ok(path) => path, + Err(err) => { + tracing::warn!( + "failed to convert repo skills root {} to URI: {err:#}", + agents_skills.display() + ); + continue; + } + }; + match fs.get_metadata(&agents_skills_uri, /*sandbox*/ None).await { Ok(metadata) if metadata.is_directory => roots.push(SkillRoot { path: agents_skills, scope: SkillScope::Repo, @@ -429,7 +440,17 @@ async fn find_project_root( for ancestor in cwd.ancestors() { for marker in project_root_markers { let marker_path = ancestor.join(marker); - match fs.get_metadata(&marker_path, /*sandbox*/ None).await { + let marker_path_uri = match PathUri::from_abs_path(&marker_path) { + Ok(path) => path, + Err(err) => { + tracing::warn!( + "failed to convert project root marker {} to URI: {err:#}", + marker_path.display() + ); + continue; + } + }; + match fs.get_metadata(&marker_path_uri, /*sandbox*/ None).await { Ok(_) => return ancestor, Err(err) if err.kind() == io::ErrorKind::NotFound => {} Err(err) => { @@ -475,8 +496,12 @@ async fn canonicalize_for_skill_identity( fs: &dyn ExecutorFileSystem, path: &AbsolutePathBuf, ) -> AbsolutePathBuf { - fs.canonicalize(path, /*sandbox*/ None) + let Ok(path_uri) = PathUri::from_abs_path(path) else { + return path.clone(); + }; + fs.canonicalize(&path_uri, /*sandbox*/ None) .await + .and_then(|path| path.to_abs_path()) .unwrap_or_else(|_| path.clone()) } @@ -494,7 +519,17 @@ async fn discover_skills_under_root( None => None, }; - match fs.get_metadata(&root, /*sandbox*/ None).await { + let root_uri = match PathUri::from_abs_path(&root) { + Ok(path) => path, + Err(err) => { + tracing::warn!( + "failed to convert skills root {} to URI: {err:#}", + root.display() + ); + return; + } + }; + match fs.get_metadata(&root_uri, /*sandbox*/ None).await { Ok(metadata) if metadata.is_directory => {} Ok(_) => return, Err(err) if err.kind() == io::ErrorKind::NotFound => return, @@ -536,7 +571,17 @@ async fn discover_skills_under_root( let mut truncated_by_dir_limit = false; while let Some((dir, depth)) = queue.pop_front() { - let entries = match fs.read_directory(&dir, /*sandbox*/ None).await { + let dir_uri = match PathUri::from_abs_path(&dir) { + Ok(path) => path, + Err(e) => { + tracing::warn!( + "failed to convert skills dir {} to URI: {e:#}", + dir.display() + ); + continue; + } + }; + let entries = match fs.read_directory(&dir_uri, /*sandbox*/ None).await { Ok(entries) => entries, Err(e) => { error!("failed to read skills dir {}: {e:#}", dir.display()); @@ -551,7 +596,17 @@ async fn discover_skills_under_root( } let path = dir.join(&file_name); - let metadata = match fs.get_metadata(&path, /*sandbox*/ None).await { + let path_uri = match PathUri::from_abs_path(&path) { + Ok(path) => path, + Err(e) => { + tracing::warn!( + "failed to convert skills path {} to URI: {e:#}", + path.display() + ); + continue; + } + }; + let metadata = match fs.get_metadata(&path_uri, /*sandbox*/ None).await { Ok(metadata) => metadata, Err(e) => { error!("failed to stat skills path {}: {e:#}", path.display()); @@ -563,7 +618,7 @@ async fn discover_skills_under_root( if !follow_symlinks { continue; } - match fs.read_directory(&path, /*sandbox*/ None).await { + match fs.read_directory(&path_uri, /*sandbox*/ None).await { Ok(_) => { let resolved_dir = canonicalize_for_skill_identity(fs, &path).await; enqueue_dir( @@ -635,8 +690,9 @@ async fn parse_skill_file( plugin_id: Option<&str>, plugin_root: Option<&AbsolutePathBuf>, ) -> Result { + let path_uri = PathUri::from_abs_path(path).map_err(SkillParseError::Read)?; let contents = fs - .read_file_text(path, /*sandbox*/ None) + .read_file_text(&path_uri, /*sandbox*/ None) .await .map_err(SkillParseError::Read)?; @@ -730,7 +786,18 @@ async fn load_skill_metadata( let metadata_path = skill_dir .join(SKILLS_METADATA_DIR) .join(SKILLS_METADATA_FILENAME); - match fs.get_metadata(&metadata_path, /*sandbox*/ None).await { + let metadata_path_uri = match PathUri::from_abs_path(&metadata_path) { + Ok(path) => path, + Err(error) => { + tracing::warn!( + "ignoring {path}: failed to convert {label} path to URI: {error}", + path = metadata_path.display(), + label = SKILLS_METADATA_FILENAME + ); + return LoadedSkillMetadata::default(); + } + }; + match fs.get_metadata(&metadata_path_uri, /*sandbox*/ None).await { Ok(metadata) if metadata.is_file => {} Ok(_) => return LoadedSkillMetadata::default(), Err(error) if error.kind() == io::ErrorKind::NotFound => { @@ -746,7 +813,10 @@ async fn load_skill_metadata( } } - let contents = match fs.read_file_text(&metadata_path, /*sandbox*/ None).await { + let contents = match fs + .read_file_text(&metadata_path_uri, /*sandbox*/ None) + .await + { Ok(contents) => contents, Err(error) => { tracing::warn!( diff --git a/codex-rs/core-skills/src/model.rs b/codex-rs/core-skills/src/model.rs index a9cc24306a..5569655adf 100644 --- a/codex-rs/core-skills/src/model.rs +++ b/codex-rs/core-skills/src/model.rs @@ -9,6 +9,7 @@ use codex_exec_server::LOCAL_FS; use codex_protocol::protocol::Product; use codex_protocol::protocol::SkillScope; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; #[derive(Debug, Clone, PartialEq)] pub struct SkillMetadata { @@ -152,8 +153,8 @@ impl HostLoadedSkills { .outcome .file_system_for_skill(skill) .unwrap_or_else(|| Arc::clone(&LOCAL_FS)); - fs.read_file_text(&skill.path_to_skills_md, /*sandbox*/ None) - .await + let path = PathUri::from_abs_path(&skill.path_to_skills_md)?; + fs.read_file_text(&path, /*sandbox*/ None).await } } diff --git a/codex-rs/core-skills/src/render.rs b/codex-rs/core-skills/src/render.rs index 1eff1f879f..1477a299b6 100644 --- a/codex-rs/core-skills/src/render.rs +++ b/codex-rs/core-skills/src/render.rs @@ -22,17 +22,17 @@ pub const SKILL_DESCRIPTION_TRUNCATED_WARNING: &str = "Skill descriptions were s pub const SKILL_DESCRIPTION_TRUNCATED_WARNING_WITH_PERCENT: &str = "Skill descriptions were shortened to fit the 2% skills context budget. Codex can still see every skill, but some descriptions are shorter. Disable unused skills or plugins to leave more room for the rest."; pub const SKILL_DESCRIPTIONS_REMOVED_WARNING_PREFIX: &str = "Exceeded skills context budget. All skill descriptions were removed and"; -pub const SKILLS_INTRO_WITH_ABSOLUTE_PATHS: &str = "A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and file path so you can open the source for full instructions when using a specific skill."; +pub const SKILLS_INTRO_WITH_ABSOLUTE_PATHS: &str = "A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that can be used. Each entry includes a name, description, and source locator. `file` locators are on the host filesystem, `environment resource` locators are owned by an execution environment, `orchestrator resource` locators are opaque non-filesystem resources, and `custom resource` locators use their provider's access mechanism."; pub const SKILLS_INTRO_WITH_ALIASES: &str = "A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list of skills that can be used. Each entry includes a name, description, and a short path that can be expanded into an absolute path using the skill roots table."; -pub const SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS: &str = r###"- Discovery: The list above is the skills available in this session (name + description + file path). Skill bodies live on disk at the listed paths. +pub const SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS: &str = r###"- Discovery: The list above is the skills available in this session (name + description + source locator). `file` entries live on the host filesystem, `environment resource` entries are owned by their execution environment, `orchestrator resource` entries must be accessed through `skills.list` and `skills.read`, and `custom resource` entries use their provider's access mechanism. - Trigger rules: If the user names a skill (with `$SkillName` or plain text) OR the task clearly matches a skill's description shown above, you must use that skill for that turn. Multiple mentions mean use them all. Do not carry skills across turns unless re-mentioned. -- Missing/blocked: If a named skill isn't in the list or the path can't be read, say so briefly and continue with the best fallback. +- Missing/blocked: If a named skill isn't in the list or its source can't be read, say so briefly and continue with the best fallback. - How to use a skill (progressive disclosure): - 1) After deciding to use a skill, the main agent must open and read its `SKILL.md` completely before taking task actions. If a read is truncated or paginated, continue until EOF. - 2) When `SKILL.md` references relative paths (e.g., `scripts/foo.py`), resolve them relative to the skill directory listed above first, and only consider other paths if needed. - 3) If `SKILL.md` points to extra folders such as `references/`, use its routing instructions to identify the files required for the task. The main agent must read each required instruction or reference file itself before acting on it. Do not delegate reading, summarizing, or interpreting skill instructions to a subagent. Subagents may still perform task work when the selected skill allows it. - 4) If `scripts/` exist, prefer running or patching them instead of retyping large code blocks. - 5) If `assets/` or templates exist, reuse them instead of recreating from scratch. + 1) After deciding to use a skill, the main agent must read its `SKILL.md` completely before taking task actions. For a `file` entry, open the listed path. For an `environment resource`, use the filesystem of the owning environment. For an `orchestrator resource`, call `skills.list` with `{"authority":{"kind":"orchestrator"}}`, select the matching package, and pass its `main_resource` to `skills.read`. If a read is truncated or paginated, continue until EOF. + 2) When `SKILL.md` references another resource, use the same access mechanism. Resolve relative paths against a filesystem-backed skill directory. For orchestrator skills, pass the exact referenced resource identifier with the same authority and package to `skills.read`; do not treat `skill://` identifiers as filesystem paths. + 3) If `SKILL.md` points to extra folders such as `references/`, use its routing instructions to identify the resources required for the task. The main agent must read each required instruction or reference file itself before acting on it. Do not delegate reading, summarizing, or interpreting skill instructions to a subagent. Subagents may still perform task work when the selected skill allows it. + 4) For filesystem-backed skills, prefer running or patching provided scripts instead of retyping large code blocks. For orchestrator skills, use `skills.read` and the available tools; do not invent a local path. + 5) Reuse provided assets or templates through the same source access mechanism instead of recreating them. - Coordination and sequencing: - If multiple skills apply, choose the minimal set that covers the request and state the order you'll use them. - Announce which skill(s) you're using and why (one short line). If you skip an obvious skill, say why. diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 352e8051e4..5174395e34 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -70,6 +70,7 @@ codex-utils-image = { workspace = true } codex-utils-home-dir = { workspace = true } codex-utils-output-truncation = { workspace = true } codex-utils-path = { workspace = true } +codex-utils-path-uri = { workspace = true } codex-utils-plugins = { workspace = true } codex-utils-pty = { workspace = true } codex-utils-string = { workspace = true } @@ -134,6 +135,7 @@ codex-shell-escalation = { workspace = true } assert_cmd = { workspace = true } assert_matches = { workspace = true } codex-image-generation-extension = { workspace = true } +codex-home = { workspace = true } codex-otel = { workspace = true } codex-test-binary-support = { workspace = true } codex-utils-cargo-bin = { workspace = true } diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index c55e54655d..b35a32ca8e 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -576,6 +576,9 @@ "request_rule": { "type": "boolean" }, + "resize_all_images": { + "type": "boolean" + }, "responses_websockets": { "type": "boolean" }, @@ -624,6 +627,9 @@ "terminal_visualization_instructions": { "type": "boolean" }, + "token_budget": { + "type": "boolean" + }, "tool_call_mcp_elicitation": { "type": "boolean" }, @@ -4742,6 +4748,9 @@ "request_rule": { "type": "boolean" }, + "resize_all_images": { + "type": "boolean" + }, "responses_websockets": { "type": "boolean" }, @@ -4790,6 +4799,9 @@ "terminal_visualization_instructions": { "type": "boolean" }, + "token_budget": { + "type": "boolean" + }, "tool_call_mcp_elicitation": { "type": "boolean" }, diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 772da10c30..ac42984b58 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -2491,7 +2491,7 @@ async fn list_agent_subtree_thread_ids_includes_anonymous_and_closed_descendants } #[tokio::test] -async fn list_agent_subtree_thread_ids_includes_live_descendants_without_state_db() { +async fn list_agent_subtree_thread_ids_finds_live_descendants_of_unloaded_root() { let (_home, config) = test_config().await; let manager = ThreadManager::with_models_provider_home_and_state_for_tests( CodexAuth::from_api_key("dummy"), @@ -2536,6 +2536,8 @@ async fn list_agent_subtree_thread_ids_includes_live_descendants_without_state_d .await .expect("grandchild spawn should succeed"); + manager.remove_thread(&parent_thread_id).await; + let mut subtree_thread_ids = manager .list_agent_subtree_thread_ids(parent_thread_id) .await diff --git a/codex-rs/core/src/agents_md.rs b/codex-rs/core/src/agents_md.rs index 5739e42f30..95ff98f515 100644 --- a/codex-rs/core/src/agents_md.rs +++ b/codex-rs/core/src/agents_md.rs @@ -21,11 +21,12 @@ use codex_config::ConfigLayerStackOrdering; use codex_config::default_project_root_markers; use codex_config::merge_toml_values; use codex_config::project_root_markers_from_config; -use codex_exec_server::Environment; use codex_exec_server::ExecutorFileSystem; +use codex_extension_api::UserInstructions; use codex_features::Feature; use codex_prompts::HIERARCHICAL_AGENTS_MESSAGE; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use std::io; use toml::Value as TomlValue; use tracing::error; @@ -39,276 +40,220 @@ pub const LOCAL_AGENTS_MD_FILENAME: &str = "AGENTS.override.md"; /// concatenated with the following separator. const AGENTS_MD_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; -/// Resolves AGENTS.md files into model-visible user instructions and source -/// paths. -pub struct AgentsMdManager<'a> { - config: &'a Config, -} - -impl<'a> AgentsMdManager<'a> { - pub fn new(config: &'a Config) -> Self { - Self { config } - } - - pub(crate) async fn load_global_instructions( - fs: &dyn ExecutorFileSystem, - codex_dir: Option<&AbsolutePathBuf>, - startup_warnings: &mut Vec, - ) -> Option { - let base = codex_dir?; - for candidate in [LOCAL_AGENTS_MD_FILENAME, DEFAULT_AGENTS_MD_FILENAME] { - let path = base.join(candidate); - let data = match fs.read_file(&path, /*sandbox*/ None).await { - Ok(data) => data, - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) if err.kind() == io::ErrorKind::IsADirectory => continue, - Err(err) => { - startup_warnings.push(format!( - "Failed to read global AGENTS.md instructions from `{}`: {err}", - path.display() - )); - continue; - } - }; - warn_invalid_utf8(&path, &data, "Global", startup_warnings); - let contents = String::from_utf8_lossy(&data); - let trimmed = contents.trim(); - if !trimmed.is_empty() { - return Some(LoadedAgentsMd::new_user(trimmed.to_string(), path)); - } - } - None - } - - /// Combines configured user instructions and AGENTS.md content into a - /// single model-visible instruction string. - pub(crate) async fn user_instructions( - &self, - environment: &Environment, - startup_warnings: &mut Vec, - ) -> Option { - let fs = environment.get_filesystem(); - self.user_instructions_with_fs(fs.as_ref(), startup_warnings) - .await - } - - async fn user_instructions_with_fs( - &self, - fs: &dyn ExecutorFileSystem, - startup_warnings: &mut Vec, - ) -> Option { - let agents_md_docs = self.read_agents_md(fs, startup_warnings).await; - - let mut loaded = self.config.user_instructions.clone().unwrap_or_default(); - - match agents_md_docs { +/// Loads project AGENTS.md content and combines it with host-provided user +/// instructions. +pub(crate) async fn load_project_instructions( + config: &mut Config, + user_instructions: Option, + fs: Option<&dyn ExecutorFileSystem>, +) -> Option { + let mut loaded = LoadedAgentsMd::from_user_instructions(user_instructions); + if let Some(fs) = fs { + match read_agents_md(config, fs).await { Ok(Some(docs)) => loaded.entries.extend(docs.entries), Ok(None) => {} Err(e) => { error!("error trying to find AGENTS.md docs: {e:#}"); } - }; - - if self.config.features.enabled(Feature::ChildAgentsMd) { - loaded.entries.push(InstructionEntry { - contents: HIERARCHICAL_AGENTS_MESSAGE.to_string(), - provenance: InstructionProvenance::Internal, - }); } - - (!loaded.is_empty()).then_some(loaded) } - /// Attempt to locate and load AGENTS.md documentation. - /// - /// On success returns `Ok(Some(loaded))` where `loaded` contains every - /// discovered doc. If no documentation file is found the function returns - /// `Ok(None)`. Unexpected I/O failures bubble up as `Err` so callers can - /// decide how to handle them. - async fn read_agents_md( - &self, - fs: &dyn ExecutorFileSystem, - startup_warnings: &mut Vec, - ) -> io::Result> { - let max_total = self.config.project_doc_max_bytes; + if config.features.enabled(Feature::ChildAgentsMd) { + loaded.entries.push(InstructionEntry { + contents: HIERARCHICAL_AGENTS_MESSAGE.to_string(), + provenance: InstructionProvenance::Internal, + }); + } - if max_total == 0 { - return Ok(None); + (!loaded.is_empty()).then_some(loaded) +} + +/// Attempt to locate and load AGENTS.md documentation. +/// +/// On success returns `Ok(Some(loaded))` where `loaded` contains every +/// discovered doc. If no documentation file is found the function returns +/// `Ok(None)`. Unexpected I/O failures bubble up as `Err` so callers can +/// decide how to handle them. +async fn read_agents_md( + config: &mut Config, + fs: &dyn ExecutorFileSystem, +) -> io::Result> { + let max_total = config.project_doc_max_bytes; + + if max_total == 0 { + return Ok(None); + } + + let paths = agents_md_paths(config, fs).await?; + if paths.is_empty() { + return Ok(None); + } + + let mut remaining: u64 = max_total as u64; + let mut loaded = LoadedAgentsMd::default(); + + for p in paths { + if remaining == 0 { + break; } - let paths = self.agents_md_paths(fs).await?; - if paths.is_empty() { - return Ok(None); + let path_uri = PathUri::from_abs_path(&p)?; + match fs.get_metadata(&path_uri, /*sandbox*/ None).await { + Ok(metadata) if !metadata.is_file => continue, + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), } - let mut remaining: u64 = max_total as u64; - let mut loaded = LoadedAgentsMd::default(); + let mut data = match fs.read_file(&path_uri, /*sandbox*/ None).await { + Ok(data) => data, + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + warn_invalid_utf8(&p, &data, "Project", &mut config.startup_warnings); - for p in paths { - if remaining == 0 { + let size = data.len() as u64; + if size > remaining { + data.truncate(remaining as usize); + } + + if size > remaining { + tracing::warn!( + "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", + p.display(), + remaining, + ); + } + + let text = String::from_utf8_lossy(&data).to_string(); + if !text.trim().is_empty() { + loaded.entries.push(InstructionEntry { + contents: text, + provenance: InstructionProvenance::Project(p), + }); + remaining = remaining.saturating_sub(data.len() as u64); + } + } + + if loaded.is_empty() { + Ok(None) + } else { + Ok(Some(loaded)) + } +} + +/// Discovers AGENTS.md files from the project root to the current working +/// directory, inclusive. Symlinks are allowed. +async fn agents_md_paths( + config: &Config, + fs: &dyn ExecutorFileSystem, +) -> io::Result> { + let dir = config.cwd.clone(); + + let mut merged = TomlValue::Table(toml::map::Map::new()); + for layer in config.config_layer_stack.get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ false, + ) { + if matches!(layer.name, ConfigLayerSource::Project { .. }) { + continue; + } + merge_toml_values(&mut merged, &layer.config); + } + let project_root_markers = match project_root_markers_from_config(&merged) { + Ok(Some(markers)) => markers, + Ok(None) => default_project_root_markers(), + Err(err) => { + tracing::warn!("invalid project_root_markers: {err}"); + default_project_root_markers() + } + }; + let mut project_root = None; + if !project_root_markers.is_empty() { + for ancestor in dir.ancestors() { + for marker in &project_root_markers { + let marker_path = ancestor.join(marker); + let marker_path_uri = PathUri::from_abs_path(&marker_path)?; + let marker_exists = match fs.get_metadata(&marker_path_uri, /*sandbox*/ None).await + { + Ok(_) => true, + Err(err) if err.kind() == io::ErrorKind::NotFound => false, + Err(err) => return Err(err), + }; + if marker_exists { + project_root = Some(ancestor.clone()); + break; + } + } + if project_root.is_some() { break; } + } + } - match fs.get_metadata(&p, /*sandbox*/ None).await { - Ok(metadata) if !metadata.is_file => continue, + let search_dirs: Vec = if let Some(root) = project_root { + let mut dirs = Vec::new(); + let mut cursor = dir.clone(); + loop { + dirs.push(cursor.clone()); + if cursor == root { + break; + } + let Some(parent) = cursor.parent() else { + break; + }; + cursor = parent; + } + dirs.reverse(); + dirs + } else { + vec![dir] + }; + + let mut found: Vec = Vec::new(); + let candidate_filenames = candidate_filenames(config); + for d in search_dirs { + for name in &candidate_filenames { + let candidate = d.join(name); + let candidate_uri = PathUri::from_abs_path(&candidate)?; + match fs.get_metadata(&candidate_uri, /*sandbox*/ None).await { + Ok(md) if md.is_file => { + found.push(candidate); + break; + } Ok(_) => {} Err(err) if err.kind() == io::ErrorKind::NotFound => continue, Err(err) => return Err(err), } - - let mut data = match fs.read_file(&p, /*sandbox*/ None).await { - Ok(data) => data, - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - }; - warn_invalid_utf8(&p, &data, "Project", startup_warnings); - - let size = data.len() as u64; - if size > remaining { - data.truncate(remaining as usize); - } - - if size > remaining { - tracing::warn!( - "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", - p.display(), - remaining, - ); - } - - let text = String::from_utf8_lossy(&data).to_string(); - if !text.trim().is_empty() { - loaded.entries.push(InstructionEntry { - contents: text, - provenance: InstructionProvenance::Project(p), - }); - remaining = remaining.saturating_sub(data.len() as u64); - } - } - - if loaded.is_empty() { - Ok(None) - } else { - Ok(Some(loaded)) } } - /// Discover the list of AGENTS.md files using the same search rules as - /// `read_agents_md`, but return the file paths instead of concatenated - /// contents. The list is ordered from project root to the current working - /// directory (inclusive). Symlinks are allowed. When `project_doc_max_bytes` - /// is zero, returns an empty list. - async fn agents_md_paths( - &self, - fs: &dyn ExecutorFileSystem, - ) -> io::Result> { - if self.config.project_doc_max_bytes == 0 { - return Ok(Vec::new()); + Ok(found) +} + +fn candidate_filenames(config: &Config) -> Vec<&str> { + let mut names: Vec<&str> = Vec::with_capacity(2 + config.project_doc_fallback_filenames.len()); + names.push(LOCAL_AGENTS_MD_FILENAME); + names.push(DEFAULT_AGENTS_MD_FILENAME); + for candidate in &config.project_doc_fallback_filenames { + let candidate = candidate.as_str(); + if candidate.is_empty() { + continue; } - - let dir = self.config.cwd.clone(); - - let mut merged = TomlValue::Table(toml::map::Map::new()); - for layer in self.config.config_layer_stack.get_layers( - ConfigLayerStackOrdering::LowestPrecedenceFirst, - /*include_disabled*/ false, - ) { - if matches!(layer.name, ConfigLayerSource::Project { .. }) { - continue; - } - merge_toml_values(&mut merged, &layer.config); + if !names.contains(&candidate) { + names.push(candidate); } - let project_root_markers = match project_root_markers_from_config(&merged) { - Ok(Some(markers)) => markers, - Ok(None) => default_project_root_markers(), - Err(err) => { - tracing::warn!("invalid project_root_markers: {err}"); - default_project_root_markers() - } - }; - let mut project_root = None; - if !project_root_markers.is_empty() { - for ancestor in dir.ancestors() { - for marker in &project_root_markers { - let marker_path = ancestor.join(marker); - let marker_exists = match fs.get_metadata(&marker_path, /*sandbox*/ None).await - { - Ok(_) => true, - Err(err) if err.kind() == io::ErrorKind::NotFound => false, - Err(err) => return Err(err), - }; - if marker_exists { - project_root = Some(ancestor.clone()); - break; - } - } - if project_root.is_some() { - break; - } - } - } - - let search_dirs: Vec = if let Some(root) = project_root { - let mut dirs = Vec::new(); - let mut cursor = dir.clone(); - loop { - dirs.push(cursor.clone()); - if cursor == root { - break; - } - let Some(parent) = cursor.parent() else { - break; - }; - cursor = parent; - } - dirs.reverse(); - dirs - } else { - vec![dir] - }; - - let mut found: Vec = Vec::new(); - let candidate_filenames = self.candidate_filenames(); - for d in search_dirs { - for name in &candidate_filenames { - let candidate = d.join(name); - match fs.get_metadata(&candidate, /*sandbox*/ None).await { - Ok(md) if md.is_file => { - found.push(candidate); - break; - } - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - } - } - } - - Ok(found) - } - - fn candidate_filenames(&self) -> Vec<&str> { - let mut names: Vec<&str> = - Vec::with_capacity(2 + self.config.project_doc_fallback_filenames.len()); - names.push(LOCAL_AGENTS_MD_FILENAME); - names.push(DEFAULT_AGENTS_MD_FILENAME); - for candidate in &self.config.project_doc_fallback_filenames { - let candidate = candidate.as_str(); - if candidate.is_empty() { - continue; - } - if !names.contains(&candidate) { - names.push(candidate); - } - } - names } + names } /// Model-visible instructions loaded from AGENTS.md files and internal /// guidance. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct LoadedAgentsMd { + /// Host-provided user instructions. + user_instructions: Option, + /// Ordered instructions and their provenance. entries: Vec, } @@ -320,10 +265,19 @@ impl LoadedAgentsMd { return Self::default(); } Self { - entries: vec![InstructionEntry { - contents, - provenance: InstructionProvenance::User(path), - }], + user_instructions: Some(UserInstructions { + text: contents, + source: path, + }), + entries: Vec::new(), + } + } + + fn from_user_instructions(user_instructions: Option) -> Self { + Self { + user_instructions: user_instructions + .filter(|instructions| !instructions.text.trim().is_empty()), + entries: Vec::new(), } } @@ -337,6 +291,7 @@ impl LoadedAgentsMd { return Self::default(); } Self { + user_instructions: None, entries: vec![InstructionEntry { contents, provenance: InstructionProvenance::Internal, @@ -345,40 +300,57 @@ impl LoadedAgentsMd { } fn is_empty(&self) -> bool { - self.entries - .iter() - .all(|entry| entry.contents.trim().is_empty()) + self.user_instructions.is_none() + && self + .entries + .iter() + .all(|entry| entry.contents.trim().is_empty()) } /// Returns the concatenated model-visible instruction text. pub fn text(&self) -> String { let mut output = String::new(); - let mut previous_provenance: Option<&InstructionProvenance> = None; + let mut has_previous = false; + let mut previous_was_project = false; + if let Some(instructions) = &self.user_instructions { + output.push_str(&instructions.text); + has_previous = true; + } for entry in &self.entries { - if let Some(previous_provenance) = previous_provenance { + let is_project = matches!(&entry.provenance, InstructionProvenance::Project(_)); + if has_previous { // The project-doc marker tells the model where workspace-scoped // instructions begin, so it is only needed on the transition // from user or internal instructions to project instructions. - let separator = match (previous_provenance, &entry.provenance) { - ( - InstructionProvenance::User(_) | InstructionProvenance::Internal, - InstructionProvenance::Project(_), - ) => AGENTS_MD_SEPARATOR, - _ => "\n\n", + let separator = if is_project && !previous_was_project { + AGENTS_MD_SEPARATOR + } else { + "\n\n" }; output.push_str(separator); } output.push_str(&entry.contents); - previous_provenance = Some(&entry.provenance); + has_previous = true; + previous_was_project = is_project; } output } + /// Returns the host-provided user instructions. + pub(crate) fn user_instructions(&self) -> Option<&UserInstructions> { + self.user_instructions.as_ref() + } + /// Returns the AGENTS.md files that supplied instruction entries. pub fn sources(&self) -> impl Iterator { - self.entries + self.user_instructions .iter() - .filter_map(|entry| entry.provenance.path()) + .map(|instructions| &instructions.source) + .chain( + self.entries + .iter() + .filter_map(|entry| entry.provenance.path()), + ) } } @@ -394,9 +366,6 @@ struct InstructionEntry { #[derive(Clone, Debug, PartialEq, Eq)] enum InstructionProvenance { - /// User-level instructions, normally loaded from CODEX_HOME. - User(AbsolutePathBuf), - /// Workspace instructions discovered from project AGENTS.md files. Project(AbsolutePathBuf), @@ -407,7 +376,7 @@ enum InstructionProvenance { impl InstructionProvenance { fn path(&self) -> Option<&AbsolutePathBuf> { match self { - Self::User(path) | Self::Project(path) => Some(path), + Self::Project(path) => Some(path), Self::Internal => None, } } diff --git a/codex-rs/core/src/agents_md_tests.rs b/codex-rs/core/src/agents_md_tests.rs index d8298dfd7d..3788a0f6dc 100644 --- a/codex-rs/core/src/agents_md_tests.rs +++ b/codex-rs/core/src/agents_md_tests.rs @@ -1,29 +1,172 @@ use super::*; use crate::config::ConfigBuilder; +use async_trait::async_trait; +use codex_config::ConfigLayerEntry; +use codex_config::ConfigLayerStack; +use codex_config::ConfigRequirements; +use codex_config::ConfigRequirementsToml; +use codex_exec_server::CopyOptions; +use codex_exec_server::CreateDirectoryOptions; +use codex_exec_server::FileMetadata; +use codex_exec_server::FileSystemSandboxContext; use codex_exec_server::LOCAL_FS; +use codex_exec_server::ReadDirectoryEntry; +use codex_exec_server::RemoveOptions; +use codex_extension_api::UserInstructions; use codex_features::Feature; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use core_test_support::PathBufExt; use core_test_support::TempDirExt; use core_test_support::create_directory_symlink; use pretty_assertions::assert_eq; use std::fs; +use std::io; +use std::ops::Deref; +use std::ops::DerefMut; use std::path::Path; use std::path::PathBuf; use tempfile::TempDir; -async fn get_user_instructions(config: &Config) -> Option { +#[derive(Clone, Copy)] +enum InjectedFailure { + Metadata(io::ErrorKind), + Read(io::ErrorKind), +} + +struct FailingFileSystem { + path: AbsolutePathBuf, + failure: InjectedFailure, +} + +#[async_trait] +impl ExecutorFileSystem for FailingFileSystem { + async fn canonicalize( + &self, + _path: &PathUri, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> io::Result { + unreachable!("canonicalize should not be called") + } + + async fn read_file( + &self, + path: &PathUri, + sandbox: Option<&FileSystemSandboxContext>, + ) -> io::Result> { + if path.to_abs_path()? == self.path + && let InjectedFailure::Read(kind) = self.failure + { + return Err(io::Error::new(kind, "injected read failure")); + } + LOCAL_FS.read_file(path, sandbox).await + } + + async fn write_file( + &self, + _path: &PathUri, + _contents: Vec, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> io::Result<()> { + unreachable!("write_file should not be called") + } + + async fn create_directory( + &self, + _path: &PathUri, + _create_directory_options: CreateDirectoryOptions, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> io::Result<()> { + unreachable!("create_directory should not be called") + } + + async fn get_metadata( + &self, + path: &PathUri, + sandbox: Option<&FileSystemSandboxContext>, + ) -> io::Result { + if path.to_abs_path()? == self.path + && let InjectedFailure::Metadata(kind) = self.failure + { + return Err(io::Error::new(kind, "injected metadata failure")); + } + LOCAL_FS.get_metadata(path, sandbox).await + } + + async fn read_directory( + &self, + _path: &PathUri, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> io::Result> { + unreachable!("read_directory should not be called") + } + + async fn remove( + &self, + _path: &PathUri, + _remove_options: RemoveOptions, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> io::Result<()> { + unreachable!("remove should not be called") + } + + async fn copy( + &self, + _source_path: &PathUri, + _destination_path: &PathUri, + _copy_options: CopyOptions, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> io::Result<()> { + unreachable!("copy should not be called") + } +} + +struct TestConfig { + config: Config, + user_instructions: Option, +} + +impl Deref for TestConfig { + type Target = Config; + + fn deref(&self) -> &Self::Target { + &self.config + } +} + +impl DerefMut for TestConfig { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.config + } +} + +async fn get_user_instructions(config: &TestConfig) -> Option { let mut warnings = Vec::new(); - AgentsMdManager::new(config) - .user_instructions_with_fs(LOCAL_FS.as_ref(), &mut warnings) + load_agents_md(config, &mut warnings) .await .map(|loaded| loaded.text()) } -async fn agents_md_paths(config: &Config) -> std::io::Result> { - AgentsMdManager::new(config) - .agents_md_paths(LOCAL_FS.as_ref()) - .await +async fn load_agents_md(config: &TestConfig, warnings: &mut Vec) -> Option { + let mut core_config = config.config.clone(); + let existing_warning_count = core_config.startup_warnings.len(); + let loaded = load_project_instructions( + &mut core_config, + config.user_instructions.clone(), + Some(LOCAL_FS.as_ref()), + ) + .await; + warnings.extend( + core_config + .startup_warnings + .into_iter() + .skip(existing_warning_count), + ); + loaded +} + +async fn agents_md_paths(config: &TestConfig) -> std::io::Result> { + super::agents_md_paths(&config.config, LOCAL_FS.as_ref()).await } fn assert_invalid_utf8_warning(warnings: &[String], source: &str, path: &Path) { @@ -44,7 +187,7 @@ fn assert_invalid_utf8_warning(warnings: &[String], source: &str, path: &Path) { /// optionally specify a custom `instructions` string – when `None` the /// value is cleared to mimic a scenario where no system instructions have /// been configured. -async fn make_config(root: &TempDir, limit: usize, instructions: Option<&str>) -> Config { +async fn make_config(root: &TempDir, limit: usize, instructions: Option<&str>) -> TestConfig { let codex_home = TempDir::new().unwrap(); let mut config = ConfigBuilder::default() .codex_home(codex_home.path().to_path_buf()) @@ -55,13 +198,14 @@ async fn make_config(root: &TempDir, limit: usize, instructions: Option<&str>) - config.cwd = root.abs(); config.project_doc_max_bytes = limit; - config.user_instructions = instructions.map(|text| { - LoadedAgentsMd::new_user( - text.to_owned(), - config.codex_home.join(DEFAULT_AGENTS_MD_FILENAME), - ) + let user_instructions = instructions.map(|text| UserInstructions { + text: text.to_owned(), + source: config.codex_home.join(DEFAULT_AGENTS_MD_FILENAME), }); - config + TestConfig { + config, + user_instructions, + } } async fn make_config_with_fallback( @@ -69,7 +213,7 @@ async fn make_config_with_fallback( limit: usize, instructions: Option<&str>, fallbacks: &[&str], -) -> Config { +) -> TestConfig { let mut config = make_config(root, limit, instructions).await; config.project_doc_fallback_filenames = fallbacks .iter() @@ -83,7 +227,7 @@ async fn make_config_with_project_root_markers( limit: usize, instructions: Option<&str>, markers: &[&str], -) -> Config { +) -> TestConfig { let codex_home = TempDir::new().unwrap(); let cli_overrides = vec![( "project_root_markers".to_string(), @@ -103,13 +247,14 @@ async fn make_config_with_project_root_markers( config.cwd = root.abs(); config.project_doc_max_bytes = limit; - config.user_instructions = instructions.map(|text| { - LoadedAgentsMd::new_user( - text.to_owned(), - config.codex_home.join(DEFAULT_AGENTS_MD_FILENAME), - ) + let user_instructions = instructions.map(|text| UserInstructions { + text: text.to_owned(), + source: config.codex_home.join(DEFAULT_AGENTS_MD_FILENAME), }); - config + TestConfig { + config, + user_instructions, + } } /// AGENTS.md missing – should yield `None`. @@ -153,12 +298,14 @@ fn empty_loaded_instructions_are_empty() { #[test] fn loaded_instructions_with_only_empty_or_whitespace_entries_are_empty() { let empty = LoadedAgentsMd { + user_instructions: None, entries: vec![InstructionEntry { contents: String::new(), provenance: InstructionProvenance::Internal, }], }; let whitespace = LoadedAgentsMd { + user_instructions: None, entries: vec![InstructionEntry { contents: " \n\t".to_string(), provenance: InstructionProvenance::Internal, @@ -186,29 +333,6 @@ async fn doc_smaller_than_limit_is_returned() { ); } -#[tokio::test] -async fn global_doc_invalid_utf8_warns_and_uses_lossy_text() { - let codex_home = tempfile::tempdir().expect("tempdir"); - let codex_home_abs = codex_home.abs(); - let path = codex_home_abs.join(DEFAULT_AGENTS_MD_FILENAME); - fs::write(&path, b"global\xFF doc").unwrap(); - - let mut warnings = Vec::new(); - let loaded = AgentsMdManager::load_global_instructions( - LOCAL_FS.as_ref(), - Some(&codex_home_abs), - &mut warnings, - ) - .await - .expect("global doc expected"); - - assert_eq!( - loaded, - LoadedAgentsMd::new_user("global\u{FFFD} doc".to_string(), path.clone()) - ); - assert_invalid_utf8_warning(&warnings, "Global", path.as_path()); -} - #[tokio::test] async fn project_doc_invalid_utf8_warns_and_uses_lossy_text() { let tmp = tempfile::tempdir().expect("tempdir"); @@ -217,8 +341,7 @@ async fn project_doc_invalid_utf8_warns_and_uses_lossy_text() { let config = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; let mut warnings = Vec::new(); - let res = AgentsMdManager::new(&config) - .user_instructions_with_fs(LOCAL_FS.as_ref(), &mut warnings) + let res = load_agents_md(&config, &mut warnings) .await .expect("doc expected") .text(); @@ -244,6 +367,92 @@ async fn doc_larger_than_limit_is_truncated() { assert_eq!(res, huge[..LIMIT]); } +#[tokio::test] +async fn total_byte_limit_truncates_later_project_docs() { + let repo = tempfile::tempdir().expect("tempdir"); + fs::write(repo.path().join(".git"), "").unwrap(); + fs::write(repo.path().join("AGENTS.md"), "root").unwrap(); + let nested = repo.path().join("nested"); + fs::create_dir(&nested).unwrap(); + fs::write(nested.join("AGENTS.md"), "abcdef").unwrap(); + + let mut config = make_config(&repo, /*limit*/ 7, /*instructions*/ None).await; + config.cwd = nested.abs(); + + let mut warnings = Vec::new(); + let loaded = load_agents_md(&config, &mut warnings) + .await + .expect("project instructions"); + let expected = LoadedAgentsMd { + user_instructions: None, + entries: vec![ + InstructionEntry { + contents: "root".to_string(), + provenance: InstructionProvenance::Project(repo.path().join("AGENTS.md").abs()), + }, + InstructionEntry { + contents: "abc".to_string(), + provenance: InstructionProvenance::Project(config.cwd.join("AGENTS.md")), + }, + ], + }; + + assert_eq!(loaded, expected); + assert_eq!(loaded.text(), "root\n\nabc"); + assert_eq!(warnings, Vec::::new()); +} + +#[tokio::test] +async fn read_agents_md_propagates_metadata_errors() { + let tmp = tempfile::tempdir().expect("tempdir"); + let mut config = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; + let marker_path = config.cwd.join(".git"); + let fs = FailingFileSystem { + path: marker_path, + failure: InjectedFailure::Metadata(io::ErrorKind::PermissionDenied), + }; + + let err = read_agents_md(&mut config.config, &fs) + .await + .expect_err("metadata error"); + + assert_eq!(err.kind(), io::ErrorKind::PermissionDenied); +} + +#[tokio::test] +async fn read_agents_md_propagates_read_errors() { + let tmp = tempfile::tempdir().expect("tempdir"); + fs::write(tmp.path().join("AGENTS.md"), "project doc").unwrap(); + let mut config = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; + let fs = FailingFileSystem { + path: config.cwd.join("AGENTS.md"), + failure: InjectedFailure::Read(io::ErrorKind::PermissionDenied), + }; + + let err = read_agents_md(&mut config.config, &fs) + .await + .expect_err("read error"); + + assert_eq!(err.kind(), io::ErrorKind::PermissionDenied); +} + +#[tokio::test] +async fn read_agents_md_ignores_files_removed_after_discovery() { + let tmp = tempfile::tempdir().expect("tempdir"); + fs::write(tmp.path().join("AGENTS.md"), "project doc").unwrap(); + let mut config = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; + let fs = FailingFileSystem { + path: config.cwd.join("AGENTS.md"), + failure: InjectedFailure::Read(io::ErrorKind::NotFound), + }; + + let loaded = read_agents_md(&mut config.config, &fs) + .await + .expect("removed file is recoverable"); + + assert_eq!(loaded, None); +} + /// When `cwd` is nested inside a repo, the search should locate AGENTS.md /// placed at the repository root (identified by `.git`). #[tokio::test] @@ -286,17 +495,6 @@ async fn zero_byte_limit_disables_docs() { ); } -#[tokio::test] -async fn zero_byte_limit_disables_discovery() { - let tmp = tempfile::tempdir().expect("tempdir"); - fs::write(tmp.path().join("AGENTS.md"), "something").unwrap(); - - let discovery = agents_md_paths(&make_config(&tmp, /*limit*/ 0, /*instructions*/ None).await) - .await - .expect("discover paths"); - assert_eq!(discovery, Vec::::new()); -} - /// When both system instructions and AGENTS.md docs are present the two /// should be concatenated with the separator. #[tokio::test] @@ -315,30 +513,6 @@ async fn merges_existing_instructions_with_agents_md() { assert_eq!(res, expected); } -#[tokio::test] -async fn sourceless_user_instructions_preserve_separator_without_reporting_a_source() { - let tmp = tempfile::tempdir().expect("tempdir"); - fs::write(tmp.path().join("AGENTS.md"), "project doc").unwrap(); - - let mut cfg = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; - cfg.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "user instructions".to_string(), - )); - - let mut warnings = Vec::new(); - let loaded = AgentsMdManager::new(&cfg) - .user_instructions_with_fs(LOCAL_FS.as_ref(), &mut warnings) - .await - .expect("instructions expected"); - let project_agents = cfg.cwd.join("AGENTS.md"); - - assert_eq!( - loaded.text(), - format!("user instructions{AGENTS_MD_SEPARATOR}project doc") - ); - assert_eq!(loaded.sources().collect::>(), vec![&project_agents]); -} - /// If there are existing system instructions but AGENTS.md docs are /// missing we expect the original instructions to be returned unchanged. #[tokio::test] @@ -378,13 +552,13 @@ async fn concatenates_root_and_cwd_docs() { cfg.cwd = nested.abs(); let mut warnings = Vec::new(); - let loaded = AgentsMdManager::new(&cfg) - .user_instructions_with_fs(LOCAL_FS.as_ref(), &mut warnings) + let loaded = load_agents_md(&cfg, &mut warnings) .await .expect("doc expected"); let root_agents = repo.path().join("AGENTS.md").abs(); let crate_agents = cfg.cwd.join("AGENTS.md"); let expected = LoadedAgentsMd { + user_instructions: None, entries: vec![ InstructionEntry { contents: "root doc".to_string(), @@ -435,6 +609,51 @@ async fn project_root_markers_are_honored_for_agents_discovery() { assert_eq!(res, "parent doc\n\nchild doc"); } +#[tokio::test] +async fn project_layers_do_not_override_project_root_markers() { + let root = tempfile::tempdir().expect("tempdir"); + fs::write(root.path().join(".git"), "").unwrap(); + fs::write(root.path().join("AGENTS.md"), "root doc").unwrap(); + let nested = root.path().join("nested"); + fs::create_dir(&nested).unwrap(); + fs::write(nested.join("AGENTS.md"), "nested doc").unwrap(); + + let mut config = make_config(&root, /*limit*/ 4096, /*instructions*/ None).await; + config.cwd = nested.abs(); + let project_layer = |dot_codex_folder: AbsolutePathBuf, marker: &str| { + ConfigLayerEntry::new( + ConfigLayerSource::Project { dot_codex_folder }, + TomlValue::Table( + [( + "project_root_markers".to_string(), + TomlValue::Array(vec![TomlValue::String(marker.to_string())]), + )] + .into_iter() + .collect(), + ), + ) + }; + config.config_layer_stack = ConfigLayerStack::new( + vec![ + project_layer(root.path().join(".codex").abs(), ".ignored-root-marker"), + project_layer(config.cwd.join(".codex"), ".ignored-nested-marker"), + ], + ConfigRequirements::default(), + ConfigRequirementsToml::default(), + ) + .expect("valid project layer ordering"); + + let discovery = agents_md_paths(&config).await.expect("discover paths"); + + assert_eq!( + discovery, + vec![ + root.path().join("AGENTS.md").abs(), + config.cwd.join("AGENTS.md"), + ] + ); +} + #[tokio::test] async fn agents_md_paths_preserve_symlinked_cwd() { let tmp = tempfile::tempdir().expect("tempdir"); @@ -462,22 +681,19 @@ async fn child_agents_message_after_global_instructions_uses_plain_separator() { cfg.features.enable(Feature::ChildAgentsMd).unwrap(); let mut warnings = Vec::new(); - let loaded = AgentsMdManager::new(&cfg) - .user_instructions_with_fs(LOCAL_FS.as_ref(), &mut warnings) + let loaded = load_agents_md(&cfg, &mut warnings) .await .expect("instructions expected"); let global_agents = cfg.codex_home.join(DEFAULT_AGENTS_MD_FILENAME); let expected = LoadedAgentsMd { - entries: vec![ - InstructionEntry { - contents: "global doc".to_string(), - provenance: InstructionProvenance::User(global_agents), - }, - InstructionEntry { - contents: HIERARCHICAL_AGENTS_MESSAGE.to_string(), - provenance: InstructionProvenance::Internal, - }, - ], + user_instructions: Some(UserInstructions { + text: "global doc".to_string(), + source: global_agents, + }), + entries: vec![InstructionEntry { + contents: HIERARCHICAL_AGENTS_MESSAGE.to_string(), + provenance: InstructionProvenance::Internal, + }], }; assert_eq!(loaded, expected); @@ -498,25 +714,23 @@ async fn instruction_sources_include_global_before_agents_md_docs() { fs::write(&global_agents, "global doc").unwrap(); let mut warnings = Vec::new(); - let loaded = AgentsMdManager::new(&cfg) - .user_instructions_with_fs(LOCAL_FS.as_ref(), &mut warnings) + let loaded = load_agents_md(&cfg, &mut warnings) .await .expect("instructions expected"); let project_agents = cfg.cwd.join("AGENTS.md"); let expected = LoadedAgentsMd { - entries: vec![ - InstructionEntry { - contents: "global doc".to_string(), - provenance: InstructionProvenance::User(global_agents.clone()), - }, - InstructionEntry { - contents: "project doc".to_string(), - provenance: InstructionProvenance::Project(project_agents.clone()), - }, - ], + user_instructions: Some(UserInstructions { + text: "global doc".to_string(), + source: global_agents.clone(), + }), + entries: vec![InstructionEntry { + contents: "project doc".to_string(), + provenance: InstructionProvenance::Project(project_agents.clone()), + }], }; assert_eq!(loaded, expected); + assert_eq!(loaded.user_instructions(), cfg.user_instructions.as_ref()); assert_eq!( loaded.sources().collect::>(), vec![&global_agents, &project_agents] @@ -539,18 +753,17 @@ async fn child_agents_message_after_project_docs_is_not_an_instruction_source() fs::write(&global_agents, "global doc").unwrap(); let mut warnings = Vec::new(); - let loaded = AgentsMdManager::new(&cfg) - .user_instructions_with_fs(LOCAL_FS.as_ref(), &mut warnings) + let loaded = load_agents_md(&cfg, &mut warnings) .await .expect("instructions expected"); let project_agents = cfg.cwd.join("AGENTS.md"); let expected = LoadedAgentsMd { + user_instructions: Some(UserInstructions { + text: "global doc".to_string(), + source: global_agents.clone(), + }), entries: vec![ - InstructionEntry { - contents: "global doc".to_string(), - provenance: InstructionProvenance::User(global_agents.clone()), - }, InstructionEntry { contents: "project doc".to_string(), provenance: InstructionProvenance::Project(project_agents.clone()), diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 87381fca7b..75ed2039ac 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -70,7 +70,6 @@ use codex_login::default_client::build_reqwest_client; use codex_otel::SessionTelemetry; use codex_otel::current_span_w3c_trace_context; -use codex_protocol::SessionId; use codex_protocol::ThreadId; use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use codex_protocol::config_types::Verbosity as VerbosityConfig; @@ -79,7 +78,6 @@ use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::InternalSessionSource; use codex_protocol::protocol::SessionSource; -use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::W3cTraceContext; use codex_rollout_trace::CompactionTraceContext; use codex_rollout_trace::InferenceTraceAttempt; @@ -111,6 +109,8 @@ use crate::client_common::Prompt; use crate::client_common::ResponseEvent; use crate::client_common::ResponseStream; use crate::feedback_tags; +use crate::responses_metadata::CodexResponsesMetadata; +use crate::responses_metadata::subagent_header_value; use crate::util::emit_feedback_auth_recovery_tags; use codex_api::map_api_error; use codex_feedback::FeedbackRequestTags; @@ -169,13 +169,10 @@ pub(crate) struct CompactConversationRequestSettings { /// configuration is per turn and is passed explicitly to streaming/unary methods. #[derive(Debug)] struct ModelClientState { - session_id: SessionId, thread_id: ThreadId, - installation_id: String, provider: SharedModelProvider, auth_env_telemetry: AuthEnvTelemetry, session_source: SessionSource, - parent_thread_id: Option, model_verbosity: Option, enable_request_compression: bool, include_timing_metrics: bool, @@ -319,12 +316,9 @@ impl ModelClient { /// are passed to [`ModelClientSession::stream`] (and other turn-scoped methods) explicitly. pub fn new( auth_manager: Option>, - session_id: SessionId, thread_id: ThreadId, - installation_id: String, provider_info: ModelProviderInfo, session_source: SessionSource, - parent_thread_id: Option, model_verbosity: Option, enable_request_compression: bool, include_timing_metrics: bool, @@ -341,13 +335,10 @@ impl ModelClient { let include_attestation = model_provider.supports_attestation(); Self { state: Arc::new(ModelClientState { - session_id, thread_id, - installation_id, provider: model_provider, auth_env_telemetry, session_source, - parent_thread_id, model_verbosity, enable_request_compression, include_timing_metrics, @@ -444,8 +435,7 @@ impl ModelClient { settings: CompactConversationRequestSettings, session_telemetry: &SessionTelemetry, compaction_trace: &CompactionTraceContext, - window_id: &str, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, ) -> Result> { if prompt.input.is_empty() { return Ok(Vec::new()); @@ -469,7 +459,7 @@ impl ModelClient { settings.effort, settings.summary, settings.service_tier, - window_id, + responses_metadata, )?; let ResponsesApiRequest { model, @@ -496,18 +486,17 @@ impl ModelClient { }; let mut extra_headers = ApiHeaderMap::new(); - if let Ok(header_value) = HeaderValue::from_str(&self.state.installation_id) { + if let Ok(header_value) = HeaderValue::from_str(&responses_metadata.installation_id) { extra_headers.insert(X_CODEX_INSTALLATION_ID_HEADER, header_value); } extra_headers.extend(build_responses_headers( self.state.beta_features_header.as_deref(), /*turn_state*/ None, - parse_turn_metadata_header(turn_metadata_header).as_ref(), )); - extra_headers.extend(self.build_responses_identity_headers(Some(window_id))); + extra_headers.extend(self.build_responses_compatibility_headers(responses_metadata)); extra_headers.extend(build_session_headers( - Some(self.state.session_id.to_string()), - Some(self.state.thread_id.to_string()), + Some(responses_metadata.session_id.to_string()), + Some(responses_metadata.thread_id.to_string()), )); if let Some(header_value) = self.generate_attestation_header_for().await { extra_headers.insert(X_OAI_ATTESTATION_HEADER, header_value); @@ -626,50 +615,29 @@ impl ModelClient { extra_headers } - fn build_responses_identity_headers(&self, window_id: Option<&str>) -> ApiHeaderMap { - let mut extra_headers = self.build_subagent_headers(); - if let Some(parent_thread_id) = parent_thread_id_header_value(self.state.parent_thread_id) - && let Ok(val) = HeaderValue::from_str(&parent_thread_id) - { - extra_headers.insert(X_CODEX_PARENT_THREAD_ID_HEADER, val); - } - if let Some(window_id) = window_id - && let Ok(val) = HeaderValue::from_str(window_id) - { - extra_headers.insert(X_CODEX_WINDOW_ID_HEADER, val); + fn build_responses_compatibility_headers( + &self, + responses_metadata: &CodexResponsesMetadata, + ) -> ApiHeaderMap { + let mut extra_headers = responses_metadata.compatibility_headers(); + if matches!( + self.state.session_source, + SessionSource::Internal(InternalSessionSource::MemoryConsolidation) + ) { + extra_headers.insert( + X_OPENAI_MEMGEN_REQUEST_HEADER, + HeaderValue::from_static("true"), + ); } extra_headers } fn build_ws_client_metadata( &self, - window_id: &str, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, use_responses_lite: bool, ) -> HashMap { - let mut client_metadata = HashMap::new(); - client_metadata.insert( - X_CODEX_INSTALLATION_ID_HEADER.to_string(), - self.state.installation_id.clone(), - ); - client_metadata.insert(X_CODEX_WINDOW_ID_HEADER.to_string(), window_id.to_string()); - if let Some(subagent) = subagent_header_value(&self.state.session_source) { - client_metadata.insert(X_OPENAI_SUBAGENT_HEADER.to_string(), subagent); - } - if let Some(parent_thread_id) = parent_thread_id_header_value(self.state.parent_thread_id) { - client_metadata.insert( - X_CODEX_PARENT_THREAD_ID_HEADER.to_string(), - parent_thread_id, - ); - } - if let Some(turn_metadata_header) = parse_turn_metadata_header(turn_metadata_header) - && let Ok(turn_metadata) = turn_metadata_header.to_str() - { - client_metadata.insert( - X_CODEX_TURN_METADATA_HEADER.to_string(), - turn_metadata.to_string(), - ); - } + let mut client_metadata = responses_metadata.client_metadata(); if use_responses_lite { client_metadata.insert( WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY.to_string(), @@ -743,10 +711,10 @@ impl ModelClient { effort: Option, summary: ReasoningSummaryConfig, service_tier: Option, - window_id: &str, + responses_metadata: &CodexResponsesMetadata, ) -> Result { let instructions = &prompt.base_instructions.text; - let input = prompt.get_formatted_input(); + let input = prompt.get_formatted_input_for_request(model_info.use_responses_lite); let tools = create_tools_json_for_responses_api(&prompt.tools)?; let reasoning = Self::build_reasoning(model_info, effort, summary); let include = if reasoning.is_some() { @@ -786,13 +754,7 @@ impl ModelClient { service_tier, prompt_cache_key, text, - client_metadata: Some(HashMap::from([ - ( - X_CODEX_INSTALLATION_ID_HEADER.to_string(), - self.state.installation_id.clone(), - ), - (X_CODEX_WINDOW_ID_HEADER.to_string(), window_id.to_string()), - ])), + client_metadata: Some(responses_metadata.client_metadata()), }; Ok(request) } @@ -835,13 +797,13 @@ impl ModelClient { session_telemetry: &SessionTelemetry, api_provider: codex_api::Provider, api_auth: SharedAuthProvider, + responses_metadata: &CodexResponsesMetadata, turn_state: Option>>, - turn_metadata_header: Option<&str>, auth_context: AuthRequestTelemetryContext, request_route_telemetry: RequestRouteTelemetry, ) -> std::result::Result { let headers = self - .build_websocket_headers(turn_state.as_ref(), turn_metadata_header) + .build_websocket_headers(responses_metadata, turn_state.as_ref()) .await; let websocket_telemetry = ModelClientSession::build_websocket_telemetry( session_telemetry, @@ -921,22 +883,19 @@ impl ModelClient { /// replayed on reconnect within the same turn. async fn build_websocket_headers( &self, + responses_metadata: &CodexResponsesMetadata, turn_state: Option<&Arc>>, - turn_metadata_header: Option<&str>, ) -> ApiHeaderMap { - let turn_metadata_header = parse_turn_metadata_header(turn_metadata_header); - let session_id = self.state.session_id.to_string(); - let thread_id = self.state.thread_id.to_string(); - let mut headers = build_responses_headers( - self.state.beta_features_header.as_deref(), - turn_state, - turn_metadata_header.as_ref(), - ); - if let Ok(header_value) = HeaderValue::from_str(&thread_id) { + let mut headers = + build_responses_headers(self.state.beta_features_header.as_deref(), turn_state); + if let Ok(header_value) = HeaderValue::from_str(&responses_metadata.thread_id) { headers.insert("x-client-request-id", header_value); } - headers.extend(build_session_headers(Some(session_id), Some(thread_id))); - headers.extend(self.build_responses_identity_headers(/*window_id*/ None)); + headers.extend(build_session_headers( + Some(responses_metadata.session_id.to_string()), + Some(responses_metadata.thread_id.to_string()), + )); + headers.extend(self.build_responses_compatibility_headers(responses_metadata)); if let Some(header_value) = self.generate_attestation_header_for().await { headers.insert(X_OAI_ATTESTATION_HEADER, header_value); } @@ -979,27 +938,22 @@ impl ModelClientSession { /// regardless of transport choice. async fn build_responses_options( &self, - window_id: &str, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, compression: Compression, use_responses_lite: bool, ) -> ApiResponsesOptions { - let turn_metadata_header = parse_turn_metadata_header(turn_metadata_header); - let session_id = self.client.state.session_id.to_string(); - let thread_id = self.client.state.thread_id.to_string(); ApiResponsesOptions { - session_id: Some(session_id), - thread_id: Some(thread_id), + session_id: Some(responses_metadata.session_id.to_string()), + thread_id: Some(responses_metadata.thread_id.to_string()), session_source: Some(self.client.state.session_source.clone()), extra_headers: { let mut headers = build_responses_headers( self.client.state.beta_features_header.as_deref(), Some(&self.turn_state), - turn_metadata_header.as_ref(), ); headers.extend( self.client - .build_responses_identity_headers(Some(window_id)), + .build_responses_compatibility_headers(responses_metadata), ); if let Some(header_value) = self.client.generate_attestation_header_for().await { headers.insert(X_OAI_ATTESTATION_HEADER, header_value); @@ -1026,12 +980,12 @@ impl ModelClientSession { let previous_request = self.websocket_session.last_request.as_ref()?; let mut previous_without_input = previous_request.clone(); previous_without_input.input.clear(); + previous_without_input.client_metadata = None; let mut request_without_input = request.clone(); request_without_input.input.clear(); + request_without_input.client_metadata = None; if previous_without_input != request_without_input { - trace!( - "incremental request failed, properties didn't match {previous_without_input:?} != {request_without_input:?}" - ); + trace!("incremental request failed, websocket reuse properties didn't match"); return None; } @@ -1100,6 +1054,7 @@ impl ModelClientSession { pub async fn preconnect_websocket( &mut self, session_telemetry: &SessionTelemetry, + responses_metadata: &CodexResponsesMetadata, ) -> std::result::Result<(), ApiError> { if !self.client.responses_websocket_enabled() { return Ok(()); @@ -1124,8 +1079,8 @@ impl ModelClientSession { session_telemetry, client_setup.api_provider, client_setup.api_auth, + responses_metadata, Some(Arc::clone(&self.turn_state)), - /*turn_metadata_header*/ None, auth_context, RequestRouteTelemetry::for_endpoint(RESPONSES_ENDPOINT), ) @@ -1145,7 +1100,7 @@ impl ModelClientSession { wire_api = %self.client.state.provider.info().wire_api, transport = "responses_websocket", api.path = "responses", - turn.has_metadata_header = params.turn_metadata_header.is_some() + turn.has_metadata_header = params.responses_metadata.has_turn_metadata() ) )] async fn websocket_connection( @@ -1156,7 +1111,7 @@ impl ModelClientSession { session_telemetry, api_provider, api_auth, - turn_metadata_header, + responses_metadata, options, auth_context, request_route_telemetry, @@ -1180,8 +1135,8 @@ impl ModelClientSession { session_telemetry, api_provider, api_auth, + responses_metadata, Some(turn_state), - turn_metadata_header, auth_context, request_route_telemetry, ) @@ -1236,19 +1191,18 @@ impl ModelClientSession { transport = "responses_http", http.method = "POST", api.path = "responses", - turn.has_metadata_header = turn_metadata_header.is_some() + turn.has_metadata_header = responses_metadata.has_turn_metadata() ) )] async fn stream_responses_api( &self, - window_id: &str, prompt: &Prompt, model_info: &ModelInfo, session_telemetry: &SessionTelemetry, effort: Option, summary: ReasoningSummaryConfig, service_tier: Option, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, inference_trace: &InferenceTraceContext, ) -> Result { let auth_manager = self.client.state.provider.auth_manager(); @@ -1273,8 +1227,7 @@ impl ModelClientSession { let compression = self.responses_request_compression(client_setup.auth.as_ref()); let mut options = self .build_responses_options( - window_id, - turn_metadata_header, + responses_metadata, compression, model_info.use_responses_lite, ) @@ -1287,7 +1240,7 @@ impl ModelClientSession { effort.clone(), summary, service_tier.clone(), - window_id, + responses_metadata, )?; let inference_trace_attempt = inference_trace.start_attempt(); inference_trace_attempt.add_request_headers(&mut options.extra_headers); @@ -1355,20 +1308,19 @@ impl ModelClientSession { wire_api = %self.client.state.provider.info().wire_api, transport = "responses_websocket", api.path = "responses", - turn.has_metadata_header = turn_metadata_header.is_some(), + turn.has_metadata_header = responses_metadata.has_turn_metadata(), websocket.warmup = warmup ) )] async fn stream_responses_websocket( &mut self, - window_id: &str, prompt: &Prompt, model_info: &ModelInfo, session_telemetry: &SessionTelemetry, effort: Option, summary: ReasoningSummaryConfig, service_tier: Option, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, warmup: bool, request_trace: Option, inference_trace: &InferenceTraceContext, @@ -1390,8 +1342,7 @@ impl ModelClientSession { let options = self .build_responses_options( - window_id, - turn_metadata_header, + responses_metadata, compression, model_info.use_responses_lite, ) @@ -1403,13 +1354,12 @@ impl ModelClientSession { effort.clone(), summary, service_tier.clone(), - window_id, + responses_metadata, )?; let mut ws_payload = ResponseCreateWsRequest { client_metadata: response_create_client_metadata( Some(self.client.build_ws_client_metadata( - window_id, - turn_metadata_header, + responses_metadata, model_info.use_responses_lite, )), request_trace.as_ref(), @@ -1425,7 +1375,7 @@ impl ModelClientSession { session_telemetry, api_provider: client_setup.api_provider, api_auth: client_setup.api_auth, - turn_metadata_header, + responses_metadata, options: &options, auth_context: request_auth_context, request_route_telemetry: RequestRouteTelemetry::for_endpoint( @@ -1544,14 +1494,13 @@ impl ModelClientSession { #[allow(clippy::too_many_arguments)] pub async fn prewarm_websocket( &mut self, - window_id: &str, prompt: &Prompt, model_info: &ModelInfo, session_telemetry: &SessionTelemetry, effort: Option, summary: ReasoningSummaryConfig, service_tier: Option, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, ) -> Result<()> { if !self.client.responses_websocket_enabled() { return Ok(()); @@ -1563,14 +1512,13 @@ impl ModelClientSession { let disabled_trace = InferenceTraceContext::disabled(); match self .stream_responses_websocket( - window_id, prompt, model_info, session_telemetry, effort, summary, service_tier, - turn_metadata_header, + responses_metadata, /*warmup*/ true, current_span_w3c_trace_context(), &disabled_trace, @@ -1607,14 +1555,13 @@ impl ModelClientSession { /// branches. pub async fn stream( &mut self, - window_id: &str, prompt: &Prompt, model_info: &ModelInfo, session_telemetry: &SessionTelemetry, effort: Option, summary: ReasoningSummaryConfig, service_tier: Option, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, inference_trace: &InferenceTraceContext, ) -> Result { let wire_api = self.client.state.provider.info().wire_api; @@ -1624,14 +1571,13 @@ impl ModelClientSession { let request_trace = current_span_w3c_trace_context(); match self .stream_responses_websocket( - window_id, prompt, model_info, session_telemetry, effort.clone(), summary, service_tier.clone(), - turn_metadata_header, + responses_metadata, /*warmup*/ false, request_trace, inference_trace, @@ -1646,14 +1592,13 @@ impl ModelClientSession { } self.stream_responses_api( - window_id, prompt, model_info, session_telemetry, effort, summary, service_tier, - turn_metadata_header, + responses_metadata, inference_trace, ) .await @@ -1680,14 +1625,6 @@ impl ModelClientSession { } } -/// Parses per-turn metadata into an HTTP header value. -/// -/// Invalid values are treated as absent so callers can compare and propagate -/// metadata with the same sanitization path used when constructing headers. -fn parse_turn_metadata_header(turn_metadata_header: Option<&str>) -> Option { - turn_metadata_header.and_then(|value| HeaderValue::from_str(value).ok()) -} - /// Stamp a ResponsesWsRequest with the current time. /// /// Meant to be called just before sending the request over the socket, to capture realistic @@ -1709,11 +1646,9 @@ fn stamp_ws_stream_request_start_ms(request: &mut ResponsesWsRequest) { /// /// - `x-codex-beta-features`: comma-separated beta feature keys enabled for the session. /// - `x-codex-turn-state`: sticky routing token captured earlier in the turn. -/// - `x-codex-turn-metadata`: optional per-turn metadata for observability. fn build_responses_headers( beta_features_header: Option<&str>, turn_state: Option<&Arc>>, - turn_metadata_header: Option<&HeaderValue>, ) -> ApiHeaderMap { let mut headers = ApiHeaderMap::new(); if let Some(value) = beta_features_header @@ -1728,9 +1663,6 @@ fn build_responses_headers( { headers.insert(X_CODEX_TURN_STATE_HEADER, header_value); } - if let Some(header_value) = turn_metadata_header { - headers.insert(X_CODEX_TURN_METADATA_HEADER, header_value.clone()); - } headers } @@ -1743,31 +1675,6 @@ fn add_responses_lite_header(headers: &mut ApiHeaderMap, use_responses_lite: boo } } -fn subagent_header_value(session_source: &SessionSource) -> Option { - match session_source { - SessionSource::SubAgent(subagent_source) => match subagent_source { - SubAgentSource::Review => Some("review".to_string()), - SubAgentSource::Compact => Some("compact".to_string()), - SubAgentSource::MemoryConsolidation => Some("memory_consolidation".to_string()), - SubAgentSource::ThreadSpawn { .. } => Some("collab_spawn".to_string()), - SubAgentSource::Other(label) => Some(label.clone()), - }, - SessionSource::Internal(InternalSessionSource::MemoryConsolidation) => { - Some("memory_consolidation".to_string()) - } - SessionSource::Cli - | SessionSource::VSCode - | SessionSource::Exec - | SessionSource::Mcp - | SessionSource::Custom(_) - | SessionSource::Unknown => None, - } -} - -fn parent_thread_id_header_value(parent_thread_id: Option) -> Option { - parent_thread_id.map(|parent_thread_id| parent_thread_id.to_string()) -} - const RESPONSE_STREAM_CHANNEL_CAPACITY: usize = 1600; const STREAM_DROPPED_REASON: &str = "response stream dropped before provider terminal event"; @@ -1985,7 +1892,7 @@ impl AuthRequestTelemetryContext { let auth_telemetry = auth_header_telemetry(api_auth); Self { auth_mode: auth_mode.map(|mode| match mode { - AuthMode::ApiKey => "ApiKey", + AuthMode::ApiKey | AuthMode::BedrockApiKey => "ApiKey", AuthMode::Chatgpt | AuthMode::ChatgptAuthTokens | AuthMode::AgentIdentity @@ -2004,7 +1911,7 @@ struct WebsocketConnectParams<'a> { session_telemetry: &'a SessionTelemetry, api_provider: codex_api::Provider, api_auth: SharedAuthProvider, - turn_metadata_header: Option<&'a str>, + responses_metadata: &'a CodexResponsesMetadata, options: &'a ApiResponsesOptions, auth_context: AuthRequestTelemetryContext, request_route_telemetry: RequestRouteTelemetry, diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 48d0523df4..4d7729d11b 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -2,6 +2,8 @@ pub use codex_api::ResponseEvent; use codex_config::types::Personality; use codex_protocol::error::Result; use codex_protocol::models::BaseInstructions; +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::InterAgentCommunication; use codex_tools::ToolSpec; @@ -71,6 +73,56 @@ impl Prompt { }) .collect() } + + pub(crate) fn get_formatted_input_for_request( + &self, + use_responses_lite: bool, + ) -> Vec { + let mut input = self.get_formatted_input(); + if use_responses_lite { + strip_image_details(&mut input); + } + input + } +} + +fn strip_image_details(items: &mut [ResponseItem]) { + for item in items { + match item { + ResponseItem::Message { content, .. } => { + for content_item in content { + if let ContentItem::InputImage { detail, .. } = content_item { + *detail = None; + } + } + } + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + if let Some(content) = output.content_items_mut() { + for content_item in content { + if let FunctionCallOutputContentItem::InputImage { detail, .. } = + content_item + { + *detail = None; + } + } + } + } + ResponseItem::Reasoning { .. } + | ResponseItem::AgentMessage { .. } + | ResponseItem::LocalShellCall { .. } + | ResponseItem::FunctionCall { .. } + | ResponseItem::ToolSearchCall { .. } + | ResponseItem::CustomToolCall { .. } + | ResponseItem::ToolSearchOutput { .. } + | ResponseItem::WebSearchCall { .. } + | ResponseItem::ImageGenerationCall { .. } + | ResponseItem::Compaction { .. } + | ResponseItem::CompactionTrigger + | ResponseItem::ContextCompaction { .. } + | ResponseItem::Other => {} + } + } } pub struct ResponseStream { diff --git a/codex-rs/core/src/client_common_tests.rs b/codex-rs/core/src/client_common_tests.rs index 45d4334222..4b218e490c 100644 --- a/codex-rs/core/src/client_common_tests.rs +++ b/codex-rs/core/src/client_common_tests.rs @@ -3,10 +3,95 @@ use codex_api::ResponsesApiRequest; use codex_api::TextControls; use codex_api::create_text_param_for_request; use codex_protocol::config_types::ServiceTier; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageDetail; use pretty_assertions::assert_eq; use super::*; +fn prompt_with_image_outputs() -> Prompt { + Prompt { + input: vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + detail: Some(ImageDetail::Original), + }], + phase: None, + }, + ResponseItem::FunctionCallOutput { + call_id: "function-call".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,function".to_string(), + detail: Some(ImageDetail::High), + }, + ]), + }, + ResponseItem::CustomToolCallOutput { + call_id: "custom-call".to_string(), + name: None, + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,custom".to_string(), + detail: Some(ImageDetail::Auto), + }, + ]), + }, + ], + ..Default::default() + } +} + +#[test] +fn responses_lite_request_copies_strip_image_details() { + let prompt = prompt_with_image_outputs(); + let original = prompt.input.clone(); + + let stripped = prompt.get_formatted_input_for_request(/*use_responses_lite*/ true); + + assert_eq!( + stripped, + vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + detail: None, + }], + phase: None, + }, + ResponseItem::FunctionCallOutput { + call_id: "function-call".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,function".to_string(), + detail: None, + }, + ]), + }, + ResponseItem::CustomToolCallOutput { + call_id: "custom-call".to_string(), + name: None, + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,custom".to_string(), + detail: None, + }, + ]), + }, + ] + ); + assert_eq!(prompt.input, original); + assert_eq!( + prompt.get_formatted_input_for_request(/*use_responses_lite*/ false), + original + ); +} + #[test] fn serializes_text_verbosity_when_set() { let input: Vec = vec![]; diff --git a/codex-rs/core/src/client_tests.rs b/codex-rs/core/src/client_tests.rs index d93bd8bf7a..18cd202236 100644 --- a/codex-rs/core/src/client_tests.rs +++ b/codex-rs/core/src/client_tests.rs @@ -10,6 +10,9 @@ use super::X_OPENAI_SUBAGENT_HEADER; use crate::AttestationContext; use crate::AttestationProvider; use crate::GenerateAttestationFuture; +use crate::responses_metadata::CodexResponsesMetadata; +use crate::test_support::TestCodexResponsesRequestKind; +use crate::test_support::responses_metadata as test_responses_metadata; use codex_api::ApiError; use codex_api::ResponseEvent; use codex_app_server_protocol::AuthMode; @@ -21,7 +24,6 @@ use codex_model_provider_info::ModelProviderInfo; use codex_model_provider_info::WireApi; use codex_model_provider_info::create_oss_provider_with_base_url; use codex_otel::SessionTelemetry; -use codex_protocol::SessionId; use codex_protocol::ThreadId; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; @@ -60,24 +62,16 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::registry::LookupSpan; use tracing_subscriber::util::SubscriberInitExt; -fn test_model_client(session_source: SessionSource) -> ModelClient { - test_model_client_with_parent(session_source, /*parent_thread_id*/ None) -} +const TEST_INSTALLATION_ID: &str = "11111111-1111-4111-8111-111111111111"; -fn test_model_client_with_parent( - session_source: SessionSource, - parent_thread_id: Option, -) -> ModelClient { +fn test_model_client(session_source: SessionSource) -> ModelClient { let provider = create_oss_provider_with_base_url("https://example.com/v1", WireApi::Responses); let thread_id = ThreadId::new(); ModelClient::new( /*auth_manager*/ None, - thread_id.into(), thread_id, - /*installation_id*/ "11111111-1111-4111-8111-111111111111".to_string(), provider, session_source, - parent_thread_id, /*model_verbosity*/ None, /*enable_request_compression*/ false, /*include_timing_metrics*/ false, @@ -86,6 +80,26 @@ fn test_model_client_with_parent( ) } +fn test_responses_metadata_for_client( + client: &ModelClient, + turn_id: Option<&str>, + window_id: String, + parent_thread_id: Option, + request_kind: TestCodexResponsesRequestKind, +) -> CodexResponsesMetadata { + let thread_id = client.state.thread_id.to_string(); + test_responses_metadata( + TEST_INSTALLATION_ID, + &thread_id, + &thread_id, + turn_id, + window_id, + &client.state.session_source, + parent_thread_id, + request_kind, + ) +} + fn test_model_info() -> ModelInfo { serde_json::from_value(json!({ "slug": "gpt-test", @@ -280,48 +294,63 @@ fn build_subagent_headers_sets_internal_memory_consolidation_label() { #[test] fn build_ws_client_metadata_includes_window_lineage_and_turn_metadata() { let parent_thread_id = ThreadId::new(); - let client = test_model_client_with_parent( - SessionSource::SubAgent(SubAgentSource::ThreadSpawn { - parent_thread_id, - depth: 2, - agent_path: None, - agent_nickname: None, - agent_role: None, - }), - Some(parent_thread_id), - ); + let client = test_model_client(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth: 2, + agent_path: None, + agent_nickname: None, + agent_role: None, + })); - let thread_id = client.state.thread_id; - let window_id = format!("{thread_id}:1"); - let client_metadata = client.build_ws_client_metadata( - &window_id, - Some(r#"{"turn_id":"turn-123"}"#), - /*use_responses_lite*/ false, + let thread_id = client.state.thread_id.to_string(); + let expected_window_id = format!("{thread_id}:1"); + let responses_metadata = test_responses_metadata_for_client( + &client, + Some("turn-123"), + expected_window_id.clone(), + Some(parent_thread_id), + TestCodexResponsesRequestKind::Turn, ); + let client_metadata = + client.build_ws_client_metadata(&responses_metadata, /*use_responses_lite*/ false); + let parent_thread_id = parent_thread_id.to_string(); + let turn_metadata: serde_json::Value = serde_json::from_str( + client_metadata + .get(X_CODEX_TURN_METADATA_HEADER) + .expect("turn metadata"), + ) + .expect("valid turn metadata"); + for (client_key, metadata_key, expected) in [ + ( + X_CODEX_INSTALLATION_ID_HEADER, + "installation_id", + "11111111-1111-4111-8111-111111111111", + ), + ("session_id", "session_id", thread_id.as_str()), + ("thread_id", "thread_id", thread_id.as_str()), + ("turn_id", "turn_id", "turn-123"), + ( + X_CODEX_WINDOW_ID_HEADER, + "window_id", + expected_window_id.as_str(), + ), + ( + X_CODEX_PARENT_THREAD_ID_HEADER, + "parent_thread_id", + parent_thread_id.as_str(), + ), + ] { + assert_eq!( + client_metadata.get(client_key).map(String::as_str), + Some(expected) + ); + assert_eq!(turn_metadata[metadata_key].as_str(), Some(expected)); + } assert_eq!( - client_metadata, - std::collections::HashMap::from([ - ( - X_CODEX_INSTALLATION_ID_HEADER.to_string(), - "11111111-1111-4111-8111-111111111111".to_string(), - ), - ( - X_CODEX_WINDOW_ID_HEADER.to_string(), - format!("{thread_id}:1"), - ), - ( - X_OPENAI_SUBAGENT_HEADER.to_string(), - "collab_spawn".to_string(), - ), - ( - X_CODEX_PARENT_THREAD_ID_HEADER.to_string(), - parent_thread_id.to_string(), - ), - ( - X_CODEX_TURN_METADATA_HEADER.to_string(), - r#"{"turn_id":"turn-123"}"#.to_string(), - ), - ]) + client_metadata + .get(X_OPENAI_SUBAGENT_HEADER) + .map(String::as_str), + Some("collab_spawn") ); } @@ -529,12 +558,9 @@ fn model_client_with_counting_attestation( }; let model_client = ModelClient::new( auth_manager, - SessionId::new(), ThreadId::new(), - /*installation_id*/ "11111111-1111-4111-8111-111111111111".to_string(), provider, SessionSource::Exec, - /*parent_thread_id*/ None, /*model_verbosity*/ None, /*enable_request_compression*/ false, /*include_timing_metrics*/ false, @@ -550,9 +576,16 @@ fn model_client_with_counting_attestation( async fn websocket_handshake_includes_attestation_for_chatgpt_codex_responses() { let (model_client, attestation_calls) = model_client_with_counting_attestation(/*include_attestation*/ true); + let responses_metadata = test_responses_metadata_for_client( + &model_client, + /*turn_id*/ None, + format!("{}:0", model_client.state.thread_id), + /*parent_thread_id*/ None, + TestCodexResponsesRequestKind::WebsocketConnection, + ); let headers = model_client - .build_websocket_headers(/*turn_state*/ None, /*turn_metadata_header*/ None) + .build_websocket_headers(&responses_metadata, /*turn_state*/ None) .await; assert_eq!( diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index 5b68ecf917..6ee0328463 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -5,6 +5,7 @@ use async_channel::Receiver; use async_channel::Sender; use codex_analytics::GuardianApprovalRequestSource; use codex_async_utils::OrCancelExt; +use codex_extension_api::LoadedUserInstructions; use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; @@ -79,8 +80,13 @@ pub(crate) async fn run_codex_thread_interactive( let (tx_ops, rx_ops) = async_channel::bounded(SUBMISSION_CHANNEL_CAPACITY); let conversation_history = initial_history.unwrap_or(InitialHistory::New); let forked_from_thread_id = conversation_history.forked_from_id(); + let user_instructions = LoadedUserInstructions { + instructions: parent_session.user_instructions().await, + warnings: Vec::new(), + }; let CodexSpawnOk { codex, .. } = Box::pin(Codex::spawn(CodexSpawnArgs { config, + user_instructions, installation_id: parent_session.installation_id.clone(), auth_manager, models_manager, diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index ab1f6df03e..c4cb1e8e12 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -159,6 +159,14 @@ pub struct CodexThread { out_of_band_elicitation_count: Mutex, } +#[derive(Debug, Eq, PartialEq)] +pub struct BackgroundTerminalInfo { + pub item_id: String, + pub process_id: String, + pub command: String, + pub cwd: AbsolutePathBuf, +} + /// Conduit for the bidirectional stream of messages that compose a thread /// (formerly called a conversation) in Codex. impl CodexThread { @@ -396,6 +404,17 @@ impl CodexThread { self.codex.agent_status().await } + pub async fn list_background_terminals(&self) -> Vec { + self.codex.session.list_background_terminals().await + } + + pub async fn terminate_background_terminal(&self, process_id: i32) -> bool { + self.codex + .session + .terminate_background_terminal(process_id) + .await + } + pub(crate) fn subscribe_status(&self) -> watch::Receiver { self.codex.agent_status.clone() } diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 96feddf138..6ffb779d38 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -8,12 +8,14 @@ use crate::hook_runtime::PostCompactHookOutcome; use crate::hook_runtime::PreCompactHookOutcome; use crate::hook_runtime::run_post_compact_hooks; use crate::hook_runtime::run_pre_compact_hooks; +use crate::responses_metadata::CodexResponsesMetadata; +use crate::responses_metadata::CodexResponsesRequestKind; +use crate::responses_metadata::CompactionTurnMetadata; #[cfg(test)] use crate::session::PreviousTurnSettings; use crate::session::session::Session; use crate::session::turn::get_last_assistant_message_from_turn; use crate::session::turn_context::TurnContext; -use crate::turn_metadata::CompactionTurnMetadata; use crate::util::backoff; use codex_analytics::CodexCompactionEvent; use codex_analytics::CompactionImplementation; @@ -143,17 +145,17 @@ async fn run_compact_task_inner( let pre_compact_outcome = run_pre_compact_hooks(&sess, &turn_context, trigger).await; match pre_compact_outcome { PreCompactHookOutcome::Continue => {} - PreCompactHookOutcome::Stopped { reason } => { - let error = reason.unwrap_or_else(|| "PreCompact hook stopped execution".to_string()); + PreCompactHookOutcome::Stopped => { + let error = CodexErr::TurnAborted; attempt .track( sess.as_ref(), CompactionStatus::Interrupted, - Some(error), + Some(&error), CompactionAnalyticsDetails::default(), ) .await; - return Err(CodexErr::TurnAborted); + return Err(error); } } let result = run_compact_task_inner_impl( @@ -165,7 +167,7 @@ async fn run_compact_task_inner( ) .await; let status = compaction_status_from_result(&result); - let error = result.as_ref().err().map(ToString::to_string); + let codex_error = result.as_ref().err(); if result.is_ok() { let post_compact_outcome = run_post_compact_hooks(&sess, &turn_context, trigger).await; if let PostCompactHookOutcome::Stopped = post_compact_outcome { @@ -173,7 +175,7 @@ async fn run_compact_task_inner( .track( sess.as_ref(), status, - error, + codex_error, CompactionAnalyticsDetails::default(), ) .await; @@ -184,7 +186,7 @@ async fn run_compact_task_inner( .track( sess.as_ref(), status, - error, + codex_error, CompactionAnalyticsDetails::default(), ) .await; @@ -215,6 +217,12 @@ async fn run_compact_task_inner_impl( // Reuse one client session so turn-scoped state (sticky routing, websocket incremental // request tracking) // survives retries within this compact turn. + let window_id = sess.current_window_id().await; + let responses_metadata = turn_context.turn_metadata_state.to_responses_metadata( + sess.installation_id.clone(), + window_id, + CodexResponsesRequestKind::Compaction(compaction_metadata), + ); loop { // Clone is required because of the loop @@ -228,16 +236,11 @@ async fn run_compact_task_inner_impl( personality: turn_context.personality, ..Default::default() }; - let window_id = sess.current_window_id().await; - let turn_metadata_header = turn_context - .turn_metadata_state - .current_header_value_for_compaction(&window_id, compaction_metadata); let attempt_result = drain_to_completed( &sess, turn_context.as_ref(), &mut client_session, - &window_id, - turn_metadata_header.as_deref(), + &responses_metadata, &prompt, ) .await; @@ -294,6 +297,7 @@ async fn run_compact_task_inner_impl( let user_messages = collect_user_messages(history_items); let mut new_history = build_compacted_history(Vec::new(), &user_messages, &summary_text); + let window_id = sess.advance_auto_compact_window_id().await; if matches!( initial_context_injection, @@ -310,7 +314,7 @@ async fn run_compact_task_inner_impl( let compacted_item = CompactedItem { message: summary_text.clone(), replacement_history: Some(new_history.clone()), - window_id: None, + window_id: Some(window_id), }; sess.replace_compacted_history(new_history, reference_context_item, compacted_item) .await; @@ -342,6 +346,7 @@ pub(crate) struct CompactionAnalyticsDetails { pub(crate) active_context_tokens_before: Option, pub(crate) retained_image_count: Option, pub(crate) compaction_summary_tokens: Option, + pub(crate) cached_input_tokens: Option, } impl CompactionAnalyticsAttempt { @@ -371,13 +376,14 @@ impl CompactionAnalyticsAttempt { self, sess: &Session, status: CompactionStatus, - error: Option, + codex_error: Option<&CodexErr>, details: CompactionAnalyticsDetails, ) { let CompactionAnalyticsDetails { active_context_tokens_before, retained_image_count, compaction_summary_tokens, + cached_input_tokens, } = details; let active_context_tokens_before = active_context_tokens_before.unwrap_or(self.active_context_tokens_before); @@ -393,11 +399,14 @@ impl CompactionAnalyticsAttempt { phase: self.phase, strategy: CompactionStrategy::Memento, status, - error, + codex_error_kind: codex_error.map(Into::into), + codex_error_http_status_code: codex_error + .and_then(CodexErr::http_status_code_value), active_context_tokens_before, active_context_tokens_after, retained_image_count, compaction_summary_tokens, + cached_input_tokens, started_at: self.started_at, completed_at: now_unix_seconds(), duration_ms: Some( @@ -581,20 +590,18 @@ async fn drain_to_completed( sess: &Session, turn_context: &TurnContext, client_session: &mut ModelClientSession, - window_id: &str, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, prompt: &Prompt, ) -> CodexResult<()> { let mut stream = client_session .stream( - window_id, prompt, &turn_context.model_info, &turn_context.session_telemetry, turn_context.reasoning_effort.clone(), turn_context.reasoning_summary, turn_context.config.service_tier.clone(), - turn_metadata_header, + responses_metadata, // Rollout tracing currently models remote compaction only; local compaction streams // are left untraced until the reducer has a first-class local compaction lifecycle. &InferenceTraceContext::disabled(), diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 767fdf7b01..0822d0f0a6 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -12,10 +12,11 @@ use crate::hook_runtime::PostCompactHookOutcome; use crate::hook_runtime::PreCompactHookOutcome; use crate::hook_runtime::run_post_compact_hooks; use crate::hook_runtime::run_pre_compact_hooks; +use crate::responses_metadata::CodexResponsesRequestKind; +use crate::responses_metadata::CompactionTurnMetadata; use crate::session::session::Session; use crate::session::turn::built_tools; use crate::session::turn_context::TurnContext; -use crate::turn_metadata::CompactionTurnMetadata; use codex_analytics::CompactionImplementation; use codex_analytics::CompactionPhase; use codex_analytics::CompactionReason; @@ -113,17 +114,17 @@ async fn run_remote_compact_task_inner( let pre_compact_outcome = run_pre_compact_hooks(sess, turn_context, trigger).await; match pre_compact_outcome { PreCompactHookOutcome::Continue => {} - PreCompactHookOutcome::Stopped { reason } => { - let error = reason.unwrap_or_else(|| "PreCompact hook stopped execution".to_string()); + PreCompactHookOutcome::Stopped => { + let error = CodexErr::TurnAborted; attempt .track( sess.as_ref(), codex_analytics::CompactionStatus::Interrupted, - Some(error), + Some(&error), analytics_details, ) .await; - return Err(CodexErr::TurnAborted); + return Err(error); } } let result = run_remote_compact_task_inner_impl( @@ -135,18 +136,18 @@ async fn run_remote_compact_task_inner( ) .await; let status = compaction_status_from_result(&result); - let error = result.as_ref().err().map(ToString::to_string); + let codex_error = result.as_ref().err(); if result.is_ok() { let post_compact_outcome = run_post_compact_hooks(sess, turn_context, trigger).await; if let PostCompactHookOutcome::Stopped = post_compact_outcome { attempt - .track(sess.as_ref(), status, error, analytics_details) + .track(sess.as_ref(), status, codex_error, analytics_details) .await; return Err(CodexErr::TurnAborted); } } attempt - .track(sess.as_ref(), status, error.clone(), analytics_details) + .track(sess.as_ref(), status, codex_error, analytics_details) .await; if let Err(err) = result { sess.track_turn_codex_error(turn_context, &err); @@ -225,9 +226,11 @@ async fn run_remote_compact_task_inner_impl( output_schema_strict: true, }; let window_id = sess.current_window_id().await; - let turn_metadata_header = turn_context - .turn_metadata_state - .current_header_value_for_compaction(&window_id, compaction_metadata); + let responses_metadata = turn_context.turn_metadata_state.to_responses_metadata( + sess.installation_id.clone(), + window_id, + CodexResponsesRequestKind::Compaction(compaction_metadata), + ); let mut new_history = sess .services .model_client @@ -245,10 +248,10 @@ async fn run_remote_compact_task_inner_impl( }, &turn_context.session_telemetry, &compaction_trace, - &window_id, - turn_metadata_header.as_deref(), + &responses_metadata, ) .await?; + let new_window_id = sess.advance_auto_compact_window_id().await; new_history = process_compacted_history( sess.as_ref(), turn_context.as_ref(), @@ -264,7 +267,7 @@ async fn run_remote_compact_task_inner_impl( let compacted_item = CompactedItem { message: String::new(), replacement_history: Some(new_history.clone()), - window_id: None, + window_id: Some(new_window_id), }; // Install is the semantic boundary where the compact endpoint's output becomes live // thread history. Keep it distinct from the later inference request so the reducer can diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index 9a14dcd963..047c2527cb 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -15,12 +15,14 @@ use crate::hook_runtime::PostCompactHookOutcome; use crate::hook_runtime::PreCompactHookOutcome; use crate::hook_runtime::run_post_compact_hooks; use crate::hook_runtime::run_pre_compact_hooks; +use crate::responses_metadata::CodexResponsesMetadata; +use crate::responses_metadata::CodexResponsesRequestKind; +use crate::responses_metadata::CompactionTurnMetadata; use crate::responses_retry::ResponsesStreamRequest; use crate::responses_retry::handle_retryable_response_stream_error; use crate::session::session::Session; use crate::session::turn::built_tools; use crate::session::turn_context::TurnContext; -use crate::turn_metadata::CompactionTurnMetadata; use codex_analytics::CompactionImplementation; use codex_analytics::CompactionPhase; use codex_analytics::CompactionReason; @@ -127,17 +129,17 @@ async fn run_remote_compact_task_inner( let pre_compact_outcome = run_pre_compact_hooks(sess, turn_context, trigger).await; match pre_compact_outcome { PreCompactHookOutcome::Continue => {} - PreCompactHookOutcome::Stopped { reason } => { - let error = reason.unwrap_or_else(|| "PreCompact hook stopped execution".to_string()); + PreCompactHookOutcome::Stopped => { + let error = CodexErr::TurnAborted; attempt .track( sess.as_ref(), codex_analytics::CompactionStatus::Interrupted, - Some(error), + Some(&error), analytics_details, ) .await; - return Err(CodexErr::TurnAborted); + return Err(error); } } let result = run_remote_compact_task_inner_impl( @@ -150,18 +152,18 @@ async fn run_remote_compact_task_inner( ) .await; let status = compaction_status_from_result(&result); - let error = result.as_ref().err().map(ToString::to_string); + let codex_error = result.as_ref().err(); if result.is_ok() { let post_compact_outcome = run_post_compact_hooks(sess, turn_context, trigger).await; if let PostCompactHookOutcome::Stopped = post_compact_outcome { attempt - .track(sess.as_ref(), status, error, analytics_details) + .track(sess.as_ref(), status, codex_error, analytics_details) .await; return Err(CodexErr::TurnAborted); } } attempt - .track(sess.as_ref(), status, error.clone(), analytics_details) + .track(sess.as_ref(), status, codex_error, analytics_details) .await; if let Err(err) = result { sess.track_turn_codex_error(turn_context, &err); @@ -241,9 +243,11 @@ async fn run_remote_compact_task_inner_impl( }; let window_id = sess.current_window_id().await; - let turn_metadata_header = turn_context - .turn_metadata_state - .current_header_value_for_compaction(&window_id, compaction_metadata); + let responses_metadata = turn_context.turn_metadata_state.to_responses_metadata( + sess.installation_id.clone(), + window_id, + CodexResponsesRequestKind::Compaction(compaction_metadata), + ); let trace_attempt = compaction_trace.start_attempt(&serde_json::json!({ "model": turn_context.model_info.slug.as_str(), "instructions": prompt.base_instructions.text.as_str(), @@ -264,8 +268,7 @@ async fn run_remote_compact_task_inner_impl( turn_context, client_session, &prompt, - &window_id, - turn_metadata_header.as_deref(), + &responses_metadata, ) .await; @@ -281,10 +284,12 @@ async fn run_remote_compact_task_inner_impl( if let Some(token_usage) = token_usage { analytics_details.active_context_tokens_before = Some(token_usage.input_tokens); analytics_details.compaction_summary_tokens = Some(token_usage.output_tokens); + analytics_details.cached_input_tokens = Some(token_usage.cached_input_tokens); } let (compacted_history, retained_images) = build_v2_compacted_history(&prompt_input, compaction_output); analytics_details.retained_image_count = Some(retained_images); + let new_window_id = sess.advance_auto_compact_window_id().await; let new_history = process_compacted_history( sess.as_ref(), turn_context.as_ref(), @@ -300,7 +305,7 @@ async fn run_remote_compact_task_inner_impl( let compacted_item = CompactedItem { message: String::new(), replacement_history: Some(new_history.clone()), - window_id: None, + window_id: Some(new_window_id), }; compaction_trace.record_installed(&CompactionCheckpointTracePayload { input_history: &trace_input_history, @@ -325,8 +330,7 @@ async fn run_remote_compaction_request_v2( turn_context: &TurnContext, client_session: &mut ModelClientSession, prompt: &Prompt, - window_id: &str, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, ) -> CodexResult { let max_retries = turn_context .provider @@ -337,14 +341,13 @@ async fn run_remote_compaction_request_v2( loop { let result = match client_session .stream( - window_id, prompt, &turn_context.model_info, &turn_context.session_telemetry, turn_context.reasoning_effort.clone(), turn_context.reasoning_summary, turn_context.config.service_tier.clone(), - turn_metadata_header, + responses_metadata, &InferenceTraceContext::disabled(), ) .await diff --git a/codex-rs/core/src/compact_tests.rs b/codex-rs/core/src/compact_tests.rs index 69bd95f2df..550a7ec846 100644 --- a/codex-rs/core/src/compact_tests.rs +++ b/codex-rs/core/src/compact_tests.rs @@ -483,6 +483,7 @@ async fn process_compacted_history_reinjects_model_switch_message() { }]; let previous_turn_settings = PreviousTurnSettings { model: "previous-regular-model".to_string(), + comp_hash: None, realtime_active: None, }; diff --git a/codex-rs/core/src/config/agent_roles.rs b/codex-rs/core/src/config/agent_roles.rs index abdef33e7d..dfcec2eac8 100644 --- a/codex-rs/core/src/config/agent_roles.rs +++ b/codex-rs/core/src/config/agent_roles.rs @@ -7,6 +7,7 @@ use codex_config::config_toml::ConfigToml; use codex_exec_server::ExecutorFileSystem; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::AbsolutePathBufGuard; +use codex_utils_path_uri::PathUri; use serde::Deserialize; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -319,7 +320,8 @@ async fn read_resolved_agent_role_file( path: &AbsolutePathBuf, role_name_hint: Option<&str>, ) -> std::io::Result { - let contents = fs.read_file_text(path, /*sandbox*/ None).await?; + let path_uri = PathUri::from_abs_path(path)?; + let contents = fs.read_file_text(&path_uri, /*sandbox*/ None).await?; let config_base_dir = path.parent().unwrap_or_else(|| path.clone()); parse_agent_role_file_contents( &contents, @@ -391,8 +393,9 @@ async fn validate_agent_role_config_file( return Ok(()); }; + let config_file_uri = PathUri::from_abs_path(config_file)?; let metadata = fs - .get_metadata(config_file, /*sandbox*/ None) + .get_metadata(&config_file_uri, /*sandbox*/ None) .await .map_err(|e| { std::io::Error::new( @@ -522,7 +525,8 @@ async fn collect_agent_role_files( let mut files = Vec::new(); let mut dirs = vec![dir.clone()]; while let Some(dir) = dirs.pop() { - let entries = match fs.read_directory(&dir, /*sandbox*/ None).await { + let dir_uri = PathUri::from_abs_path(&dir)?; + let entries = match fs.read_directory(&dir_uri, /*sandbox*/ None).await { Ok(entries) => entries, Err(err) if err.kind() == ErrorKind::NotFound => continue, Err(err) => return Err(err), diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index bd187120d2..bbefa53ac4 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -1,5 +1,3 @@ -use crate::agents_md::DEFAULT_AGENTS_MD_FILENAME; -use crate::agents_md::LOCAL_AGENTS_MD_FILENAME; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config::edit::apply_blocking; @@ -207,61 +205,6 @@ async fn load_config_normalizes_relative_cwd_override() -> std::io::Result<()> { Ok(()) } -#[tokio::test] -async fn load_config_loads_global_agents_instructions() -> std::io::Result<()> { - let codex_home = tempdir()?; - let global_agents_path = codex_home.abs().join(DEFAULT_AGENTS_MD_FILENAME); - std::fs::write(&global_agents_path, "\n global instructions \n")?; - - let mut config = Config::load_from_base_config_with_overrides( - ConfigToml::default(), - ConfigOverrides::default(), - codex_home.abs(), - ) - .await?; - let _ = config.features.enable(Feature::MemoryTool); - - let user_instructions = config - .user_instructions - .as_ref() - .expect("global instructions expected"); - assert_eq!(user_instructions.text(), "global instructions"); - assert_eq!( - user_instructions.sources().collect::>(), - vec![&global_agents_path] - ); - Ok(()) -} - -#[tokio::test] -async fn load_config_prefers_global_agents_override_instructions() -> std::io::Result<()> { - let codex_home = tempdir()?; - std::fs::write( - codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME), - "global instructions", - )?; - let global_agents_override_path = codex_home.abs().join(LOCAL_AGENTS_MD_FILENAME); - std::fs::write(&global_agents_override_path, "local override instructions")?; - - let config = Config::load_from_base_config_with_overrides( - ConfigToml::default(), - ConfigOverrides::default(), - codex_home.abs(), - ) - .await?; - - let user_instructions = config - .user_instructions - .as_ref() - .expect("global override instructions expected"); - assert_eq!(user_instructions.text(), "local override instructions"); - assert_eq!( - user_instructions.sources().collect::>(), - vec![&global_agents_override_path] - ); - Ok(()) -} - #[tokio::test] async fn test_toml_parsing() { let history_with_persistence = r#" @@ -4548,13 +4491,14 @@ async fn rebuild_preserving_session_layers_refreshes_plugin_derived_mcp_config() .await?; let plugins_manager = PluginsManager::new(codex_home.path().to_path_buf()); let mcp_config = config.to_mcp_config(&plugins_manager).await; + let configured_servers = mcp_config.mcp_server_catalog.configured_servers(); assert_eq!( - mcp_config.configured_mcp_servers.get("sample"), + configured_servers.get("sample"), Some(&http_mcp("https://sample.example/mcp")) ); assert_eq!( - mcp_config.plugin_ids_by_mcp_server_name, + mcp_config.mcp_server_catalog.plugin_ids_by_server_name(), HashMap::from([("sample".to_string(), "sample@test".to_string())]) ); @@ -4604,12 +4548,18 @@ enabled = true .await?; let plugins_manager = PluginsManager::new(codex_home.path().to_path_buf()); let mcp_config = config.to_mcp_config(&plugins_manager).await; + let configured_servers = mcp_config.mcp_server_catalog.configured_servers(); assert_eq!( - mcp_config.configured_mcp_servers.get("sample"), + configured_servers.get("sample"), Some(&http_mcp("https://user.example/mcp")) ); - assert!(mcp_config.plugin_ids_by_mcp_server_name.is_empty()); + assert!( + mcp_config + .mcp_server_catalog + .plugin_ids_by_server_name() + .is_empty() + ); Ok(()) } @@ -4666,17 +4616,16 @@ url = "https://sample.example/mcp" .await?; let plugins_manager = PluginsManager::new(codex_home.path().to_path_buf()); let mcp_config = config.to_mcp_config(&plugins_manager).await; + let configured_servers = mcp_config.mcp_server_catalog.configured_servers(); assert_eq!( - mcp_config - .configured_mcp_servers + configured_servers .get("sample") .map(|server| (server.enabled, server.disabled_reason.clone())), Some((true, None)) ); assert_eq!( - mcp_config - .configured_mcp_servers + configured_servers .get("unlisted") .map(|server| (server.enabled, server.disabled_reason.clone())), Some(( @@ -4739,10 +4688,10 @@ enabled = true .await?; let plugins_manager = PluginsManager::new(codex_home.path().to_path_buf()); let mcp_config = config.to_mcp_config(&plugins_manager).await; + let configured_servers = mcp_config.mcp_server_catalog.configured_servers(); assert_eq!( - mcp_config - .configured_mcp_servers + configured_servers .get("sample") .map(|server| (server.enabled, server.disabled_reason.clone())), Some(( @@ -10076,10 +10025,7 @@ enabled = true .build() .await?; - assert_eq!(config.multi_agent_v2.max_concurrent_threads_per_session, 4); - assert_eq!(config.multi_agent_v2.min_wait_timeout_ms, 10_000); - assert_eq!(config.multi_agent_v2.max_wait_timeout_ms, 3_600_000); - assert_eq!(config.multi_agent_v2.default_wait_timeout_ms, 30_000); + assert_eq!(config.multi_agent_v2, MultiAgentV2Config::default()); assert_eq!( ( config.agent_max_threads, @@ -10087,36 +10033,32 @@ enabled = true ), (None, Some(3)) ); - assert_eq!( - config.multi_agent_v2.root_agent_usage_hint_text.as_deref(), - Some(DEFAULT_MULTI_AGENT_V2_ROOT_AGENT_USAGE_HINT_TEXT) - ); - assert!( - !config - .multi_agent_v2 - .root_agent_usage_hint_text - .as_deref() - .unwrap_or_default() - .contains("maximum concurrency"), - ); - assert_eq!( - config.multi_agent_v2.subagent_usage_hint_text.as_deref(), - Some(DEFAULT_MULTI_AGENT_V2_SUBAGENT_USAGE_HINT_TEXT) - ); - assert!( - !config - .multi_agent_v2 - .subagent_usage_hint_text - .as_deref() - .unwrap_or_default() - .contains("maximum concurrency"), - ); - assert!(config.multi_agent_v2.hide_spawn_agent_metadata); - assert!(config.multi_agent_v2.non_code_mode_only); Ok(()) } +#[test] +fn multi_agent_v2_default_usage_hints_use_configured_thread_cap() { + let config_toml = toml::from_str( + r#"[features.multi_agent_v2] +enabled = true +max_concurrent_threads_per_session = 17 +"#, + ) + .expect("multi-agent v2 config should parse"); + + let config = resolve_multi_agent_v2_config(&config_toml); + let concurrency_guidance = "There are 17 available concurrency slots, meaning that up to 17 agents can be active at once, including you."; + assert!( + [ + config.root_agent_usage_hint_text, + config.subagent_usage_hint_text, + ] + .into_iter() + .all(|hint| hint.is_some_and(|hint| hint.ends_with(concurrency_guidance))) + ); +} + #[tokio::test] async fn multi_agent_v2_empty_usage_hint_overrides_clear_default_hints() -> std::io::Result<()> { let codex_home = TempDir::new()?; diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 74c6c93e5b..62714d7e78 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1,5 +1,3 @@ -use crate::agents_md::AgentsMdManager; -pub use crate::agents_md::LoadedAgentsMd; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::path_utils::normalize_for_native_workdir; @@ -75,6 +73,8 @@ use codex_login::AuthManagerConfig; use codex_login::AuthRouteConfig; use codex_login::auth_route_config_from_system_proxy_config; use codex_mcp::McpConfig; +use codex_mcp::McpServerRegistration; +use codex_mcp::ResolvedMcpCatalog; use codex_memories_read::memory_root; use codex_model_provider_info::LEGACY_OLLAMA_CHAT_PROVIDER_ID; use codex_model_provider_info::ModelProviderInfo; @@ -109,6 +109,7 @@ use codex_protocol::protocol::SandboxPolicy; pub use codex_thread_store::ExtraConfig; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::AbsolutePathBufGuard; +use codex_utils_path_uri::PathUri; use rmcp::model::ElicitationCapability; use rmcp::model::FormElicitationCapability; use rmcp::model::UrlElicitationCapability; @@ -117,7 +118,6 @@ use serde::Serialize; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; -use std::collections::hash_map::Entry; use std::io::ErrorKind; use std::path::Path; use std::path::PathBuf; @@ -237,6 +237,13 @@ Payload: ``` You may also see them addressed as to=/root/..., which indicates your identity is /root/... "#; + +fn default_multi_agent_v2_usage_hint_text(usage_hint_text: &str, max_concurrency: usize) -> String { + format!( + "{usage_hint_text}\nThere are {max_concurrency} available concurrency slots, meaning that up to {max_concurrency} agents can be active at once, including you." + ) +} + pub(crate) const HARD_MIN_MULTI_AGENT_V2_TIMEOUT_MS: i64 = 0; pub(crate) const HARD_MAX_MULTI_AGENT_V2_TIMEOUT_MS: i64 = DEFAULT_MULTI_AGENT_V2_MAX_WAIT_TIMEOUT_MS; @@ -650,9 +657,6 @@ pub struct Config { /// Defaults to `false`. pub show_raw_agent_reasoning: bool, - /// User-provided instructions from AGENTS.md. - pub user_instructions: Option, - /// Base instructions override. pub base_instructions: Option, @@ -1062,22 +1066,23 @@ pub struct MultiAgentV2Config { pub non_code_mode_only: bool, } -impl Default for MultiAgentV2Config { - fn default() -> Self { +impl MultiAgentV2Config { + fn defaults_for_max_concurrency(max_concurrent_threads_per_session: usize) -> Self { Self { - max_concurrent_threads_per_session: - DEFAULT_MULTI_AGENT_V2_MAX_CONCURRENT_THREADS_PER_SESSION, + max_concurrent_threads_per_session, min_wait_timeout_ms: DEFAULT_MULTI_AGENT_V2_MIN_WAIT_TIMEOUT_MS, max_wait_timeout_ms: DEFAULT_MULTI_AGENT_V2_MAX_WAIT_TIMEOUT_MS, default_wait_timeout_ms: DEFAULT_MULTI_AGENT_V2_DEFAULT_WAIT_TIMEOUT_MS, usage_hint_enabled: true, usage_hint_text: None, - root_agent_usage_hint_text: Some( - DEFAULT_MULTI_AGENT_V2_ROOT_AGENT_USAGE_HINT_TEXT.to_string(), - ), - subagent_usage_hint_text: Some( - DEFAULT_MULTI_AGENT_V2_SUBAGENT_USAGE_HINT_TEXT.to_string(), - ), + root_agent_usage_hint_text: Some(default_multi_agent_v2_usage_hint_text( + DEFAULT_MULTI_AGENT_V2_ROOT_AGENT_USAGE_HINT_TEXT, + max_concurrent_threads_per_session, + )), + subagent_usage_hint_text: Some(default_multi_agent_v2_usage_hint_text( + DEFAULT_MULTI_AGENT_V2_SUBAGENT_USAGE_HINT_TEXT, + max_concurrent_threads_per_session, + )), tool_namespace: None, hide_spawn_agent_metadata: true, non_code_mode_only: true, @@ -1085,6 +1090,14 @@ impl Default for MultiAgentV2Config { } } +impl Default for MultiAgentV2Config { + fn default() -> Self { + Self::defaults_for_max_concurrency( + DEFAULT_MULTI_AGENT_V2_MAX_CONCURRENT_THREADS_PER_SESSION, + ) + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum TerminalResizeReflowMaxRows { /// Use the runtime terminal detector to choose a scrollback-sized cap. @@ -1392,12 +1405,18 @@ impl Config { ) -> McpConfig { let plugins_input = self.plugins_config_input(); let loaded_plugins = plugins_manager.plugins_for_config(&plugins_input).await; - let mut configured_mcp_servers = self.mcp_servers.get().clone(); - let mut plugin_ids_by_mcp_server_name = HashMap::new(); - for plugin in loaded_plugins + let mut catalog = ResolvedMcpCatalog::builder(); + let empty_mcp_allowlist = self + .config_layer_stack + .requirements() + .mcp_servers + .as_ref() + .filter(|requirements| requirements.value.is_empty()); + for (plugin_order, plugin) in loaded_plugins .plugins() .iter() .filter(|plugin| plugin.is_active()) + .enumerate() { let mut plugin_mcp_servers = plugin.mcp_servers.clone(); filter_plugin_mcp_servers_by_requirements( @@ -1405,22 +1424,22 @@ impl Config { &mut plugin_mcp_servers, self.config_layer_stack.requirements().plugins.as_ref(), ); + filter_mcp_servers_by_requirements(&mut plugin_mcp_servers, empty_mcp_allowlist); for (name, plugin_server) in plugin_mcp_servers { - if let Entry::Vacant(entry) = configured_mcp_servers.entry(name.clone()) { - entry.insert(plugin_server); - plugin_ids_by_mcp_server_name.insert(name, plugin.config_name.clone()); - } + catalog.register(McpServerRegistration::from_plugin( + name, + plugin.config_name.clone(), + plugin_order, + plugin_server, + )); } } - if let Some(mcp_requirements) = self.config_layer_stack.requirements().mcp_servers.as_ref() - && mcp_requirements.value.is_empty() - { - // A present empty allowlist bans configurable MCPs, including plugin MCPs merged - // above. - filter_mcp_servers_by_requirements(&mut configured_mcp_servers, Some(mcp_requirements)); + for (name, server) in self.mcp_servers.get() { + catalog.register(McpServerRegistration::from_config( + name.clone(), + server.clone(), + )); } - plugin_ids_by_mcp_server_name - .retain(|server_name, _| configured_mcp_servers.contains_key(server_name)); McpConfig { chatgpt_base_url: self.chatgpt_base_url.clone(), @@ -1447,8 +1466,7 @@ impl Config { // indicates this should be an empty object. ElicitationCapability::default() }, - configured_mcp_servers, - plugin_ids_by_mcp_server_name, + mcp_server_catalog: catalog.build(), plugin_capability_summaries: loaded_plugins.capability_summaries().to_vec(), } } @@ -2347,11 +2365,11 @@ fn resolve_code_mode_config(config_toml: &ConfigToml) -> CodeModeConfig { fn resolve_multi_agent_v2_config(config_toml: &ConfigToml) -> MultiAgentV2Config { let base = multi_agent_v2_toml_config(config_toml.features.as_ref()); - let default = MultiAgentV2Config::default(); - let max_concurrent_threads_per_session = base .and_then(|config| config.max_concurrent_threads_per_session) - .unwrap_or(default.max_concurrent_threads_per_session); + .unwrap_or(DEFAULT_MULTI_AGENT_V2_MAX_CONCURRENT_THREADS_PER_SESSION); + let default = + MultiAgentV2Config::defaults_for_max_concurrency(max_concurrent_threads_per_session); let min_wait_timeout_ms = base .and_then(|config| config.min_wait_timeout_ms) .unwrap_or(default.min_wait_timeout_ms); @@ -2686,12 +2704,6 @@ impl Config { .startup_warnings() .unwrap_or_default() .to_vec(); - let user_instructions = AgentsMdManager::load_global_instructions( - LOCAL_FS.as_ref(), - Some(&codex_home), - &mut startup_warnings, - ) - .await; // Destructure ConfigOverrides fully to ensure all overrides are applied. let ConfigOverrides { @@ -3534,7 +3546,6 @@ impl Config { approvals_reviewer: constrained_approvals_reviewer.value(), enforce_residency: enforce_residency.value, notify: cfg.notify, - user_instructions, base_instructions, personality, developer_instructions, @@ -3750,8 +3761,9 @@ impl Config { return Ok(None); }; + let path_uri = PathUri::from_abs_path(path)?; let contents = fs - .read_file_text(path, /*sandbox*/ None) + .read_file_text(&path_uri, /*sandbox*/ None) .await .map_err(|e| { std::io::Error::new( diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index d8df728554..8ccf51ef5f 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -43,6 +43,7 @@ use codex_mcp::codex_apps_tools_cache_key; use codex_mcp::compute_auth_statuses; use codex_mcp::effective_mcp_servers; use codex_mcp::host_owned_codex_apps_enabled; +use codex_mcp::tool_plugin_provenance; const CONNECTORS_READY_TIMEOUT_ON_EMPTY_TOOLS: Duration = Duration::from_secs(30); @@ -251,7 +252,8 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_mcp_manager( }); } let cache_key = accessible_connectors_cache_key(config, auth.as_ref()); - let tool_plugin_provenance = mcp_manager.tool_plugin_provenance(config).await; + let mcp_config = mcp_manager.runtime_config(config).await; + let tool_plugin_provenance = tool_plugin_provenance(&mcp_config); if !force_refetch && let Some(cached_connectors) = read_cached_accessible_connectors(&cache_key) { let cached_connectors = codex_connectors::filter::filter_disallowed_connectors( @@ -265,7 +267,6 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_mcp_manager( }); } - let mcp_config = mcp_manager.runtime_config(config).await; let mut mcp_servers = effective_mcp_servers(&mcp_config, auth.as_ref()); mcp_servers.retain(|name, _| name == CODEX_APPS_MCP_SERVER_NAME); let host_owned_codex_apps_enabled = host_owned_codex_apps_enabled(&mcp_config, auth.as_ref()); diff --git a/codex-rs/core/src/context/available_skills_instructions.rs b/codex-rs/core/src/context/available_skills_instructions.rs index f7921072b2..4e166d436c 100644 --- a/codex-rs/core/src/context/available_skills_instructions.rs +++ b/codex-rs/core/src/context/available_skills_instructions.rs @@ -5,12 +5,23 @@ use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; use super::ContextualUserFragment; +/// Model-context fragment describing the skills available to Codex. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct AvailableSkillsInstructions { +pub struct AvailableSkillsInstructions { skill_root_lines: Vec, skill_lines: Vec, } +impl AvailableSkillsInstructions { + /// Creates a skills context fragment from pre-rendered catalog lines. + pub fn from_skill_lines(skill_lines: Vec) -> Self { + Self { + skill_root_lines: Vec::new(), + skill_lines, + } + } +} + impl From for AvailableSkillsInstructions { fn from(available_skills: AvailableSkills) -> Self { Self { diff --git a/codex-rs/core/src/context/environment_context_tests.rs b/codex-rs/core/src/context/environment_context_tests.rs index 49c3fc0c58..c8e491b68c 100644 --- a/codex-rs/core/src/context/environment_context_tests.rs +++ b/codex-rs/core/src/context/environment_context_tests.rs @@ -178,6 +178,7 @@ fn turn_context_item_filesystem_uses_workspace_roots_instead_of_cwd() { network: None, file_system_sandbox_policy: None, model: "gpt-5".to_string(), + comp_hash: None, personality: None, collaboration_mode: None, multi_agent_version: None, diff --git a/codex-rs/core/src/context/mod.rs b/codex-rs/core/src/context/mod.rs index 78fc70fd49..a470c3afb7 100644 --- a/codex-rs/core/src/context/mod.rs +++ b/codex-rs/core/src/context/mod.rs @@ -23,6 +23,7 @@ mod realtime_end_instructions; mod realtime_start_instructions; mod realtime_start_with_instructions; mod subagent_notification; +mod token_budget_context; mod turn_aborted; mod user_instructions; mod user_shell_command; @@ -30,7 +31,7 @@ mod user_shell_command; pub(crate) use approved_command_prefix_saved::ApprovedCommandPrefixSaved; pub(crate) use apps_instructions::AppsInstructions; pub(crate) use available_plugins_instructions::AvailablePluginsInstructions; -pub(crate) use available_skills_instructions::AvailableSkillsInstructions; +pub use available_skills_instructions::AvailableSkillsInstructions; pub(crate) use codex_context_fragments::AdditionalContextDeveloperFragment; pub(crate) use codex_context_fragments::AdditionalContextUserFragment; pub use codex_context_fragments::ContextualUserFragment; @@ -60,6 +61,8 @@ pub(crate) use realtime_end_instructions::RealtimeEndInstructions; pub(crate) use realtime_start_instructions::RealtimeStartInstructions; pub(crate) use realtime_start_with_instructions::RealtimeStartWithInstructions; pub(crate) use subagent_notification::SubagentNotification; +pub(crate) use token_budget_context::TokenBudgetContext; +pub(crate) use token_budget_context::TokenBudgetRemainingContext; pub(crate) use turn_aborted::TurnAborted; pub(crate) use user_instructions::UserInstructions; pub(crate) use user_shell_command::UserShellCommand; diff --git a/codex-rs/core/src/context/token_budget_context.rs b/codex-rs/core/src/context/token_budget_context.rs new file mode 100644 index 0000000000..df32ae4d71 --- /dev/null +++ b/codex-rs/core/src/context/token_budget_context.rs @@ -0,0 +1,78 @@ +use super::ContextualUserFragment; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TokenBudgetContext { + window_id: u64, + tokens_left: i64, +} + +impl TokenBudgetContext { + pub(crate) fn new(window_id: u64, tokens_left: i64) -> Self { + Self { + window_id, + tokens_left, + } + } +} + +impl ContextualUserFragment for TokenBudgetContext { + fn role(&self) -> &'static str { + "developer" + } + + fn markers(&self) -> (&'static str, &'static str) { + Self::type_markers() + } + + fn type_markers() -> (&'static str, &'static str) { + ("\n", "\n") + } + + fn body(&self) -> String { + let window_id = self.window_id; + let tokens_left = self.tokens_left; + format!( + "Current context window {window_id}.\nYou have {tokens_left} tokens left in this context window." + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TokenBudgetRemainingContext { + tokens_left: Option, +} + +impl TokenBudgetRemainingContext { + pub(crate) fn new(tokens_left: i64) -> Self { + Self { + tokens_left: Some(tokens_left), + } + } + + pub(crate) fn unknown() -> Self { + Self { tokens_left: None } + } +} + +impl ContextualUserFragment for TokenBudgetRemainingContext { + fn role(&self) -> &'static str { + "developer" + } + + fn markers(&self) -> (&'static str, &'static str) { + Self::type_markers() + } + + fn type_markers() -> (&'static str, &'static str) { + ("\n", "\n") + } + + fn body(&self) -> String { + match self.tokens_left { + Some(tokens_left) => { + format!("You have {tokens_left} tokens left in this context window.") + } + None => "You have unknown tokens left in this context window.".to_string(), + } + } +} diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index bb22749cb3..f4bc87f5b8 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -130,6 +130,7 @@ fn reference_context_item() -> TurnContextItem { network: None, file_system_sandbox_policy: None, model: "gpt-test".to_string(), + comp_hash: None, personality: None, collaboration_mode: None, multi_agent_version: None, diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index d3cae7feed..e9383aef41 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -16,6 +16,7 @@ use codex_protocol::models::is_local_image_close_tag_text; use codex_protocol::models::is_local_image_open_tag_text; use codex_protocol::protocol::COLLABORATION_MODE_OPEN_TAG; use codex_protocol::protocol::REALTIME_CONVERSATION_OPEN_TAG; +use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::user_input::UserInput; use tracing::warn; use uuid::Uuid; @@ -29,7 +30,9 @@ const CONTEXTUAL_DEVELOPER_PREFIXES: &[&str] = &[ "", COLLABORATION_MODE_OPEN_TAG, REALTIME_CONVERSATION_OPEN_TAG, + SKILLS_INSTRUCTIONS_OPEN_TAG, "", + "", ]; pub(crate) fn is_contextual_user_message_content(message: &[ContentItem]) -> bool { diff --git a/codex-rs/core/src/event_mapping_tests.rs b/codex-rs/core/src/event_mapping_tests.rs index 5f2ed66830..6f9dad64bb 100644 --- a/codex-rs/core/src/event_mapping_tests.rs +++ b/codex-rs/core/src/event_mapping_tests.rs @@ -1,3 +1,5 @@ +use super::has_non_contextual_dev_message_content; +use super::is_contextual_dev_message_content; use super::parse_turn_item; use crate::context::ContextualUserFragment; use crate::context::InternalContextSource; @@ -13,9 +15,30 @@ use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::models::ResponseItem; use codex_protocol::models::WebSearchAction; +use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::user_input::UserInput; use pretty_assertions::assert_eq; +#[test] +fn recognizes_skills_instructions_as_contextual_developer_content() { + assert!(is_contextual_dev_message_content(&[ + ContentItem::InputText { + text: format!("{SKILLS_INSTRUCTIONS_OPEN_TAG}\n## Skills"), + }, + ])); +} + +#[test] +fn recognizes_token_budget_as_contextual_developer_content() { + let content = vec![ContentItem::InputText { + text: "\nYou have 710 tokens left in this context window.\n" + .to_string(), + }]; + + assert!(is_contextual_dev_message_content(&content)); + assert!(!has_non_contextual_dev_message_content(&content)); +} + #[test] fn parses_user_message_with_text_and_two_images() { let img1 = "https://example.com/one.png".to_string(); diff --git a/codex-rs/core/src/guardian/mod.rs b/codex-rs/core/src/guardian/mod.rs index b4920f1ff6..b06a2daad3 100644 --- a/codex-rs/core/src/guardian/mod.rs +++ b/codex-rs/core/src/guardian/mod.rs @@ -168,7 +168,7 @@ use prompt::render_guardian_transcript_entries; #[cfg(test)] use review::GuardianReviewOutcome; #[cfg(test)] -use review::run_guardian_review_session as run_guardian_review_session_for_test; +use review::run_guardian_review_session_with_retry as run_guardian_review_session_for_test; #[cfg(test)] use review_session::build_guardian_review_session_config as build_guardian_review_session_config_for_test; diff --git a/codex-rs/core/src/guardian/review.rs b/codex-rs/core/src/guardian/review.rs index 934e68042f..99f9c5d999 100644 --- a/codex-rs/core/src/guardian/review.rs +++ b/codex-rs/core/src/guardian/review.rs @@ -7,6 +7,7 @@ use codex_analytics::GuardianReviewTrackContext; use codex_analytics::GuardianReviewedAction; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::GuardianAssessmentDecisionSource; use codex_protocol::protocol::GuardianAssessmentEvent; @@ -19,11 +20,14 @@ use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::WarningEvent; use std::sync::Arc; use tokio::sync::oneshot; +use tokio::time::Instant; +use tokio::time::sleep_until; use tokio_util::sync::CancellationToken; use crate::session::session::Session; use crate::session::turn_context::TurnContext; use crate::turn_timing::now_unix_timestamp_ms; +use crate::util::backoff; use super::AUTO_REVIEW_DENIAL_WINDOW_SIZE; use super::GUARDIAN_REVIEW_TIMEOUT; @@ -58,6 +62,8 @@ const GUARDIAN_TIMEOUT_INSTRUCTIONS: &str = concat!( "You may retry once, or ask the user for guidance or explicit approval.", ); +const GUARDIAN_REVIEW_MAX_ATTEMPTS: i64 = 3; + pub(crate) fn new_guardian_review_id() -> String { uuid::Uuid::new_v4().to_string() } @@ -95,9 +101,16 @@ pub(super) enum GuardianReviewOutcome { #[derive(Debug)] pub(super) enum GuardianReviewError { - PromptBuild { message: String }, - Session { message: String }, - Parse { message: String }, + PromptBuild { + message: String, + }, + Session { + message: String, + error_info: Option, + }, + Parse { + message: String, + }, Timeout, Cancelled, } @@ -112,6 +125,14 @@ impl GuardianReviewError { fn session(err: anyhow::Error) -> Self { Self::Session { message: err.to_string(), + error_info: None, + } + } + + fn session_with_error_info(err: anyhow::Error, error_info: CodexErrorInfo) -> Self { + Self::Session { + message: err.to_string(), + error_info: Some(error_info), } } @@ -335,13 +356,14 @@ async fn run_guardian_review( let schema = guardian_output_schema(); let terminal_action = action_summary.clone(); - let (outcome, analytics_result) = Box::pin(run_guardian_review_session( + let (outcome, analytics_result) = Box::pin(run_guardian_review_session_with_retry( session.clone(), turn.clone(), request, retry_reason.clone(), schema, external_cancel, + GUARDIAN_REVIEW_MAX_ATTEMPTS, )) .await; @@ -464,7 +486,7 @@ async fn run_guardian_review( | GuardianReviewError::Parse { .. } => { let message = match &error { GuardianReviewError::PromptBuild { message } - | GuardianReviewError::Session { message } + | GuardianReviewError::Session { message, .. } | GuardianReviewError::Parse { message } => message, GuardianReviewError::Timeout | GuardianReviewError::Cancelled => { "guardian review failed" @@ -657,13 +679,14 @@ pub(crate) fn spawn_approval_request_review( /// context. It may still reuse the parent's managed-network allowlist for /// read-only checks, but it intentionally runs without inherited exec-policy /// rules. -pub(super) async fn run_guardian_review_session( +async fn run_guardian_review_session_before_deadline( session: Arc, turn: Arc, request: GuardianApprovalRequest, retry_reason: Option, schema: serde_json::Value, external_cancel: Option, + deadline: Instant, ) -> (GuardianReviewOutcome, GuardianReviewAnalyticsResult) { let network_proxy = session.services.network_proxy.load_full(); let live_network_config = match network_proxy.as_ref() { @@ -753,6 +776,7 @@ pub(super) async fn run_guardian_review_session( reasoning_summary: turn.reasoning_summary, personality: turn.personality, external_cancel, + deadline, }), ) .await; @@ -787,10 +811,16 @@ pub(super) async fn run_guardian_review_session( GuardianReviewOutcome::Error(GuardianReviewError::prompt_build(err)), session_analytics_result, ), - GuardianReviewSessionOutcome::SessionFailed(err) => ( - GuardianReviewOutcome::Error(GuardianReviewError::session(err)), - session_analytics_result, - ), + GuardianReviewSessionOutcome::SessionFailed { error, error_info } => { + let error = match error_info { + Some(error_info) => GuardianReviewError::session_with_error_info(error, error_info), + None => GuardianReviewError::session(error), + }; + ( + GuardianReviewOutcome::Error(error), + session_analytics_result, + ) + } GuardianReviewSessionOutcome::TimedOut => ( GuardianReviewOutcome::Error(GuardianReviewError::Timeout), session_analytics_result, @@ -802,9 +832,85 @@ pub(super) async fn run_guardian_review_session( } } +pub(super) async fn run_guardian_review_session_with_retry( + session: Arc, + turn: Arc, + request: GuardianApprovalRequest, + retry_reason: Option, + schema: serde_json::Value, + external_cancel: Option, + max_attempts: i64, +) -> (GuardianReviewOutcome, GuardianReviewAnalyticsResult) { + assert!(max_attempts > 0, "guardian review must run at least once"); + let deadline = Instant::now() + GUARDIAN_REVIEW_TIMEOUT; + let mut attempt_count = 1; + loop { + let (outcome, mut analytics_result) = run_guardian_review_session_before_deadline( + Arc::clone(&session), + Arc::clone(&turn), + request.clone(), + retry_reason.clone(), + schema.clone(), + external_cancel.clone(), + deadline, + ) + .await; + analytics_result.attempt_count = attempt_count; + if attempt_count >= max_attempts || !should_retry_guardian_review(&outcome) { + return (outcome, analytics_result); + } + if let Some(error) = + wait_before_guardian_retry(attempt_count, deadline, external_cancel.as_ref()).await + { + return (GuardianReviewOutcome::Error(error), analytics_result); + } + attempt_count += 1; + } +} + +async fn wait_before_guardian_retry( + attempt_count: i64, + deadline: Instant, + external_cancel: Option<&CancellationToken>, +) -> Option { + let retry_delay = backoff(attempt_count as u64); + let retry_at = (Instant::now() + retry_delay).min(deadline); + tokio::select! { + _ = sleep_until(retry_at) => { + (Instant::now() >= deadline).then_some(GuardianReviewError::Timeout) + } + _ = async { + if let Some(cancel_token) = external_cancel { + cancel_token.cancelled().await; + } else { + std::future::pending::<()>().await; + } + } => Some(GuardianReviewError::Cancelled), + } +} + +fn should_retry_guardian_review(outcome: &GuardianReviewOutcome) -> bool { + matches!( + outcome, + GuardianReviewOutcome::Error( + GuardianReviewError::Session { + error_info: Some( + CodexErrorInfo::ServerOverloaded + | CodexErrorInfo::HttpConnectionFailed { .. } + | CodexErrorInfo::ResponseStreamConnectionFailed { .. } + | CodexErrorInfo::InternalServerError + | CodexErrorInfo::ResponseStreamDisconnected { .. } + ), + .. + } | GuardianReviewError::Parse { .. } + ) + ) +} + #[cfg(test)] mod review_tests { use super::*; + use std::time::Duration; #[test] fn guardian_review_error_reason_distinguishes_error_kinds() { @@ -812,6 +918,10 @@ mod review_tests { let prompt_error = GuardianReviewError::prompt_build(anyhow::anyhow!("bad prompt/config")); let session_error = GuardianReviewError::session(anyhow::anyhow!("guardian runtime failed")); + let structured_session_error = GuardianReviewError::session_with_error_info( + anyhow::anyhow!("temporary guardian failure"), + CodexErrorInfo::ServerOverloaded, + ); assert!(matches!( parse_error.failure_reason(), @@ -825,5 +935,109 @@ mod review_tests { session_error.failure_reason(), GuardianReviewFailureReason::SessionError )); + assert!(matches!( + structured_session_error.failure_reason(), + GuardianReviewFailureReason::SessionError + )); + } + + #[test] + fn guardian_review_retry_only_retries_transient_session_and_parse_errors() { + let assessment = GuardianAssessment { + risk_level: GuardianRiskLevel::High, + user_authorization: GuardianUserAuthorization::Unknown, + outcome: GuardianAssessmentOutcome::Deny, + rationale: "deny".to_string(), + }; + let transient_error_info = [ + CodexErrorInfo::ServerOverloaded, + CodexErrorInfo::HttpConnectionFailed { + http_status_code: Some(502), + }, + CodexErrorInfo::ResponseStreamConnectionFailed { + http_status_code: Some(503), + }, + CodexErrorInfo::InternalServerError, + CodexErrorInfo::ResponseStreamDisconnected { + http_status_code: None, + }, + ]; + let mut outcomes = transient_error_info + .into_iter() + .map(|error_info| { + ( + GuardianReviewOutcome::Error(GuardianReviewError::session_with_error_info( + anyhow::anyhow!("transient session"), + error_info, + )), + true, + ) + }) + .collect::>(); + outcomes.extend([ + (GuardianReviewOutcome::Completed(assessment), false), + ( + GuardianReviewOutcome::Error(GuardianReviewError::prompt_build(anyhow::anyhow!( + "prompt" + ))), + false, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::session(anyhow::anyhow!( + "session" + ))), + false, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::session_with_error_info( + anyhow::anyhow!("bad request"), + CodexErrorInfo::BadRequest, + )), + false, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::parse(anyhow::anyhow!("parse"))), + true, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::Timeout), + false, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::Cancelled), + false, + ), + ]); + + for (outcome, expected) in outcomes { + assert_eq!(should_retry_guardian_review(&outcome), expected); + } + } + + #[tokio::test] + async fn guardian_review_retry_wait_honors_cancellation() { + let cancel_token = CancellationToken::new(); + cancel_token.cancel(); + + let error = wait_before_guardian_retry( + /*attempt_count*/ 1, + Instant::now() + Duration::from_secs(/*secs*/ 1), + Some(&cancel_token), + ) + .await; + + assert!(matches!(error, Some(GuardianReviewError::Cancelled))); + } + + #[tokio::test] + async fn guardian_review_retry_wait_honors_deadline() { + let error = wait_before_guardian_retry( + /*attempt_count*/ 1, + Instant::now(), + /*external_cancel*/ None, + ) + .await; + + assert!(matches!(error, Some(GuardianReviewError::Timeout))); } } diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index fcda9a5a23..63b778838e 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -7,6 +7,7 @@ use std::time::Duration; use anyhow::anyhow; use codex_analytics::GuardianReviewAnalyticsResult; use codex_analytics::GuardianReviewSessionKind; +use codex_extension_api::UserInstructions; use codex_protocol::ThreadId; use codex_protocol::config_types::AutoCompactTokenLimitScope; use codex_protocol::config_types::Personality; @@ -15,6 +16,8 @@ use codex_protocol::models::PermissionProfile; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::CodexErrorInfo; +use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::InitialHistory; @@ -29,7 +32,6 @@ use tokio::sync::Semaphore; use tokio_util::sync::CancellationToken; use tracing::warn; -use crate::LoadedAgentsMd; use crate::codex_delegate::run_codex_thread_interactive; use crate::config::Config; use crate::config::Constrained; @@ -46,7 +48,6 @@ use codex_features::Feature; use codex_model_provider_info::ModelProviderInfo; use codex_utils_absolute_path::AbsolutePathBuf; -use super::GUARDIAN_REVIEW_TIMEOUT; use super::GUARDIAN_REVIEWER_NAME; use super::GuardianApprovalRequest; use super::prompt::GuardianPromptMode; @@ -60,7 +61,10 @@ const GUARDIAN_INTERRUPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(5); pub(crate) enum GuardianReviewSessionOutcome { Completed(anyhow::Result>), PromptBuildFailed(anyhow::Error), - SessionFailed(anyhow::Error), + SessionFailed { + error: anyhow::Error, + error_info: Option, + }, TimedOut, Aborted, } @@ -77,6 +81,7 @@ pub(crate) struct GuardianReviewSessionParams { pub(crate) reasoning_summary: ReasoningSummaryConfig, pub(crate) personality: Option, pub(crate) external_cancel: Option, + pub(crate) deadline: tokio::time::Instant, } #[derive(Default)] @@ -147,7 +152,7 @@ struct GuardianReviewSessionReuseKey { permissions: Permissions, developer_instructions: Option, base_instructions: Option, - user_instructions: Option, + user_instructions: Option, compact_prompt: Option, cwd: AbsolutePathBuf, mcp_servers: Constrained>, @@ -159,7 +164,10 @@ struct GuardianReviewSessionReuseKey { } impl GuardianReviewSessionReuseKey { - fn from_spawn_config(spawn_config: &Config) -> Self { + fn from_spawn_config( + spawn_config: &Config, + user_instructions: Option, + ) -> Self { Self { model: spawn_config.model.clone(), model_provider_id: spawn_config.model_provider_id.clone(), @@ -172,7 +180,7 @@ impl GuardianReviewSessionReuseKey { permissions: spawn_config.permissions.clone(), developer_instructions: spawn_config.developer_instructions.clone(), base_instructions: spawn_config.base_instructions.clone(), - user_instructions: spawn_config.user_instructions.clone(), + user_instructions, compact_prompt: spawn_config.compact_prompt.clone(), cwd: spawn_config.cwd.clone(), mcp_servers: spawn_config.mcp_servers.clone(), @@ -312,8 +320,11 @@ impl GuardianReviewSessionManager { &self, params: GuardianReviewSessionParams, ) -> (GuardianReviewSessionOutcome, GuardianReviewAnalyticsResult) { - let deadline = tokio::time::Instant::now() + GUARDIAN_REVIEW_TIMEOUT; - let next_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config(¶ms.spawn_config); + let deadline = params.deadline; + let next_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config( + ¶ms.spawn_config, + params.parent_session.user_instructions().await, + ); let mut stale_trunk_to_shutdown = None; let mut spawned_trunk = false; let trunk_candidate = match run_before_review_deadline( @@ -364,7 +375,9 @@ impl GuardianReviewSessionManager { state.trunk.as_ref().cloned() } - Err(outcome) => return (outcome, GuardianReviewAnalyticsResult::without_session()), + Err(outcome) => { + return (outcome, GuardianReviewAnalyticsResult::without_session()); + } }; if let Some(review_session) = stale_trunk_to_shutdown { @@ -434,6 +447,7 @@ impl GuardianReviewSessionManager { pub(crate) async fn cache_for_test(&self, codex: Codex) { let reuse_key = GuardianReviewSessionReuseKey::from_spawn_config( codex.session.get_config().await.as_ref(), + codex.session.user_instructions().await, ); self.state.lock().await.trunk = Some(Arc::new(GuardianReviewSession { reuse_key, @@ -452,6 +466,7 @@ impl GuardianReviewSessionManager { pub(crate) async fn register_ephemeral_for_test(&self, codex: Codex) { let reuse_key = GuardianReviewSessionReuseKey::from_spawn_config( codex.session.get_config().await.as_ref(), + codex.session.user_instructions().await, ); self.state .lock() @@ -560,7 +575,9 @@ impl GuardianReviewSessionManager { GuardianReviewAnalyticsResult::without_session(), ); } - Err(outcome) => return (outcome, GuardianReviewAnalyticsResult::without_session()), + Err(outcome) => { + return (outcome, GuardianReviewAnalyticsResult::without_session()); + } }; self.register_active_ephemeral(Arc::clone(&review_session)) .await; @@ -764,7 +781,10 @@ async fn run_review_on_session( Ok(Ok(child_turn_id)) => child_turn_id, Ok(Err(err)) => { return ( - GuardianReviewSessionOutcome::SessionFailed(err.into()), + GuardianReviewSessionOutcome::SessionFailed { + error: err.into(), + error_info: None, + }, false, analytics_result, ); @@ -825,7 +845,7 @@ async fn wait_for_guardian_review( ) -> (GuardianReviewSessionOutcome, bool, bool) { let timeout = tokio::time::sleep_until(deadline); tokio::pin!(timeout); - let mut last_error_message: Option = None; + let mut last_error: Option = None; loop { tokio::select! { @@ -862,10 +882,13 @@ async fn wait_for_guardian_review( .time_to_first_token_ms .and_then(|ms| u64::try_from(ms).ok()); if turn_complete.last_agent_message.is_none() - && let Some(error_message) = last_error_message + && let Some(error) = last_error { return ( - GuardianReviewSessionOutcome::Completed(Err(anyhow!(error_message))), + GuardianReviewSessionOutcome::SessionFailed { + error: anyhow!(error.message), + error_info: error.codex_error_info, + }, true, true, ); @@ -877,7 +900,7 @@ async fn wait_for_guardian_review( ); } EventMsg::Error(error) => { - last_error_message = Some(error.message); + last_error = Some(error); } EventMsg::TurnAborted(_) => { return (GuardianReviewSessionOutcome::Aborted, true, false); @@ -920,6 +943,8 @@ pub(crate) fn build_guardian_review_session_config( let mut guardian_config = parent_config.clone(); guardian_config.model = Some(active_model.to_string()); guardian_config.model_reasoning_effort = reasoning_effort; + guardian_config.model_provider.request_max_retries = Some(1); + guardian_config.model_provider.stream_max_retries = Some(1); guardian_config.include_skill_instructions = false; guardian_config.base_instructions = Some( parent_config @@ -1058,8 +1083,10 @@ mod tests { let (tx_event, rx_event) = async_channel::unbounded(); let (_agent_status_tx, agent_status) = tokio::sync::watch::channel(AgentStatus::PendingInit); - let reuse_key = - GuardianReviewSessionReuseKey::from_spawn_config(session.get_config().await.as_ref()); + let reuse_key = GuardianReviewSessionReuseKey::from_spawn_config( + session.get_config().await.as_ref(), + session.user_instructions().await, + ); ( GuardianReviewSession { @@ -1148,6 +1175,7 @@ mod tests { reasoning_summary, personality, external_cancel: None, + deadline: tokio::time::Instant::now() + Duration::from_secs(30), } } @@ -1161,8 +1189,10 @@ mod tests { /*reasoning_effort*/ None, ) .expect("cached guardian config"); - let cached_reuse_key = - GuardianReviewSessionReuseKey::from_spawn_config(&cached_spawn_config); + let cached_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config( + &cached_spawn_config, + /*user_instructions*/ None, + ); let mut changed_parent_config = parent_config; changed_parent_config.model_provider.base_url = @@ -1174,12 +1204,18 @@ mod tests { /*reasoning_effort*/ None, ) .expect("next guardian config"); - let next_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config(&next_spawn_config); + let next_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config( + &next_spawn_config, + /*user_instructions*/ None, + ); assert_ne!(cached_reuse_key, next_reuse_key); assert_eq!( cached_reuse_key, - GuardianReviewSessionReuseKey::from_spawn_config(&cached_spawn_config) + GuardianReviewSessionReuseKey::from_spawn_config( + &cached_spawn_config, + /*user_instructions*/ None, + ) ); } @@ -1233,8 +1269,10 @@ mod tests { /*reasoning_effort*/ None, ) .expect("cached guardian config"); - let cached_reuse_key = - GuardianReviewSessionReuseKey::from_spawn_config(&cached_spawn_config); + let cached_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config( + &cached_spawn_config, + /*user_instructions*/ None, + ); let mut changed_parent_config = parent_config; changed_parent_config.model_auto_compact_token_limit_scope = @@ -1246,7 +1284,10 @@ mod tests { /*reasoning_effort*/ None, ) .expect("next guardian config"); - let next_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config(&next_spawn_config); + let next_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config( + &next_spawn_config, + /*user_instructions*/ None, + ); assert_ne!(cached_reuse_key, next_reuse_key); } @@ -1473,6 +1514,31 @@ mod tests { assert!(keep_review_session); } + #[tokio::test] + async fn run_review_removes_trunk_when_event_stream_is_broken() { + let (mut review_session, tx_event, _rx_sub) = test_review_session().await; + let params = test_review_params().await; + review_session.reuse_key = GuardianReviewSessionReuseKey::from_spawn_config( + ¶ms.spawn_config, + params.parent_session.user_instructions().await, + ); + let manager = GuardianReviewSessionManager { + state: Arc::new(Mutex::new(GuardianReviewSessionState { + trunk: Some(Arc::new(review_session)), + ephemeral_reviews: Vec::new(), + })), + }; + drop(tx_event); + + let (outcome, _) = manager.run_review(params).await; + + assert!(matches!( + outcome, + GuardianReviewSessionOutcome::Completed(Err(_)) + )); + assert!(manager.state.lock().await.trunk.is_none()); + } + #[tokio::test] async fn wait_for_guardian_review_ignores_prior_turn_completion() { let (review_session, tx_event, _rx_sub) = test_review_session().await; @@ -1545,6 +1611,47 @@ mod tests { assert!(capture_token_usage); } + #[tokio::test] + async fn wait_for_guardian_review_preserves_structured_session_error() { + let (review_session, tx_event, _rx_sub) = test_review_session().await; + tx_event + .send(Event { + id: "current-turn".to_string(), + msg: EventMsg::Error(ErrorEvent { + message: "temporary failure".to_string(), + codex_error_info: Some(CodexErrorInfo::ServerOverloaded), + }), + }) + .await + .expect("queue guardian error"); + tx_event + .send(turn_complete_event( + "current-turn", + /*last_agent_message*/ None, + Some(42), + )) + .await + .expect("queue current turn completion"); + + let mut analytics_result = GuardianReviewAnalyticsResult::without_session(); + let (outcome, keep_review_session, capture_token_usage) = wait_for_guardian_review( + &review_session, + "current-turn", + tokio::time::Instant::now() + Duration::from_secs(1), + /*external_cancel*/ None, + &mut analytics_result, + ) + .await; + + let GuardianReviewSessionOutcome::SessionFailed { error, error_info } = outcome else { + panic!("expected structured session failure"); + }; + assert_eq!(error.to_string(), "temporary failure"); + assert_eq!(error_info, Some(CodexErrorInfo::ServerOverloaded)); + assert!(keep_review_session); + assert!(capture_token_usage); + } + #[tokio::test] async fn wait_for_guardian_review_ignores_prior_turn_aborts() { let (review_session, tx_event, _rx_sub) = test_review_session().await; diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs index ffadd801ca..1698ea0c68 100644 --- a/codex-rs/core/src/guardian/tests.rs +++ b/codex-rs/core/src/guardian/tests.rs @@ -55,10 +55,11 @@ use core_test_support::context_snapshot::ContextSnapshotOptions; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_response_created; -use core_test_support::responses::mount_response_once; +use core_test_support::responses::mount_response_sequence; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; +use core_test_support::responses::sse_failed; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; use core_test_support::streaming_sse::StreamingSseChunk; @@ -172,6 +173,50 @@ async fn guardian_test_session_and_turn( guardian_test_session_and_turn_with_base_url(server.uri().as_str()).await } +async fn guardian_test_session_turn_and_rx( + server: &wiremock::MockServer, +) -> ( + Arc, + Arc, + async_channel::Receiver, +) { + let (mut session, mut turn, rx) = + crate::session::tests::make_session_and_context_with_rx().await; + Arc::get_mut(&mut session) + .expect("session should be uniquely owned") + .thread_id = fixed_guardian_parent_session_id(); + let mut config = (*turn.config).clone(); + config.model_provider.base_url = Some(format!("{}/v1", server.uri())); + let config = Arc::new(config); + let models_manager = test_support::models_manager_with_provider( + config.codex_home.to_path_buf(), + Arc::clone(&session.services.auth_manager), + config.model_provider.clone(), + ); + Arc::get_mut(&mut session) + .expect("session should be uniquely owned") + .services + .models_manager = models_manager; + let turn_mut = Arc::get_mut(&mut turn).expect("turn should be uniquely owned"); + turn_mut.config = Arc::clone(&config); + turn_mut.provider = + create_model_provider(config.model_provider.clone(), turn_mut.auth_manager.clone()); + turn_mut.user_instructions = None; + + (session, turn, rx) +} + +fn guardian_shell_request(id: &str) -> GuardianApprovalRequest { + GuardianApprovalRequest::Shell { + id: id.to_string(), + command: vec!["git".to_string(), "push".to_string()], + cwd: test_path_buf("/repo/codex-rs/core").abs(), + sandbox_permissions: crate::sandboxing::SandboxPermissions::UseDefault, + additional_permissions: None, + justification: Some("Need to push the reviewed docs fix.".to_string()), + } +} + async fn guardian_test_session_and_turn_with_base_url( base_url: &str, ) -> (Arc, Arc) { @@ -179,7 +224,6 @@ async fn guardian_test_session_and_turn_with_base_url( session.thread_id = fixed_guardian_parent_session_id(); let mut config = (*turn.config).clone(); config.model_provider.base_url = Some(format!("{base_url}/v1")); - config.user_instructions = None; let config = Arc::new(config); let models_manager = test_support::models_manager_with_provider( config.codex_home.to_path_buf(), @@ -1343,6 +1387,7 @@ async fn guardian_request_model_for_auto_review_override( Some("Sandbox denied outbound git push to github.com.".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; let (GuardianReviewOutcome::Completed(_), _) = outcome else { @@ -1454,6 +1499,7 @@ async fn guardian_review_request_layout_matches_model_visible_request_snapshot() Some("Sandbox denied outbound git push to github.com.".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; let (GuardianReviewOutcome::Completed(assessment), metadata) = outcome else { @@ -1632,6 +1678,7 @@ async fn guardian_reuses_prompt_cache_key_and_appends_prior_reviews() -> anyhow: Some("First retry reason".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; session @@ -1676,6 +1723,7 @@ async fn guardian_reuses_prompt_cache_key_and_appends_prior_reviews() -> anyhow: Some("Second retry reason".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; session @@ -1716,6 +1764,7 @@ async fn guardian_reuses_prompt_cache_key_and_appends_prior_reviews() -> anyhow: Some("Third retry reason".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; @@ -1907,6 +1956,7 @@ async fn guardian_reused_trunk_ignores_stale_prior_turn_completion() -> anyhow:: /*retry_reason*/ None, guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; let (GuardianReviewOutcome::Completed(first_assessment), first_metadata) = first_outcome else { @@ -1949,6 +1999,7 @@ async fn guardian_reused_trunk_ignores_stale_prior_turn_completion() -> anyhow:: /*retry_reason*/ None, guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; let (GuardianReviewOutcome::Completed(second_assessment), second_metadata) = second_outcome @@ -1978,15 +2029,17 @@ async fn guardian_review_surfaces_responses_api_errors_in_rejection_reason() -> let server = start_mock_server().await; let error_message = "Item 'rs_test' of type 'reasoning' was provided without its required following item."; - let _request_log = mount_response_once( + let request_log = mount_response_sequence( &server, - wiremock::ResponseTemplate::new(400).set_body_json(serde_json::json!({ - "error": { - "message": error_message, - "type": "invalid_request_error", - "param": "input" - } - })), + vec![ + wiremock::ResponseTemplate::new(400).set_body_json(serde_json::json!({ + "error": { + "message": error_message, + "type": "invalid_request_error", + "param": "input" + } + })), + ], ) .await; @@ -1994,7 +2047,6 @@ async fn guardian_review_surfaces_responses_api_errors_in_rejection_reason() -> crate::session::tests::make_session_and_context_with_rx().await; let mut config = (*turn.config).clone(); config.model_provider.base_url = Some(format!("{}/v1", server.uri())); - config.user_instructions = None; let config = Arc::new(config); let models_manager = test_support::models_manager_with_provider( config.codex_home.to_path_buf(), @@ -2030,6 +2082,7 @@ async fn guardian_review_surfaces_responses_api_errors_in_rejection_reason() -> .await; assert_eq!(decision, ReviewDecision::Denied); + assert_eq!(request_log.requests().len(), 1); let mut warnings = Vec::new(); let mut denial_rationales = Vec::new(); @@ -2080,13 +2133,255 @@ async fn guardian_review_surfaces_responses_api_errors_in_rejection_reason() -> Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_retries_transient_session_failure_then_approves() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let approval = serde_json::json!({ + "risk_level": "low", + "user_authorization": "high", + "outcome": "allow", + "rationale": "retry succeeded", + }) + .to_string(); + let request_log = mount_sse_sequence( + &server, + vec![ + sse_failed( + "resp-session-failure", + "server_is_overloaded", + "temporary reviewer overload", + ), + sse(vec![ + ev_response_created("resp-approved"), + ev_assistant_message("msg-approved", &approval), + ev_completed("resp-approved"), + ]), + ], + ) + .await; + let (session, turn) = guardian_test_session_and_turn(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let (outcome, metadata) = run_guardian_review_session_for_test( + Arc::clone(&session), + Arc::clone(&turn), + guardian_shell_request("shell-session-retry"), + /*retry_reason*/ None, + guardian_output_schema(), + /*external_cancel*/ None, + /*max_attempts*/ 3, + ) + .await; + + let GuardianReviewOutcome::Completed(assessment) = outcome else { + panic!("expected guardian assessment"); + }; + assert_eq!(assessment.outcome, GuardianAssessmentOutcome::Allow); + assert_eq!(assessment.rationale, "retry succeeded"); + assert_eq!(metadata.attempt_count, 2); + assert!(matches!( + metadata.guardian_session_kind, + Some(codex_analytics::GuardianReviewSessionKind::TrunkReused) + )); + assert_eq!(request_log.requests().len(), 2); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_does_not_retry_missing_assessment_payload() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let request_log = mount_sse_sequence( + &server, + vec![sse(vec![ + ev_response_created("resp-missing-assessment"), + ev_completed("resp-missing-assessment"), + ])], + ) + .await; + let (session, turn) = guardian_test_session_and_turn(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let decision = review_approval_request( + &session, + &turn, + "review-missing-assessment".to_string(), + guardian_shell_request("shell-missing-assessment"), + /*retry_reason*/ None, + ) + .await; + + assert_eq!(decision, ReviewDecision::Denied); + assert_eq!(request_log.requests().len(), 1); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_retries_two_parse_failures_then_approves() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let approval = serde_json::json!({ + "risk_level": "low", + "user_authorization": "high", + "outcome": "allow", + "rationale": "retry succeeded", + }) + .to_string(); + let request_log = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-parse-failure-1"), + ev_assistant_message("msg-parse-failure-1", "not valid guardian json"), + ev_completed("resp-parse-failure-1"), + ]), + sse(vec![ + ev_response_created("resp-parse-failure-2"), + ev_assistant_message("msg-parse-failure-2", "still not valid guardian json"), + ev_completed("resp-parse-failure-2"), + ]), + sse(vec![ + ev_response_created("resp-approved"), + ev_assistant_message("msg-approved", &approval), + ev_completed("resp-approved"), + ]), + ], + ) + .await; + let (session, turn) = guardian_test_session_and_turn(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let (outcome, metadata) = run_guardian_review_session_for_test( + Arc::clone(&session), + Arc::clone(&turn), + guardian_shell_request("shell-parse-retry"), + /*retry_reason*/ None, + guardian_output_schema(), + /*external_cancel*/ None, + /*max_attempts*/ 3, + ) + .await; + + let GuardianReviewOutcome::Completed(assessment) = outcome else { + panic!("expected guardian assessment"); + }; + assert_eq!(assessment.outcome, GuardianAssessmentOutcome::Allow); + assert_eq!(assessment.rationale, "retry succeeded"); + assert_eq!(metadata.attempt_count, 3); + assert!(matches!( + metadata.guardian_session_kind, + Some(codex_analytics::GuardianReviewSessionKind::TrunkReused) + )); + assert_eq!(request_log.requests().len(), 3); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_exhausts_three_failures_with_one_terminal_event() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let request_log = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-parse-failure-1"), + ev_assistant_message("msg-parse-failure-1", "invalid one"), + ev_completed("resp-parse-failure-1"), + ]), + sse(vec![ + ev_response_created("resp-parse-failure-2"), + ev_assistant_message("msg-parse-failure-2", "invalid two"), + ev_completed("resp-parse-failure-2"), + ]), + sse(vec![ + ev_response_created("resp-parse-failure-3"), + ev_assistant_message("msg-parse-failure-3", "invalid three"), + ev_completed("resp-parse-failure-3"), + ]), + ], + ) + .await; + let (session, turn, rx) = guardian_test_session_turn_and_rx(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let decision = review_approval_request( + &session, + &turn, + "review-exhausted-retry".to_string(), + guardian_shell_request("shell-exhausted-retry"), + /*retry_reason*/ None, + ) + .await; + + assert_eq!(decision, ReviewDecision::Denied); + assert_eq!(request_log.requests().len(), 3); + let mut statuses = Vec::new(); + while let Ok(event) = rx.try_recv() { + if let EventMsg::GuardianAssessment(event) = event.msg { + statuses.push(event.status); + } + } + assert_eq!( + statuses, + vec![ + GuardianAssessmentStatus::InProgress, + GuardianAssessmentStatus::Denied, + ] + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_does_not_retry_valid_denial() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let denial = serde_json::json!({ + "risk_level": "high", + "user_authorization": "unknown", + "outcome": "deny", + "rationale": "unsafe", + }) + .to_string(); + let request_log = mount_sse_sequence( + &server, + vec![sse(vec![ + ev_response_created("resp-denied"), + ev_assistant_message("msg-denied", &denial), + ev_completed("resp-denied"), + ])], + ) + .await; + let (session, turn) = guardian_test_session_and_turn(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let decision = review_approval_request( + &session, + &turn, + "review-valid-denial".to_string(), + guardian_shell_request("shell-valid-denial"), + /*retry_reason*/ None, + ) + .await; + + assert_eq!(decision, ReviewDecision::Denied); + assert_eq!(request_log.requests().len(), 1); + Ok(()) +} + #[tokio::test] -async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> anyhow::Result<()> { +async fn guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history() -> anyhow::Result<()> +{ const TEST_STACK_SIZE_BYTES: usize = 4 * 1024 * 1024; let handle = std::thread::Builder::new() - .name("guardian_parallel_reviews_fork_from_last_committed_trunk_history".to_string()) + .name("guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history".to_string()) .stack_size(TEST_STACK_SIZE_BYTES) .spawn(|| -> anyhow::Result<()> { let runtime = tokio::runtime::Builder::new_current_thread() @@ -2141,10 +2436,18 @@ async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> a gate: None, body: sse(vec![ ev_response_created("resp-guardian-3"), - ev_assistant_message("msg-guardian-3", &third_assessment), + ev_assistant_message("msg-guardian-3", "not valid guardian json"), ev_completed("resp-guardian-3"), ]), }], + vec![StreamingSseChunk { + gate: None, + body: sse(vec![ + ev_response_created("resp-guardian-4"), + ev_assistant_message("msg-guardian-4", &third_assessment), + ev_completed("resp-guardian-4"), + ]), + }], ]) .await; @@ -2271,20 +2574,28 @@ async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> a .await; assert_eq!(third_decision, ReviewDecision::Approved); let requests = server.requests().await; - assert_eq!(requests.len(), 3); + assert_eq!(requests.len(), 4); let second_request_body = serde_json::from_slice::(&requests[1])?; - let third_request_body = serde_json::from_slice::(&requests[2])?; + let failed_ephemeral_request_body = + serde_json::from_slice::(&requests[2])?; + let retried_ephemeral_request_body = + serde_json::from_slice::(&requests[3])?; assert_eq!( second_request_body["prompt_cache_key"], - third_request_body["prompt_cache_key"], + failed_ephemeral_request_body["prompt_cache_key"], "forked guardian review should reuse the trunk guardian prompt cache key" ); - let third_request_body_text = third_request_body.to_string(); + assert_eq!( + failed_ephemeral_request_body["prompt_cache_key"], + retried_ephemeral_request_body["prompt_cache_key"], + "retried ephemeral review should preserve the guardian prompt cache key" + ); + let third_request_body_text = retried_ephemeral_request_body.to_string(); assert!( third_request_body_text.contains("first guardian rationale"), "forked guardian review should include the last committed trunk assessment" ); - let third_user_message = last_user_message_text_from_body(&third_request_body); + let third_user_message = last_user_message_text_from_body(&retried_ephemeral_request_body); assert!(third_user_message.contains(">>> TRANSCRIPT DELTA START\n")); assert!( third_user_message.contains("[5] user: Please inspect pending changes before pushing.") @@ -2315,7 +2626,7 @@ async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> a match handle.join() { Ok(result) => result, Err(_) => Err(anyhow::anyhow!( - "guardian_parallel_reviews_fork_from_last_committed_trunk_history thread panicked" + "guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history thread panicked" )), } } @@ -2541,6 +2852,10 @@ async fn guardian_review_session_config_keeps_bedrock_provider_for_bedrock_gpt_5 ) .expect("guardian config"); + let mut expected_model_provider = + ModelProviderInfo::create_amazon_bedrock_provider(/*aws*/ None); + expected_model_provider.request_max_retries = Some(1); + expected_model_provider.stream_max_retries = Some(1); assert_eq!( ( guardian_config.model, @@ -2550,7 +2865,7 @@ async fn guardian_review_session_config_keeps_bedrock_provider_for_bedrock_gpt_5 ( Some(AMAZON_BEDROCK_GPT_5_4_MODEL_ID.to_string()), AMAZON_BEDROCK_PROVIDER_ID.to_string(), - ModelProviderInfo::create_amazon_bedrock_provider(/*aws*/ None), + expected_model_provider, ) ); } diff --git a/codex-rs/core/src/hook_runtime.rs b/codex-rs/core/src/hook_runtime.rs index 8501c6259c..1097f7bc78 100644 --- a/codex-rs/core/src/hook_runtime.rs +++ b/codex-rs/core/src/hook_runtime.rs @@ -385,9 +385,7 @@ pub(crate) async fn run_pre_compact_hooks( let outcome = sess.hooks().run_pre_compact(request).await; emit_hook_completed_events(sess, turn_context, outcome.hook_events).await; if outcome.should_stop { - PreCompactHookOutcome::Stopped { - reason: outcome.stop_reason, - } + PreCompactHookOutcome::Stopped } else { PreCompactHookOutcome::Continue } @@ -395,7 +393,7 @@ pub(crate) async fn run_pre_compact_hooks( pub(crate) enum PreCompactHookOutcome { Continue, - Stopped { reason: Option }, + Stopped, } pub(crate) enum PostCompactHookOutcome { diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs new file mode 100644 index 0000000000..d992adb809 --- /dev/null +++ b/codex-rs/core/src/image_preparation.rs @@ -0,0 +1,122 @@ +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ImageDetail; +use codex_protocol::models::ResponseItem; +use codex_utils_image::ImageProcessingError; +use codex_utils_image::PromptImageMode; +use codex_utils_image::PromptImageResizeLimits; +use codex_utils_image::load_data_url_for_prompt; +use tracing::warn; + +pub(crate) const IMAGE_PROCESSING_ERROR_PLACEHOLDER: &str = + "image content omitted because it could not be processed"; +const IMAGE_TOO_LARGE_PLACEHOLDER: &str = + "image content omitted because it exceeded the supported size limit; use a smaller image"; +const UNSUPPORTED_LOW_DETAIL_PLACEHOLDER: &str = "image content omitted because detail 'low' is not supported; use 'high', 'original', or 'auto'"; + +const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits { + max_dimension: 2048, + max_patches: 2_500, +}; +const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits { + max_dimension: 6000, + max_patches: 10_000, +}; +#[derive(Debug, thiserror::Error)] +enum ImagePreparationError { + #[error("image detail `low` is not supported")] + UnsupportedLowDetail, + #[error(transparent)] + Processing(#[from] ImageProcessingError), +} + +impl ImagePreparationError { + fn placeholder(&self) -> &'static str { + match self { + ImagePreparationError::UnsupportedLowDetail => UNSUPPORTED_LOW_DETAIL_PLACEHOLDER, + ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge { .. }) => { + IMAGE_TOO_LARGE_PLACEHOLDER + } + ImagePreparationError::Processing(_) => IMAGE_PROCESSING_ERROR_PLACEHOLDER, + } + } +} + +pub(crate) fn prepare_response_items(items: &mut [ResponseItem]) { + for item in items { + match item { + ResponseItem::Message { content, .. } => prepare_message_content(content), + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + if let Some(content) = output.content_items_mut() { + prepare_tool_output_content(content); + } + } + ResponseItem::Reasoning { .. } + | ResponseItem::AgentMessage { .. } + | ResponseItem::LocalShellCall { .. } + | ResponseItem::FunctionCall { .. } + | ResponseItem::ToolSearchCall { .. } + | ResponseItem::CustomToolCall { .. } + | ResponseItem::ToolSearchOutput { .. } + | ResponseItem::WebSearchCall { .. } + | ResponseItem::ImageGenerationCall { .. } + | ResponseItem::Compaction { .. } + | ResponseItem::CompactionTrigger + | ResponseItem::ContextCompaction { .. } + | ResponseItem::Other => {} + } + } +} + +fn prepare_message_content(items: &mut [ContentItem]) { + for item in items { + if let ContentItem::InputImage { image_url, detail } = item + && is_data_url(image_url) + && let Err(error) = prepare_image(image_url, *detail) + { + warn!(%error, "failed to prepare message image"); + *item = ContentItem::InputText { + text: error.placeholder().to_string(), + }; + } + } +} + +fn prepare_tool_output_content(items: &mut [FunctionCallOutputContentItem]) { + for item in items { + if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item + && is_data_url(image_url) + && let Err(error) = prepare_image(image_url, *detail) + { + warn!(%error, "failed to prepare tool output image"); + *item = FunctionCallOutputContentItem::InputText { + text: error.placeholder().to_string(), + }; + } + } +} + +fn is_data_url(image_url: &str) -> bool { + image_url + .get(.."data:".len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) +} + +fn prepare_image( + image_url: &mut String, + detail: Option, +) -> Result<(), ImagePreparationError> { + let limits = match detail { + None | Some(ImageDetail::Auto | ImageDetail::High) => HIGH_DETAIL_LIMITS, + Some(ImageDetail::Original) => ORIGINAL_DETAIL_LIMITS, + Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail), + }; + let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?; + *image_url = image.into_data_url(); + Ok(()) +} + +#[cfg(test)] +#[path = "image_preparation_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rs new file mode 100644 index 0000000000..1d54973cc2 --- /dev/null +++ b/codex-rs/core/src/image_preparation_tests.rs @@ -0,0 +1,193 @@ +use std::io::Cursor; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_utils_image::data_url_from_bytes; +use image::DynamicImage; +use image::GenericImageView; +use image::ImageBuffer; +use image::ImageFormat; +use image::Rgba; +use pretty_assertions::assert_eq; + +use super::*; + +fn png_data_url(width: u32, height: u32) -> (String, Vec) { + let image = ImageBuffer::from_pixel(width, height, Rgba([10u8, 20, 30, 255])); + let mut encoded = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(image) + .write_to(&mut encoded, ImageFormat::Png) + .expect("encode PNG"); + let bytes = encoded.into_inner(); + (data_url_from_bytes("image/png", &bytes), bytes) +} + +fn decoded_image(image_url: &str) -> (Vec, DynamicImage) { + let (_, payload) = image_url.split_once(',').expect("data URL payload"); + let bytes = BASE64_STANDARD.decode(payload).expect("decode image URL"); + let image = image::load_from_memory(&bytes).expect("decode processed image"); + (bytes, image) +} + +#[test] +fn preparation_preserves_small_image_bytes_and_non_data_urls() { + let (data_url, original_bytes) = png_data_url(/*width*/ 64, /*height*/ 32); + let http_url = "https://example.com/image.png".to_string(); + let mut items = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputImage { + image_url: data_url, + detail: Some(ImageDetail::High), + }, + ContentItem::InputImage { + image_url: http_url.clone(), + detail: Some(ImageDetail::Low), + }, + ], + phase: None, + }]; + + prepare_response_items(&mut items); + + let ResponseItem::Message { content, .. } = &items[0] else { + panic!("expected message"); + }; + let [ + ContentItem::InputImage { image_url, .. }, + ContentItem::InputImage { + image_url: preserved_http_url, + detail: Some(ImageDetail::Low), + }, + ] = content.as_slice() + else { + panic!("expected two images"); + }; + assert_eq!(decoded_image(image_url).0, original_bytes); + assert_eq!(preserved_http_url, &http_url); +} + +#[test] +fn detail_policies_apply_the_expected_budgets() { + for (detail, input_dimensions, expected_dimensions) in [ + (Some(ImageDetail::High), (2048, 2048), (1600, 1600)), + (Some(ImageDetail::Original), (6401, 100), (6000, 94)), + (Some(ImageDetail::Original), (3201, 3201), (3200, 3200)), + (Some(ImageDetail::Auto), (2048, 2048), (1600, 1600)), + (None, (2048, 2048), (1600, 1600)), + ] { + let (image_url, _) = png_data_url(input_dimensions.0, input_dimensions.1); + let mut items = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputImage { image_url, detail }], + phase: None, + }]; + + prepare_response_items(&mut items); + + let ResponseItem::Message { content, .. } = &items[0] else { + panic!("expected message"); + }; + let [ContentItem::InputImage { image_url, .. }] = content.as_slice() else { + panic!("expected image"); + }; + assert_eq!(decoded_image(image_url).1.dimensions(), expected_dimensions); + } +} + +#[test] +fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() { + let (valid_image_url, _) = png_data_url(/*width*/ 64, /*height*/ 32); + let expected_valid_image_url = valid_image_url.clone(); + let mut items = vec![ResponseItem::CustomToolCallOutput { + call_id: "call-1".to_string(), + name: None, + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "before".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,%%%".to_string(), + detail: Some(ImageDetail::High), + }, + FunctionCallOutputContentItem::InputImage { + image_url: data_url_from_bytes("image/png", b"not an image"), + detail: Some(ImageDetail::High), + }, + FunctionCallOutputContentItem::InputImage { + image_url: valid_image_url.clone(), + detail: Some(ImageDetail::Low), + }, + FunctionCallOutputContentItem::InputImage { + image_url: valid_image_url, + detail: Some(ImageDetail::High), + }, + ]), + success: Some(true), + }, + }]; + + prepare_response_items(&mut items); + + assert_eq!( + items, + vec![ResponseItem::CustomToolCallOutput { + call_id: "call-1".to_string(), + name: None, + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "before".to_string(), + }, + FunctionCallOutputContentItem::InputText { + text: IMAGE_PROCESSING_ERROR_PLACEHOLDER.to_string(), + }, + FunctionCallOutputContentItem::InputText { + text: IMAGE_PROCESSING_ERROR_PLACEHOLDER.to_string(), + }, + FunctionCallOutputContentItem::InputText { + text: UNSUPPORTED_LOW_DETAIL_PLACEHOLDER.to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: expected_valid_image_url, + detail: Some(ImageDetail::High), + }, + ]), + success: Some(true), + }, + }] + ); +} + +#[test] +fn preparation_errors_use_bounded_actionable_placeholders() { + let cases = [ + ( + ImagePreparationError::UnsupportedLowDetail, + UNSUPPORTED_LOW_DETAIL_PLACEHOLDER, + ), + ( + ImagePreparationError::Processing(ImageProcessingError::ImageTooLarge { + representation: "decoded input", + size: 2, + max: 1, + }), + IMAGE_TOO_LARGE_PLACEHOLDER, + ), + ( + ImagePreparationError::Processing(ImageProcessingError::InvalidDataUrl { + reason: "details remain in logs".to_string(), + }), + IMAGE_PROCESSING_ERROR_PLACEHOLDER, + ), + ]; + + for (error, expected) in cases { + assert_eq!(error.placeholder(), expected); + } +} diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 400c803680..073c06cede 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -12,13 +12,17 @@ mod client_common; mod realtime_context; mod realtime_conversation; mod realtime_prompt; +mod responses_metadata; mod responses_retry; pub(crate) mod session; +pub use responses_metadata::CodexResponsesMetadata; pub use session::SteerInputError; +pub use turn_metadata::detached_memory_responses_metadata; mod codex_thread; mod compact_remote; mod compact_remote_v2; mod config_lock; +pub use codex_thread::BackgroundTerminalInfo; pub use codex_thread::CodexThread; pub use codex_thread::CodexThreadSettingsOverrides; pub use codex_thread::ThreadConfigSnapshot; @@ -41,6 +45,7 @@ mod exec_policy; mod git_info_tests; mod guardian; mod hook_runtime; +mod image_preparation; mod installation_id; pub(crate) mod landlock; pub use landlock::spawn_command_under_linux_sandbox; @@ -124,7 +129,6 @@ pub type NewConversation = NewThread; #[deprecated(note = "use CodexThread")] pub type CodexConversation = CodexThread; pub(crate) mod agents_md; -pub use agents_md::AgentsMdManager; pub use agents_md::DEFAULT_AGENTS_MD_FILENAME; pub use agents_md::LOCAL_AGENTS_MD_FILENAME; pub use agents_md::LoadedAgentsMd; @@ -189,7 +193,6 @@ pub use exec_policy::check_execpolicy_for_warnings; pub use exec_policy::format_exec_policy_error_with_source; pub use exec_policy::load_exec_policy; pub use installation_id::resolve_installation_id; -pub use turn_metadata::build_turn_metadata_header; pub mod compact; mod memory_usage; pub mod otel_init; diff --git a/codex-rs/core/src/mcp.rs b/codex-rs/core/src/mcp.rs index 71571deaec..5af247b690 100644 --- a/codex-rs/core/src/mcp.rs +++ b/codex-rs/core/src/mcp.rs @@ -10,11 +10,12 @@ use codex_login::CodexAuth; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::EffectiveMcpServer; use codex_mcp::McpConfig; -use codex_mcp::ToolPluginProvenance; +use codex_mcp::McpServerRegistration; use codex_mcp::codex_apps_mcp_server_config; use codex_mcp::configured_mcp_servers; use codex_mcp::effective_mcp_servers; -use codex_mcp::tool_plugin_provenance as collect_tool_plugin_provenance; + +const LEGACY_CODEX_APPS_REGISTRATION_ID: &str = "legacy_codex_apps"; #[derive(Clone)] pub struct McpManager { @@ -45,21 +46,53 @@ impl McpManager { /// runtime-only extension overlays. pub async fn runtime_config(&self, config: &Config) -> McpConfig { let mut mcp_config = config.to_mcp_config(self.plugins_manager.as_ref()).await; + let mut catalog = mcp_config.mcp_server_catalog.to_builder(); if mcp_config.apps_enabled { - mcp_config.configured_mcp_servers.insert( + catalog.register(McpServerRegistration::from_compatibility( CODEX_APPS_MCP_SERVER_NAME.to_string(), + LEGACY_CODEX_APPS_REGISTRATION_ID, codex_apps_mcp_server_config( &mcp_config.chatgpt_base_url, mcp_config.apps_mcp_product_sku.as_deref(), ), - ); + )); } else { - mcp_config - .configured_mcp_servers - .remove(CODEX_APPS_MCP_SERVER_NAME); + catalog.remove_compatibility( + CODEX_APPS_MCP_SERVER_NAME.to_string(), + LEGACY_CODEX_APPS_REGISTRATION_ID, + ); } - let contributions = self.contributions(config).await; - Self::apply_to_configured_servers(&contributions, &mut mcp_config.configured_mcp_servers); + + let mut contribution_order = 0; + for contributor in self.extensions.mcp_server_contributors() { + for contribution in contributor.contribute(config).await { + match contribution { + McpServerContribution::Set { + name, + config: server_config, + } => catalog.register(McpServerRegistration::from_extension( + name, + contributor.id(), + contribution_order, + *server_config, + )), + McpServerContribution::Remove { name } => { + catalog.remove_extension(name, contributor.id(), contribution_order) + } + } + contribution_order += 1; + } + } + let catalog = catalog.build(); + for conflict in catalog.conflicts() { + tracing::warn!( + server = conflict.name, + outcome = ?conflict.outcome, + contenders = ?conflict.contenders, + "conflicting MCP server actions; using resolved catalog outcome" + ); + } + mcp_config.mcp_server_catalog = catalog; mcp_config } @@ -84,34 +117,4 @@ impl McpManager { let mcp_config = self.runtime_config(config).await; effective_mcp_servers(&mcp_config, auth) } - - /// Returns provenance for plugin-owned servers in the configured view. - pub async fn tool_plugin_provenance(&self, config: &Config) -> ToolPluginProvenance { - let mcp_config = config.to_mcp_config(self.plugins_manager.as_ref()).await; - collect_tool_plugin_provenance(&mcp_config) - } - - async fn contributions(&self, config: &Config) -> Vec { - let mut contributions = Vec::new(); - for contributor in self.extensions.mcp_server_contributors() { - contributions.extend(contributor.contribute(config).await); - } - contributions - } - - fn apply_to_configured_servers( - contributions: &[McpServerContribution], - servers: &mut HashMap, - ) { - for contribution in contributions { - match contribution { - McpServerContribution::Set { name, config } => { - servers.insert(name.clone(), config.as_ref().clone()); - } - McpServerContribution::Remove { name } => { - servers.remove(name); - } - } - } - } } diff --git a/codex-rs/core/src/plugins/discoverable.rs b/codex-rs/core/src/plugins/discoverable.rs index 8ddec7056e..fc77ae21e6 100644 --- a/codex-rs/core/src/plugins/discoverable.rs +++ b/codex-rs/core/src/plugins/discoverable.rs @@ -41,6 +41,7 @@ pub(crate) async fn list_tool_suggest_discoverable_plugins( .into_iter() .map(|plugin| DiscoverablePluginInfo { id: plugin.id, + remote_plugin_id: plugin.remote_plugin_id, name: plugin.name, description: plugin.description, has_skills: plugin.has_skills, diff --git a/codex-rs/core/src/plugins/discoverable_tests.rs b/codex-rs/core/src/plugins/discoverable_tests.rs index c1dfe8e639..a6da04e77c 100644 --- a/codex-rs/core/src/plugins/discoverable_tests.rs +++ b/codex-rs/core/src/plugins/discoverable_tests.rs @@ -393,6 +393,7 @@ remote_plugin = true remote_plugins, vec![DiscoverablePluginInfo { id: "github@openai-curated-remote".to_string(), + remote_plugin_id: Some("plugins~Plugin_remote_github".to_string()), name: "Remote GitHub".to_string(), description: Some("Remote GitHub short".to_string()), has_skills: true, @@ -680,6 +681,7 @@ async fn list_tool_suggest_discoverable_plugins_normalizes_description() { discoverable_plugins, vec![DiscoverablePluginInfo { id: "slack@openai-curated".to_string(), + remote_plugin_id: None, name: "slack".to_string(), description: Some("Plugin with extra spacing".to_string()), has_skills: true, @@ -816,6 +818,7 @@ discoverables = [{ type = "plugin", id = "sample@openai-curated" }] discoverable_plugins, vec![DiscoverablePluginInfo { id: "sample@openai-curated".to_string(), + remote_plugin_id: None, name: "sample".to_string(), description: Some( "Plugin that includes skills, MCP servers, and app connectors".to_string(), diff --git a/codex-rs/core/src/prompt_debug.rs b/codex-rs/core/src/prompt_debug.rs index f6bf7d0022..74fe19f38e 100644 --- a/codex-rs/core/src/prompt_debug.rs +++ b/codex-rs/core/src/prompt_debug.rs @@ -2,10 +2,10 @@ use std::sync::Arc; use codex_exec_server::EnvironmentManager; use codex_exec_server::ExecServerRuntimePaths; +use codex_extension_api::UserInstructionsProvider; use codex_login::AuthManager; use codex_protocol::error::CodexErr; use codex_protocol::error::Result as CodexResult; -use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::SessionSource; use codex_protocol::user_input::UserInput; @@ -27,6 +27,7 @@ pub async fn build_prompt_input( mut config: Config, input: Vec, state_db: Option, + user_instructions_provider: Arc, ) -> CodexResult> { config.ephemeral = true; @@ -53,6 +54,7 @@ pub async fn build_prompt_input( .map_err(|err| CodexErr::Fatal(err.to_string()))?, ), empty_extension_registry(), + user_instructions_provider, /*analytics_events_client*/ None, thread_store, state_db.clone(), @@ -78,8 +80,7 @@ pub(crate) async fn build_prompt_input_from_session( .await; if !input.is_empty() { - let input_item = ResponseInputItem::from(input); - let response_item = ResponseItem::from(input_item); + let response_item = sess.response_item_from_user_input(turn_context.as_ref(), input); sess.record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&response_item)) .await; } diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 66dc653e56..33568f6f38 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -1,5 +1,6 @@ use crate::client::ModelClient; use crate::realtime_context::build_realtime_startup_context; +use crate::realtime_context::truncate_realtime_text_to_token_budget; use crate::realtime_prompt::prepare_realtime_backend_prompt; use crate::session::session::Session; use anyhow::Context; @@ -64,6 +65,8 @@ const USER_TEXT_IN_QUEUE_CAPACITY: usize = 64; const HANDOFF_OUT_QUEUE_CAPACITY: usize = 64; const OUTPUT_EVENTS_QUEUE_CAPACITY: usize = 256; const REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET: usize = 5_300; +const REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET: usize = 1_000; +const STANDALONE_HANDOFF_ID: &str = "codex"; const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5"; pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] "; pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] "; @@ -106,6 +109,9 @@ struct RealtimeHandoffState { #[derive(Debug, PartialEq, Eq)] enum HandoffOutput { + StandaloneAssistantOutput { + output_text: String, + }, ProgressUpdate { handoff_id: String, output_text: String, @@ -460,22 +466,38 @@ impl RealtimeConversationManager { state.handoff.clone() }; - let Some(handoff_id) = handoff.active_handoff.lock().await.clone() else { - return Ok(()); + let active_handoff = handoff.active_handoff.lock().await.clone(); + let output = match active_handoff { + Some(handoff_id) => { + let output_text = prefix_realtime_text( + output_text, + REALTIME_BACKEND_TEXT_PREFIX, + handoff.session_kind, + ); + *handoff.last_output_text.lock().await = Some(output_text.clone()); + HandoffOutput::ProgressUpdate { + handoff_id, + output_text, + } + } + None if output_text.trim().is_empty() => return Ok(()), + None => { + let output_text = prefix_realtime_text( + output_text, + REALTIME_BACKEND_TEXT_PREFIX, + handoff.session_kind, + ); + HandoffOutput::StandaloneAssistantOutput { + output_text: truncate_realtime_text_to_token_budget( + &output_text, + REALTIME_ASSISTANT_OUTPUT_TOKEN_BUDGET, + ), + } + } }; - - let output_text = prefix_realtime_text( - output_text, - REALTIME_BACKEND_TEXT_PREFIX, - handoff.session_kind, - ); - *handoff.last_output_text.lock().await = Some(output_text.clone()); handoff .output_tx - .send(HandoffOutput::ProgressUpdate { - handoff_id, - output_text, - }) + .send(output) .await .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string()))?; Ok(()) @@ -511,14 +533,6 @@ impl RealtimeConversationManager { .map_err(|_| CodexErr::InvalidRequest("conversation is not running".to_string())) } - pub(crate) async fn active_handoff_id(&self) -> Option { - let handoff = { - let guard = self.state.lock().await; - guard.as_ref().map(|state| state.handoff.clone()) - }?; - handoff.active_handoff.lock().await.clone() - } - pub(crate) async fn clear_active_handoff(&self) { let handoff = { let guard = self.state.lock().await; @@ -1188,6 +1202,15 @@ async fn handle_handoff_output( let result = match event_parser { RealtimeEventParser::V1 => match handoff_output { + HandoffOutput::StandaloneAssistantOutput { output_text } => { + // TODO(guinness): Use the new client event for standalone handoffs once the API changes are complete. + writer + .send_conversation_handoff_append( + STANDALONE_HANDOFF_ID.to_string(), + output_text, + ) + .await + } HandoffOutput::ProgressUpdate { handoff_id, output_text, @@ -1202,6 +1225,15 @@ async fn handle_handoff_output( } }, RealtimeEventParser::RealtimeV2 => match handoff_output { + HandoffOutput::StandaloneAssistantOutput { output_text } => { + if let Err(err) = writer.send_conversation_item_create(output_text).await { + Err(err) + } else { + return response_create_queue + .request_create(writer, events_tx, "standalone assistant output") + .await; + } + } HandoffOutput::ProgressUpdate { handoff_id, output_text, diff --git a/codex-rs/core/src/responses_metadata.rs b/codex-rs/core/src/responses_metadata.rs new file mode 100644 index 0000000000..6e86ed8eb3 --- /dev/null +++ b/codex-rs/core/src/responses_metadata.rs @@ -0,0 +1,366 @@ +use std::collections::BTreeMap; +use std::collections::HashMap; + +use codex_analytics::CompactionImplementation; +use codex_analytics::CompactionPhase; +use codex_analytics::CompactionReason; +use codex_analytics::CompactionStrategy; +use codex_analytics::CompactionTrigger; +use codex_protocol::ThreadId; +use codex_protocol::protocol::InternalSessionSource; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SubAgentSource; +use codex_utils_string::to_ascii_json_string; +use http::HeaderMap as ApiHeaderMap; +use http::HeaderValue; +use serde::Serialize; +use serde_json::Value; + +use crate::client::X_CODEX_INSTALLATION_ID_HEADER; +use crate::client::X_CODEX_PARENT_THREAD_ID_HEADER; +use crate::client::X_CODEX_TURN_METADATA_HEADER; +use crate::client::X_CODEX_WINDOW_ID_HEADER; +use crate::client::X_OPENAI_SUBAGENT_HEADER; + +pub(crate) const INSTALLATION_ID_KEY: &str = "installation_id"; +pub(crate) const SESSION_ID_KEY: &str = "session_id"; +pub(crate) const THREAD_ID_KEY: &str = "thread_id"; +pub(crate) const TURN_ID_KEY: &str = "turn_id"; +pub(crate) const WINDOW_ID_KEY: &str = "window_id"; +pub(crate) const REQUEST_KIND_KEY: &str = "request_kind"; +pub(crate) const COMPACTION_KEY: &str = "compaction"; +pub(crate) const TURN_STARTED_AT_UNIX_MS_KEY: &str = "turn_started_at_unix_ms"; + +pub(crate) const FORKED_FROM_THREAD_ID_KEY: &str = "forked_from_thread_id"; +pub(crate) const PARENT_THREAD_ID_KEY: &str = "parent_thread_id"; +pub(crate) const SUBAGENT_KIND_KEY: &str = "subagent_kind"; +pub(crate) const SANDBOX_KEY: &str = "sandbox"; +pub(crate) const WORKSPACES_KEY: &str = "workspaces"; + +// App-server clients can specify additional metadata in the `responsesapi_client_metadata` param +// when submitting a turn, but they must not override fields owned by core. +const RESERVED_METADATA_KEYS: &[&str] = &[ + INSTALLATION_ID_KEY, + X_CODEX_INSTALLATION_ID_HEADER, + SESSION_ID_KEY, + THREAD_ID_KEY, + TURN_ID_KEY, + WINDOW_ID_KEY, + X_CODEX_WINDOW_ID_HEADER, + X_CODEX_TURN_METADATA_HEADER, + X_CODEX_PARENT_THREAD_ID_HEADER, + X_OPENAI_SUBAGENT_HEADER, + REQUEST_KIND_KEY, + COMPACTION_KEY, + TURN_STARTED_AT_UNIX_MS_KEY, + FORKED_FROM_THREAD_ID_KEY, + PARENT_THREAD_ID_KEY, + SUBAGENT_KIND_KEY, + SANDBOX_KEY, + WORKSPACES_KEY, +]; + +/// Metadata attached to model requests whose purpose is conversation compaction. +/// +/// This covers both local compaction requests sent through the normal `/responses` path and remote +/// compaction requests sent through `/responses/compact`. These fields describe the operation at +/// dispatch time. Post-response outcomes such as status, error, duration, and token deltas remain +/// in compaction analytics events. +#[derive(Clone, Copy, Debug, Serialize)] +pub(crate) struct CompactionTurnMetadata { + trigger: CompactionTrigger, + reason: CompactionReason, + implementation: CompactionImplementation, + phase: CompactionPhase, + strategy: CompactionStrategy, +} + +impl CompactionTurnMetadata { + pub(crate) fn new( + trigger: CompactionTrigger, + reason: CompactionReason, + implementation: CompactionImplementation, + phase: CompactionPhase, + ) -> Self { + Self { + trigger, + reason, + implementation, + phase, + strategy: CompactionStrategy::Memento, + } + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum CodexResponsesRequestKind { + Turn, + Prewarm, + Compaction(CompactionTurnMetadata), + Memory, +} + +impl CodexResponsesRequestKind { + fn metadata(self) -> (&'static str, Option) { + match self { + CodexResponsesRequestKind::Turn => ("turn", None), + CodexResponsesRequestKind::Prewarm => ("prewarm", None), + CodexResponsesRequestKind::Compaction(metadata) => ("compaction", Some(metadata)), + CodexResponsesRequestKind::Memory => ("memory", None), + } + } + + fn has_turn_identity(self) -> bool { + !matches!(self, CodexResponsesRequestKind::Memory) + } +} + +#[derive(Clone, Debug, Serialize, Default)] +pub(crate) struct TurnMetadataWorkspace { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) associated_remote_urls: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) latest_git_commit_hash: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) has_changes: Option, +} + +/// Caller-owned snapshot of Codex metadata sent to ResponsesAPI. +/// +/// The full Codex turn metadata blob is transported canonically as +/// `client_metadata["x-codex-turn-metadata"]`. Flat `client_metadata` keys and direct HTTP/ws +/// headers are generated compatibility projections of this snapshot, not separate sources of +/// truth. +#[derive(Clone, Debug)] +pub struct CodexResponsesMetadata { + pub(crate) installation_id: String, + pub(crate) session_id: String, + pub(crate) thread_id: String, + pub(crate) turn_id: Option, + pub(crate) window_id: String, + pub(crate) request_kind: Option, + pub(crate) forked_from_thread_id: Option, + pub(crate) parent_thread_id: Option, + pub(crate) subagent_header: Option, + pub(crate) subagent_kind: Option, + pub(crate) sandbox: Option, + pub(crate) workspaces: BTreeMap, + pub(crate) turn_started_at_unix_ms: Option, + pub(crate) extra: BTreeMap, +} + +impl CodexResponsesMetadata { + pub(crate) fn new( + installation_id: String, + session_id: String, + thread_id: String, + window_id: String, + ) -> Self { + Self { + installation_id, + session_id, + thread_id, + turn_id: None, + window_id, + request_kind: None, + forked_from_thread_id: None, + parent_thread_id: None, + subagent_header: None, + subagent_kind: None, + sandbox: None, + workspaces: BTreeMap::new(), + turn_started_at_unix_ms: None, + extra: BTreeMap::new(), + } + } + + pub(crate) fn has_turn_metadata(&self) -> bool { + self.request_kind.is_some() + } + + pub(crate) fn turn_metadata_json(&self) -> Option { + to_ascii_json_string(&self.turn_metadata_payload()).ok() + } + + pub(crate) fn turn_metadata_value(&self) -> Option { + serde_json::to_value(self.turn_metadata_payload()).ok() + } + + pub(crate) fn client_metadata(&self) -> HashMap { + let mut client_metadata = HashMap::from([ + ( + X_CODEX_INSTALLATION_ID_HEADER.to_string(), + self.installation_id.clone(), + ), + (SESSION_ID_KEY.to_string(), self.session_id.clone()), + (THREAD_ID_KEY.to_string(), self.thread_id.clone()), + (X_CODEX_WINDOW_ID_HEADER.to_string(), self.window_id.clone()), + ]); + if let Some(turn_id) = &self.turn_id { + client_metadata.insert(TURN_ID_KEY.to_string(), turn_id.clone()); + } + if let Some(subagent_header) = &self.subagent_header { + client_metadata.insert( + X_OPENAI_SUBAGENT_HEADER.to_string(), + subagent_header.clone(), + ); + } + if let Some(parent_thread_id) = self.parent_thread_id { + client_metadata.insert( + X_CODEX_PARENT_THREAD_ID_HEADER.to_string(), + parent_thread_id.to_string(), + ); + } + if self.has_turn_metadata() + && let Some(turn_metadata_json) = self.turn_metadata_json() + { + client_metadata.insert(X_CODEX_TURN_METADATA_HEADER.to_string(), turn_metadata_json); + } + client_metadata + } + + pub(crate) fn compatibility_headers(&self) -> ApiHeaderMap { + let mut headers = ApiHeaderMap::new(); + insert_header(&mut headers, X_CODEX_WINDOW_ID_HEADER, &self.window_id); + // Direct x-codex-turn-metadata is compatibility output. New per-request consumers should + // prefer client_metadata["x-codex-turn-metadata"], which is rendered from this same object. + if self.has_turn_metadata() + && let Some(turn_metadata_json) = self.turn_metadata_json() + { + insert_header( + &mut headers, + X_CODEX_TURN_METADATA_HEADER, + &turn_metadata_json, + ); + } + if let Some(parent_thread_id) = self.parent_thread_id { + insert_header( + &mut headers, + X_CODEX_PARENT_THREAD_ID_HEADER, + &parent_thread_id.to_string(), + ); + } + if let Some(subagent_header) = &self.subagent_header { + insert_header(&mut headers, X_OPENAI_SUBAGENT_HEADER, subagent_header); + } + headers + } + + fn turn_metadata_payload(&self) -> CodexTurnMetadataPayload<'_> { + let request_kind = self.request_kind; + let (request_kind_value, compaction) = request_kind.map_or((None, None), |request_kind| { + let (request_kind, compaction) = request_kind.metadata(); + (Some(request_kind), compaction) + }); + let has_turn_identity = + request_kind.is_none_or(CodexResponsesRequestKind::has_turn_identity); + let has_request_identity = + request_kind.is_some_and(CodexResponsesRequestKind::has_turn_identity); + CodexTurnMetadataPayload { + installation_id: has_request_identity.then_some(self.installation_id.as_str()), + session_id: has_turn_identity.then_some(self.session_id.as_str()), + thread_id: has_turn_identity.then_some(self.thread_id.as_str()), + turn_id: has_turn_identity + .then_some(self.turn_id.as_deref()) + .flatten(), + window_id: has_request_identity.then_some(self.window_id.as_str()), + request_kind: request_kind_value, + forked_from_thread_id: self.forked_from_thread_id, + parent_thread_id: self.parent_thread_id, + subagent_kind: self.subagent_kind.as_deref(), + sandbox: self.sandbox.as_deref(), + workspaces: non_empty_workspaces(&self.workspaces), + turn_started_at_unix_ms: self.turn_started_at_unix_ms, + compaction, + // responsesapi_client_metadata enriches the Codex turn metadata blob, not literal + // top-level Responses client_metadata. Reserved Codex-owned keys are filtered when + // these extras enter turn state. + extra: &self.extra, + } + } +} + +pub(crate) fn subagent_header_value(session_source: &SessionSource) -> Option { + match session_source { + SessionSource::SubAgent(subagent_source) => match subagent_source { + SubAgentSource::Review => Some("review".to_string()), + SubAgentSource::Compact => Some("compact".to_string()), + SubAgentSource::MemoryConsolidation => Some("memory_consolidation".to_string()), + SubAgentSource::ThreadSpawn { .. } => Some("collab_spawn".to_string()), + SubAgentSource::Other(label) => Some(label.clone()), + }, + SessionSource::Internal(InternalSessionSource::MemoryConsolidation) => { + Some("memory_consolidation".to_string()) + } + SessionSource::Cli + | SessionSource::VSCode + | SessionSource::Exec + | SessionSource::Mcp + | SessionSource::Custom(_) + | SessionSource::Unknown => None, + } +} + +pub(crate) fn subagent_metadata_kind(session_source: &SessionSource) -> Option { + match session_source { + SessionSource::SubAgent(subagent_source) => Some(subagent_source.kind().to_string()), + SessionSource::Cli + | SessionSource::VSCode + | SessionSource::Exec + | SessionSource::Mcp + | SessionSource::Custom(_) + | SessionSource::Internal(_) + | SessionSource::Unknown => None, + } +} + +fn insert_header(headers: &mut ApiHeaderMap, name: &'static str, value: &str) { + if let Ok(header_value) = HeaderValue::from_str(value) { + headers.insert(name, header_value); + } +} + +pub(crate) fn filter_extra_metadata(extra: HashMap) -> BTreeMap { + extra + .into_iter() + .filter(|(key, _)| !RESERVED_METADATA_KEYS.contains(&key.as_str())) + .collect() +} + +fn non_empty_workspaces( + workspaces: &BTreeMap, +) -> Option<&BTreeMap> { + (!workspaces.is_empty()).then_some(workspaces) +} + +#[derive(Serialize)] +struct CodexTurnMetadataPayload<'a> { + #[serde(default, skip_serializing_if = "Option::is_none")] + installation_id: Option<&'a str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + session_id: Option<&'a str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + thread_id: Option<&'a str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + turn_id: Option<&'a str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + window_id: Option<&'a str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + request_kind: Option<&'static str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + forked_from_thread_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + parent_thread_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + subagent_kind: Option<&'a str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + sandbox: Option<&'a str>, + #[serde(default, skip_serializing_if = "Option::is_none")] + workspaces: Option<&'a BTreeMap>, + #[serde(default, skip_serializing_if = "Option::is_none")] + turn_started_at_unix_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + compaction: Option, + #[serde(flatten)] + extra: &'a BTreeMap, +} diff --git a/codex-rs/core/src/session/mcp.rs b/codex-rs/core/src/session/mcp.rs index 39333801c0..7f52da219b 100644 --- a/codex-rs/core/src/session/mcp.rs +++ b/codex-rs/core/src/session/mcp.rs @@ -294,11 +294,7 @@ impl Session { .mcp_manager .runtime_config(config.as_ref()) .await; - let tool_plugin_provenance = self - .services - .mcp_manager - .tool_plugin_provenance(config.as_ref()) - .await; + let tool_plugin_provenance = codex_mcp::tool_plugin_provenance(&mcp_config); let mcp_servers = effective_mcp_servers_from_configured(mcp_servers, &mcp_config, auth.as_ref()); let host_owned_codex_apps_enabled = diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index ddb31a9a52..e6a4d03dda 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; @@ -13,6 +14,7 @@ use crate::agent::AgentControl; use crate::agent::AgentStatus; use crate::agent::agent_status_from_event; use crate::agent::status::is_final; +use crate::agents_md::LoadedAgentsMd; use crate::attestation::AttestationProvider; use crate::build_available_skills; use crate::compact; @@ -31,6 +33,7 @@ use crate::context::PersonalitySpecInstructions; use crate::default_skill_metadata_budget; use crate::environment_selection::ResolvedTurnEnvironments; use crate::exec_policy::ExecPolicyManager; +use crate::image_preparation::prepare_response_items; use crate::parse_turn_item; use crate::realtime_conversation::RealtimeConversationManager; use crate::session_prefix::format_subagent_notification_message; @@ -52,6 +55,7 @@ use codex_exec_server::Environment; use codex_exec_server::EnvironmentManager; use codex_exec_server::FileSystemSandboxContext; use codex_extension_api::ExtensionDataInit; +use codex_extension_api::LoadedUserInstructions; use codex_extension_api::PromptSlot; use codex_features::FEATURES; use codex_features::Feature; @@ -63,6 +67,7 @@ use codex_login::CodexAuth; use codex_login::auth_env_telemetry::collect_auth_env_telemetry; use codex_login::default_client::originator; use codex_mcp::McpConnectionManager; +use codex_mcp::McpResourceClient; use codex_mcp::McpRuntimeContext; use codex_mcp::codex_apps_tools_cache_key; use codex_models_manager::manager::RefreshStrategy; @@ -205,6 +210,7 @@ mod review; mod rollout_reconstruction; #[allow(clippy::module_inception)] pub(crate) mod session; +mod token_budget; pub(crate) mod turn; pub(crate) mod turn_context; use self::config_lock::export_config_lock_if_configured; @@ -273,10 +279,12 @@ impl SteerInputError { /// Conceptually this is the same role that `previous_model` used to fill, but /// it can carry other prior-turn settings that matter when constructing /// sensible state-change diffs or full-context reinjection, such as model -/// switches or detecting a prior `realtime_active -> false` transition. +/// switches, compaction compatibility, or detecting a prior +/// `realtime_active -> false` transition. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct PreviousTurnSettings { pub(crate) model: String, + pub(crate) comp_hash: Option, pub(crate) realtime_active: Option, } @@ -285,7 +293,7 @@ use crate::SkillLoadOutcome; #[cfg(test)] use crate::SkillMetadata; use crate::SkillsManager; -use crate::agents_md::AgentsMdManager; +use crate::agents_md::load_project_instructions; use crate::context::UserInstructions; use crate::exec_policy::ExecPolicyUpdateError; use crate::guardian::GuardianReviewSessionManager; @@ -326,6 +334,7 @@ use codex_protocol::config_types::CollaborationMode; use codex_protocol::config_types::Personality; use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use codex_protocol::config_types::WindowsSandboxLevel; +use codex_protocol::models::LocalImagePreparation; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; @@ -392,6 +401,7 @@ pub struct CodexSpawnOk { pub(crate) struct CodexSpawnArgs { pub(crate) config: Config, + pub(crate) user_instructions: LoadedUserInstructions, pub(crate) installation_id: String, pub(crate) auth_manager: Arc, pub(crate) models_manager: SharedModelsManager, @@ -477,6 +487,7 @@ impl Codex { async fn spawn_internal(args: CodexSpawnArgs) -> CodexResult { let CodexSpawnArgs { mut config, + user_instructions, installation_id, auth_manager, models_manager, @@ -508,16 +519,21 @@ impl Codex { let (tx_sub, rx_sub) = async_channel::bounded(SUBMISSION_CHANNEL_CAPACITY); let (tx_event, rx_event) = async_channel::unbounded(); + let LoadedUserInstructions { + instructions: user_instructions, + warnings: user_instruction_provider_warnings, + } = user_instructions; + // TODO(anp) pull startup_warnings out of Config + config + .startup_warnings + .extend(user_instruction_provider_warnings); + // TODO(anp) assemble instructions from multiple environments let primary_environment = environment_selections.primary_environment(); - let mut user_instruction_warnings = Vec::new(); - let user_instructions = if let Some(primary_environment) = primary_environment { - AgentsMdManager::new(&config) - .user_instructions(primary_environment.as_ref(), &mut user_instruction_warnings) - .await - } else { - None - }; - config.startup_warnings.extend(user_instruction_warnings); + let primary_fs = primary_environment + .as_ref() + .map(|environment| environment.get_filesystem()); + let loaded_agents_md = + load_project_instructions(&mut config, user_instructions, primary_fs.as_deref()).await; let exec_policy = if crate::guardian::is_guardian_reviewer_source(&session_source) { // Guardian review should rely on the built-in shell safety checks, @@ -597,7 +613,7 @@ impl Codex { model_reasoning_summary: config.model_reasoning_summary, service_tier, developer_instructions: config.developer_instructions.clone(), - user_instructions, + loaded_agents_md, personality: config.personality, base_instructions, compact_prompt: config.compact_prompt.clone(), @@ -811,7 +827,7 @@ impl Codex { let state = self.session.state.lock().await; state .session_configuration - .user_instructions + .loaded_agents_md .as_ref() .map_or_else(Vec::new, |instructions| { instructions.sources().cloned().collect() @@ -1300,13 +1316,20 @@ impl Session { rollout_items: &[RolloutItem], ) -> Option { let rollout_reconstruction::RolloutReconstruction { - history, + mut history, previous_turn_settings, reference_context_item, window_id, } = self .reconstruct_history_from_rollout(turn_context, rollout_items) .await; + if turn_context.features.enabled(Feature::ResizeAllImages) { + // Keep the recorded rollout unchanged. Prepare its reconstructed history before + // installing it, so legacy images are processed once for this resume or fork and + // will be processed again if the rollout is reconstructed in a future session. + // This meets image resizing requirements without modifying persisted rollouts. + prepare_response_items(&mut history); + } { let mut state = self.state.lock().await; state.replace_history(history, reference_context_item); @@ -1493,6 +1516,16 @@ impl Session { .clone() } + pub(crate) async fn user_instructions(&self) -> Option { + let state = self.state.lock().await; + state + .session_configuration + .loaded_agents_md + .as_ref() + .and_then(LoadedAgentsMd::user_instructions) + .cloned() + } + pub(crate) async fn provider(&self) -> ModelProviderInfo { let state = self.state.lock().await; state.session_configuration.provider.clone() @@ -1796,9 +1829,7 @@ impl Session { let Some(text) = realtime_text_for_event(msg) else { return; }; - if self.conversation.running_state().await.is_none() - || self.conversation.active_handoff_id().await.is_none() - { + if self.conversation.running_state().await.is_none() { return; } if let Err(err) = self.conversation.handoff_out(text).await { @@ -2585,11 +2616,43 @@ impl Session { /// Records conversation items: append to history, persist to rollout, and /// notify clients observing raw response items. + pub(crate) fn prepare_conversation_items_for_history<'a>( + &self, + turn_context: &TurnContext, + items: &'a [ResponseItem], + ) -> Cow<'a, [ResponseItem]> { + if !turn_context.features.enabled(Feature::ResizeAllImages) { + return Cow::Borrowed(items); + } + + let mut prepared_items = items.to_vec(); + prepare_response_items(&mut prepared_items); + Cow::Owned(prepared_items) + } + + pub(crate) fn response_item_from_user_input( + &self, + turn_context: &TurnContext, + input: Vec, + ) -> ResponseItem { + let local_image_preparation = if turn_context.features.enabled(Feature::ResizeAllImages) { + LocalImagePreparation::Defer + } else { + LocalImagePreparation::Process + }; + ResponseItem::from(ResponseInputItem::from_user_input( + input, + local_image_preparation, + )) + } + pub(crate) async fn record_conversation_items( &self, turn_context: &TurnContext, items: &[ResponseItem], ) { + let items = self.prepare_conversation_items_for_history(turn_context, items); + let items = items.as_ref(); { let mut state = self.state.lock().await; state.record_items(items.iter(), turn_context.truncation_policy); @@ -2672,15 +2735,13 @@ impl Session { &self, items: Vec, reference_context_item: Option, - mut compacted_item: CompactedItem, + compacted_item: CompactedItem, ) { { let mut state = self.state.lock().await; state.replace_history(items, reference_context_item.clone()); } - compacted_item.window_id = Some(self.advance_auto_compact_window_id().await); - self.persist_rollout_items(&[RolloutItem::Compacted(compacted_item)]) .await; if let Some(turn_context_item) = reference_context_item { @@ -2765,6 +2826,7 @@ impl Session { collaboration_mode, base_instructions, session_source, + auto_compact_window_id, ) = { let state = self.state.lock().await; ( @@ -2773,6 +2835,7 @@ impl Session { state.session_configuration.collaboration_mode.clone(), state.session_configuration.base_instructions.clone(), state.session_configuration.session_source.clone(), + state.auto_compact_window_id(), ) }; if let Some(model_switch_message) = @@ -2922,6 +2985,18 @@ impl Session { .render(), ); } + // This is full-context metadata. Steady-state context diffs should not re-emit it. + if turn_context.features.enabled(Feature::TokenBudget) + && let Some(model_context_window) = turn_context.model_context_window() + { + developer_sections.push( + crate::context::TokenBudgetContext::new( + auto_compact_window_id, + model_context_window, + ) + .render(), + ); + } if turn_context.config.include_environment_context { let shell = self.user_shell(); let subagents = self @@ -3000,11 +3075,49 @@ impl Session { format!("{thread_id}:{window_id}") } - async fn advance_auto_compact_window_id(&self) -> u64 { + pub(crate) async fn advance_auto_compact_window_id(&self) -> u64 { let mut state = self.state.lock().await; state.advance_auto_compact_window_id() } + pub(crate) async fn request_new_context_window(&self) { + let mut state = self.state.lock().await; + state.request_new_context_window(); + } + + pub(crate) async fn maybe_start_new_context_window( + &self, + turn_context: &TurnContext, + ) -> Option { + let window_id = { + let mut state = self.state.lock().await; + state.start_new_context_window_if_requested() + }; + let window_id = window_id?; + let context_items = self.build_initial_context(turn_context).await; + let turn_context_item = turn_context.to_turn_context_item(); + let replacement_history = context_items; + { + let mut state = self.state.lock().await; + state.replace_history(replacement_history.clone(), Some(turn_context_item.clone())); + }; + self.persist_rollout_items(&[ + RolloutItem::Compacted(CompactedItem { + message: String::new(), + replacement_history: Some(replacement_history), + window_id: Some(window_id), + }), + RolloutItem::TurnContext(turn_context_item), + ]) + .await; + { + let mut state = self.state.lock().await; + state.queue_pending_session_start_source(codex_hooks::SessionStartSource::Compact); + } + self.recompute_token_usage(turn_context).await; + Some(window_id) + } + pub(crate) async fn reference_context_item(&self) -> Option { let state = self.state.lock().await; state.reference_context_item() @@ -3213,7 +3326,7 @@ impl Session { // Persist the user message to history, but emit the turn item from `UserInput` so // UI-only `text_elements` are preserved. `ResponseItem::Message` does not carry // those spans, and `record_response_item_and_emit_turn_item` would drop them. - let response_item = ResponseItem::from(ResponseInputItem::from(input.to_vec())); + let response_item = self.response_item_from_user_input(turn_context, input.to_vec()); self.record_conversation_items(turn_context, std::slice::from_ref(&response_item)) .await; let mut user_message_item = UserMessageItem::new(input); diff --git a/codex-rs/core/src/session/review.rs b/codex-rs/core/src/session/review.rs index 00a89b2b5c..9a7dfc8298 100644 --- a/codex-rs/core/src/session/review.rs +++ b/codex-rs/core/src/session/review.rs @@ -91,7 +91,6 @@ pub(super) async fn spawn_review_thread( forked_from_thread_id, parent_turn_context.parent_thread_id, &session_source, - parent_turn_context.thread_source.clone(), review_turn_id.clone(), #[allow(deprecated)] parent_turn_context.cwd.clone(), @@ -114,6 +113,7 @@ pub(super) async fn spawn_review_thread( config: per_turn_config, auth_manager: auth_manager_for_context, model_info: model_info.clone(), + comp_hash: model_info.comp_hash.clone(), tool_mode, session_telemetry: session_telemetry_for_context, provider: provider_for_context, diff --git a/codex-rs/core/src/session/rollout_reconstruction.rs b/codex-rs/core/src/session/rollout_reconstruction.rs index 2d0327bf40..e886648512 100644 --- a/codex-rs/core/src/session/rollout_reconstruction.rs +++ b/codex-rs/core/src/session/rollout_reconstruction.rs @@ -184,6 +184,7 @@ impl Session { ) { active_segment.previous_turn_settings = Some(PreviousTurnSettings { model: ctx.model.clone(), + comp_hash: ctx.comp_hash.clone(), realtime_active: ctx.realtime_active, }); if matches!( diff --git a/codex-rs/core/src/session/rollout_reconstruction_tests.rs b/codex-rs/core/src/session/rollout_reconstruction_tests.rs index c84ee7f5f3..ca8494168c 100644 --- a/codex-rs/core/src/session/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/session/rollout_reconstruction_tests.rs @@ -70,6 +70,7 @@ async fn record_initial_history_resumed_bare_turn_context_does_not_hydrate_previ network: None, file_system_sandbox_policy: None, model: previous_model.to_string(), + comp_hash: None, personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), multi_agent_version: None, @@ -109,6 +110,7 @@ async fn record_initial_history_resumed_hydrates_previous_turn_settings_from_lif network: None, file_system_sandbox_policy: None, model: previous_model.to_string(), + comp_hash: Some("comp-hash-a".to_string()), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), multi_agent_version: None, @@ -166,6 +168,7 @@ async fn record_initial_history_resumed_hydrates_previous_turn_settings_from_lif session.previous_turn_settings().await, Some(PreviousTurnSettings { model: previous_model.to_string(), + comp_hash: Some("comp-hash-a".to_string()), realtime_active: Some(turn_context.realtime_active), }) ); @@ -271,6 +274,7 @@ async fn reconstruct_history_rollback_keeps_history_and_metadata_in_sync_for_com reconstructed.previous_turn_settings, Some(PreviousTurnSettings { model: turn_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -364,6 +368,7 @@ async fn reconstruct_history_rollback_keeps_history_and_metadata_in_sync_for_inc reconstructed.previous_turn_settings, Some(PreviousTurnSettings { model: turn_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -489,6 +494,7 @@ async fn reconstruct_history_rollback_skips_non_user_turns_for_history_and_metad reconstructed.previous_turn_settings, Some(PreviousTurnSettings { model: turn_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -589,6 +595,7 @@ async fn reconstruct_history_rollback_counts_inter_agent_assistant_turns() { reconstructed.previous_turn_settings, Some(PreviousTurnSettings { model: turn_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -810,6 +817,7 @@ async fn record_initial_history_resumed_rollback_drops_incomplete_user_turn_comp session.previous_turn_settings().await, Some(PreviousTurnSettings { model: turn_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -963,6 +971,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis network: None, file_system_sandbox_policy: None, model: previous_model.to_string(), + comp_hash: None, personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), multi_agent_version: None, @@ -1024,6 +1033,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis session.previous_turn_settings().await, Some(PreviousTurnSettings { model: previous_model.to_string(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -1043,6 +1053,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis network: None, file_system_sandbox_policy: None, model: previous_model.to_string(), + comp_hash: None, personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), multi_agent_version: None, @@ -1072,6 +1083,7 @@ async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_tu network: None, file_system_sandbox_policy: None, model: previous_model.to_string(), + comp_hash: None, personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), multi_agent_version: None, @@ -1161,6 +1173,7 @@ async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_tu session.previous_turn_settings().await, Some(PreviousTurnSettings { model: previous_model.to_string(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -1192,6 +1205,7 @@ async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_fo network: None, file_system_sandbox_policy: None, model: current_model.to_string(), + comp_hash: None, personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), multi_agent_version: None, @@ -1281,6 +1295,7 @@ async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_fo session.previous_turn_settings().await, Some(PreviousTurnSettings { model: current_model.to_string(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -1310,6 +1325,7 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clea network: None, file_system_sandbox_policy: None, model: previous_model.to_string(), + comp_hash: None, personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), multi_agent_version: None, @@ -1391,6 +1407,7 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clea session.previous_turn_settings().await, Some(PreviousTurnSettings { model: previous_model.to_string(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -1441,6 +1458,7 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_preserves_turn_ session.previous_turn_settings().await, Some(PreviousTurnSettings { model: turn_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -1470,6 +1488,7 @@ async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clear network: None, file_system_sandbox_policy: None, model: previous_model.to_string(), + comp_hash: None, personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), multi_agent_version: None, @@ -1563,6 +1582,7 @@ async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clear session.previous_turn_settings().await, Some(PreviousTurnSettings { model: previous_model.to_string(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index e42cf202e8..2d14ceb5f5 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -55,9 +55,9 @@ pub(crate) struct SessionConfiguration { /// Developer instructions that supplement the base instructions. pub(super) developer_instructions: Option, - /// Model instructions that are appended to the base instructions and the - /// files that supplied them. - pub(super) user_instructions: Option, + /// Model instructions assembled from provider instructions and discovered + /// AGENTS.md files. + pub(super) loaded_agents_md: Option, /// Personality preference for the model. pub(super) personality: Option, @@ -605,16 +605,16 @@ impl Session { let mcp_manager_for_mcp = Arc::clone(&mcp_manager); let auth_and_mcp_fut = async move { let auth = auth_manager_clone.auth().await; - let mcp_servers = mcp_manager_for_mcp - .effective_servers(&config_for_mcp, auth.as_ref()) - .await; + let mcp_config = mcp_manager_for_mcp.runtime_config(&config_for_mcp).await; + let mcp_servers = codex_mcp::effective_mcp_servers(&mcp_config, auth.as_ref()); + let tool_plugin_provenance = codex_mcp::tool_plugin_provenance(&mcp_config); let auth_statuses = compute_auth_statuses( mcp_servers.iter(), config_for_mcp.mcp_oauth_credentials_store_mode, auth.as_ref(), ) .await; - (auth, mcp_servers, auth_statuses) + (auth, mcp_servers, auth_statuses, tool_plugin_provenance) } .instrument(info_span!( "session_init.auth_mcp", @@ -637,7 +637,7 @@ impl Session { let ( thread_persistence_result, state_db_ctx, - (auth, mcp_servers, auth_statuses), + (auth, mcp_servers, auth_statuses, tool_plugin_provenance), plugin_skill_errors, ) = tokio::join!( thread_persistence_fut, @@ -946,8 +946,20 @@ impl Session { .effective_agent_max_threads(MultiAgentVersion::V2) .unwrap_or(usize::MAX), ); + // Keep one stable manager handle for the session so extension resource clients + // automatically observe the manager installed at startup and on later refreshes. + let mcp_connection_manager = Arc::new(arc_swap::ArcSwap::from_pointee( + McpConnectionManager::new_uninitialized_with_permission_profile( + &config.permissions.approval_policy, + config.permissions.permission_profile(), + config.prefix_mcp_tool_names(), + ), + )); let session_extension_data = codex_extension_api::ExtensionData::new(session_id.to_string()); + session_extension_data.insert(McpResourceClient::new(Arc::clone( + &mcp_connection_manager, + ))); let thread_extension_data = codex_extension_api::ExtensionData::new_with_init( thread_id.to_string(), thread_extension_init, @@ -970,13 +982,7 @@ impl Session { // before any MCP-related events. It is reasonable to consider // changing this to use Option or OnceCell, though the current // setup is straightforward enough and performs well. - mcp_connection_manager: arc_swap::ArcSwap::from_pointee( - McpConnectionManager::new_uninitialized_with_permission_profile( - &config.permissions.approval_policy, - config.permissions.permission_profile(), - config.prefix_mcp_tool_names(), - ), - ), + mcp_connection_manager, mcp_startup_cancellation_token: Mutex::new(CancellationToken::new()), unified_exec_manager: UnifiedExecProcessManager::new( config.background_terminal_max_timeout, @@ -1015,12 +1021,9 @@ impl Session { attestation_provider: attestation_provider.clone(), model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), - session_id, thread_id, - installation_id.clone(), session_configuration.provider.clone(), session_configuration.session_source.clone(), - session_configuration.parent_thread_id, config.model_verbosity, config.features.enabled(Feature::EnableRequestCompression), config.features.enabled(Feature::RuntimeMetrics), @@ -1098,7 +1101,6 @@ impl Session { sess.send_event_raw(event).await; } - let tool_plugin_provenance = mcp_manager.tool_plugin_provenance(config.as_ref()).await; let host_owned_codex_apps_enabled = config .features .apps_enabled_for_auth(auth.as_ref().is_some_and(|auth| auth.uses_codex_backend())); diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 29de7ca662..0dbb13fcd2 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -42,7 +42,9 @@ use codex_protocol::models::ActivePermissionProfile; use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE; use codex_protocol::models::FileSystemPermissions; use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ImageDetail; use codex_protocol::models::PermissionProfile; use codex_protocol::models::SandboxEnforcement; use codex_protocol::openai_models::ModelServiceTier; @@ -421,12 +423,9 @@ fn test_model_client_session() -> crate::client::ModelClientSession { .expect("test thread id should be valid"); crate::client::ModelClient::new( /*auth_manager*/ None, - thread_id.into(), thread_id, - /*installation_id*/ "11111111-1111-4111-8111-111111111111".to_string(), ModelProviderInfo::create_openai_provider(/* base_url */ /*base_url*/ None), codex_protocol::protocol::SessionSource::Exec, - /*parent_thread_id*/ None, /*model_verbosity*/ None, /*enable_request_compression*/ false, /*include_timing_metrics*/ false, @@ -1641,6 +1640,114 @@ async fn record_initial_history_reconstructs_resumed_transcript() { assert_eq!(expected, history.raw_items()); } +#[tokio::test] +async fn resize_all_images_prepares_failures_before_history_insertion() { + let (session, turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx( + CodexAuth::from_api_key("Test API Key"), + Vec::new(), + |config| { + let _ = config.features.enable(Feature::ResizeAllImages); + }, + ) + .await; + let item = ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "before".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,%%%".to_string(), + detail: Some(ImageDetail::High), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + detail: Some(ImageDetail::High), + }, + ]), + success: Some(true), + }, + }; + + session + .record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&item)) + .await; + + assert_eq!( + session.state.lock().await.clone_history().raw_items(), + &[ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "before".to_string(), + }, + FunctionCallOutputContentItem::InputText { + text: "image content omitted because it could not be processed".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + detail: Some(ImageDetail::High), + }, + ]), + success: Some(true), + }, + }] + ); +} + +#[tokio::test] +async fn resize_all_images_prepares_resumed_history_before_installing_it() { + let (session, _turn_context, _rx) = make_session_and_context_with_auth_and_config_and_rx( + CodexAuth::from_api_key("Test API Key"), + Vec::new(), + |config| { + let _ = config.features.enable(Feature::ResizeAllImages); + }, + ) + .await; + let resumed_item = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputImage { + image_url: "data:image/png;base64,%%%".to_string(), + detail: Some(ImageDetail::High), + }, + ContentItem::InputText { + text: "keep me".to_string(), + }, + ], + phase: None, + }; + + session + .record_initial_history(InitialHistory::Resumed(ResumedHistory { + conversation_id: ThreadId::default(), + history: vec![RolloutItem::ResponseItem(resumed_item)], + rollout_path: Some(PathBuf::from("/tmp/resume.jsonl")), + })) + .await; + + assert_eq!( + session.state.lock().await.clone_history().raw_items(), + &[ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputText { + text: "image content omitted because it could not be processed".to_string(), + }, + ContentItem::InputText { + text: "keep me".to_string(), + }, + ], + phase: None, + }] + ); +} + #[test] fn resolve_multi_agent_version_handles_unset_and_legacy_history() { let thread_id = ThreadId::default(); @@ -2559,6 +2666,7 @@ async fn record_initial_history_forked_hydrates_previous_turn_settings() { network: None, file_system_sandbox_policy: None, model: previous_model.to_string(), + comp_hash: None, personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), multi_agent_version: None, @@ -2611,6 +2719,7 @@ async fn record_initial_history_forked_hydrates_previous_turn_settings() { session.previous_turn_settings().await, Some(PreviousTurnSettings { model: previous_model.to_string(), + comp_hash: None, realtime_active: Some(turn_context.realtime_active), }) ); @@ -2653,6 +2762,7 @@ async fn thread_rollback_drops_last_turn_from_history() { sess.persist_rollout_items(&rollout_items).await; sess.set_previous_turn_settings(Some(PreviousTurnSettings { model: "stale-model".to_string(), + comp_hash: None, realtime_active: Some(tc.realtime_active), })) .await; @@ -2835,6 +2945,7 @@ async fn thread_rollback_recomputes_previous_turn_settings_and_reference_context .await; sess.set_previous_turn_settings(Some(PreviousTurnSettings { model: "stale-model".to_string(), + comp_hash: None, realtime_active: None, })) .await; @@ -2851,6 +2962,7 @@ async fn thread_rollback_recomputes_previous_turn_settings_and_reference_context sess.previous_turn_settings().await, Some(PreviousTurnSettings { model: tc.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(tc.realtime_active), }) ); @@ -2951,6 +3063,7 @@ async fn thread_rollback_restores_cleared_reference_context_item_after_compactio RolloutItem::TurnContext(TurnContextItem { turn_id: Some(rolled_back_turn_id.clone()), model: "rolled-back-model".to_string(), + comp_hash: None, ..first_context_item.clone() }), RolloutItem::ResponseItem(user_message("turn 2 user")), @@ -3170,7 +3283,7 @@ async fn set_rate_limits_retains_previous_credits() { collaboration_mode, model_reasoning_summary: config.model_reasoning_summary, developer_instructions: config.developer_instructions.clone(), - user_instructions: config.user_instructions.clone(), + loaded_agents_md: None, service_tier: None, personality: config.personality, base_instructions: config @@ -3277,7 +3390,7 @@ async fn set_rate_limits_updates_plan_type_when_present() { collaboration_mode, model_reasoning_summary: config.model_reasoning_summary, developer_instructions: config.developer_instructions.clone(), - user_instructions: config.user_instructions.clone(), + loaded_agents_md: None, service_tier: None, personality: config.personality, base_instructions: config @@ -3809,7 +3922,7 @@ pub(crate) async fn make_session_configuration_for_tests() -> SessionConfigurati collaboration_mode, model_reasoning_summary: config.model_reasoning_summary, developer_instructions: config.developer_instructions.clone(), - user_instructions: config.user_instructions.clone(), + loaded_agents_md: None, service_tier: None, personality: config.personality, base_instructions: config @@ -4661,7 +4774,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_packaged_zsh() { collaboration_mode, model_reasoning_summary: config.model_reasoning_summary, developer_instructions: config.developer_instructions.clone(), - user_instructions: config.user_instructions.clone(), + loaded_agents_md: None, service_tier: None, personality: config.personality, base_instructions: config @@ -4769,7 +4882,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { collaboration_mode, model_reasoning_summary: config.model_reasoning_summary, developer_instructions: config.developer_instructions.clone(), - user_instructions: config.user_instructions.clone(), + loaded_agents_md: None, service_tier: None, personality: config.personality, base_instructions: config @@ -4824,13 +4937,13 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { ); let services = SessionServices { - mcp_connection_manager: arc_swap::ArcSwap::from_pointee( + mcp_connection_manager: Arc::new(arc_swap::ArcSwap::from_pointee( McpConnectionManager::new_uninitialized_with_permission_profile( &config.permissions.approval_policy, config.permissions.permission_profile(), config.prefix_mcp_tool_names(), ), - ), + )), mcp_startup_cancellation_token: Mutex::new(CancellationToken::new()), unified_exec_manager: UnifiedExecProcessManager::new( config.background_terminal_max_timeout, @@ -4880,12 +4993,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { attestation_provider: None, model_client: ModelClient::new( Some(auth_manager.clone()), - thread_id.into(), thread_id, - /*installation_id*/ "11111111-1111-4111-8111-111111111111".to_string(), session_configuration.provider.clone(), session_configuration.session_source.clone(), - session_configuration.parent_thread_id, config.model_verbosity, config.features.enabled(Feature::EnableRequestCompression), config.features.enabled(Feature::RuntimeMetrics), @@ -5001,7 +5111,7 @@ async fn make_session_with_config_and_rx( collaboration_mode, model_reasoning_summary: config.model_reasoning_summary, developer_instructions: config.developer_instructions.clone(), - user_instructions: config.user_instructions.clone(), + loaded_agents_md: None, service_tier: None, personality: config.personality, base_instructions: config @@ -5103,7 +5213,7 @@ async fn make_session_with_history_source_and_agent_control_and_rx( collaboration_mode, model_reasoning_summary: config.model_reasoning_summary, developer_instructions: config.developer_instructions.clone(), - user_instructions: config.user_instructions.clone(), + loaded_agents_md: None, service_tier: None, personality: config.personality, base_instructions: config @@ -6847,7 +6957,7 @@ where collaboration_mode, model_reasoning_summary: config.model_reasoning_summary, developer_instructions: config.developer_instructions.clone(), - user_instructions: config.user_instructions.clone(), + loaded_agents_md: None, service_tier: None, personality: config.personality, base_instructions: config @@ -6902,13 +7012,13 @@ where ); let services = SessionServices { - mcp_connection_manager: arc_swap::ArcSwap::from_pointee( + mcp_connection_manager: Arc::new(arc_swap::ArcSwap::from_pointee( McpConnectionManager::new_uninitialized_with_permission_profile( &config.permissions.approval_policy, config.permissions.permission_profile(), config.prefix_mcp_tool_names(), ), - ), + )), mcp_startup_cancellation_token: Mutex::new(CancellationToken::new()), unified_exec_manager: UnifiedExecProcessManager::new( config.background_terminal_max_timeout, @@ -6958,12 +7068,9 @@ where attestation_provider: None, model_client: ModelClient::new( Some(Arc::clone(&auth_manager)), - thread_id.into(), thread_id, - /*installation_id*/ "11111111-1111-4111-8111-111111111111".to_string(), session_configuration.provider.clone(), session_configuration.session_source.clone(), - session_configuration.parent_thread_id, config.model_verbosity, config.features.enabled(Feature::EnableRequestCompression), config.features.enabled(Feature::RuntimeMetrics), @@ -7347,6 +7454,7 @@ async fn build_settings_update_items_uses_previous_turn_settings_for_realtime_en previous_context_item.realtime_active = None; let previous_turn_settings = PreviousTurnSettings { model: previous_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(true), }; let mut current_context = previous_context @@ -7940,6 +8048,7 @@ async fn build_initial_context_uses_previous_turn_settings_for_realtime_end() { let (session, turn_context) = make_session_and_context().await; let previous_turn_settings = PreviousTurnSettings { model: turn_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(true), }; @@ -7962,6 +8071,7 @@ async fn build_initial_context_restates_realtime_start_when_reference_context_is turn_context.realtime_active = true; let previous_turn_settings = PreviousTurnSettings { model: turn_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(true), }; @@ -7995,6 +8105,18 @@ fn file_system_policy_with_unreadable_glob(turn_context: &TurnContext) -> FileSy policy } +#[tokio::test] +async fn turn_context_item_uses_turn_context_comp_hash_snapshot() { + let (_session, mut turn_context) = make_session_and_context().await; + turn_context.comp_hash = Some("turn-context-hash".to_string()); + turn_context.model_info.comp_hash = Some("model-info-hash".to_string()); + + assert_eq!( + turn_context.to_turn_context_item().comp_hash.as_deref(), + Some("turn-context-hash") + ); +} + #[tokio::test] async fn turn_context_item_omits_legacy_equivalent_file_system_sandbox_policy() { let (_session, turn_context) = make_session_and_context().await; @@ -8186,6 +8308,7 @@ async fn build_initial_context_prepends_model_switch_message() { let (session, turn_context) = make_session_and_context().await; let previous_turn_settings = PreviousTurnSettings { model: "previous-regular-model".to_string(), + comp_hash: None, realtime_active: None, }; @@ -8238,6 +8361,7 @@ async fn record_context_updates_and_set_reference_context_item_persists_full_rei session .set_previous_turn_settings(Some(PreviousTurnSettings { model: previous_context.model_info.slug.clone(), + comp_hash: None, realtime_active: Some(previous_context.realtime_active), })) .await; diff --git a/codex-rs/core/src/session/tests/guardian_tests.rs b/codex-rs/core/src/session/tests/guardian_tests.rs index c73add2920..0d3c9c033b 100644 --- a/codex-rs/core/src/session/tests/guardian_tests.rs +++ b/codex-rs/core/src/session/tests/guardian_tests.rs @@ -705,6 +705,7 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() { let CodexSpawnOk { codex, .. } = Codex::spawn(CodexSpawnArgs { config, + user_instructions: Default::default(), installation_id: "11111111-1111-4111-8111-111111111111".to_string(), auth_manager, models_manager, diff --git a/codex-rs/core/src/session/token_budget.rs b/codex-rs/core/src/session/token_budget.rs new file mode 100644 index 0000000000..b8f9f82fbc --- /dev/null +++ b/codex-rs/core/src/session/token_budget.rs @@ -0,0 +1,44 @@ +use super::session::Session; +use super::turn_context::TurnContext; +use crate::context::ContextualUserFragment; +use codex_features::Feature; + +const TOKEN_BUDGET_USAGE_THRESHOLDS: [i64; 3] = [25, 50, 75]; + +pub(super) async fn maybe_record_token_budget_remaining_context( + sess: &Session, + turn_context: &TurnContext, + tokens_before_sampling: i64, + tokens_after_sampling: i64, +) { + if !turn_context.features.enabled(Feature::TokenBudget) { + return; + } + let Some(model_context_window) = turn_context.model_context_window() else { + return; + }; + if model_context_window <= 0 || tokens_after_sampling <= tokens_before_sampling { + return; + } + + let tokens_before_sampling = tokens_before_sampling.max(0); + let tokens_after_sampling = tokens_after_sampling.max(0); + let crossed_threshold = TOKEN_BUDGET_USAGE_THRESHOLDS.iter().any(|threshold| { + tokens_before_sampling.saturating_mul(100) < model_context_window.saturating_mul(*threshold) + && tokens_after_sampling.saturating_mul(100) + >= model_context_window.saturating_mul(*threshold) + }); + if !crossed_threshold { + return; + } + + let tokens_left = model_context_window + .saturating_sub(tokens_after_sampling) + .max(0); + + let response_item = ContextualUserFragment::into( + crate::context::TokenBudgetRemainingContext::new(tokens_left), + ); + sess.record_conversation_items(turn_context, std::slice::from_ref(&response_item)) + .await; +} diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index f65ada6112..7bb34aec87 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -35,6 +35,8 @@ use crate::mentions::collect_explicit_app_ids; use crate::mentions::collect_explicit_plugin_mentions; use crate::mentions::collect_tool_mentions_from_messages; use crate::plugins::build_plugin_injections; +use crate::responses_metadata::CodexResponsesMetadata; +use crate::responses_metadata::CodexResponsesRequestKind; use crate::responses_retry::ResponsesStreamRequest; use crate::responses_retry::handle_retryable_response_stream_error; use crate::session::PreviousTurnSettings; @@ -172,6 +174,7 @@ pub(crate) async fn run_turn( .await; sess.set_previous_turn_settings(Some(PreviousTurnSettings { model: turn_context.model_info.slug.clone(), + comp_hash: turn_context.comp_hash.clone(), realtime_active: Some(turn_context.realtime_active), })) .await; @@ -222,17 +225,19 @@ pub(crate) async fn run_turn( .await; let window_id = sess.current_window_id().await; - let turn_metadata_header = turn_context - .turn_metadata_state - .current_header_value_for_model_request(&window_id); + let responses_metadata = turn_context.turn_metadata_state.to_responses_metadata( + sess.installation_id.clone(), + window_id, + CodexResponsesRequestKind::Turn, + ); + let tokens_before_sampling = sess.get_total_token_usage().await; match run_sampling_request( Arc::clone(&sess), Arc::clone(&turn_context), Arc::clone(&turn_extension_data), Arc::clone(&turn_diff_tracker), &mut client_session, - &window_id, - turn_metadata_header.as_deref(), + &responses_metadata, sampling_request_input.clone(), cancellation_token.child_token(), ) @@ -275,6 +280,24 @@ pub(crate) async fn run_turn( "post sampling token usage" ); + let tokens_after_sampling = token_status.active_context_tokens; + super::token_budget::maybe_record_token_budget_remaining_context( + sess.as_ref(), + turn_context.as_ref(), + tokens_before_sampling, + tokens_after_sampling, + ) + .await; + + let started_new_context_window = sess + .maybe_start_new_context_window(turn_context.as_ref()) + .await + .is_some(); + if started_new_context_window && needs_follow_up { + can_drain_pending_input = !model_needs_follow_up; + continue; + } + // as long as compaction works well in getting us way below the token limit, we shouldn't worry about being in an infinite loop. if token_limit_reached && needs_follow_up { if let Err(err) = run_auto_compact( @@ -779,8 +802,16 @@ async fn run_pre_sampling_compact( Ok(()) } -/// Runs pre-sampling compaction against the previous model when switching to a smaller -/// context-window model. +/// Returns true only when both turns declare compaction compatibility hashes and they differ. +/// A missing hash does not provide enough information to trigger compaction. +fn comp_hash_changed(previous: Option<&str>, current: Option<&str>) -> bool { + previous + .zip(current) + .is_some_and(|(previous, current)| previous != current) +} + +/// Runs pre-sampling compaction against the previous model when its compaction compatibility +/// hash changed or when switching to a smaller context-window model. /// /// Returns `Err(_)` only when compaction was attempted and failed. async fn maybe_run_previous_model_inline_compact( @@ -791,12 +822,29 @@ async fn maybe_run_previous_model_inline_compact( let Some(previous_turn_settings) = sess.previous_turn_settings().await else { return Ok(()); }; + let should_compact_for_comp_hash_change = comp_hash_changed( + previous_turn_settings.comp_hash.as_deref(), + turn_context.comp_hash.as_deref(), + ); let previous_model_turn_context = Arc::new( turn_context .with_model(previous_turn_settings.model, &sess.services.models_manager) .await, ); + if should_compact_for_comp_hash_change { + run_auto_compact( + sess, + &previous_model_turn_context, + client_session, + InitialContextInjection::DoNotInject, + CompactionReason::CompHashChanged, + CompactionPhase::PreTurn, + ) + .await?; + return Ok(()); + } + let Some(old_context_window) = previous_model_turn_context.model_context_window() else { return Ok(()); }; @@ -984,8 +1032,7 @@ async fn run_sampling_request( turn_store: Arc, turn_diff_tracker: SharedTurnDiffTracker, client_session: &mut ModelClientSession, - window_id: &str, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, input: Vec, cancellation_token: CancellationToken, ) -> CodexResult { @@ -1028,8 +1075,7 @@ async fn run_sampling_request( Arc::clone(&turn_context), Arc::clone(&turn_store), client_session, - window_id, - turn_metadata_header, + responses_metadata, Arc::clone(&turn_diff_tracker), &prompt, cancellation_token.child_token(), @@ -1758,8 +1804,7 @@ async fn try_run_sampling_request( turn_context: Arc, turn_store: Arc, client_session: &mut ModelClientSession, - window_id: &str, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, turn_diff_tracker: SharedTurnDiffTracker, prompt: &Prompt, cancellation_token: CancellationToken, @@ -1780,14 +1825,13 @@ async fn try_run_sampling_request( let sampling_timing_guard = turn_context.turn_timing_state.begin_sampling(); let mut stream = client_session .stream( - window_id, prompt, &turn_context.model_info, &turn_context.session_telemetry, turn_context.reasoning_effort.clone(), turn_context.reasoning_summary, turn_context.config.service_tier.clone(), - turn_metadata_header, + responses_metadata, &inference_trace, ) .instrument(trace_span!("stream_request")) diff --git a/codex-rs/core/src/session/turn_context.rs b/codex-rs/core/src/session/turn_context.rs index 10eff3f3b4..5e5d5b7609 100644 --- a/codex-rs/core/src/session/turn_context.rs +++ b/codex-rs/core/src/session/turn_context.rs @@ -61,6 +61,7 @@ pub struct TurnContext { pub config: Arc, pub(crate) auth_manager: Option>, pub(crate) model_info: ModelInfo, + pub(crate) comp_hash: Option, pub(crate) tool_mode: ToolMode, pub(crate) session_telemetry: SessionTelemetry, pub(crate) provider: SharedModelProvider, @@ -229,6 +230,7 @@ impl TurnContext { config: Arc::new(config), auth_manager: self.auth_manager.clone(), model_info: model_info.clone(), + comp_hash: model_info.comp_hash.clone(), tool_mode, session_telemetry: self .session_telemetry @@ -355,6 +357,7 @@ impl TurnContext { network: self.turn_context_network_item(), file_system_sandbox_policy: self.non_legacy_file_system_sandbox_policy(), model: self.model_info.slug.clone(), + comp_hash: self.comp_hash.clone(), personality: self.personality, collaboration_mode: Some(self.collaboration_mode.clone()), multi_agent_version: Some(self.multi_agent_version), @@ -514,7 +517,6 @@ impl Session { session_configuration.forked_from_thread_id, session_configuration.parent_thread_id, &session_configuration.session_source, - session_configuration.thread_source.clone(), sub_id.clone(), cwd.clone(), &session_configuration.permission_profile(), @@ -531,6 +533,7 @@ impl Session { config: per_turn_config.clone(), auth_manager: auth_manager_for_context, model_info: model_info.clone(), + comp_hash: model_info.comp_hash.clone(), tool_mode, session_telemetry: session_telemetry_for_context, provider: provider_for_context, @@ -548,7 +551,7 @@ impl Session { developer_instructions: session_configuration.developer_instructions.clone(), compact_prompt: session_configuration.compact_prompt.clone(), user_instructions: session_configuration - .user_instructions + .loaded_agents_md .as_ref() .map(LoadedAgentsMd::text), collaboration_mode: session_configuration.collaboration_mode.clone(), diff --git a/codex-rs/core/src/session_startup_prewarm.rs b/codex-rs/core/src/session_startup_prewarm.rs index fda4982545..4fee5e7747 100644 --- a/codex-rs/core/src/session_startup_prewarm.rs +++ b/codex-rs/core/src/session_startup_prewarm.rs @@ -9,6 +9,7 @@ use tracing::info; use tracing::warn; use crate::client::ModelClientSession; +use crate::responses_metadata::CodexResponsesRequestKind; use crate::session::INITIAL_SUBMIT_ID; use crate::session::session::Session; use crate::session::turn::build_prompt; @@ -267,21 +268,24 @@ async fn schedule_startup_prewarm_inner( /*status*/ None, ); let window_id = session.current_window_id().await; - let startup_turn_metadata_header = startup_turn_context + let responses_metadata = startup_turn_context .turn_metadata_state - .current_header_value_for_prewarm(&window_id); + .to_responses_metadata( + session.installation_id.clone(), + window_id, + CodexResponsesRequestKind::Prewarm, + ); let mut client_session = session.services.model_client.new_session(); let websocket_warmup_started_at = Instant::now(); client_session .prewarm_websocket( - &window_id, &startup_prompt, &startup_turn_context.model_info, &startup_turn_context.session_telemetry, startup_turn_context.reasoning_effort.clone(), startup_turn_context.reasoning_summary, startup_turn_context.config.service_tier.clone(), - startup_turn_metadata_header.as_deref(), + &responses_metadata, ) .await?; startup_turn_context.session_telemetry.record_startup_phase( diff --git a/codex-rs/core/src/state/auto_compact_window.rs b/codex-rs/core/src/state/auto_compact_window.rs index f6ebd325f3..1e3eceadad 100644 --- a/codex-rs/core/src/state/auto_compact_window.rs +++ b/codex-rs/core/src/state/auto_compact_window.rs @@ -14,6 +14,7 @@ enum AutoCompactWindowPrefill { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct AutoCompactWindow { window_id: u64, + new_context_window_requested: bool, /// Absolute input-token baseline for the current compaction window. /// /// `body_after_prefix` subtracts this from later active-context usage. It is @@ -26,6 +27,7 @@ impl AutoCompactWindow { pub(super) fn new() -> Self { Self { window_id: 0, + new_context_window_requested: false, prefill_input_tokens: None, } } @@ -44,9 +46,20 @@ impl AutoCompactWindow { pub(super) fn advance_window_id(&mut self) -> u64 { self.window_id = self.window_id.saturating_add(1); + self.new_context_window_requested = false; self.window_id } + pub(super) fn request_new_context_window(&mut self) { + self.new_context_window_requested = true; + } + + pub(super) fn take_new_context_window_request(&mut self) -> bool { + let requested = self.new_context_window_requested; + self.new_context_window_requested = false; + requested + } + /// Records the request-input side of the first server usage sample. The /// sampled output from that response is body growth and should remain /// counted against the scoped auto-compact budget. @@ -98,8 +111,13 @@ mod tests { assert_eq!(window.window_id(), 0); window.set_window_id(/*window_id*/ 3); assert_eq!(window.window_id(), 3); + window.request_new_context_window(); + assert!(window.take_new_context_window_request()); + assert!(!window.take_new_context_window_request()); + window.request_new_context_window(); assert_eq!(window.advance_window_id(), 4); assert_eq!(window.window_id(), 4); + assert!(!window.take_new_context_window_request()); assert_eq!( window.snapshot(), diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index b4c499118d..d8d91e85a8 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -40,7 +40,7 @@ use tokio_util::sync::CancellationToken; pub(crate) struct SessionServices { /// The latest manager; callers retain an owned handle while performing MCP I/O. - pub(crate) mcp_connection_manager: ArcSwap, + pub(crate) mcp_connection_manager: Arc>, pub(crate) mcp_startup_cancellation_token: Mutex, pub(crate) unified_exec_manager: UnifiedExecProcessManager, #[cfg_attr(not(unix), allow(dead_code))] diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index ced1428a31..269d3e0f60 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -156,6 +156,20 @@ impl SessionState { self.auto_compact_window.advance_window_id() } + pub(crate) fn request_new_context_window(&mut self) { + self.auto_compact_window.request_new_context_window(); + } + + pub(crate) fn start_new_context_window_if_requested(&mut self) -> Option { + if !self.auto_compact_window.take_new_context_window_request() { + return None; + } + + let window_id = self.auto_compact_window.advance_window_id(); + self.auto_compact_window.clear_prefill(); + Some(window_id) + } + pub(crate) fn token_info(&self) -> Option { self.history.token_info() } diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 911f30b319..52af0d04db 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -21,6 +21,7 @@ use tracing::info_span; use tracing::trace; use tracing::warn; +use crate::codex_thread::BackgroundTerminalInfo; use crate::config::Config; use crate::context::ContextualUserFragment; use crate::hook_runtime::inspect_pending_input; @@ -782,6 +783,17 @@ impl Session { .await; } + pub(crate) async fn list_background_terminals(&self) -> Vec { + self.services.unified_exec_manager.list_processes().await + } + + pub(crate) async fn terminate_background_terminal(&self, process_id: i32) -> bool { + self.services + .unified_exec_manager + .terminate_process(process_id) + .await + } + async fn handle_task_abort(self: &Arc, task: RunningTask, reason: TurnAbortReason) { let sub_id = task.turn_context.sub_id.clone(); if task.cancellation_token.is_cancelled() { diff --git a/codex-rs/core/src/test_support.rs b/codex-rs/core/src/test_support.rs index e491ca7413..60e80ca290 100644 --- a/codex-rs/core/src/test_support.rs +++ b/codex-rs/core/src/test_support.rs @@ -8,6 +8,9 @@ use std::path::PathBuf; use std::sync::Arc; use codex_exec_server::EnvironmentManager; +use codex_extension_api::LoadUserInstructionsFuture; +use codex_extension_api::LoadedUserInstructions; +use codex_extension_api::UserInstructionsProvider; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_model_provider::create_model_provider; @@ -17,13 +20,19 @@ use codex_models_manager::collaboration_mode_presets; use codex_models_manager::manager::SharedModelsManager; use codex_models_manager::test_support::construct_model_info_offline_for_tests; use codex_models_manager::test_support::get_model_offline_for_tests; +use codex_protocol::ThreadId; use codex_protocol::config_types::CollaborationModeMask; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelPreset; +use codex_protocol::protocol::SessionSource; use once_cell::sync::Lazy; use crate::ThreadManager; use crate::config::Config; +use crate::responses_metadata::CodexResponsesMetadata; +use crate::responses_metadata::CodexResponsesRequestKind; +use crate::responses_metadata::subagent_header_value; +use crate::responses_metadata::subagent_metadata_kind; use crate::thread_manager; use crate::unified_exec; @@ -36,6 +45,16 @@ static TEST_MODEL_PRESETS: Lazy> = Lazy::new(|| { presets }); +/// Test-only provider that supplies no user instructions. +#[derive(Debug, Default)] +pub struct EmptyUserInstructionsProvider; + +impl UserInstructionsProvider for EmptyUserInstructionsProvider { + fn load_user_instructions(&self) -> LoadUserInstructionsFuture<'_> { + Box::pin(async { LoadedUserInstructions::default() }) + } +} + pub fn set_thread_manager_test_mode(enabled: bool) { thread_manager::set_thread_manager_test_mode_for_tests(enabled); } @@ -133,6 +152,44 @@ pub fn construct_model_info_offline(model: &str, config: &Config) -> ModelInfo { construct_model_info_offline_for_tests(model, &config.to_models_manager_config()) } +#[derive(Clone, Copy)] +pub enum TestCodexResponsesRequestKind { + Turn, + Prewarm, + WebsocketConnection, +} + +#[allow(clippy::too_many_arguments)] +pub fn responses_metadata( + installation_id: &str, + session_id: &str, + thread_id: &str, + turn_id: Option<&str>, + window_id: String, + session_source: &SessionSource, + parent_thread_id: Option, + request_kind: TestCodexResponsesRequestKind, +) -> CodexResponsesMetadata { + let request_kind = match request_kind { + TestCodexResponsesRequestKind::Turn => Some(CodexResponsesRequestKind::Turn), + TestCodexResponsesRequestKind::Prewarm => Some(CodexResponsesRequestKind::Prewarm), + TestCodexResponsesRequestKind::WebsocketConnection => None, + }; + CodexResponsesMetadata { + turn_id: request_kind.and(turn_id.map(ToString::to_string)), + request_kind, + parent_thread_id, + subagent_header: subagent_header_value(session_source), + subagent_kind: request_kind.and_then(|_| subagent_metadata_kind(session_source)), + ..CodexResponsesMetadata::new( + installation_id.to_string(), + session_id.to_string(), + thread_id.to_string(), + window_id, + ) + } +} + pub fn all_model_presets() -> &'static Vec { &TEST_MODEL_PRESETS } diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index d5f54d555e..b82e63947e 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -23,6 +23,8 @@ use codex_core_plugins::PluginsManager; use codex_exec_server::EnvironmentManager; use codex_extension_api::ExtensionDataInit; use codex_extension_api::ExtensionRegistry; +use codex_extension_api::LoadedUserInstructions; +use codex_extension_api::UserInstructionsProvider; use codex_extension_api::empty_extension_registry; use codex_features::Feature; use codex_login::AuthManager; @@ -209,6 +211,7 @@ pub(crate) struct ThreadManagerState { plugins_manager: Arc, mcp_manager: Arc, extensions: Arc>, + user_instructions_provider: Arc, thread_store: Arc, attestation_provider: Option>, session_source: SessionSource, @@ -259,6 +262,7 @@ impl ThreadManager { session_source: SessionSource, environment_manager: Arc, extensions: Arc>, + user_instructions_provider: Arc, analytics_events_client: Option, thread_store: Arc, state_db: Option, @@ -272,6 +276,7 @@ impl ThreadManager { codex_home.to_path_buf(), restriction_product, )); + plugins_manager.set_auth_mode(auth_manager.get_api_auth_mode()); let mcp_manager = Arc::new(McpManager::new_with_extensions( Arc::clone(&plugins_manager), Arc::clone(&extensions), @@ -291,6 +296,7 @@ impl ThreadManager { plugins_manager, mcp_manager, extensions, + user_instructions_provider, thread_store, attestation_provider, auth_manager, @@ -365,6 +371,7 @@ impl ThreadManager { codex_home.clone(), restriction_product, )); + plugins_manager.set_auth_mode(auth_manager.get_api_auth_mode()); let mcp_manager = Arc::new(McpManager::new(Arc::clone(&plugins_manager))); let skills_manager = Arc::new(SkillsManager::new_with_restriction_product( skills_codex_home, @@ -392,6 +399,9 @@ impl ThreadManager { plugins_manager, mcp_manager, extensions: empty_extension_registry(), + user_instructions_provider: Arc::new( + crate::test_support::EmptyUserInstructionsProvider, + ), thread_store, attestation_provider: None, auth_manager, @@ -515,14 +525,12 @@ impl ThreadManager { &self, thread_id: ThreadId, ) -> CodexResult> { - let thread = self.state.get_thread(thread_id).await?; - let mut subtree_thread_ids = Vec::new(); let mut seen_thread_ids = HashSet::new(); subtree_thread_ids.push(thread_id); seen_thread_ids.insert(thread_id); - if let Some(state_db_ctx) = thread.state_db() { + if let Some(state_db_ctx) = self.state.state_db() { for status in [ DirectionalThreadSpawnEdgeStatus::Open, DirectionalThreadSpawnEdgeStatus::Closed, @@ -541,11 +549,8 @@ impl ThreadManager { } } - for descendant_id in thread - .codex - .session - .services - .agent_control + for descendant_id in self + .agent_control() .list_live_agent_subtree_thread_ids(thread_id) .await? { @@ -1094,6 +1099,56 @@ impl ThreadManagerState { resolve_multi_agent_version(initial_history, inherited_multi_agent_version) } + /// Resolves the provider snapshot for a newly spawned runtime. + /// + /// Loads a fresh provider snapshot for: + /// - fresh root threads; + /// - cold resumes; + /// - root forks. + /// + /// Uses an existing snapshot for: + /// - subagents, which inherit from their parent without invoking the + /// provider; + /// - running resumes and compaction paths, which retain the live session. + /// + /// Provider warnings only apply to fresh loads. If a parent runtime is no + /// longer available, its child starts without provider instructions rather + /// than loading independently. + async fn user_instructions_for_spawn( + &self, + session_source: &SessionSource, + parent_thread_id: Option, + forked_from_thread_id: Option, + ) -> LoadedUserInstructions { + let is_root_agent = !session_source.is_non_root_agent(); + if is_root_agent { + return self + .user_instructions_provider + .load_user_instructions() + .await; + } + + let inherited_thread_id = match session_source { + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, .. + }) => Some(*parent_thread_id), + _ => parent_thread_id.or(forked_from_thread_id), + }; + let instructions = match inherited_thread_id { + // The spawn path retains only thread IDs, so look up the live + // runtime again here to inherit its user instructions. + Some(thread_id) => match self.get_thread(thread_id).await { + Ok(thread) => thread.codex.session.user_instructions().await, + Err(_) => None, + }, + None => None, + }; + LoadedUserInstructions { + instructions, + warnings: Vec::new(), + } + } + /// Spawn a new thread with no history using a provided config. pub(crate) async fn spawn_new_thread( &self, @@ -1311,6 +1366,9 @@ impl ThreadManagerState { } let environment_selections = resolve_environment_selections(self.environment_manager.as_ref(), &environments)?; + let user_instructions = self + .user_instructions_for_spawn(&session_source, parent_thread_id, forked_from_thread_id) + .await; let parent_rollout_thread_trace = self .parent_rollout_thread_trace_for_source(&session_source, &initial_history) .await; @@ -1327,6 +1385,7 @@ impl ThreadManagerState { codex, thread_id, .. } = Box::pin(Codex::spawn(CodexSpawnArgs { config, + user_instructions, installation_id: self.installation_id.clone(), auth_manager, models_manager: Arc::clone(&self.models_manager), diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 48ca2598ab..d7afebfb33 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -432,6 +432,7 @@ async fn start_thread_seeds_extension_data_before_lifecycle_contributors_run() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), Arc::new(extensions.build()), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, /*state_db*/ None), /*state_db*/ None, @@ -481,6 +482,7 @@ async fn resume_and_fork_do_not_restore_thread_environments_from_rollout() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, /*state_db*/ None), /*state_db*/ None, @@ -598,6 +600,7 @@ async fn explicit_installation_id_skips_codex_home_file() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store, state_db.clone(), @@ -637,6 +640,7 @@ async fn resume_active_thread_from_rollout_returns_running_thread() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, /*state_db*/ None), /*state_db*/ None, @@ -694,6 +698,7 @@ async fn resume_stopped_thread_from_rollout_spawns_new_thread() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, /*state_db*/ None), /*state_db*/ None, @@ -758,6 +763,7 @@ async fn resume_stopped_thread_from_rollout_preserves_thread_source() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store, state_db.clone(), @@ -848,6 +854,7 @@ async fn rollout_path_resume_and_fork_read_history_through_thread_store() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store.clone(), state_db, @@ -949,6 +956,7 @@ async fn new_uses_active_provider_for_model_refresh() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, /*state_db*/ None), /*state_db*/ None, @@ -1169,6 +1177,7 @@ async fn interrupted_fork_snapshot_does_not_synthesize_turn_id_for_legacy_histor SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, state_db.clone()), state_db.clone(), @@ -1275,6 +1284,7 @@ async fn interrupted_fork_snapshot_preserves_explicit_turn_id() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, state_db.clone()), state_db.clone(), @@ -1371,6 +1381,7 @@ async fn interrupted_fork_snapshot_uses_persisted_mid_turn_history_without_live_ SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, state_db.clone()), state_db.clone(), diff --git a/codex-rs/core/src/tools/events.rs b/codex-rs/core/src/tools/events.rs index e286af53d4..2751780d58 100644 --- a/codex-rs/core/src/tools/events.rs +++ b/codex-rs/core/src/tools/events.rs @@ -588,7 +588,7 @@ async fn emit_patch_end( if let Some(tracker) = ctx.turn_diff_tracker { let (should_emit_turn_diff, unified_diff) = { let mut guard = tracker.lock().await; - let previous_diff = guard.get_unified_diff(); + let had_unified_diff = guard.has_unified_diff(); let tracker_changed = match tracker_update { TurnDiffTrackerUpdate::Track { environment_id, @@ -605,7 +605,7 @@ async fn emit_patch_end( }; let unified_diff = guard.get_unified_diff(); ( - tracker_changed && (previous_diff.is_some() || unified_diff.is_some()), + tracker_changed && (had_unified_diff || unified_diff.is_some()), unified_diff.unwrap_or_default(), ) }; @@ -718,4 +718,98 @@ mod tests { ) .await; } + + #[tokio::test] + async fn net_zero_patch_emits_empty_turn_diff() { + let (session, turn, rx_event) = + make_session_and_context_with_dynamic_tools_and_rx(Vec::new()).await; + let tracker = Arc::new(Mutex::new(TurnDiffTracker::new())); + let dir = tempdir().expect("tempdir"); + let cwd = AbsolutePathBuf::from_absolute_path(dir.path()).expect("absolute cwd"); + + for patch in [ + "*** Begin Patch\n*** Add File: a.txt\n+one\n*** End Patch", + "*** Begin Patch\n*** Delete File: a.txt\n*** End Patch", + ] { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let delta = codex_apply_patch::apply_patch( + patch, + &cwd, + &mut stdout, + &mut stderr, + LOCAL_FS.as_ref(), + /*sandbox*/ None, + ) + .await + .expect("apply patch"); + + emit_patch_end( + ToolEventCtx::new(session.as_ref(), turn.as_ref(), "call-id", Some(&tracker)), + HashMap::new(), + String::new(), + String::new(), + PatchApplyStatus::Completed, + TurnDiffTrackerUpdate::Track { + environment_id: None, + delta: &delta, + }, + ) + .await; + + rx_event.recv().await.expect("item completed event"); + let unified_diff = loop { + let event = rx_event.recv().await.expect("turn diff event"); + if let EventMsg::TurnDiff(TurnDiffEvent { unified_diff }) = event.msg { + break unified_diff; + } + }; + if patch.contains("Delete File") { + assert_eq!(unified_diff, ""); + } else { + assert!(unified_diff.contains("+one")); + } + } + } + + #[tokio::test] + async fn invalidation_emits_empty_turn_diff() { + let (session, turn, rx_event) = + make_session_and_context_with_dynamic_tools_and_rx(Vec::new()).await; + let tracker = Arc::new(Mutex::new(TurnDiffTracker::new())); + let dir = tempdir().expect("tempdir"); + let cwd = AbsolutePathBuf::from_absolute_path(dir.path()).expect("absolute cwd"); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let delta = codex_apply_patch::apply_patch( + "*** Begin Patch\n*** Add File: a.txt\n+one\n*** End Patch", + &cwd, + &mut stdout, + &mut stderr, + LOCAL_FS.as_ref(), + /*sandbox*/ None, + ) + .await + .expect("apply patch"); + tracker.lock().await.track_delta("", &delta); + + emit_patch_end( + ToolEventCtx::new(session.as_ref(), turn.as_ref(), "call-id", Some(&tracker)), + HashMap::new(), + String::new(), + String::new(), + PatchApplyStatus::Completed, + TurnDiffTrackerUpdate::Invalidate, + ) + .await; + + rx_event.recv().await.expect("item completed event"); + loop { + let event = rx_event.recv().await.expect("turn diff event"); + if let EventMsg::TurnDiff(TurnDiffEvent { unified_diff }) = event.msg { + assert_eq!(unified_diff, ""); + break; + } + } + } } diff --git a/codex-rs/core/src/tools/handlers/dynamic.rs b/codex-rs/core/src/tools/handlers/dynamic.rs index dabcea8e5a..62eef88c30 100644 --- a/codex-rs/core/src/tools/handlers/dynamic.rs +++ b/codex-rs/core/src/tools/handlers/dynamic.rs @@ -33,7 +33,6 @@ pub struct DynamicToolHandler { tool_name: ToolName, spec: ToolSpec, exposure: ToolExposure, - search_text: String, } impl DynamicToolHandler { @@ -56,7 +55,6 @@ impl DynamicToolHandler { } else { ToolExposure::Direct }, - search_text: build_dynamic_search_text(tool), }) } } @@ -75,8 +73,7 @@ impl ToolExecutor for DynamicToolHandler { } fn search_info(&self) -> Option { - ToolSearchInfo::from_spec( - self.search_text.clone(), + ToolSearchInfo::from_tool_spec( self.spec(), Some(ToolSearchSourceInfo { name: "Dynamic tools".to_string(), @@ -217,27 +214,3 @@ async fn request_dynamic_tool( response } - -fn build_dynamic_search_text(tool: &DynamicToolSpec) -> String { - let mut schema_properties = tool - .input_schema - .get("properties") - .and_then(serde_json::Value::as_object) - .map(|map| map.keys().cloned().collect::>()) - .unwrap_or_default(); - schema_properties.sort(); - let mut parts = vec![ - tool.name.clone(), - tool.name.replace('_', " "), - tool.description.clone(), - ]; - if let Some(namespace) = &tool.namespace { - parts.push(namespace.clone()); - } - parts.extend(schema_properties); - parts.join(" ") -} - -#[cfg(test)] -#[path = "dynamic_tests.rs"] -mod tests; diff --git a/codex-rs/core/src/tools/handlers/dynamic_tests.rs b/codex-rs/core/src/tools/handlers/dynamic_tests.rs deleted file mode 100644 index 201ef45855..0000000000 --- a/codex-rs/core/src/tools/handlers/dynamic_tests.rs +++ /dev/null @@ -1,36 +0,0 @@ -use super::*; -use codex_tools::ToolSearchSourceInfo; -use pretty_assertions::assert_eq; -use serde_json::json; - -#[test] -fn search_info_uses_dynamic_tool_metadata_and_parameter_names() { - let handler = DynamicToolHandler::new(&DynamicToolSpec { - namespace: Some("codex_app".to_string()), - name: "automation_update".to_string(), - description: "Create or update automations.".to_string(), - input_schema: json!({ - "type": "object", - "properties": { - "timezone": { "type": "string" }, - "mode": { "type": "string" } - } - }), - defer_loading: true, - }) - .expect("dynamic handler should be created"); - - let search_info = handler.search_info().expect("dynamic search info"); - - assert_eq!( - search_info.entry.search_text, - "automation_update automation update Create or update automations. codex_app mode timezone" - ); - assert_eq!( - search_info.source_info, - Some(ToolSearchSourceInfo { - name: "Dynamic tools".to_string(), - description: Some("Tools provided by the current Codex thread.".to_string()), - }) - ); -} diff --git a/codex-rs/core/src/tools/handlers/get_context_remaining.rs b/codex-rs/core/src/tools/handlers/get_context_remaining.rs new file mode 100644 index 0000000000..a49ab65394 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/get_context_remaining.rs @@ -0,0 +1,54 @@ +use crate::context::ContextualUserFragment; +use crate::function_tool::FunctionCallError; +use crate::tools::context::FunctionToolOutput; +use crate::tools::context::ToolInvocation; +use crate::tools::context::ToolPayload; +use crate::tools::context::boxed_tool_output; +use crate::tools::handlers::get_context_remaining_spec::GET_CONTEXT_REMAINING_TOOL_NAME; +use crate::tools::handlers::get_context_remaining_spec::create_get_context_remaining_tool; +use crate::tools::registry::CoreToolRuntime; +use crate::tools::registry::ToolExecutor; +use codex_tools::ToolName; +use codex_tools::ToolSpec; + +pub struct GetContextRemainingHandler; + +impl ToolExecutor for GetContextRemainingHandler { + fn tool_name(&self) -> ToolName { + ToolName::plain(GET_CONTEXT_REMAINING_TOOL_NAME) + } + + fn spec(&self) -> ToolSpec { + create_get_context_remaining_tool() + } + + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { + if !matches!(invocation.payload, ToolPayload::Function { .. }) { + return Err(FunctionCallError::RespondToModel( + "get_context_remaining handler received unsupported payload".to_string(), + )); + } + + let Some(model_context_window) = invocation.turn.model_context_window() else { + let fragment = crate::context::TokenBudgetRemainingContext::unknown().render(); + return Ok(boxed_tool_output(FunctionToolOutput::from_text( + fragment, + Some(true), + ))); + }; + let active_context_tokens = invocation.session.get_total_token_usage().await.max(0); + let tokens_left = model_context_window + .saturating_sub(active_context_tokens) + .max(0); + let fragment = crate::context::TokenBudgetRemainingContext::new(tokens_left).render(); + + Ok(boxed_tool_output(FunctionToolOutput::from_text( + fragment, + Some(true), + ))) + }) + } +} + +impl CoreToolRuntime for GetContextRemainingHandler {} diff --git a/codex-rs/core/src/tools/handlers/get_context_remaining_spec.rs b/codex-rs/core/src/tools/handlers/get_context_remaining_spec.rs new file mode 100644 index 0000000000..988f2492ff --- /dev/null +++ b/codex-rs/core/src/tools/handlers/get_context_remaining_spec.rs @@ -0,0 +1,17 @@ +use codex_tools::JsonSchema; +use codex_tools::ResponsesApiTool; +use codex_tools::ToolSpec; +use std::collections::BTreeMap; + +pub(crate) const GET_CONTEXT_REMAINING_TOOL_NAME: &str = "get_context_remaining"; + +pub fn create_get_context_remaining_tool() -> ToolSpec { + ToolSpec::Function(ResponsesApiTool { + name: GET_CONTEXT_REMAINING_TOOL_NAME.to_string(), + description: "Get the remaining tokens in the current context window.".to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::object(BTreeMap::new(), /*required*/ None, Some(false.into())), + output_schema: None, + }) +} diff --git a/codex-rs/core/src/tools/handlers/mod.rs b/codex-rs/core/src/tools/handlers/mod.rs index fdfb809965..258c9914dd 100644 --- a/codex-rs/core/src/tools/handlers/mod.rs +++ b/codex-rs/core/src/tools/handlers/mod.rs @@ -4,6 +4,8 @@ pub(crate) mod apply_patch; pub(crate) mod apply_patch_spec; mod dynamic; pub(crate) mod extension_tools; +mod get_context_remaining; +pub(crate) mod get_context_remaining_spec; mod list_available_plugins_to_install; pub(crate) mod list_available_plugins_to_install_spec; mod mcp; @@ -13,6 +15,8 @@ pub(crate) mod multi_agents; pub(crate) mod multi_agents_common; pub(crate) mod multi_agents_spec; pub(crate) mod multi_agents_v2; +mod new_context_window; +pub(crate) mod new_context_window_spec; mod plan; pub(crate) mod plan_spec; mod request_permissions; @@ -51,11 +55,13 @@ pub use apply_patch::ApplyPatchHandler; use codex_protocol::models::AdditionalPermissionProfile; use codex_protocol::protocol::AskForApproval; pub use dynamic::DynamicToolHandler; +pub use get_context_remaining::GetContextRemainingHandler; pub use list_available_plugins_to_install::ListAvailablePluginsToInstallHandler; pub use mcp::McpHandler; pub use mcp_resource::ListMcpResourceTemplatesHandler; pub use mcp_resource::ListMcpResourcesHandler; pub use mcp_resource::ReadMcpResourceHandler; +pub use new_context_window::NewContextWindowHandler; pub use plan::PlanHandler; pub use request_permissions::RequestPermissionsHandler; pub use request_plugin_install::RequestPluginInstallHandler; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index 8f0d9d56b9..5fa2cea70a 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -26,7 +26,6 @@ pub struct SpawnAgentToolOptions { pub hide_agent_type_model_reasoning: bool, pub include_usage_hint: bool, pub usage_hint_text: Option, - pub max_concurrent_threads_per_session: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -102,7 +101,6 @@ pub fn create_spawn_agent_tool_v2(options: SpawnAgentToolOptions) -> ToolSpec { inherited_model_guidance, options.include_usage_hint, options.usage_hint_text, - options.max_concurrent_threads_per_session, ), strict: false, defer_loading: None, @@ -722,17 +720,9 @@ fn spawn_agent_tool_description_v2( inherited_model_guidance: Option<&str>, include_usage_hint: bool, usage_hint_text: Option, - max_concurrent_threads_per_session: Option, ) -> String { let agent_role_guidance = available_models_description.unwrap_or_default(); let inherited_model_guidance = inherited_model_guidance.unwrap_or_default(); - let concurrency_guidance = max_concurrent_threads_per_session - .map(|limit| { - format!( - "This session is configured with `max_concurrent_threads_per_session = {limit}` for concurrently open agent threads." - ) - }) - .unwrap_or_default(); let tool_description = format!( r#" @@ -743,8 +733,7 @@ The spawned agent will have the same tools as you and the ability to spawn its o {inherited_model_guidance} Only call this tool for a concrete, bounded subtask that can run independently alongside useful local work; otherwise continue locally. It will be able to send you and other running agents messages, and its final answer will be provided to you when it finishes. -The new agent's canonical task name will be provided to it along with the message. -{concurrency_guidance}"# +The new agent's canonical task name will be provided to it along with the message."# ); if !include_usage_hint { diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs index 512e318a82..1ef93b77f0 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec_tests.rs @@ -47,7 +47,6 @@ fn spawn_agent_tool_v2_requires_task_name_and_lists_visible_models() { hide_agent_type_model_reasoning: false, include_usage_hint: true, usage_hint_text: None, - max_concurrent_threads_per_session: Some(4), }); let ToolSpec::Function(ResponsesApiTool { @@ -69,7 +68,7 @@ fn spawn_agent_tool_v2_requires_task_name_and_lists_visible_models() { .expect("spawn_agent should use object params"); assert!(description.contains("Spawns an agent to work on the specified task.")); assert!(description.contains("The spawned agent will have the same tools as you")); - assert!(description.contains("`max_concurrent_threads_per_session = 4`")); + assert!(!description.contains("max_concurrent_threads_per_session")); assert!(description.contains(SPAWN_AGENT_INHERITED_MODEL_GUIDANCE)); assert!( description @@ -124,7 +123,6 @@ fn spawn_agent_tool_v1_keeps_legacy_fork_context_field() { hide_agent_type_model_reasoning: false, include_usage_hint: true, usage_hint_text: None, - max_concurrent_threads_per_session: None, }); let ToolSpec::Namespace(namespace) = tool else { @@ -182,7 +180,6 @@ fn spawn_agent_tool_caps_visible_model_summaries() { hide_agent_type_model_reasoning: false, include_usage_hint: true, usage_hint_text: None, - max_concurrent_threads_per_session: Some(4), }); let ToolSpec::Function(ResponsesApiTool { description, .. }) = tool else { @@ -227,7 +224,6 @@ fn spawn_agent_tool_hides_service_tier_with_spawn_metadata() { hide_agent_type_model_reasoning: true, include_usage_hint: true, usage_hint_text: None, - max_concurrent_threads_per_session: Some(4), }); let ToolSpec::Function(ResponsesApiTool { diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index f55d9c0876..88a8f07dcb 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -1,5 +1,4 @@ use super::*; -use crate::LoadedAgentsMd; use crate::ThreadManager; use crate::config::AgentRoleConfig; use crate::config::DEFAULT_AGENT_MAX_DEPTH; @@ -4241,6 +4240,7 @@ async fn tool_handlers_cascade_close_and_resume_and_keep_explicitly_closed_subtr SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(crate::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, state_db.clone()), state_db.clone(), @@ -4520,25 +4520,6 @@ async fn build_agent_spawn_config_uses_turn_context_values() { assert_eq!(config, expected); } -#[tokio::test] -async fn build_agent_spawn_config_preserves_base_user_instructions() { - let (_session, mut turn) = make_session_and_context().await; - let mut base_config = (*turn.config).clone(); - base_config.user_instructions = Some(LoadedAgentsMd::new_user( - "base-user".to_string(), - base_config.codex_home.join("AGENTS.md"), - )); - turn.user_instructions = Some("resolved-user".to_string()); - turn.config = Arc::new(base_config.clone()); - let base_instructions = BaseInstructions { - text: "base".to_string(), - }; - - let config = build_agent_spawn_config(&base_instructions, &turn).expect("spawn config"); - - assert_eq!(config.user_instructions, base_config.user_instructions); -} - #[tokio::test] async fn build_agent_resume_config_clears_base_instructions() { let (_session, mut turn) = make_session_and_context().await; diff --git a/codex-rs/core/src/tools/handlers/new_context_window.rs b/codex-rs/core/src/tools/handlers/new_context_window.rs new file mode 100644 index 0000000000..e1cd656399 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/new_context_window.rs @@ -0,0 +1,45 @@ +use crate::function_tool::FunctionCallError; +use crate::tools::context::FunctionToolOutput; +use crate::tools::context::ToolInvocation; +use crate::tools::context::ToolPayload; +use crate::tools::context::boxed_tool_output; +use crate::tools::handlers::new_context_window_spec::NEW_CONTEXT_WINDOW_TOOL_NAME; +use crate::tools::handlers::new_context_window_spec::create_new_context_window_tool; +use crate::tools::registry::CoreToolRuntime; +use crate::tools::registry::ToolExecutor; +use codex_tools::ToolName; +use codex_tools::ToolSpec; + +pub(crate) const NEW_CONTEXT_WINDOW_MESSAGE: &str = + "A new context window will start without summarizing conversation history."; + +pub struct NewContextWindowHandler; + +impl ToolExecutor for NewContextWindowHandler { + fn tool_name(&self) -> ToolName { + ToolName::plain(NEW_CONTEXT_WINDOW_TOOL_NAME) + } + + fn spec(&self) -> ToolSpec { + create_new_context_window_tool() + } + + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { + if !matches!(invocation.payload, ToolPayload::Function { .. }) { + return Err(FunctionCallError::RespondToModel( + "new_context handler received unsupported payload".to_string(), + )); + } + + invocation.session.request_new_context_window().await; + + Ok(boxed_tool_output(FunctionToolOutput::from_text( + NEW_CONTEXT_WINDOW_MESSAGE.to_string(), + Some(true), + ))) + }) + } +} + +impl CoreToolRuntime for NewContextWindowHandler {} diff --git a/codex-rs/core/src/tools/handlers/new_context_window_spec.rs b/codex-rs/core/src/tools/handlers/new_context_window_spec.rs new file mode 100644 index 0000000000..f58b66e589 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/new_context_window_spec.rs @@ -0,0 +1,17 @@ +use codex_tools::JsonSchema; +use codex_tools::ResponsesApiTool; +use codex_tools::ToolSpec; +use std::collections::BTreeMap; + +pub(crate) const NEW_CONTEXT_WINDOW_TOOL_NAME: &str = "new_context"; + +pub fn create_new_context_window_tool() -> ToolSpec { + ToolSpec::Function(ResponsesApiTool { + name: NEW_CONTEXT_WINDOW_TOOL_NAME.to_string(), + description: "Start a new context window.".to_string(), + strict: false, + defer_loading: None, + parameters: JsonSchema::object(BTreeMap::new(), /*required*/ None, Some(false.into())), + output_schema: None, + }) +} diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install.rs b/codex-rs/core/src/tools/handlers/request_plugin_install.rs index 801031348f..55906185ab 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install.rs @@ -359,7 +359,7 @@ fn verified_plugin_install_completed( ) -> bool { let plugins_input = config.plugins_config_input(); plugins_manager - .list_marketplaces_for_config(&plugins_input, &[]) + .list_marketplaces_for_config(&plugins_input, &[], /*include_openai_curated*/ true) .ok() .into_iter() .flat_map(|outcome| outcome.marketplaces) diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install_tests.rs b/codex-rs/core/src/tools/handlers/request_plugin_install_tests.rs index 830e9d408e..cacc25ff95 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install_tests.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install_tests.rs @@ -130,6 +130,7 @@ async fn persist_disabled_install_request_writes_plugin_config() { let codex_home = tempdir().expect("tempdir should succeed"); let tool = DiscoverableTool::Plugin(Box::new(DiscoverablePluginInfo { id: "slack@openai-curated".to_string(), + remote_plugin_id: None, name: "Slack".to_string(), description: None, has_skills: true, diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs index 0432ad338f..17708564bb 100644 --- a/codex-rs/core/src/tools/handlers/view_image.rs +++ b/codex-rs/core/src/tools/handlers/view_image.rs @@ -1,3 +1,4 @@ +use codex_features::Feature; use codex_protocol::items::ImageViewItem; use codex_protocol::items::TurnItem; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; @@ -8,6 +9,7 @@ use codex_protocol::models::ImageDetail; use codex_protocol::models::ResponseInputItem; use codex_protocol::openai_models::InputModality; use codex_utils_image::PromptImageMode; +use codex_utils_image::data_url_from_bytes; use codex_utils_image::load_for_prompt_bytes; use serde::Deserialize; @@ -25,6 +27,7 @@ use crate::tools::registry::CoreToolRuntime; use crate::tools::registry::ToolExecutor; use codex_tools::ToolName; use codex_tools::ToolSpec; +use codex_utils_path_uri::PathUri; pub struct ViewImageHandler { options: ViewImageToolOptions, @@ -144,9 +147,15 @@ impl ViewImageHandler { let abs_path = cwd.join(path); let sandbox = turn.file_system_sandbox_context(/*additional_permissions*/ None, &cwd); let fs = turn_environment.environment.get_filesystem(); + let path_uri = PathUri::from_abs_path(&abs_path).map_err(|error| { + FunctionCallError::RespondToModel(format!( + "unable to locate image at `{}`: {error}", + abs_path.display() + )) + })?; let metadata = fs - .get_metadata(&abs_path, Some(&sandbox)) + .get_metadata(&path_uri, Some(&sandbox)) .await .map_err(|error| { FunctionCallError::RespondToModel(format!( @@ -162,7 +171,7 @@ impl ViewImageHandler { ))); } let file_bytes = fs - .read_file(&abs_path, Some(&sandbox)) + .read_file(&path_uri, Some(&sandbox)) .await .map_err(|error| { FunctionCallError::RespondToModel(format!( @@ -175,25 +184,30 @@ impl ViewImageHandler { let can_request_original_detail = can_request_original_image_detail(&turn.model_info); let use_original_detail = can_request_original_detail && matches!(detail, Some(ViewImageDetail::Original)); - let image_mode = if use_original_detail { - PromptImageMode::Original - } else { - PromptImageMode::ResizeToFit - }; let image_detail = if use_original_detail { ImageDetail::Original } else { DEFAULT_IMAGE_DETAIL }; - let image = - load_for_prompt_bytes(abs_path.as_path(), file_bytes, image_mode).map_err(|error| { - FunctionCallError::RespondToModel(format!( - "unable to process image at `{}`: {error}", - abs_path.display() - )) - })?; - let image_url = image.into_data_url(); + let image_url = if turn.features.enabled(Feature::ResizeAllImages) { + // The history insertion path owns image decoding and resizing when this is enabled. + data_url_from_bytes("application/octet-stream", &file_bytes) + } else { + let image_mode = if use_original_detail { + PromptImageMode::Original + } else { + PromptImageMode::ResizeToFit + }; + load_for_prompt_bytes(abs_path.as_path(), file_bytes, image_mode) + .map_err(|error| { + FunctionCallError::RespondToModel(format!( + "unable to process image at `{}`: {error}", + abs_path.display() + )) + })? + .into_data_url() + }; let item = TurnItem::ImageView(ImageViewItem { id: call_id, @@ -218,7 +232,7 @@ pub struct ViewImageOutput { impl ToolOutput for ViewImageOutput { fn log_preview(&self) -> String { - self.image_url.clone() + format!("", self.image_url.len()) } fn success_for_logging(&self) -> bool { @@ -264,6 +278,16 @@ mod tests { use std::sync::Arc; use tokio::sync::Mutex; + #[test] + fn log_preview_omits_image_data() { + let output = ViewImageOutput { + image_url: "data:image/png;base64,AAA".to_string(), + image_detail: DEFAULT_IMAGE_DETAIL, + }; + + assert_eq!(output.log_preview(), ""); + } + #[test] fn code_mode_result_returns_image_url_object() { let output = ViewImageOutput { diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index 697eba1d4a..51a0da7ad8 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -9,10 +9,12 @@ use crate::tools::handlers::CodeModeWaitHandler; use crate::tools::handlers::DynamicToolHandler; use crate::tools::handlers::ExecCommandHandler; use crate::tools::handlers::ExecCommandHandlerOptions; +use crate::tools::handlers::GetContextRemainingHandler; use crate::tools::handlers::ListAvailablePluginsToInstallHandler; use crate::tools::handlers::ListMcpResourceTemplatesHandler; use crate::tools::handlers::ListMcpResourcesHandler; use crate::tools::handlers::McpHandler; +use crate::tools::handlers::NewContextWindowHandler; use crate::tools::handlers::PlanHandler; use crate::tools::handlers::ReadMcpResourceHandler; use crate::tools::handlers::RequestPermissionsHandler; @@ -386,15 +388,6 @@ fn wait_agent_timeout_options(turn_context: &TurnContext) -> WaitAgentTimeoutOpt } } -fn max_concurrent_threads_per_session(turn_context: &TurnContext) -> Option { - multi_agent_v2_enabled(turn_context).then_some( - turn_context - .config - .multi_agent_v2 - .max_concurrent_threads_per_session, - ) -} - fn agent_type_description( turn_context: &TurnContext, default_agent_type_description: &str, @@ -659,6 +652,11 @@ fn add_core_utility_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mut planned_tools.add(RequestPermissionsHandler); } + if features.enabled(Feature::TokenBudget) { + planned_tools.add_with_exposure(NewContextWindowHandler, ToolExposure::DirectModelOnly); + planned_tools.add_with_exposure(GetContextRemainingHandler, ToolExposure::DirectModelOnly); + } + if tool_suggest_enabled(turn_context) && let Some(discoverable_tools) = context.discoverable_tools.filter(|tools| !tools.is_empty()) @@ -722,9 +720,6 @@ fn add_collaboration_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mu .hide_spawn_agent_metadata, include_usage_hint: turn_context.config.multi_agent_v2.usage_hint_enabled, usage_hint_text: turn_context.config.multi_agent_v2.usage_hint_text.clone(), - max_concurrent_threads_per_session: max_concurrent_threads_per_session( - turn_context, - ), }), tool_namespace, ), @@ -769,9 +764,6 @@ fn add_collaboration_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mu hide_agent_type_model_reasoning: false, include_usage_hint: turn_context.config.multi_agent_v2.usage_hint_enabled, usage_hint_text: turn_context.config.multi_agent_v2.usage_hint_text.clone(), - max_concurrent_threads_per_session: max_concurrent_threads_per_session( - turn_context, - ), }), exposure, ); diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index a9eb5012d5..2f7f23947b 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -400,6 +400,7 @@ fn dynamic_tool(namespace: Option<&str>, name: &str, defer_loading: bool) -> Dyn fn discoverable_plugin(id: &str, name: &str) -> DiscoverableTool { DiscoverablePluginInfo { id: id.to_string(), + remote_plugin_id: None, name: name.to_string(), description: Some(format!("{name} plugin")), has_skills: false, @@ -1014,7 +1015,7 @@ async fn multi_agent_feature_selects_one_agent_tool_family() { ToolSpec::Function(tool) => tool.description.as_str(), other => panic!("expected spawn_agent function spec, got {other:?}"), }; - assert!(spawn_agent_description.contains("max_concurrent_threads_per_session = 17")); + assert!(!spawn_agent_description.contains("max_concurrent_threads_per_session")); let direct_model_only = probe(|turn| { set_features( diff --git a/codex-rs/core/src/turn_diff_tracker.rs b/codex-rs/core/src/turn_diff_tracker.rs index 388c2210bd..ed8e9684ad 100644 --- a/codex-rs/core/src/turn_diff_tracker.rs +++ b/codex-rs/core/src/turn_diff_tracker.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; +use std::time::Duration; use sha1::digest::Output; @@ -12,6 +13,14 @@ use codex_apply_patch::AppliedPatchFileChange; const ZERO_OID: &str = "0000000000000000000000000000000000000000"; const DEV_NULL: &str = "/dev/null"; const REGULAR_FILE_MODE: &str = "100644"; +// Normal edits finish well within 100 ms; pathological inputs fall back to a coarse, +// content-exact diff without stalling tool completion. +const DIFF_TIMEOUT: Duration = Duration::from_millis(100); + +struct TrackedContent { + content: String, + revision: u64, +} #[derive(Clone, Debug, Eq, Hash, PartialEq)] struct TrackedPath { @@ -28,14 +37,27 @@ impl TrackedPath { } } +#[derive(Eq, Hash, PartialEq)] +struct DiffCacheKey { + left_path: TrackedPath, + left_revision: Option, + right_path: TrackedPath, + right_revision: Option, +} + /// Tracks the net text diff for the current turn from committed apply_patch /// mutations, without rereading the workspace filesystem. pub struct TurnDiffTracker { valid: bool, display_roots_by_environment: HashMap, - baseline_by_path: HashMap, - current_by_path: HashMap, + baseline_by_path: HashMap, + current_by_path: HashMap, origin_by_current_path: HashMap, + next_revision: u64, + rendered_diffs: HashMap>, + unified_diff: Option, + #[cfg(test)] + rendered_diff_count: std::cell::Cell, } impl Default for TurnDiffTracker { @@ -46,6 +68,11 @@ impl Default for TurnDiffTracker { baseline_by_path: HashMap::new(), current_by_path: HashMap::new(), origin_by_current_path: HashMap::new(), + next_revision: 0, + rendered_diffs: HashMap::new(), + unified_diff: None, + #[cfg(test)] + rendered_diff_count: std::cell::Cell::new(0), } } } @@ -64,6 +91,10 @@ impl TurnDiffTracker { } pub fn track_delta(&mut self, environment_id: &str, delta: &AppliedPatchDelta) { + if !self.valid { + return; + } + if !delta.is_exact() { self.invalidate(); return; @@ -72,17 +103,24 @@ impl TurnDiffTracker { for change in delta.changes() { self.apply_change(environment_id, change); } + self.refresh_unified_diff(); } pub fn invalidate(&mut self) { self.valid = false; + self.rendered_diffs.clear(); + self.unified_diff = None; } pub fn get_unified_diff(&self) -> Option { - if !self.valid { - return None; - } + self.unified_diff.clone() + } + pub(crate) fn has_unified_diff(&self) -> bool { + self.unified_diff.is_some() + } + + fn refresh_unified_diff(&mut self) { let rename_pairs = self.rename_pairs(); let paired_destinations = rename_pairs.values().cloned().collect::>(); let mut handled = HashSet::new(); @@ -95,6 +133,8 @@ impl TurnDiffTracker { paths.sort_by_key(|path| self.display_path(path)); paths.dedup(); + let mut previous_diffs = std::mem::take(&mut self.rendered_diffs); + let mut rendered_diffs = HashMap::new(); let mut aggregated = String::new(); for path in paths { if !handled.insert(path.clone()) { @@ -105,22 +145,41 @@ impl TurnDiffTracker { continue; } - let diff = if let Some(dest) = rename_pairs.get(&path) { + let (left_path, right_path) = if let Some(dest) = rename_pairs.get(&path) { handled.insert(dest.clone()); - self.render_rename_diff(&path, dest) + (&path, dest) } else { - self.render_path_diff(&path) + (&path, &path) }; - if let Some(diff) = diff { - aggregated.push_str(&diff); + let left_content = self.baseline_by_path.get(left_path); + let right_content = self.current_by_path.get(right_path); + let key = DiffCacheKey { + left_path: left_path.clone(), + left_revision: left_content.map(|content| content.revision), + right_path: right_path.clone(), + right_revision: right_content.map(|content| content.revision), + }; + let rendered = previous_diffs.remove(&key).unwrap_or_else(|| { + self.render_diff( + left_path, + left_content.map(|content| content.content.as_str()), + right_path, + right_content.map(|content| content.content.as_str()), + ) + }); + + if let Some(diff) = rendered.as_deref() { + aggregated.push_str(diff); if !aggregated.ends_with('\n') { aggregated.push('\n'); } } + rendered_diffs.insert(key, rendered); } - (!aggregated.is_empty()).then_some(aggregated) + self.rendered_diffs = rendered_diffs; + self.unified_diff = (!aggregated.is_empty()).then_some(aggregated); } fn apply_change(&mut self, environment_id: &str, change: &AppliedPatchChange) { @@ -157,18 +216,20 @@ impl TurnDiffTracker { && !self.baseline_by_path.contains_key(&path) && let Some(overwritten_content) = overwritten_content { + let overwritten_content = self.tracked_content(overwritten_content); self.baseline_by_path - .insert(path.clone(), overwritten_content.to_string()); + .insert(path.clone(), overwritten_content); } - self.current_by_path.insert(path, content.to_string()); + let content = self.tracked_content(content); + self.current_by_path.insert(path, content); } fn apply_delete(&mut self, path: TrackedPath, content: &str) { if self.current_by_path.remove(&path).is_none() && !self.baseline_by_path.contains_key(&path) { - self.baseline_by_path - .insert(path.clone(), content.to_string()); + let content = self.tracked_content(content); + self.baseline_by_path.insert(path.clone(), content); } self.origin_by_current_path.remove(&path); } @@ -184,8 +245,9 @@ impl TurnDiffTracker { if !self.current_by_path.contains_key(&source_path) && !self.baseline_by_path.contains_key(&source_path) { + let old_content = self.tracked_content(old_content); self.baseline_by_path - .insert(source_path.clone(), old_content.to_string()); + .insert(source_path.clone(), old_content); } match move_path { @@ -194,28 +256,38 @@ impl TurnDiffTracker { && !self.baseline_by_path.contains_key(&dest_path) && let Some(overwritten_move_content) = overwritten_move_content { + let overwritten_move_content = self.tracked_content(overwritten_move_content); self.baseline_by_path - .insert(dest_path.clone(), overwritten_move_content.to_string()); + .insert(dest_path.clone(), overwritten_move_content); } let origin = self .origin_by_current_path .remove(&source_path) .unwrap_or_else(|| source_path.clone()); self.current_by_path.remove(&source_path); - self.current_by_path - .insert(dest_path.clone(), new_content.to_string()); + let new_content = self.tracked_content(new_content); + self.current_by_path.insert(dest_path.clone(), new_content); self.origin_by_current_path.remove(&dest_path); if dest_path != origin { self.origin_by_current_path.insert(dest_path, origin); } } None => { - self.current_by_path - .insert(source_path, new_content.to_string()); + let new_content = self.tracked_content(new_content); + self.current_by_path.insert(source_path, new_content); } } } + fn tracked_content(&mut self, content: &str) -> TrackedContent { + let revision = self.next_revision; + self.next_revision += 1; + TrackedContent { + content: content.to_string(), + revision, + } + } + fn rename_pairs(&self) -> HashMap { self.origin_by_current_path .iter() @@ -234,28 +306,6 @@ impl TurnDiffTracker { .collect() } - fn render_path_diff(&self, path: &TrackedPath) -> Option { - self.render_diff( - path, - self.baseline_by_path.get(path).map(String::as_str), - path, - self.current_by_path.get(path).map(String::as_str), - ) - } - - fn render_rename_diff( - &self, - source_path: &TrackedPath, - dest_path: &TrackedPath, - ) -> Option { - self.render_diff( - source_path, - self.baseline_by_path.get(source_path).map(String::as_str), - dest_path, - self.current_by_path.get(dest_path).map(String::as_str), - ) - } - fn render_diff( &self, left_path: &TrackedPath, @@ -267,6 +317,10 @@ impl TurnDiffTracker { return None; } + #[cfg(test)] + self.rendered_diff_count + .set(self.rendered_diff_count.get() + 1); + let left_display = self.display_path(left_path); let right_display = self.display_path(right_path); let left_oid = left_content.map_or_else( @@ -299,16 +353,23 @@ impl TurnDiffTracker { DEV_NULL.to_string() }; - let unified = - similar::TextDiff::from_lines(left_content.unwrap_or(""), right_content.unwrap_or("")) - .unified_diff() - .context_radius(3) - .header(&old_header, &new_header) - .to_string(); + let mut config = similar::TextDiff::configure(); + config.timeout(DIFF_TIMEOUT); + let unified = config + .diff_lines(left_content.unwrap_or(""), right_content.unwrap_or("")) + .unified_diff() + .context_radius(3) + .header(&old_header, &new_header) + .to_string(); diff.push_str(&unified); Some(diff) } + #[cfg(test)] + fn rendered_diff_count(&self) -> usize { + self.rendered_diff_count.get() + } + fn display_path(&self, path: &TrackedPath) -> String { let display = self .display_roots_by_environment diff --git a/codex-rs/core/src/turn_diff_tracker_tests.rs b/codex-rs/core/src/turn_diff_tracker_tests.rs index 6454430404..a7be3d2b0c 100644 --- a/codex-rs/core/src/turn_diff_tracker_tests.rs +++ b/codex-rs/core/src/turn_diff_tracker_tests.rs @@ -2,10 +2,15 @@ use super::*; use codex_apply_patch::AppliedPatchDelta; use codex_apply_patch::MaybeApplyPatchVerified; use codex_exec_server::LOCAL_FS; +use codex_git_utils::ApplyGitRequest; +use codex_git_utils::apply_git_patch; use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; use std::fs; use std::path::Path; +use std::process::Command; +use std::time::Duration; +use std::time::Instant; use tempfile::tempdir; fn git_blob_sha1_hex(data: &str) -> String { @@ -369,3 +374,122 @@ index {left_oid_b}..{right_oid_b} ); assert_eq!(tracker.get_unified_diff(), Some(expected)); } + +#[tokio::test] +async fn reuses_rendered_diffs_for_unchanged_paths() { + let dir = tempdir().expect("tempdir"); + let mut tracker = tracker_with_root(dir.path()); + + let add_a = apply_verified_patch( + dir.path(), + "*** Begin Patch\n*** Add File: a.txt\n+one\n*** End Patch", + ) + .await; + tracker.track_delta("", &add_a); + assert_eq!(tracker.rendered_diff_count(), 1); + + let add_b = apply_verified_patch( + dir.path(), + "*** Begin Patch\n*** Add File: b.txt\n+two\n*** End Patch", + ) + .await; + tracker.track_delta("", &add_b); + + assert_eq!(tracker.rendered_diff_count(), 2); + assert_eq!( + tracker.get_unified_diff(), + tracker.get_unified_diff(), + "reading the cached aggregate must not render file diffs", + ); + assert_eq!(tracker.rendered_diff_count(), 2); +} + +#[tokio::test] +async fn repeated_updates_only_rerender_the_touched_path() { + let dir = tempdir().expect("tempdir"); + let mut tracker = tracker_with_root(dir.path()); + + for patch in [ + "*** Begin Patch\n*** Add File: stable.txt\n+stable\n*** End Patch".to_string(), + "*** Begin Patch\n*** Add File: hot.txt\n+value 0\n*** End Patch".to_string(), + ] { + tracker.track_delta("", &apply_verified_patch(dir.path(), &patch).await); + } + + for value in 1..=40 { + let patch = format!( + "*** Begin Patch\n*** Update File: hot.txt\n@@\n-value {}\n+value {value}\n*** End Patch", + value - 1, + ); + tracker.track_delta("", &apply_verified_patch(dir.path(), &patch).await); + } + + assert_eq!(tracker.rendered_diff_count(), 42); +} + +#[test] +fn large_rewrite_returns_promptly_and_preserves_exact_content() { + let dir = tempdir().expect("tempdir"); + assert!( + Command::new("git") + .args(["init", "--quiet"]) + .current_dir(dir.path()) + .status() + .expect("run git init") + .success() + ); + assert!( + Command::new("git") + .args(["config", "core.autocrlf", "false"]) + .current_dir(dir.path()) + .status() + .expect("disable line ending conversion") + .success() + ); + let old_content = (0..48_000) + .map(|line| format!("old line {line:05}\n")) + .collect::(); + let new_content = (0..48_000) + .map(|line| format!("new line {line:05}\n")) + .collect::(); + let path = dir.path().join("large.txt"); + fs::write(&path, &old_content).expect("seed large file"); + assert!( + Command::new("git") + .args(["add", "large.txt"]) + .current_dir(dir.path()) + .status() + .expect("run git add") + .success() + ); + let tracker = tracker_with_root(dir.path()); + let tracked_path = TrackedPath::new("", &path); + + let started = Instant::now(); + let diff = tracker + .render_diff( + &tracked_path, + Some(&old_content), + &tracked_path, + Some(&new_content), + ) + .expect("complete rewrite should produce a diff"); + + assert!( + started.elapsed() < Duration::from_secs(2), + "large rewrite took {:?}", + started.elapsed(), + ); + let result = apply_git_patch(&ApplyGitRequest { + cwd: dir.path().to_path_buf(), + diff, + revert: false, + preflight: false, + }) + .expect("apply generated diff"); + assert_eq!(result.exit_code, 0, "{}", result.stderr); + assert_eq!( + fs::read_to_string(path).expect("read large file"), + new_content + ); +} diff --git a/codex-rs/core/src/turn_metadata.rs b/codex-rs/core/src/turn_metadata.rs index e66a1fbb50..d13c081817 100644 --- a/codex-rs/core/src/turn_metadata.rs +++ b/codex-rs/core/src/turn_metadata.rs @@ -6,16 +6,15 @@ use std::sync::RwLock; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; -use codex_analytics::CompactionImplementation; -use codex_analytics::CompactionPhase; -use codex_analytics::CompactionReason; -use codex_analytics::CompactionStrategy; -use codex_analytics::CompactionTrigger; -use codex_utils_string::to_ascii_json_string; -use serde::Serialize; use serde_json::Value; use tokio::task::JoinHandle; +use crate::responses_metadata::CodexResponsesMetadata; +use crate::responses_metadata::CodexResponsesRequestKind; +use crate::responses_metadata::TurnMetadataWorkspace; +use crate::responses_metadata::filter_extra_metadata; +use crate::responses_metadata::subagent_header_value; +use crate::responses_metadata::subagent_metadata_kind; use crate::sandbox_tags::permission_profile_sandbox_tag; use codex_git_utils::get_git_remote_urls_assume_git_repo; use codex_git_utils::get_git_repo_root; @@ -26,62 +25,18 @@ use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::models::PermissionProfile; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::SessionSource; -use codex_protocol::protocol::ThreadSource; use codex_utils_absolute_path::AbsolutePathBuf; const MODEL_KEY: &str = "model"; const REASONING_EFFORT_KEY: &str = "reasoning_effort"; -const TURN_STARTED_AT_UNIX_MS_KEY: &str = "turn_started_at_unix_ms"; const USER_INPUT_REQUESTED_DURING_TURN_KEY: &str = "user_input_requested_during_turn"; const WORKSPACE_KIND_KEY: &str = "workspace_kind"; -const REQUEST_KIND_KEY: &str = "request_kind"; -const COMPACTION_KEY: &str = "compaction"; -const WINDOW_ID_KEY: &str = "window_id"; pub(crate) struct McpTurnMetadataContext<'a> { pub(crate) model: &'a str, pub(crate) reasoning_effort: Option, } -/// Metadata present only on outbound model requests that perform compaction. -/// -/// These fields describe the operation at dispatch time. Post-response outcomes such as status, -/// error, duration, and token deltas remain in compaction analytics events. -#[derive(Clone, Copy, Debug, Serialize)] -pub(crate) struct CompactionTurnMetadata { - trigger: CompactionTrigger, - reason: CompactionReason, - implementation: CompactionImplementation, - phase: CompactionPhase, - strategy: CompactionStrategy, -} - -impl CompactionTurnMetadata { - pub(crate) fn new( - trigger: CompactionTrigger, - reason: CompactionReason, - implementation: CompactionImplementation, - phase: CompactionPhase, - ) -> Self { - Self { - trigger, - reason, - implementation, - phase, - strategy: CompactionStrategy::Memento, - } - } -} - -#[derive(Clone, Copy, Debug, Serialize)] -#[serde(rename_all = "snake_case")] -enum TurnMetadataRequestKind { - Turn, - Prewarm, - Compaction, - Memory, -} - #[derive(Clone, Debug, Default)] struct WorkspaceGitMetadata { associated_remote_urls: Option>, @@ -97,16 +52,6 @@ impl WorkspaceGitMetadata { } } -#[derive(Clone, Debug, Serialize, Default)] -struct TurnMetadataWorkspace { - #[serde(default, skip_serializing_if = "Option::is_none")] - associated_remote_urls: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - latest_git_commit_hash: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - has_changes: Option, -} - impl From for TurnMetadataWorkspace { fn from(value: WorkspaceGitMetadata) -> Self { Self { @@ -117,141 +62,40 @@ impl From for TurnMetadataWorkspace { } } -/// Base payload for the outbound model request `x-codex-turn-metadata` header. -/// -/// Turn-owned state populates identity fields, including optional fork and subagent lineage. A -/// concrete request kind is added at outbound model dispatch so turns, startup prewarm, and -/// compaction remain distinguishable. Detached memory requests are constructed as `memory` -/// directly. -#[derive(Clone, Debug, Serialize)] -pub(crate) struct TurnMetadataBag { - #[serde(default, skip_serializing_if = "Option::is_none")] - request_kind: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - session_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - thread_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - forked_from_thread_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - parent_thread_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - subagent_kind: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - thread_source: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - turn_id: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - workspaces: BTreeMap, - #[serde(default, skip_serializing_if = "Option::is_none")] - sandbox: Option, -} - -impl TurnMetadataBag { - fn with_workspace_git_metadata( - mut self, - repo_root: Option, - workspace_git_metadata: Option, - ) -> Self { - if let (Some(repo_root), Some(workspace_git_metadata)) = (repo_root, workspace_git_metadata) - && !workspace_git_metadata.is_empty() - { - self.workspaces - .insert(repo_root, workspace_git_metadata.into()); - } - self - } - - fn to_header_value(&self) -> Option { - to_ascii_json_string(self).ok() - } -} - -fn merge_turn_metadata( - header: &str, - turn_started_at_unix_ms: Option, - responsesapi_client_metadata: Option<&HashMap>, -) -> Option { - if turn_started_at_unix_ms.is_none() && responsesapi_client_metadata.is_none() { - return None; - } - - let mut metadata = serde_json::from_str::>(header).ok()?; - if let Some(turn_started_at_unix_ms) = turn_started_at_unix_ms { - metadata.insert( - TURN_STARTED_AT_UNIX_MS_KEY.to_string(), - Value::Number(turn_started_at_unix_ms.into()), - ); - } - if let Some(responsesapi_client_metadata) = responsesapi_client_metadata { - for (key, value) in responsesapi_client_metadata { - if matches!( - key.as_str(), - "session_id" - | "thread_id" - | "turn_id" - | TURN_STARTED_AT_UNIX_MS_KEY - | "forked_from_thread_id" - | "parent_thread_id" - | "subagent_kind" - | REQUEST_KIND_KEY - | COMPACTION_KEY - | WINDOW_ID_KEY - ) { - continue; - } - metadata - .entry(key.clone()) - .or_insert_with(|| Value::String(value.clone())); - } - } - to_ascii_json_string(&metadata).ok() -} - -pub async fn build_turn_metadata_header( +#[allow(clippy::too_many_arguments)] +pub async fn detached_memory_responses_metadata( + installation_id: String, + session_id: String, + thread_id: String, + window_id: String, + session_source: &SessionSource, cwd: &AbsolutePathBuf, sandbox: Option<&str>, -) -> Option { - let repo_root = get_git_repo_root(cwd).map(|root| root.to_string_lossy().into_owned()); - - let (head_commit_hash, associated_remote_urls, has_changes) = tokio::join!( - get_head_commit_hash(cwd), - get_git_remote_urls_assume_git_repo(cwd), - get_has_changes(cwd), - ); - let latest_git_commit_hash = head_commit_hash.map(|sha| sha.0); - TurnMetadataBag { - request_kind: Some(TurnMetadataRequestKind::Memory), - session_id: None, - thread_id: None, - forked_from_thread_id: None, - parent_thread_id: None, - subagent_kind: None, - thread_source: None, - turn_id: None, - workspaces: BTreeMap::new(), +) -> CodexResponsesMetadata { + CodexResponsesMetadata { + request_kind: Some(CodexResponsesRequestKind::Memory), + subagent_header: subagent_header_value(session_source), sandbox: sandbox.map(ToString::to_string), + workspaces: memory_workspaces(cwd).await, + ..CodexResponsesMetadata::new(installation_id, session_id, thread_id, window_id) } - .with_workspace_git_metadata( - repo_root, - Some(WorkspaceGitMetadata { - associated_remote_urls, - latest_git_commit_hash, - has_changes, - }), - ) - .to_header_value() } #[derive(Clone, Debug)] pub(crate) struct TurnMetadataState { cwd: AbsolutePathBuf, repo_root: Option, - base_metadata: TurnMetadataBag, - base_header: Option, - enriched_header: Arc>>, + session_id: String, + thread_id: String, + forked_from_thread_id: Option, + parent_thread_id: Option, + subagent_header: Option, + subagent_kind: Option, + turn_id: String, + sandbox: Option, + enriched_workspaces: Arc>>>, turn_started_at_unix_ms: Arc>>, - responsesapi_client_metadata: Arc>>>, + responsesapi_client_metadata: Arc>>, user_input_requested_during_turn: Arc, enrichment_task: Arc>>>, } @@ -264,7 +108,6 @@ impl TurnMetadataState { forked_from_thread_id: Option, parent_thread_id: Option, session_source: &SessionSource, - thread_source: Option, turn_id: String, cwd: AbsolutePathBuf, permission_profile: &PermissionProfile, @@ -280,79 +123,34 @@ impl TurnMetadataState { ) .to_string(), ); - let subagent_kind = match session_source { - SessionSource::SubAgent(subagent_source) => Some(subagent_source.kind().to_string()), - SessionSource::Cli - | SessionSource::VSCode - | SessionSource::Exec - | SessionSource::Mcp - | SessionSource::Custom(_) - | SessionSource::Internal(_) - | SessionSource::Unknown => None, - }; - let base_metadata = TurnMetadataBag { - request_kind: None, - session_id: Some(session_id), - thread_id: Some(thread_id), - forked_from_thread_id, - parent_thread_id, - subagent_kind, - thread_source, - turn_id: Some(turn_id), - workspaces: BTreeMap::new(), - sandbox, - }; - let base_header = base_metadata.to_header_value(); - Self { cwd, repo_root, - base_metadata, - base_header, - enriched_header: Arc::new(RwLock::new(None)), + session_id, + thread_id, + forked_from_thread_id, + parent_thread_id, + subagent_header: subagent_header_value(session_source), + subagent_kind: subagent_metadata_kind(session_source), + turn_id, + sandbox, + enriched_workspaces: Arc::new(RwLock::new(None)), turn_started_at_unix_ms: Arc::new(RwLock::new(None)), - responsesapi_client_metadata: Arc::new(RwLock::new(None)), + responsesapi_client_metadata: Arc::new(RwLock::new(BTreeMap::new())), user_input_requested_during_turn: Arc::new(AtomicBool::new(false)), enrichment_task: Arc::new(Mutex::new(None)), } } - pub(crate) fn current_header_value(&self) -> Option { - let header = if let Some(header) = self - .enriched_header - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .cloned() - { - header - } else { - self.base_header.clone()? - }; - let turn_started_at_unix_ms = *self - .turn_started_at_unix_ms - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let responsesapi_client_metadata = self - .responsesapi_client_metadata - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clone(); - merge_turn_metadata( - &header, - turn_started_at_unix_ms, - responsesapi_client_metadata.as_ref(), - ) - .or(Some(header)) - } - pub(crate) fn current_meta_value_for_mcp_request( &self, context: McpTurnMetadataContext<'_>, ) -> Option { - let header = self.current_header_value()?; - let mut metadata = serde_json::from_str::>(&header).ok()?; - metadata.remove(REQUEST_KIND_KEY); + let Value::Object(mut metadata) = + self.responses_metadata_template().turn_metadata_value()? + else { + return None; + }; metadata.insert( MODEL_KEY.to_string(), Value::String(context.model.to_string()), @@ -382,50 +180,18 @@ impl TurnMetadataState { Some(Value::Object(metadata)) } - fn current_header_value_for_model_request_kind( + pub(crate) fn to_responses_metadata( &self, - window_id: &str, - request_kind: TurnMetadataRequestKind, - ) -> Option { - let header = self.current_header_value()?; - let mut metadata = serde_json::from_str::>(&header).ok()?; - metadata.insert( - REQUEST_KIND_KEY.to_string(), - serde_json::to_value(request_kind).ok()?, - ); - metadata.insert( - WINDOW_ID_KEY.to_string(), - Value::String(window_id.to_string()), - ); - to_ascii_json_string(&metadata).ok() - } - - pub(crate) fn current_header_value_for_model_request(&self, window_id: &str) -> Option { - self.current_header_value_for_model_request_kind(window_id, TurnMetadataRequestKind::Turn) - } - - pub(crate) fn current_header_value_for_prewarm(&self, window_id: &str) -> Option { - self.current_header_value_for_model_request_kind( + installation_id: String, + window_id: String, + request_kind: CodexResponsesRequestKind, + ) -> CodexResponsesMetadata { + CodexResponsesMetadata { + installation_id, window_id, - TurnMetadataRequestKind::Prewarm, - ) - } - - pub(crate) fn current_header_value_for_compaction( - &self, - window_id: &str, - compaction: CompactionTurnMetadata, - ) -> Option { - let header = self.current_header_value_for_model_request_kind( - window_id, - TurnMetadataRequestKind::Compaction, - )?; - let mut metadata = serde_json::from_str::>(&header).ok()?; - metadata.insert( - COMPACTION_KEY.to_string(), - serde_json::to_value(compaction).ok()?, - ); - to_ascii_json_string(&metadata).ok() + request_kind: Some(request_kind), + ..self.responses_metadata_template() + } } pub(crate) fn mark_user_input_requested_during_turn(&self) { @@ -441,15 +207,54 @@ impl TurnMetadataState { .responsesapi_client_metadata .write() .unwrap_or_else(std::sync::PoisonError::into_inner) = - Some(responsesapi_client_metadata); + filter_extra_metadata(responsesapi_client_metadata); } pub(crate) fn workspace_kind(&self) -> Option { self.responsesapi_client_metadata .read() .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .and_then(|metadata| metadata.get(WORKSPACE_KIND_KEY).cloned()) + .get(WORKSPACE_KIND_KEY) + .cloned() + } + + fn responses_metadata_template(&self) -> CodexResponsesMetadata { + CodexResponsesMetadata { + turn_id: Some(self.turn_id.clone()), + forked_from_thread_id: self.forked_from_thread_id, + parent_thread_id: self.parent_thread_id, + subagent_header: self.subagent_header.clone(), + subagent_kind: self.subagent_kind.clone(), + sandbox: self.sandbox.clone(), + workspaces: self.current_workspaces(), + turn_started_at_unix_ms: self.current_turn_started_at_unix_ms(), + extra: self + .responsesapi_client_metadata + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(), + ..CodexResponsesMetadata::new( + String::new(), + self.session_id.clone(), + self.thread_id.clone(), + String::new(), + ) + } + } + + fn current_workspaces(&self) -> BTreeMap { + self.enriched_workspaces + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + .unwrap_or_default() + } + + fn current_turn_started_at_unix_ms(&self) -> Option { + *self + .turn_started_at_unix_ms + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) } pub(crate) fn set_turn_started_at_unix_ms(&self, turn_started_at_unix_ms: i64) { @@ -479,20 +284,16 @@ impl TurnMetadataState { return; }; - let enriched_metadata = state - .base_metadata - .clone() - .with_workspace_git_metadata(Some(repo_root), Some(workspace_git_metadata)); - if enriched_metadata.workspaces.is_empty() { + if workspace_git_metadata.is_empty() { return; } - if let Some(header_value) = enriched_metadata.to_header_value() { - *state - .enriched_header - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(header_value); - } + let mut workspaces = BTreeMap::new(); + workspaces.insert(repo_root, workspace_git_metadata.into()); + *state + .enriched_workspaces + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(workspaces); })); } @@ -522,6 +323,27 @@ impl TurnMetadataState { } } +async fn memory_workspaces(cwd: &AbsolutePathBuf) -> BTreeMap { + let repo_root = get_git_repo_root(cwd).map(|root| root.to_string_lossy().into_owned()); + let (head_commit_hash, associated_remote_urls, has_changes) = tokio::join!( + get_head_commit_hash(cwd), + get_git_remote_urls_assume_git_repo(cwd), + get_has_changes(cwd), + ); + let workspace_git_metadata = WorkspaceGitMetadata { + associated_remote_urls, + latest_git_commit_hash: head_commit_hash.map(|sha| sha.0), + has_changes, + }; + let mut workspaces = BTreeMap::new(); + if let Some(repo_root) = repo_root + && !workspace_git_metadata.is_empty() + { + workspaces.insert(repo_root, workspace_git_metadata.into()); + } + workspaces +} + #[cfg(test)] #[path = "turn_metadata_tests.rs"] mod tests; diff --git a/codex-rs/core/src/turn_metadata_tests.rs b/codex-rs/core/src/turn_metadata_tests.rs index 32a559848a..5e6d1f32bb 100644 --- a/codex-rs/core/src/turn_metadata_tests.rs +++ b/codex-rs/core/src/turn_metadata_tests.rs @@ -1,11 +1,18 @@ use super::*; +use crate::responses_metadata::CodexResponsesRequestKind; +use crate::responses_metadata::CompactionTurnMetadata; +use crate::responses_metadata::INSTALLATION_ID_KEY; +use crate::responses_metadata::WINDOW_ID_KEY; use crate::sandbox_tags::permission_profile_sandbox_tag; +use codex_analytics::CompactionImplementation; +use codex_analytics::CompactionPhase; +use codex_analytics::CompactionReason; +use codex_analytics::CompactionTrigger; use codex_protocol::models::PermissionProfile; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; -use codex_protocol::protocol::ThreadSource; use codex_utils_absolute_path::AbsolutePathBuf; use core_test_support::PathBufExt; use core_test_support::PathExt; @@ -23,6 +30,44 @@ fn test_mcp_turn_metadata_context() -> McpTurnMetadataContext<'static> { } } +fn test_responses_metadata_json( + state: &TurnMetadataState, + window_id: &str, + request_kind: CodexResponsesRequestKind, +) -> String { + state + .to_responses_metadata( + "installation-a".to_string(), + window_id.to_string(), + request_kind, + ) + .turn_metadata_json() + .expect("turn metadata json") +} + +fn test_turn_responses_metadata_json(state: &TurnMetadataState, window_id: &str) -> String { + test_responses_metadata_json(state, window_id, CodexResponsesRequestKind::Turn) +} + +fn test_compaction_responses_metadata_json( + state: &TurnMetadataState, + window_id: &str, + compaction: CompactionTurnMetadata, +) -> String { + test_responses_metadata_json( + state, + window_id, + CodexResponsesRequestKind::Compaction(compaction), + ) +} + +fn test_turn_metadata_header(state: &TurnMetadataState) -> String { + state + .responses_metadata_template() + .turn_metadata_json() + .expect("header") +} + async fn create_clean_git_repo(repo_name: &str) -> (TempDir, AbsolutePathBuf) { let temp_dir = TempDir::new().expect("temp dir"); let repo_path = temp_dir.path().join(repo_name).abs(); @@ -64,12 +109,21 @@ async fn create_clean_git_repo(repo_name: &str) -> (TempDir, AbsolutePathBuf) { } #[tokio::test] -async fn build_turn_metadata_header_marks_detached_memory_without_turn_identity() { +async fn detached_memory_responses_metadata_omits_turn_identity() { let (_temp_dir, repo_path) = create_clean_git_repo("repo-東京").await; - let header = build_turn_metadata_header(&repo_path, Some("none")) - .await - .expect("header"); + let header = detached_memory_responses_metadata( + String::new(), + String::new(), + String::new(), + String::new(), + &SessionSource::Unknown, + &repo_path, + Some("none"), + ) + .await + .turn_metadata_json() + .expect("header"); assert!(header.is_ascii()); assert!(!header.contains("東京")); let parsed: Value = serde_json::from_str(&header).expect("valid json"); @@ -100,13 +154,22 @@ async fn build_turn_metadata_header_marks_detached_memory_without_turn_identity( } #[tokio::test] -async fn build_turn_metadata_header_marks_memory_without_workspace_metadata() { +async fn detached_memory_responses_metadata_omits_empty_workspace_metadata() { let temp_dir = TempDir::new().expect("temp dir"); let cwd = temp_dir.path().abs(); - let header = build_turn_metadata_header(&cwd, /*sandbox*/ None) - .await - .expect("detached memory should emit its request kind"); + let header = detached_memory_responses_metadata( + String::new(), + String::new(), + String::new(), + String::new(), + &SessionSource::Unknown, + &cwd, + /*sandbox*/ None, + ) + .await + .turn_metadata_json() + .expect("detached memory should emit its request kind"); let parsed: Value = serde_json::from_str(&header).expect("valid json"); assert_eq!(parsed, serde_json::json!({"request_kind": "memory"})); @@ -124,7 +187,6 @@ fn turn_metadata_state_uses_platform_sandbox_tag() { /*forked_from_thread_id*/ None, /*parent_thread_id*/ None, &SessionSource::Exec, - Some(ThreadSource::User), "turn-a".to_string(), cwd, &permission_profile, @@ -132,12 +194,11 @@ fn turn_metadata_state_uses_platform_sandbox_tag() { /*enforce_managed_network*/ false, ); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let json: Value = serde_json::from_str(&header).expect("json"); let sandbox_name = json.get("sandbox").and_then(Value::as_str); let session_id = json.get("session_id").and_then(Value::as_str); let thread_id = json.get("thread_id").and_then(Value::as_str); - let thread_source = json.get("thread_source").and_then(Value::as_str); assert!(json.get("request_kind").is_none()); let expected_sandbox = permission_profile_sandbox_tag( @@ -148,39 +209,12 @@ fn turn_metadata_state_uses_platform_sandbox_tag() { assert_eq!(sandbox_name, Some(expected_sandbox)); assert_eq!(session_id, Some("session-a")); assert_eq!(thread_id, Some("thread-a")); - assert_eq!(thread_source, Some("user")); assert!(json.get("forked_from_thread_id").is_none()); assert!(json.get("parent_thread_id").is_none()); assert!(json.get("subagent_kind").is_none()); assert!(json.get("session_source").is_none()); } -#[test] -fn turn_metadata_state_uses_explicit_subagent_thread_source() { - let temp_dir = TempDir::new().expect("temp dir"); - let cwd = temp_dir.path().abs(); - let permission_profile = PermissionProfile::read_only(); - let state = TurnMetadataState::new( - "session-a".to_string(), - "thread-a".to_string(), - /*forked_from_thread_id*/ None, - /*parent_thread_id*/ None, - &SessionSource::Exec, - Some(ThreadSource::Subagent), - "turn-a".to_string(), - cwd, - &permission_profile, - WindowsSandboxLevel::Disabled, - /*enforce_managed_network*/ false, - ); - - let header = state.current_header_value().expect("header"); - let json: Value = serde_json::from_str(&header).expect("json"); - - assert_eq!(json["thread_source"].as_str(), Some("subagent")); - assert!(json.get("session_source").is_none()); -} - #[test] fn turn_metadata_state_includes_root_fork_lineage() { let temp_dir = TempDir::new().expect("temp dir"); @@ -195,7 +229,6 @@ fn turn_metadata_state_includes_root_fork_lineage() { Some(source_thread_id), /*parent_thread_id*/ None, &SessionSource::Exec, - Some(ThreadSource::User), "turn-a".to_string(), cwd, &permission_profile, @@ -203,7 +236,7 @@ fn turn_metadata_state_includes_root_fork_lineage() { /*enforce_managed_network*/ false, ); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let json: Value = serde_json::from_str(&header).expect("json"); assert_eq!( @@ -234,7 +267,6 @@ fn turn_metadata_state_includes_thread_spawn_subagent_parent_without_fork() { agent_nickname: None, agent_role: None, }), - Some(ThreadSource::Subagent), "turn-a".to_string(), cwd, &permission_profile, @@ -242,7 +274,7 @@ fn turn_metadata_state_includes_thread_spawn_subagent_parent_without_fork() { /*enforce_managed_network*/ false, ); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let json: Value = serde_json::from_str(&header).expect("json"); assert!(json.get("forked_from_thread_id").is_none()); @@ -273,7 +305,6 @@ fn turn_metadata_state_includes_forked_thread_spawn_subagent_lineage() { agent_nickname: None, agent_role: None, }), - Some(ThreadSource::Subagent), "turn-a".to_string(), cwd, &permission_profile, @@ -281,7 +312,7 @@ fn turn_metadata_state_includes_forked_thread_spawn_subagent_lineage() { /*enforce_managed_network*/ false, ); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let json: Value = serde_json::from_str(&header).expect("json"); assert_eq!( @@ -318,7 +349,6 @@ fn turn_metadata_state_includes_known_parent_for_non_thread_spawn_subagents_with /*forked_from_thread_id*/ None, Some(parent_thread_id), &SessionSource::SubAgent(subagent_source), - Some(ThreadSource::Subagent), "turn-a".to_string(), cwd.clone(), &permission_profile, @@ -326,7 +356,7 @@ fn turn_metadata_state_includes_known_parent_for_non_thread_spawn_subagents_with /*enforce_managed_network*/ false, ); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let json: Value = serde_json::from_str(&header).expect("json"); assert!(json.get("forked_from_thread_id").is_none()); @@ -350,7 +380,6 @@ fn turn_metadata_state_includes_turn_started_at_unix_ms_after_start() { /*forked_from_thread_id*/ None, /*parent_thread_id*/ None, &SessionSource::Exec, - Some(ThreadSource::User), "turn-a".to_string(), cwd, &permission_profile, @@ -359,7 +388,7 @@ fn turn_metadata_state_includes_turn_started_at_unix_ms_after_start() { ); state.set_turn_started_at_unix_ms(/*turn_started_at_unix_ms*/ 1_700_000_000_123); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let json: Value = serde_json::from_str(&header).expect("json"); assert_eq!( @@ -380,7 +409,6 @@ fn turn_metadata_state_includes_model_and_reasoning_effort_only_in_request_meta( /*forked_from_thread_id*/ None, /*parent_thread_id*/ None, &SessionSource::Exec, - /*thread_source*/ None, "turn-a".to_string(), cwd, &permission_profile, @@ -388,7 +416,7 @@ fn turn_metadata_state_includes_model_and_reasoning_effort_only_in_request_meta( /*enforce_managed_network*/ false, ); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let header_json: Value = serde_json::from_str(&header).expect("json"); assert!(header_json.get("model").is_none()); assert!(header_json.get("reasoning_effort").is_none()); @@ -429,7 +457,6 @@ fn turn_metadata_state_marks_user_input_requested_during_turn_only_for_mcp_reque /*forked_from_thread_id*/ None, /*parent_thread_id*/ None, &SessionSource::Exec, - /*thread_source*/ None, "turn-a".to_string(), cwd, &permission_profile, @@ -437,7 +464,7 @@ fn turn_metadata_state_marks_user_input_requested_during_turn_only_for_mcp_reque /*enforce_managed_network*/ false, ); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let header_json: Value = serde_json::from_str(&header).expect("json"); assert!( header_json @@ -452,7 +479,7 @@ fn turn_metadata_state_marks_user_input_requested_during_turn_only_for_mcp_reque state.mark_user_input_requested_during_turn(); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let header_json: Value = serde_json::from_str(&header).expect("json"); assert!( header_json @@ -482,7 +509,6 @@ fn turn_metadata_state_ignores_client_reserved_metadata_before_start() { /*forked_from_thread_id*/ None, /*parent_thread_id*/ None, &SessionSource::Exec, - Some(ThreadSource::User), "turn-a".to_string(), cwd, &permission_profile, @@ -505,7 +531,7 @@ fn turn_metadata_state_ignores_client_reserved_metadata_before_start() { ("subagent_kind".to_string(), "client-supplied".to_string()), ])); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let json: Value = serde_json::from_str(&header).expect("json"); assert!(json.get("turn_started_at_unix_ms").is_none()); @@ -536,7 +562,6 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( agent_nickname: None, agent_role: None, }), - Some(ThreadSource::User), "turn-a".to_string(), cwd, &permission_profile, @@ -554,6 +579,19 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( ), ("session_id".to_string(), "client-supplied".to_string()), ("thread_id".to_string(), "client-supplied".to_string()), + ("installation_id".to_string(), "client-supplied".to_string()), + ( + "x-codex-installation-id".to_string(), + "client-supplied".to_string(), + ), + ( + "x-codex-parent-thread-id".to_string(), + "client-supplied".to_string(), + ), + ( + "x-openai-subagent".to_string(), + "client-supplied".to_string(), + ), ( "forked_from_thread_id".to_string(), "client-supplied".to_string(), @@ -574,7 +612,7 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( ])); state.set_turn_started_at_unix_ms(/*turn_started_at_unix_ms*/ 1_700_000_000_123); - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); assert!(header.is_ascii()); assert!(!header.contains("東京")); let json: Value = serde_json::from_str(&header).expect("json"); @@ -586,6 +624,10 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( assert_eq!(json["reasoning_effort"].as_str(), Some("client-supplied")); assert_eq!(json["session_id"].as_str(), Some("session-a")); assert_eq!(json["thread_id"].as_str(), Some("thread-a")); + assert!(json.get(INSTALLATION_ID_KEY).is_none()); + assert!(json.get("x-codex-installation-id").is_none()); + assert!(json.get("x-codex-parent-thread-id").is_none()); + assert!(json.get("x-openai-subagent").is_none()); assert_eq!( json["forked_from_thread_id"].as_str(), Some("44444444-4444-4444-8444-444444444444") @@ -595,7 +637,7 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( Some("55555555-5555-4555-8555-555555555555") ); assert_eq!(json["subagent_kind"].as_str(), Some("thread_spawn")); - assert_eq!(json["thread_source"].as_str(), Some("user")); + assert_eq!(json["thread_source"].as_str(), Some("client-supplied")); assert_eq!(json["turn_id"].as_str(), Some("turn-a")); assert!(json.get("request_kind").is_none()); assert!(json.get(WINDOW_ID_KEY).is_none()); @@ -604,12 +646,14 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( Some(1_700_000_000_123) ); - let model_request_header = state - .current_header_value_for_model_request("thread-a:1") - .expect("model request header"); + let model_request_header = test_turn_responses_metadata_json(&state, "thread-a:1"); let model_request_json: Value = serde_json::from_str(&model_request_header).expect("model request json"); assert_eq!(model_request_json["request_kind"].as_str(), Some("turn")); + assert_eq!( + model_request_json[INSTALLATION_ID_KEY].as_str(), + Some("installation-a") + ); assert_eq!( model_request_json[WINDOW_ID_KEY].as_str(), Some("thread-a:1") @@ -635,7 +679,6 @@ fn turn_metadata_state_overlays_compaction_only_on_compaction_requests() { /*forked_from_thread_id*/ None, /*parent_thread_id*/ None, &SessionSource::Exec, - Some(ThreadSource::User), "turn-a".to_string(), cwd, &permission_profile, @@ -647,17 +690,16 @@ fn turn_metadata_state_overlays_compaction_only_on_compaction_requests() { "client-supplied".to_string(), )])); - let compact_header = state - .current_header_value_for_compaction( - "thread-a:2", - CompactionTurnMetadata::new( - CompactionTrigger::Auto, - CompactionReason::ContextLimit, - CompactionImplementation::ResponsesCompactionV2, - CompactionPhase::MidTurn, - ), - ) - .expect("compact header"); + let compact_header = test_compaction_responses_metadata_json( + &state, + "thread-a:2", + CompactionTurnMetadata::new( + CompactionTrigger::Auto, + CompactionReason::ContextLimit, + CompactionImplementation::ResponsesCompactionV2, + CompactionPhase::MidTurn, + ), + ); let compact_json: Value = serde_json::from_str(&compact_header).expect("json"); assert_eq!(compact_json["request_kind"].as_str(), Some("compaction")); assert_eq!(compact_json["turn_id"].as_str(), Some("turn-a")); @@ -673,9 +715,7 @@ fn turn_metadata_state_overlays_compaction_only_on_compaction_requests() { }) ); - let regular_header = state - .current_header_value_for_model_request("thread-a:3") - .expect("regular header"); + let regular_header = test_turn_responses_metadata_json(&state, "thread-a:3"); let regular_json: Value = serde_json::from_str(®ular_header).expect("json"); assert_eq!(regular_json["request_kind"].as_str(), Some("turn")); assert_eq!(regular_json[WINDOW_ID_KEY].as_str(), Some("thread-a:3")); @@ -701,7 +741,6 @@ async fn turn_metadata_state_preserves_lineage_after_git_enrichment() { agent_nickname: None, agent_role: None, }), - Some(ThreadSource::Subagent), "turn-a".to_string(), repo_path, &permission_profile, @@ -713,7 +752,7 @@ async fn turn_metadata_state_preserves_lineage_after_git_enrichment() { let json = tokio::time::timeout(Duration::from_secs(2), async { loop { - let header = state.current_header_value().expect("header"); + let header = test_turn_metadata_header(&state); let json: Value = serde_json::from_str(&header).expect("json"); if json .get("workspaces") diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index 94a7ab7c73..65b18b064d 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -155,6 +155,8 @@ struct ProcessEntry { process: Arc, call_id: String, process_id: i32, + cwd: AbsolutePathBuf, + initial_exec_command_active: Arc, hook_command: String, tty: bool, network_approval: Option, diff --git a/codex-rs/core/src/unified_exec/mod_tests.rs b/codex-rs/core/src/unified_exec/mod_tests.rs index 9f24314da2..d994354130 100644 --- a/codex-rs/core/src/unified_exec/mod_tests.rs +++ b/codex-rs/core/src/unified_exec/mod_tests.rs @@ -1,5 +1,6 @@ use super::head_tail_buffer::HeadTailBuffer; use super::*; +use crate::codex_thread::BackgroundTerminalInfo; use crate::exec::ExecCapturePolicy; use crate::exec::ExecExpiration; use crate::sandboxing::ExecRequest; @@ -9,6 +10,16 @@ use crate::session::turn_context::TurnContext; use crate::tools::context::ExecCommandToolOutput; use crate::unified_exec::WriteStdinRequest; use crate::unified_exec::process::OutputHandles; +use async_trait::async_trait; +use codex_exec_server::ExecProcess; +use codex_exec_server::ExecProcessEventReceiver; +use codex_exec_server::ExecServerError; +use codex_exec_server::ProcessId; +use codex_exec_server::ProcessSignal; +use codex_exec_server::ReadResponse; +use codex_exec_server::StartedExecProcess; +use codex_exec_server::WriteResponse; +use codex_exec_server::WriteStatus; use codex_sandboxing::SandboxType; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::approx_token_count; @@ -19,6 +30,8 @@ use pretty_assertions::assert_eq; use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; +use tokio::sync::Notify; +use tokio::sync::watch; use tokio::time::Duration; use tokio::time::Instant; @@ -116,6 +129,8 @@ async fn exec_command_with_tty( process: Arc::clone(&process), call_id: context.call_id.clone(), process_id, + cwd: cwd.clone(), + initial_exec_command_active: Arc::new(std::sync::atomic::AtomicBool::new(true)), hook_command: cmd.to_string(), tty, network_approval: None, @@ -158,6 +173,18 @@ async fn exec_command_with_tty( manager.release_process_id(process_id).await; None }; + if response_process_id.is_some() + && let Some(entry) = manager + .process_store + .lock() + .await + .processes + .get_mut(&process_id) + { + entry + .initial_exec_command_active + .store(false, std::sync::atomic::Ordering::Release); + } Ok(ExecCommandToolOutput { event_call_id: context.call_id, @@ -184,6 +211,82 @@ impl SpawnLifecycle for TestSpawnLifecycle { } } +struct BlockingTerminateExecProcess { + process_id: ProcessId, + terminate_started: watch::Sender, + allow_terminate: Arc, + wake_tx: watch::Sender, +} + +#[async_trait] +impl ExecProcess for BlockingTerminateExecProcess { + fn process_id(&self) -> &ProcessId { + &self.process_id + } + + fn subscribe_wake(&self) -> watch::Receiver { + self.wake_tx.subscribe() + } + + fn subscribe_events(&self) -> ExecProcessEventReceiver { + ExecProcessEventReceiver::empty() + } + + async fn read( + &self, + _after_seq: Option, + _max_bytes: Option, + _wait_ms: Option, + ) -> Result { + Ok(ReadResponse { + chunks: Vec::new(), + next_seq: 1, + exited: false, + exit_code: None, + closed: false, + failure: None, + }) + } + + async fn write(&self, _chunk: Vec) -> Result { + Ok(WriteResponse { + status: WriteStatus::Accepted, + }) + } + + async fn signal(&self, _signal: ProcessSignal) -> Result<(), ExecServerError> { + Ok(()) + } + + async fn terminate(&self) -> Result<(), ExecServerError> { + let _ = self.terminate_started.send(true); + self.allow_terminate.notified().await; + Ok(()) + } +} + +async fn blocking_terminate_unified_process( + process_id: i32, + terminate_started: watch::Sender, + allow_terminate: Arc, +) -> anyhow::Result> { + let (wake_tx, _wake_rx) = watch::channel(0); + Ok(Arc::new( + UnifiedExecProcess::from_exec_server_started( + StartedExecProcess { + process: Arc::new(BlockingTerminateExecProcess { + process_id: process_id.to_string().into(), + terminate_started, + allow_terminate, + wake_tx, + }), + }, + SandboxType::None, + ) + .await?, + )) +} + async fn write_stdin( session: &Arc, process_id: i32, @@ -241,12 +344,23 @@ async fn unified_exec_persists_across_requests() -> anyhow::Result<()> { skip_if_sandbox!(Ok(())); let (session, turn) = test_session_and_turn().await; + #[allow(deprecated)] + let cwd = turn.cwd.clone(); let open_shell = exec_command( &session, &turn, "bash -i", /*yield_time_ms*/ 2_500, /*workdir*/ None, ) .await?; let process_id = open_shell.process_id.expect("expected process_id"); + assert_eq!( + session.list_background_terminals().await, + vec![BackgroundTerminalInfo { + item_id: "call".to_string(), + process_id: process_id.to_string(), + command: "bash -i".to_string(), + cwd, + }] + ); write_stdin( &session, @@ -270,6 +384,10 @@ async fn unified_exec_persists_across_requests() -> anyhow::Result<()> { "expected environment variable output" ); + assert!(session.terminate_background_terminal(process_id).await); + assert!(!session.terminate_background_terminal(process_id).await); + assert!(session.list_background_terminals().await.is_empty()); + Ok(()) } @@ -523,6 +641,148 @@ async fn reusing_completed_process_returns_unknown_process() -> anyhow::Result<( Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn terminating_initial_exec_command_rechecks_initial_response_state() -> anyhow::Result<()> { + let (session, turn) = test_session_and_turn().await; + let manager = &session.services.unified_exec_manager; + let process_id = manager.allocate_process_id().await; + let (terminate_started_tx, mut terminate_started_rx) = watch::channel(false); + let allow_terminate = Arc::new(Notify::new()); + let process = blocking_terminate_unified_process( + process_id, + terminate_started_tx, + Arc::clone(&allow_terminate), + ) + .await?; + #[allow(deprecated)] + let cwd = turn.cwd.clone(); + manager.process_store.lock().await.processes.insert( + process_id, + ProcessEntry { + process, + call_id: "call".to_string(), + process_id, + cwd, + initial_exec_command_active: Arc::new(std::sync::atomic::AtomicBool::new(true)), + hook_command: "sleep 60".to_string(), + tty: true, + network_approval: None, + session: Arc::downgrade(&session), + last_used: Instant::now(), + }, + ); + + let terminate_task = tokio::spawn({ + let session = Arc::clone(&session); + async move { session.terminate_background_terminal(process_id).await } + }); + tokio::time::timeout( + Duration::from_secs(2), + terminate_started_rx.wait_for(|started| *started), + ) + .await + .expect("terminate should start") + .expect("terminate signal sender should stay open"); + + { + let mut store = manager.process_store.lock().await; + let entry = store + .processes + .get_mut(&process_id) + .expect("process should remain stored until initial response returns"); + entry + .initial_exec_command_active + .store(false, std::sync::atomic::Ordering::Release); + } + + allow_terminate.notify_waiters(); + let terminated = tokio::time::timeout(Duration::from_secs(2), terminate_task) + .await + .expect("terminate should finish") + .expect("terminate task should not panic"); + assert!(terminated); + assert!( + !manager + .process_store + .lock() + .await + .processes + .contains_key(&process_id) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn terminating_during_stdin_poll_returns_exited_response() -> anyhow::Result<()> { + let (session, turn) = test_session_and_turn().await; + let manager = &session.services.unified_exec_manager; + let process_id = manager.allocate_process_id().await; + let (terminate_started_tx, _terminate_started_rx) = watch::channel(false); + let allow_terminate = Arc::new(Notify::new()); + let process = blocking_terminate_unified_process( + process_id, + terminate_started_tx, + Arc::clone(&allow_terminate), + ) + .await?; + #[allow(deprecated)] + let cwd = turn.cwd.clone(); + let last_used = Instant::now() - Duration::from_secs(1); + manager.process_store.lock().await.processes.insert( + process_id, + ProcessEntry { + process: Arc::clone(&process), + call_id: "call".to_string(), + process_id, + cwd, + initial_exec_command_active: Arc::new(std::sync::atomic::AtomicBool::new(false)), + hook_command: "sleep 60".to_string(), + tty: true, + network_approval: None, + session: Arc::downgrade(&session), + last_used, + }, + ); + + let poll_task = tokio::spawn({ + let session = Arc::clone(&session); + async move { + write_stdin(&session, process_id, "", /*yield_time_ms*/ 60_000).await + } + }); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let poll_started = manager + .process_store + .lock() + .await + .processes + .get(&process_id) + .is_some_and(|entry| entry.last_used != last_used); + if poll_started { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("poll should clone process handles"); + + manager.release_process_id(process_id).await; + allow_terminate.notify_one(); + process.terminate_confirmed().await?; + + let output = tokio::time::timeout(Duration::from_secs(2), poll_task) + .await + .expect("poll should finish") + .expect("poll task should not panic")?; + assert_eq!(output.process_id, None); + assert!(manager.process_store.lock().await.processes.is_empty()); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn completed_pipe_commands_preserve_exit_code() -> anyhow::Result<()> { let (_, turn) = make_session_and_context().await; diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs index 229e5b52f8..725be9eed2 100644 --- a/codex-rs/core/src/unified_exec/process.rs +++ b/codex-rs/core/src/unified_exec/process.rs @@ -195,9 +195,16 @@ impl UnifiedExecProcess { } } - pub(super) fn terminate(&self) { + fn finish_termination(&self) { self.output_closed.store(true, Ordering::Release); self.output_closed_notify.notify_waiters(); + self.cancellation_token.cancel(); + if let Some(output_task) = &self.output_task { + output_task.abort(); + } + } + + pub(super) fn terminate(&self) { match &self.process_handle { ProcessHandle::Local(process_handle) => process_handle.terminate(), ProcessHandle::ExecServer(process_handle) => { @@ -207,10 +214,22 @@ impl UnifiedExecProcess { }); } } - self.cancellation_token.cancel(); - if let Some(output_task) = &self.output_task { - output_task.abort(); + self.finish_termination(); + } + + pub(super) async fn terminate_confirmed(&self) -> Result<(), UnifiedExecError> { + match &self.process_handle { + ProcessHandle::Local(process_handle) => process_handle.terminate(), + ProcessHandle::ExecServer(process_handle) => { + process_handle + .terminate() + .await + .map_err(|err| UnifiedExecError::process_failed(err.to_string()))?; + } } + self.signal_exit(self.exit_code()); + self.finish_termination(); + Ok(()) } pub(super) async fn interrupt(&self) -> Result<(), UnifiedExecError> { diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index c9e6f3fdca..d744a72c95 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -11,6 +11,7 @@ use tokio::time::Duration; use tokio::time::Instant; use tokio_util::sync::CancellationToken; +use crate::codex_thread::BackgroundTerminalInfo; use crate::exec_env::CODEX_THREAD_ID_ENV_VAR; use crate::exec_env::create_env; use crate::exec_policy::ExecApprovalRequest; @@ -175,11 +176,22 @@ struct PreparedProcessHandles { pause_state: Option>, session: Option>, network_approval: Option, + call_id: String, hook_command: String, process_id: i32, tty: bool, } +struct InitialExecCommandGuard { + active: Arc, +} + +impl Drop for InitialExecCommandGuard { + fn drop(&mut self) { + self.active.store(false, Ordering::Release); + } +} + fn exec_server_process_id(process_id: i32) -> String { process_id.to_string() } @@ -414,7 +426,8 @@ impl UnifiedExecProcessManager { // Persist live sessions before the initial yield wait so interrupting the // turn cannot drop the last Arc and terminate the background process. let process_started_alive = !process.has_exited() && process.exit_code().is_none(); - if process_started_alive { + let _initial_exec_command_guard = if process_started_alive { + let initial_exec_command_active = Arc::new(AtomicBool::new(true)); self.store_process( Arc::clone(&process), context, @@ -426,9 +439,15 @@ impl UnifiedExecProcessManager { request.tty, deferred_network_approval.clone(), Arc::clone(&transcript), + Arc::clone(&initial_exec_command_active), ) .await; - } + Some(InitialExecCommandGuard { + active: initial_exec_command_active, + }) + } else { + None + }; let yield_time_ms = clamp_yield_time(request.yield_time_ms); // For the initial exec_command call, we both stream output to events @@ -609,6 +628,7 @@ impl UnifiedExecProcessManager { pause_state, session, network_approval, + call_id, hook_command, process_id, tty, @@ -721,9 +741,13 @@ impl UnifiedExecProcessManager { (None, exit_code, call_id) } ProcessStatus::Unknown => { - return Err(UnifiedExecError::UnknownProcessId { - process_id: request.process_id, - }); + if process.has_exited() { + (None, process.exit_code(), call_id) + } else { + return Err(UnifiedExecError::UnknownProcessId { + process_id: request.process_id, + }); + } } }; @@ -744,29 +768,27 @@ impl UnifiedExecProcessManager { } async fn refresh_process_state(&self, process_id: i32) -> ProcessStatus { - { - let mut store = self.process_store.lock().await; - let Some(entry) = store.processes.get(&process_id) else { + let mut store = self.process_store.lock().await; + let Some(entry) = store.processes.get_mut(&process_id) else { + return ProcessStatus::Unknown; + }; + + let exit_code = entry.process.exit_code(); + let process_id = entry.process_id; + + if entry.process.has_exited() { + let Some(entry) = store.remove(process_id) else { return ProcessStatus::Unknown; }; - - let exit_code = entry.process.exit_code(); - let process_id = entry.process_id; - - if entry.process.has_exited() { - let Some(entry) = store.remove(process_id) else { - return ProcessStatus::Unknown; - }; - ProcessStatus::Exited { - exit_code, - entry: Box::new(entry), - } - } else { - ProcessStatus::Alive { - exit_code, - call_id: entry.call_id.clone(), - process_id, - } + ProcessStatus::Exited { + exit_code, + entry: Box::new(entry), + } + } else { + ProcessStatus::Alive { + exit_code, + call_id: entry.call_id.clone(), + process_id, } } } @@ -804,6 +826,7 @@ impl UnifiedExecProcessManager { pause_state, session, network_approval: entry.network_approval.clone(), + call_id: entry.call_id.clone(), hook_command: entry.hook_command.clone(), process_id: entry.process_id, tty: entry.tty, @@ -823,11 +846,14 @@ impl UnifiedExecProcessManager { tty: bool, network_approval: Option, transcript: Arc>, + initial_exec_command_active: Arc, ) { let entry = ProcessEntry { process: Arc::clone(&process), call_id: context.call_id.clone(), process_id, + cwd: cwd.clone(), + initial_exec_command_active, hook_command, tty, network_approval, @@ -1274,6 +1300,59 @@ impl UnifiedExecProcessManager { entry.process.terminate(); } } + + pub(crate) async fn list_processes(&self) -> Vec { + let store = self.process_store.lock().await; + let mut entries = store + .processes + .values() + .filter(|entry| !entry.process.has_exited()) + .collect::>(); + entries.sort_by_key(|entry| entry.process_id); + entries + .into_iter() + .map(|entry| BackgroundTerminalInfo { + item_id: entry.call_id.clone(), + process_id: entry.process_id.to_string(), + command: entry.hook_command.clone(), + cwd: entry.cwd.clone(), + }) + .collect() + } + + pub(crate) async fn terminate_process(&self, process_id: i32) -> bool { + let (process, already_exited) = { + let store = self.process_store.lock().await; + let Some(entry) = store.processes.get(&process_id) else { + return false; + }; + (Arc::clone(&entry.process), entry.process.has_exited()) + }; + + if !already_exited && process.terminate_confirmed().await.is_err() { + return false; + } + + let entry = { + let mut store = self.process_store.lock().await; + let Some(entry) = store.processes.get(&process_id) else { + return true; + }; + if !Arc::ptr_eq(&entry.process, &process) { + return true; + } + if entry.initial_exec_command_active.load(Ordering::Acquire) { + return true; + } + let Some(entry) = store.remove(process_id) else { + return false; + }; + entry + }; + + unregister_network_approval_for_entry(&entry).await; + true + } } enum ProcessStatus { diff --git a/codex-rs/core/src/unified_exec/process_tests.rs b/codex-rs/core/src/unified_exec/process_tests.rs index 42db18ff78..814de95af4 100644 --- a/codex-rs/core/src/unified_exec/process_tests.rs +++ b/codex-rs/core/src/unified_exec/process_tests.rs @@ -22,6 +22,7 @@ struct MockExecProcess { process_id: ProcessId, write_response: WriteResponse, read_responses: Mutex>, + terminate_error: Option, wake_tx: watch::Sender, } @@ -69,11 +70,17 @@ impl ExecProcess for MockExecProcess { } async fn terminate(&self) -> Result<(), ExecServerError> { + if let Some(message) = &self.terminate_error { + return Err(ExecServerError::Protocol(message.clone())); + } Ok(()) } } -async fn remote_process(write_status: WriteStatus) -> UnifiedExecProcess { +async fn remote_process( + write_status: WriteStatus, + terminate_error: Option, +) -> UnifiedExecProcess { let (wake_tx, _wake_rx) = watch::channel(0); let started = StartedExecProcess { process: Arc::new(MockExecProcess { @@ -82,6 +89,7 @@ async fn remote_process(write_status: WriteStatus) -> UnifiedExecProcess { status: write_status, }, read_responses: Mutex::new(VecDeque::new()), + terminate_error, wake_tx, }), }; @@ -93,7 +101,7 @@ async fn remote_process(write_status: WriteStatus) -> UnifiedExecProcess { #[tokio::test] async fn remote_write_unknown_process_marks_process_exited() { - let process = remote_process(WriteStatus::UnknownProcess).await; + let process = remote_process(WriteStatus::UnknownProcess, /*terminate_error*/ None).await; let err = process .write(b"hello") @@ -106,7 +114,7 @@ async fn remote_write_unknown_process_marks_process_exited() { #[tokio::test] async fn remote_write_closed_stdin_marks_process_exited() { - let process = remote_process(WriteStatus::StdinClosed).await; + let process = remote_process(WriteStatus::StdinClosed, /*terminate_error*/ None).await; let err = process .write(b"hello") @@ -119,7 +127,7 @@ async fn remote_write_closed_stdin_marks_process_exited() { #[tokio::test] async fn fail_and_terminate_preserves_failure_message() { - let process = remote_process(WriteStatus::Accepted).await; + let process = remote_process(WriteStatus::Accepted, /*terminate_error*/ None).await; process.fail_and_terminate("network denied".to_string()); process.fail_and_terminate("second failure".to_string()); @@ -131,6 +139,32 @@ async fn fail_and_terminate_preserves_failure_message() { ); } +#[tokio::test] +async fn remote_terminate_confirmed_updates_state_on_success_only() { + let process = remote_process( + WriteStatus::Accepted, + Some("terminate unavailable".to_string()), + ) + .await; + + let err = process + .terminate_confirmed() + .await + .expect_err("expected terminate failure"); + + assert!(matches!(err, UnifiedExecError::ProcessFailed { .. })); + assert!(!process.has_exited()); + + let process = remote_process(WriteStatus::Accepted, /*terminate_error*/ None).await; + + process + .terminate_confirmed() + .await + .expect("terminate should succeed"); + + assert!(process.has_exited()); +} + #[tokio::test] async fn remote_process_waits_for_early_exit_event() { let (wake_tx, _wake_rx) = watch::channel(0); @@ -148,6 +182,7 @@ async fn remote_process_waits_for_early_exit_event() { closed: true, failure: None, }])), + terminate_error: None, wake_tx: wake_tx.clone(), }), }; diff --git a/codex-rs/core/tests/common/Cargo.toml b/codex-rs/core/tests/common/Cargo.toml index fb6221cfce..76f06df1ed 100644 --- a/codex-rs/core/tests/common/Cargo.toml +++ b/codex-rs/core/tests/common/Cargo.toml @@ -20,6 +20,7 @@ codex-config = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } codex-exec-server = { workspace = true } +codex-home = { workspace = true } codex-features = { workspace = true } codex-hooks = { workspace = true } codex-login = { workspace = true } @@ -28,6 +29,7 @@ codex-models-manager = { workspace = true } codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-cargo-bin = { workspace = true } +codex-utils-path-uri = { workspace = true } ctor = { workspace = true } futures = { workspace = true } notify = { workspace = true } diff --git a/codex-rs/core/tests/common/context_snapshot.rs b/codex-rs/core/tests/common/context_snapshot.rs index 8aaefbbf15..7a165214c3 100644 --- a/codex-rs/core/tests/common/context_snapshot.rs +++ b/codex-rs/core/tests/common/context_snapshot.rs @@ -297,9 +297,11 @@ fn canonicalize_json_snapshot_value(value: &mut Value, options: &ContextSnapshot fn format_snapshot_json_string(text: &str, options: &ContextSnapshotOptions) -> String { let normalized = match options.render_mode { ContextSnapshotRenderMode::RedactedText - | ContextSnapshotRenderMode::KindWithTextPrefix { .. } => normalize_snapshot_uuids( - &normalize_snapshot_line_endings(&canonicalize_snapshot_text(text)), - ), + | ContextSnapshotRenderMode::KindWithTextPrefix { .. } => { + normalize_snapshot_dynamic_values(&normalize_snapshot_line_endings( + &canonicalize_snapshot_text(text), + )) + } ContextSnapshotRenderMode::FullText => normalize_snapshot_line_endings(text), ContextSnapshotRenderMode::KindOnly => unreachable!(), }; @@ -440,7 +442,7 @@ fn normalize_dynamic_snapshot_paths(text: &str) -> String { .into_owned() } -fn normalize_snapshot_uuids(text: &str) -> String { +fn normalize_snapshot_dynamic_values(text: &str) -> String { static UUID_RE: OnceLock = OnceLock::new(); let uuid_re = UUID_RE.get_or_init(|| { Regex::new( @@ -448,7 +450,20 @@ fn normalize_snapshot_uuids(text: &str) -> String { ) .expect("uuid regex should compile") }); - uuid_re.replace_all(text, "").into_owned() + static TURN_STARTED_AT_UNIX_MS_RE: OnceLock = OnceLock::new(); + let turn_started_at_unix_ms_re = TURN_STARTED_AT_UNIX_MS_RE.get_or_init(|| { + Regex::new(r#""turn_started_at_unix_ms":\d+"#) + .expect("turn_started_at_unix_ms regex should compile") + }); + static SANDBOX_RE: OnceLock = OnceLock::new(); + let sandbox_re = SANDBOX_RE + .get_or_init(|| Regex::new(r#""sandbox":"[^"]+""#).expect("sandbox regex should compile")); + let text = uuid_re.replace_all(text, ""); + let text = + turn_started_at_unix_ms_re.replace_all(&text, r#""turn_started_at_unix_ms":"#); + sandbox_re + .replace_all(&text, r#""sandbox":"""#) + .into_owned() } #[cfg(test)] @@ -456,6 +471,7 @@ mod tests { use super::ContextSnapshotOptions; use super::ContextSnapshotRenderMode; use super::format_response_items_snapshot; + use super::format_snapshot_json_string; use pretty_assertions::assert_eq; use serde_json::json; @@ -708,4 +724,17 @@ mod tests { "00:message/developer:## Skills\\n- openai-docs: helper (file: /openai-docs/SKILL.md)" ); } + + #[test] + fn redacted_text_mode_normalizes_turn_metadata_dynamic_json_strings() { + let rendered = format_snapshot_json_string( + r#"{"turn_id":"019eaded-ba5c-7d40-8a81-a4dcebc4679e","sandbox":"seccomp","turn_started_at_unix_ms":1781035793002}"#, + &ContextSnapshotOptions::default(), + ); + + assert_eq!( + rendered, + r#"{"turn_id":"","sandbox":"","turn_started_at_unix_ms":}"# + ); + } } diff --git a/codex-rs/core/tests/common/lib.rs b/codex-rs/core/tests/common/lib.rs index 94d72d619b..f9732b1b50 100644 --- a/codex-rs/core/tests/common/lib.rs +++ b/codex-rs/core/tests/common/lib.rs @@ -15,6 +15,8 @@ use codex_core::CodexThread; use codex_core::config::Config; use codex_core::config::ConfigBuilder; use codex_core::config::ConfigOverrides; +pub use codex_core::test_support::TestCodexResponsesRequestKind; +pub use codex_core::test_support::responses_metadata; use codex_utils_absolute_path::AbsolutePathBuf; pub use codex_utils_absolute_path::test_support::PathBufExt; pub use codex_utils_absolute_path::test_support::PathExt; diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index 62f422ba1c..1582b44d7d 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use std::process::Command; use std::sync::Arc; use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::time::Duration; @@ -24,7 +25,10 @@ use codex_exec_server::CreateDirectoryOptions; use codex_exec_server::ExecutorFileSystem; use codex_exec_server::RemoveOptions; use codex_extension_api::ExtensionRegistry; +use codex_extension_api::LoadUserInstructionsFuture; +use codex_extension_api::UserInstructionsProvider; use codex_extension_api::empty_extension_registry; +use codex_home::CodexHomeUserInstructionsProvider; use codex_login::CodexAuth; use codex_model_provider_info::ModelProviderInfo; use codex_model_provider_info::built_in_model_providers; @@ -43,6 +47,7 @@ use codex_protocol::protocol::TurnEnvironmentSelection; use codex_protocol::protocol::TurnEnvironmentSelections; use codex_protocol::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use futures::future::BoxFuture; use serde_json::Value; use tempfile::TempDir; @@ -71,6 +76,31 @@ const REMOTE_EXEC_SERVER_URL_ENV_VAR: &str = "CODEX_TEST_REMOTE_EXEC_SERVER_URL" static REMOTE_TEST_INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); const SUBMIT_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); +pub struct RecordingUserInstructionsProvider { + inner: Arc, + load_count: AtomicUsize, +} + +impl RecordingUserInstructionsProvider { + pub fn new(inner: Arc) -> Self { + Self { + inner, + load_count: AtomicUsize::new(0), + } + } + + pub fn load_count(&self) -> usize { + self.load_count.load(Ordering::SeqCst) + } +} + +impl UserInstructionsProvider for RecordingUserInstructionsProvider { + fn load_user_instructions(&self) -> LoadUserInstructionsFuture<'_> { + self.load_count.fetch_add(1, Ordering::SeqCst); + self.inner.load_user_instructions() + } +} + pub fn local(cwd: AbsolutePathBuf) -> TurnEnvironmentSelection { TurnEnvironmentSelection { environment_id: codex_exec_server::LOCAL_ENVIRONMENT_ID.to_string(), @@ -135,10 +165,11 @@ pub async fn test_env() -> Result { let environment = codex_exec_server::Environment::create_for_tests(Some(websocket_url.clone()))?; let cwd = remote_aware_cwd_path(); + let cwd_uri = PathUri::from_path(&cwd)?; environment .get_filesystem() .create_directory( - &cwd, + &cwd_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) @@ -231,6 +262,7 @@ pub struct TestCodexBuilder { user_shell_override: Option, exec_server_url: Option, extensions: Arc>, + user_instructions_provider: Option>, } impl TestCodexBuilder { @@ -320,6 +352,14 @@ impl TestCodexBuilder { self } + pub fn with_user_instructions_provider( + mut self, + provider: Arc, + ) -> Self { + self.user_instructions_provider = Some(provider); + self + } + pub fn with_windows_cmd_shell(self) -> Self { if cfg!(windows) { self.with_user_shell(get_shell_by_model_provided_path(&PathBuf::from("cmd.exe"))) @@ -508,12 +548,19 @@ impl TestCodexBuilder { let state_db = codex_core::init_state_db(&config).await; let thread_store = thread_store_from_config(&config, state_db.clone()); let installation_id = resolve_installation_id(&config.codex_home).await?; + let user_instructions_provider = + self.user_instructions_provider.clone().unwrap_or_else(|| { + Arc::new(CodexHomeUserInstructionsProvider::new( + config.codex_home.clone(), + )) + }); let thread_manager = ThreadManager::new( &config, codex_core::test_support::auth_manager_from_auth(auth.clone()), SessionSource::Exec, Arc::clone(&environment_manager), Arc::clone(&self.extensions), + user_instructions_provider, /*analytics_events_client*/ None, thread_store, state_db.clone(), @@ -907,35 +954,45 @@ impl TestCodexHarness { ) -> Result<()> { let abs_path = self.path_abs(rel); if let Some(parent) = abs_path.parent() { + let parent_uri = PathUri::from_path(&parent)?; self.test .fs() .create_directory( - &parent, + &parent_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) .await?; } + let abs_path_uri = PathUri::from_path(&abs_path)?; self.test .fs() - .write_file(&abs_path, contents.as_ref().to_vec(), /*sandbox*/ None) + .write_file( + &abs_path_uri, + contents.as_ref().to_vec(), + /*sandbox*/ None, + ) .await?; Ok(()) } pub async fn read_file_text(&self, rel: impl AsRef) -> Result { + let path = self.path_abs(rel); + let path_uri = PathUri::from_path(&path)?; Ok(self .test .fs() - .read_file_text(&self.path_abs(rel), /*sandbox*/ None) + .read_file_text(&path_uri, /*sandbox*/ None) .await?) } pub async fn create_dir_all(&self, rel: impl AsRef) -> Result<()> { + let path = self.path_abs(rel); + let path_uri = PathUri::from_path(&path)?; self.test .fs() .create_directory( - &self.path_abs(rel), + &path_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) @@ -948,10 +1005,11 @@ impl TestCodexHarness { } pub async fn remove_abs_path(&self, path: &AbsolutePathBuf) -> Result<()> { + let path_uri = PathUri::from_abs_path(path)?; self.test .fs() .remove( - path, + &path_uri, RemoveOptions { recursive: false, force: true, @@ -963,7 +1021,13 @@ impl TestCodexHarness { } pub async fn abs_path_exists(&self, path: &AbsolutePathBuf) -> Result { - match self.test.fs().get_metadata(path, /*sandbox*/ None).await { + let path_uri = PathUri::from_abs_path(path)?; + match self + .test + .fs() + .get_metadata(&path_uri, /*sandbox*/ None) + .await + { Ok(_) => Ok(true), Err(err) if err.kind() == ErrorKind::NotFound => Ok(false), Err(err) => Err(err.into()), @@ -1084,6 +1148,7 @@ pub fn test_codex() -> TestCodexBuilder { user_shell_override: None, exec_server_url: None, extensions: empty_extension_registry(), + user_instructions_provider: None, } } diff --git a/codex-rs/core/tests/responses_headers.rs b/codex-rs/core/tests/responses_headers.rs index af87762c04..cda5ffcdb0 100644 --- a/codex-rs/core/tests/responses_headers.rs +++ b/codex-rs/core/tests/responses_headers.rs @@ -15,8 +15,10 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; +use core_test_support::TestCodexResponsesRequestKind; use core_test_support::load_default_config_for_test; use core_test_support::responses; +use core_test_support::responses_metadata as test_responses_metadata; use core_test_support::test_codex::test_codex; use futures::StreamExt; use pretty_assertions::assert_eq; @@ -32,6 +34,23 @@ fn normalize_git_remote_url(url: &str) -> String { } const TEST_INSTALLATION_ID: &str = "11111111-1111-4111-8111-111111111111"; +fn test_turn_responses_metadata( + _client: &ModelClient, + thread_id: ThreadId, + session_source: &SessionSource, +) -> codex_core::CodexResponsesMetadata { + let thread_id = thread_id.to_string(); + test_responses_metadata( + TEST_INSTALLATION_ID, + &thread_id, + &thread_id, + /*turn_id*/ None, + format!("{thread_id}:0"), + session_source, + /*parent_thread_id*/ None, + TestCodexResponsesRequestKind::Turn, + ) +} #[tokio::test] async fn responses_stream_includes_subagent_header_on_review() { @@ -101,18 +120,16 @@ async fn responses_stream_includes_subagent_header_on_review() { let client = ModelClient::new( /*auth_manager*/ None, - thread_id.into(), thread_id, - /*installation_id*/ TEST_INSTALLATION_ID.to_string(), provider.clone(), - session_source, - /*parent_thread_id*/ None, + session_source.clone(), config.model_verbosity, /*enable_request_compression*/ false, /*include_timing_metrics*/ false, /*beta_features_header*/ None, /*attestation_provider*/ None, ); + let responses_metadata = test_turn_responses_metadata(&client, thread_id, &session_source); let mut client_session = client.new_session(); let mut prompt = Prompt::default(); @@ -127,14 +144,13 @@ async fn responses_stream_includes_subagent_header_on_review() { let mut stream = client_session .stream( - &expected_window_id, &prompt, &model_info, &session_telemetry, effort, summary.unwrap_or(model_info.default_reasoning_summary), /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -218,7 +234,6 @@ async fn responses_stream_includes_subagent_header_on_other() { let session_source = SessionSource::SubAgent(SubAgentSource::Other("my-task".to_string())); let model_info = codex_core::test_support::construct_model_info_offline(model.as_str(), &config); - let window_id = format!("{thread_id}:0"); let session_telemetry = SessionTelemetry::new( thread_id, @@ -235,18 +250,16 @@ async fn responses_stream_includes_subagent_header_on_other() { let client = ModelClient::new( /*auth_manager*/ None, - thread_id.into(), thread_id, - /*installation_id*/ TEST_INSTALLATION_ID.to_string(), provider.clone(), - session_source, - /*parent_thread_id*/ None, + session_source.clone(), config.model_verbosity, /*enable_request_compression*/ false, /*include_timing_metrics*/ false, /*beta_features_header*/ None, /*attestation_provider*/ None, ); + let responses_metadata = test_turn_responses_metadata(&client, thread_id, &session_source); let mut client_session = client.new_session(); let mut prompt = Prompt::default(); @@ -261,14 +274,13 @@ async fn responses_stream_includes_subagent_header_on_other() { let mut stream = client_session .stream( - &window_id, &prompt, &model_info, &session_telemetry, effort, summary.unwrap_or(model_info.default_reasoning_summary), /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -339,7 +351,6 @@ async fn responses_respects_model_info_overrides_from_config() { SessionSource::SubAgent(SubAgentSource::Other("override-check".to_string())); let model_info = codex_core::test_support::construct_model_info_offline(model.as_str(), &config); - let window_id = format!("{thread_id}:0"); let session_telemetry = SessionTelemetry::new( thread_id, model.as_str(), @@ -355,18 +366,16 @@ async fn responses_respects_model_info_overrides_from_config() { let client = ModelClient::new( /*auth_manager*/ None, - thread_id.into(), thread_id, - /*installation_id*/ TEST_INSTALLATION_ID.to_string(), provider.clone(), - session_source, - /*parent_thread_id*/ None, + session_source.clone(), config.model_verbosity, /*enable_request_compression*/ false, /*include_timing_metrics*/ false, /*beta_features_header*/ None, /*attestation_provider*/ None, ); + let responses_metadata = test_turn_responses_metadata(&client, thread_id, &session_source); let mut client_session = client.new_session(); let mut prompt = Prompt::default(); @@ -381,14 +390,13 @@ async fn responses_respects_model_info_overrides_from_config() { let mut stream = client_session .stream( - &window_id, &prompt, &model_info, &session_telemetry, effort, summary.unwrap_or(model_info.default_reasoning_summary), /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await diff --git a/codex-rs/core/tests/suite/agents_md.rs b/codex-rs/core/tests/suite/agents_md.rs index 88f014f757..585fd9e9b3 100644 --- a/codex-rs/core/tests/suite/agents_md.rs +++ b/codex-rs/core/tests/suite/agents_md.rs @@ -1,12 +1,16 @@ use anyhow::Result; use anyhow::anyhow; use codex_core::ForkSnapshot; +use codex_core::StartThreadOptions; use codex_exec_server::CreateDirectoryOptions; use codex_features::Feature; +use codex_home::CodexHomeUserInstructionsProvider; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::Op; use codex_protocol::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use core_test_support::PathBufExt; use core_test_support::create_directory_symlink; use core_test_support::load_default_config_for_test; @@ -17,6 +21,7 @@ use core_test_support::responses::mount_sse_once; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; +use core_test_support::test_codex::RecordingUserInstructionsProvider; use core_test_support::test_codex::TestCodexBuilder; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; @@ -112,10 +117,12 @@ async fn agents_override_is_preferred_over_agents_md() -> Result<()> { agents_instructions(test_codex().with_workspace_setup(|cwd, fs| async move { let agents_md = cwd.join("AGENTS.md"); let override_md = cwd.join("AGENTS.override.md"); - fs.write_file(&agents_md, b"base doc".to_vec(), /*sandbox*/ None) + let agents_md_uri = PathUri::from_path(&agents_md)?; + let override_md_uri = PathUri::from_path(&override_md)?; + fs.write_file(&agents_md_uri, b"base doc".to_vec(), /*sandbox*/ None) .await?; fs.write_file( - &override_md, + &override_md_uri, b"override doc".to_vec(), /*sandbox*/ None, ) @@ -146,14 +153,20 @@ async fn configured_fallback_is_used_when_agents_candidate_is_directory() -> Res .with_workspace_setup(|cwd, fs| async move { let agents_dir = cwd.join("AGENTS.md"); let fallback = cwd.join("WORKFLOW.md"); + let agents_dir_uri = PathUri::from_path(&agents_dir)?; + let fallback_uri = PathUri::from_path(&fallback)?; fs.create_directory( - &agents_dir, + &agents_dir_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) .await?; - fs.write_file(&fallback, b"fallback doc".to_vec(), /*sandbox*/ None) - .await?; + fs.write_file( + &fallback_uri, + b"fallback doc".to_vec(), + /*sandbox*/ None, + ) + .await?; Ok::<(), anyhow::Error>(()) }), ) @@ -183,23 +196,35 @@ async fn agents_docs_are_concatenated_from_project_root_to_cwd() -> Result<()> { let root_agents = root.join("AGENTS.md"); let git_marker = root.join(".git"); let nested_agents = nested.join("AGENTS.md"); + let nested_uri = PathUri::from_path(&nested)?; + let root_agents_uri = PathUri::from_path(&root_agents)?; + let git_marker_uri = PathUri::from_path(&git_marker)?; + let nested_agents_uri = PathUri::from_path(&nested_agents)?; fs.create_directory( - &nested, + &nested_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) .await?; - fs.write_file(&root_agents, b"root doc".to_vec(), /*sandbox*/ None) - .await?; fs.write_file( - &git_marker, + &root_agents_uri, + b"root doc".to_vec(), + /*sandbox*/ None, + ) + .await?; + fs.write_file( + &git_marker_uri, b"gitdir: /tmp/mock-git-dir\n".to_vec(), /*sandbox*/ None, ) .await?; - fs.write_file(&nested_agents, b"child doc".to_vec(), /*sandbox*/ None) - .await?; + fs.write_file( + &nested_agents_uri, + b"child doc".to_vec(), + /*sandbox*/ None, + ) + .await?; Ok::<(), anyhow::Error>(()) }), ) @@ -314,8 +339,9 @@ async fn selected_environment_sources_match_model_visible_instructions() -> Resu let mut builder = test_codex() .with_home(home) .with_workspace_setup(|cwd, fs| async move { + let agents_md_uri = PathUri::from_path(cwd.join("AGENTS.md"))?; fs.write_file( - &cwd.join("AGENTS.md"), + &agents_md_uri, b"project doc".to_vec(), /*sandbox*/ None, ) @@ -342,6 +368,89 @@ async fn selected_environment_sources_match_model_visible_instructions() -> Resu Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn loads_user_instructions_without_a_primary_environment() -> Result<()> { + let server = start_mock_server().await; + let response_mock = mount_sse_once( + &server, + sse(vec![ + ev_response_created("no-primary-environment-response"), + ev_completed("no-primary-environment-response"), + ]), + ) + .await; + let home = Arc::new(TempDir::new()?); + let global_source = + write_global_file(home.as_ref(), GLOBAL_AGENTS_FILENAME, GLOBAL_INSTRUCTIONS)?; + let provider = Arc::new(RecordingUserInstructionsProvider::new(Arc::new( + CodexHomeUserInstructionsProvider::new(AbsolutePathBuf::try_from( + home.path().to_path_buf(), + )?), + ))); + + let mut builder = test_codex() + .with_home(Arc::clone(&home)) + .with_user_instructions_provider(provider.clone()) + .with_workspace_setup(|cwd, fs| async move { + let project_agents_uri = PathUri::from_path(cwd.join(GLOBAL_AGENTS_FILENAME))?; + fs.write_file( + &project_agents_uri, + PROJECT_INSTRUCTIONS.as_bytes().to_vec(), + /*sandbox*/ None, + ) + .await?; + Ok(()) + }); + let test = builder.build_with_remote_env(&server).await?; + assert_eq!(provider.load_count(), 1); + + let no_environment_thread = test + .thread_manager + .start_thread_with_options(StartThreadOptions { + config: test.config.clone(), + initial_history: InitialHistory::New, + session_source: None, + thread_source: None, + dynamic_tools: Vec::new(), + metrics_service_name: None, + parent_trace: None, + environments: Vec::new(), + thread_extension_init: Default::default(), + }) + .await?; + assert_eq!(provider.load_count(), 2); + assert_eq!( + no_environment_thread.thread.instruction_sources().await, + vec![global_source] + ); + + no_environment_thread + .thread + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "inspect global instructions without an environment".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + wait_for_event(&no_environment_thread.thread, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let instruction_fragments = instruction_fragments(&response_mock.single_request()); + assert_eq!(instruction_fragments.len(), 1); + assert!(instruction_fragments[0].contains(GLOBAL_INSTRUCTIONS)); + assert!(!instruction_fragments[0].contains(PROJECT_INSTRUCTIONS)); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn fresh_thread_composes_global_before_project_and_reports_sources() -> Result<()> { // Set up one global source, one project source, and two ordinary model turns. @@ -367,8 +476,9 @@ async fn fresh_thread_composes_global_before_project_and_reports_sources() -> Re let mut builder = test_codex() .with_home(Arc::clone(&home)) .with_workspace_setup(|cwd, fs| async move { + let agents_md_uri = PathUri::from_path(cwd.join("AGENTS.md"))?; fs.write_file( - &cwd.join("AGENTS.md"), + &agents_md_uri, PROJECT_INSTRUCTIONS.as_bytes().to_vec(), /*sandbox*/ None, ) @@ -392,7 +502,7 @@ async fn fresh_thread_composes_global_before_project_and_reports_sources() -> Re )?; test.fs() .write_file( - &project_source, + &PathUri::from_path(&project_source)?, NEW_PROJECT_INSTRUCTIONS.as_bytes().to_vec(), /*sandbox*/ None, ) diff --git a/codex-rs/core/tests/suite/apply_patch_cli.rs b/codex-rs/core/tests/suite/apply_patch_cli.rs index a9ebadcebc..f320063a66 100644 --- a/codex-rs/core/tests/suite/apply_patch_cli.rs +++ b/codex-rs/core/tests/suite/apply_patch_cli.rs @@ -38,6 +38,7 @@ use codex_protocol::user_input::UserInput; #[cfg(target_os = "linux")] use codex_sandboxing::landlock::CODEX_LINUX_SANDBOX_ARG0; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use core_test_support::PathBufExt; use core_test_support::assert_regex_match; use core_test_support::get_remote_test_env; @@ -1329,25 +1330,24 @@ async fn apply_patch_turn_diff_paths_stay_repo_relative_when_session_cwd_is_nest config.cwd = config.cwd.join("subdir"); }) .with_workspace_setup(|cwd, fs| async move { + let cwd_uri = PathUri::from_path(&cwd)?; fs.create_directory( - &cwd, + &cwd_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) .await?; let repo_root = cwd.parent().expect("nested cwd should have parent"); + let git_uri = PathUri::from_path(repo_root.join(".git"))?; + let repo_file_uri = PathUri::from_path(repo_root.join("repo.txt"))?; fs.write_file( - &repo_root.join(".git"), + &git_uri, b"gitdir: /tmp/fake-worktree\n".to_vec(), /*sandbox*/ None, ) .await?; - fs.write_file( - &repo_root.join("repo.txt"), - b"before\n".to_vec(), - /*sandbox*/ None, - ) - .await?; + fs.write_file(&repo_file_uri, b"before\n".to_vec(), /*sandbox*/ None) + .await?; Ok(()) }) }) @@ -1582,10 +1582,11 @@ async fn apply_patch_turn_diff_tracks_local_and_remote_environment_paths() -> Re SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() )) .abs(); + let shared_cwd_uri = PathUri::from_path(&shared_cwd)?; let _ = fs::remove_dir_all(shared_cwd.as_path()); test.fs() .remove( - &shared_cwd, + &shared_cwd_uri, RemoveOptions { recursive: true, force: true, @@ -1596,7 +1597,7 @@ async fn apply_patch_turn_diff_tracks_local_and_remote_environment_paths() -> Re fs::create_dir_all(shared_cwd.as_path())?; test.fs() .create_directory( - &shared_cwd, + &shared_cwd_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) @@ -1683,7 +1684,10 @@ async fn apply_patch_turn_diff_tracks_local_and_remote_environment_paths() -> Re assert_eq!(fs::read_to_string(shared_cwd.join(file_name))?, "local\n"); assert_eq!( test.fs() - .read_file_text(&shared_cwd.join(file_name), /*sandbox*/ None) + .read_file_text( + &PathUri::from_path(shared_cwd.join(file_name))?, + /*sandbox*/ None, + ) .await?, "remote\n" ); @@ -1710,7 +1714,7 @@ index 0000000000000000000000000000000000000000..9c998f7b995a7327177b38a90d138517 let _ = fs::remove_dir_all(shared_cwd.as_path()); test.fs() .remove( - &shared_cwd, + &shared_cwd_uri, RemoveOptions { recursive: true, force: true, @@ -1851,8 +1855,9 @@ async fn apply_patch_clears_aggregated_diff_after_inexact_delta() -> Result<()> let harness = apply_patch_harness_with(|builder| { builder.with_workspace_setup(|cwd, fs| async move { + let binary_path_uri = PathUri::from_path(cwd.join("binary.dat"))?; fs.write_file( - &cwd.join("binary.dat"), + &binary_path_uri, vec![0xff, 0xfe, 0xfd], /*sandbox*/ None, ) diff --git a/codex-rs/core/tests/suite/auto_review.rs b/codex-rs/core/tests/suite/auto_review.rs index 5fe1cbc2f2..7dae7c4c62 100644 --- a/codex-rs/core/tests/suite/auto_review.rs +++ b/codex-rs/core/tests/suite/auto_review.rs @@ -256,6 +256,7 @@ fn remote_model_with_auto_review_override(slug: &str, review_model: &str) -> Mod context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), } diff --git a/codex-rs/core/tests/suite/cli_stream.rs b/codex-rs/core/tests/suite/cli_stream.rs index fc9b7d2569..1ef7c82c39 100644 --- a/codex-rs/core/tests/suite/cli_stream.rs +++ b/codex-rs/core/tests/suite/cli_stream.rs @@ -1,4 +1,3 @@ -use assert_cmd::Command as AssertCommand; use codex_git_utils::collect_git_info; use codex_login::CODEX_ACCESS_TOKEN_ENV_VAR; use codex_login::CODEX_API_KEY_ENV_VAR; @@ -7,6 +6,14 @@ use core_test_support::fs_wait; use core_test_support::responses; use core_test_support::skip_if_no_network; use pretty_assertions::assert_eq; +use std::io; +#[cfg(unix)] +use std::os::unix::process::CommandExt as _; +use std::process::Command; +use std::process::Output; +use std::process::Stdio; +use std::sync::mpsc; +use std::thread; use std::time::Duration; use tempfile::TempDir; use uuid::Uuid; @@ -22,6 +29,7 @@ const PERSONAL_ACCESS_TOKEN_AUTHORIZATION: &str = "Bearer at-cli-test"; const PERSONAL_ACCESS_TOKEN_ACCOUNT_ID: &str = "account-pat"; const WHOAMI_PATH: &str = "/v1/user-auth-credential/whoami"; const CLOUD_CONFIG_BUNDLE_PATH: &str = "/backend-api/wham/config/bundle"; +const CLI_TIMEOUT: Duration = Duration::from_secs(30); fn repo_root() -> std::path::PathBuf { #[expect(clippy::expect_used)] @@ -60,10 +68,9 @@ async fn mount_personal_access_token_startup(server: &MockServer) { } #[expect(clippy::unwrap_used)] -fn personal_access_token_exec_command(server: &MockServer, home: &TempDir) -> AssertCommand { +fn personal_access_token_exec_command(server: &MockServer, home: &TempDir) -> Command { let bin = codex_utils_cargo_bin::cargo_bin("codex").unwrap(); - let mut cmd = AssertCommand::new(bin); - cmd.timeout(Duration::from_secs(30)); + let mut cmd = Command::new(bin); cmd.arg("exec") .arg("--skip-git-repo-check") .arg("-c") @@ -81,6 +88,62 @@ fn personal_access_token_exec_command(server: &MockServer, home: &TempDir) -> As cmd } +struct ChildProcessCleanupGuard(u32); + +impl Drop for ChildProcessCleanupGuard { + fn drop(&mut self) { + #[cfg(unix)] + { + let _ = codex_utils_pty::process_group::kill_process_group(self.0); + } + + #[cfg(windows)] + { + let _ = Command::new("taskkill") + .args(["/PID", &self.0.to_string(), "/T", "/F"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + + #[cfg(not(any(unix, windows)))] + { + let _ = self.0; + } + } +} + +// Use this for new `codex exec` subprocess tests in this file. These commands +// can spawn shell/Python grandchildren, so the timeout path must reap the whole +// process group instead of only the direct CLI child. +fn run_cli_command(command: &mut Command) -> io::Result { + #[cfg(unix)] + command.process_group(0); + + command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let child = command.spawn()?; + let _cleanup = ChildProcessCleanupGuard(child.id()); + let (sender, receiver) = mpsc::sync_channel(1); + let _waiter = thread::spawn(move || { + let _ = sender.send(child.wait_with_output()); + }); + + match receiver.recv_timeout(CLI_TIMEOUT) { + Ok(output) => output, + Err(mpsc::RecvTimeoutError::Timeout) => { + Err(io::Error::new(io::ErrorKind::TimedOut, "process timed out")) + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + Err(io::Error::other("process output reader thread exited")) + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn responses_mode_stream_cli_supports_personal_access_tokens() { skip_if_no_network!(); @@ -90,9 +153,8 @@ async fn responses_mode_stream_cli_supports_personal_access_tokens() { let resp_mock = responses::mount_sse_once(&server, cli_sse_response()).await; let home = TempDir::new().unwrap(); - let output = personal_access_token_exec_command(&server, &home) - .output() - .unwrap(); + let mut cmd = personal_access_token_exec_command(&server, &home); + let output = run_cli_command(&mut cmd).unwrap(); assert!( output.status.success(), @@ -140,9 +202,8 @@ async fn responses_mode_stream_cli_does_not_attempt_oauth_refresh_for_personal_a .await; let home = TempDir::new().unwrap(); - let output = personal_access_token_exec_command(&server, &home) - .output() - .unwrap(); + let mut cmd = personal_access_token_exec_command(&server, &home); + let output = run_cli_command(&mut cmd).unwrap(); assert!(!output.status.success()); server.verify().await; @@ -168,8 +229,7 @@ async fn responses_mode_stream_cli() { server.uri() ); let bin = codex_utils_cargo_bin::cargo_bin("codex").unwrap(); - let mut cmd = AssertCommand::new(bin); - cmd.timeout(Duration::from_secs(30)); + let mut cmd = Command::new(bin); cmd.arg("exec") .arg("--skip-git-repo-check") .arg("-c") @@ -182,7 +242,7 @@ async fn responses_mode_stream_cli() { cmd.env("CODEX_HOME", home.path()) .env("OPENAI_API_KEY", "dummy"); - let output = cmd.output().unwrap(); + let output = run_cli_command(&mut cmd).unwrap(); println!("Status: {}", output.status); println!("Stdout:\n{}", String::from_utf8_lossy(&output.stdout)); println!("Stderr:\n{}", String::from_utf8_lossy(&output.stderr)); @@ -211,8 +271,7 @@ async fn responses_mode_stream_cli_supports_openai_base_url_config_override() { let home = TempDir::new().unwrap(); let bin = codex_utils_cargo_bin::cargo_bin("codex").unwrap(); - let mut cmd = AssertCommand::new(bin); - cmd.timeout(Duration::from_secs(30)); + let mut cmd = Command::new(bin); cmd.arg("exec") .arg("--skip-git-repo-check") .arg("-c") @@ -223,7 +282,7 @@ async fn responses_mode_stream_cli_supports_openai_base_url_config_override() { cmd.env("CODEX_HOME", home.path()) .env("OPENAI_API_KEY", "dummy"); - let output = cmd.output().unwrap(); + let output = run_cli_command(&mut cmd).unwrap(); assert!(output.status.success()); let request = resp_mock.single_request(); @@ -264,7 +323,7 @@ async fn exec_cli_applies_model_instructions_file() { let home = TempDir::new().unwrap(); let repo_root = repo_root(); let bin = codex_utils_cargo_bin::cargo_bin("codex").unwrap(); - let mut cmd = AssertCommand::new(bin); + let mut cmd = Command::new(bin); cmd.arg("exec") .arg("--skip-git-repo-check") .arg("-c") @@ -279,7 +338,7 @@ async fn exec_cli_applies_model_instructions_file() { cmd.env("CODEX_HOME", home.path()) .env("OPENAI_API_KEY", "dummy"); - let output = cmd.output().unwrap(); + let output = run_cli_command(&mut cmd).unwrap(); println!("Status: {}", output.status); println!("Stdout:\n{}", String::from_utf8_lossy(&output.stdout)); println!("Stderr:\n{}", String::from_utf8_lossy(&output.stderr)); @@ -334,7 +393,7 @@ async fn exec_cli_profile_applies_model_instructions_file() { let repo_root = repo_root(); let bin = codex_utils_cargo_bin::cargo_bin("codex").unwrap(); - let mut cmd = AssertCommand::new(bin); + let mut cmd = Command::new(bin); cmd.arg("exec") .arg("--skip-git-repo-check") .arg("--profile") @@ -349,7 +408,7 @@ async fn exec_cli_profile_applies_model_instructions_file() { cmd.env("CODEX_HOME", home.path()) .env("OPENAI_API_KEY", "dummy"); - let output = cmd.output().unwrap(); + let output = run_cli_command(&mut cmd).unwrap(); println!("Status: {}", output.status); println!("Stdout:\n{}", String::from_utf8_lossy(&output.stdout)); println!("Stderr:\n{}", String::from_utf8_lossy(&output.stderr)); @@ -379,8 +438,7 @@ async fn responses_api_stream_cli() { let home = TempDir::new().unwrap(); let bin = codex_utils_cargo_bin::cargo_bin("codex").unwrap(); - let mut cmd = AssertCommand::new(bin); - cmd.timeout(Duration::from_secs(30)); + let mut cmd = Command::new(bin); cmd.arg("exec") .arg("--skip-git-repo-check") .arg("-c") @@ -391,7 +449,7 @@ async fn responses_api_stream_cli() { cmd.env("CODEX_HOME", home.path()) .env("OPENAI_API_KEY", "dummy"); - let output = cmd.output().unwrap(); + let output = run_cli_command(&mut cmd).unwrap(); assert!(output.status.success()); let stdout = String::from_utf8_lossy(&output.stdout); assert!(stdout.contains("fixture hello")); @@ -421,8 +479,7 @@ async fn integration_creates_and_checks_session_file() -> anyhow::Result<()> { // 4. Run the codex CLI and invoke `exec`, which is what records a session. let bin = codex_utils_cargo_bin::cargo_bin("codex").unwrap(); - let mut cmd = AssertCommand::new(bin); - cmd.timeout(Duration::from_secs(30)); + let mut cmd = Command::new(bin); cmd.arg("exec") .arg("--skip-git-repo-check") .arg("-c") @@ -433,7 +490,7 @@ async fn integration_creates_and_checks_session_file() -> anyhow::Result<()> { cmd.env("CODEX_HOME", home.path()) .env(CODEX_API_KEY_ENV_VAR, "dummy"); - let output = cmd.output().unwrap(); + let output = run_cli_command(&mut cmd).unwrap(); assert!( output.status.success(), "codex-cli exec failed: {}", @@ -542,8 +599,7 @@ async fn integration_creates_and_checks_session_file() -> anyhow::Result<()> { let marker2 = format!("integration-resume-{}", Uuid::new_v4()); let prompt2 = format!("echo {marker2}"); let bin2 = codex_utils_cargo_bin::cargo_bin("codex").unwrap(); - let mut cmd2 = AssertCommand::new(bin2); - cmd2.timeout(Duration::from_secs(30)); + let mut cmd2 = Command::new(bin2); cmd2.arg("exec") .arg("--skip-git-repo-check") .arg("-c") @@ -556,7 +612,7 @@ async fn integration_creates_and_checks_session_file() -> anyhow::Result<()> { cmd2.env("CODEX_HOME", home.path()) .env("OPENAI_API_KEY", "dummy"); - let output2 = cmd2.output().unwrap(); + let output2 = run_cli_command(&mut cmd2).unwrap(); assert!(output2.status.success(), "resume codex-cli run failed"); assert_eq!(resp_mock.requests().len(), 2); diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index a522e041c2..cf8d2ec17d 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -1,6 +1,5 @@ use codex_config::ConfigLayerStack; use codex_config::types::AuthCredentialsStoreMode; -use codex_core::LoadedAgentsMd; use codex_core::ModelClient; use codex_core::NewThread; use codex_core::Prompt; @@ -50,6 +49,7 @@ use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_protocol::user_input::UserInput; use core_test_support::PathBufExt; +use core_test_support::TestCodexResponsesRequestKind; use core_test_support::apps_test_server::AppsTestServer; use core_test_support::load_default_config_for_test; use core_test_support::responses::ResponsesRequest; @@ -64,6 +64,7 @@ use core_test_support::responses::mount_sse_once_match; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::responses::sse_failed; +use core_test_support::responses_metadata as test_responses_metadata; use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::local_selections; @@ -91,6 +92,24 @@ use wiremock::matchers::query_param; const INSTALLATION_ID_FILENAME: &str = "installation_id"; const TEST_WINDOW_ID: &str = "test-thread:0"; +const TEST_INSTALLATION_ID: &str = "11111111-1111-4111-8111-111111111111"; + +fn test_turn_responses_metadata( + _client: &ModelClient, + thread_id: ThreadId, +) -> codex_core::CodexResponsesMetadata { + let thread_id = thread_id.to_string(); + test_responses_metadata( + TEST_INSTALLATION_ID, + &thread_id, + &thread_id, + /*turn_id*/ None, + TEST_WINDOW_ID.to_string(), + &SessionSource::Exec, + /*parent_thread_id*/ None, + TestCodexResponsesRequestKind::Turn, + ) +} #[expect(clippy::unwrap_used)] fn assert_message_role(request_body: &serde_json::Value, role: &str) { @@ -114,6 +133,41 @@ fn message_input_text_contains(request: &ResponsesRequest, role: &str, needle: & .any(|text| text.contains(needle)) } +fn assert_codex_client_metadata( + request_body: &serde_json::Value, + installation_id: &str, + session_id: &str, + thread_id: &str, +) { + let client_metadata = &request_body["client_metadata"]; + assert_eq!( + client_metadata["x-codex-installation-id"].as_str(), + Some(installation_id) + ); + assert_eq!(client_metadata["session_id"].as_str(), Some(session_id)); + assert_eq!(client_metadata["thread_id"].as_str(), Some(thread_id)); + let Some(turn_metadata_str) = client_metadata["x-codex-turn-metadata"].as_str() else { + panic!("missing x-codex-turn-metadata client metadata"); + }; + let Ok(turn_metadata) = serde_json::from_str::(turn_metadata_str) else { + panic!("invalid x-codex-turn-metadata json"); + }; + assert_eq!( + turn_metadata["installation_id"].as_str(), + Some(installation_id) + ); + assert_eq!(turn_metadata["session_id"].as_str(), Some(session_id)); + assert_eq!(turn_metadata["thread_id"].as_str(), Some(thread_id)); + assert_eq!( + client_metadata["turn_id"].as_str(), + turn_metadata["turn_id"].as_str() + ); + assert_eq!( + client_metadata["x-codex-window-id"].as_str(), + turn_metadata["window_id"].as_str() + ); +} + /// Writes an `auth.json` into the provided `codex_home` with the specified parameters. /// Returns the fake JWT string written to `tokens.id_token`. #[expect(clippy::unwrap_used)] @@ -365,9 +419,8 @@ async fn resume_includes_initial_messages_and_sends_prior_items() { let codex_home = Arc::new(TempDir::new().unwrap()); let mut builder = test_codex() .with_home(codex_home.clone()) - .with_config(|config| { - // Ensure user instructions are NOT delivered on resume. - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing("be nice")); + .with_pre_build_hook(|home| { + std::fs::write(home.join("AGENTS.md"), "be nice").expect("write global instructions"); }); let test = builder .resume(&server, codex_home, session_path.clone()) @@ -782,9 +835,10 @@ async fn includes_session_id_thread_id_and_model_headers_in_request() { let installation_id = std::fs::read_to_string(test.codex_home_path().join(INSTALLATION_ID_FILENAME)) .expect("read installation id"); + let session_id_string = expected_session_id.to_string(); let thread_id_string = expected_thread_id.to_string(); - assert_eq!(request_session_id, expected_session_id.to_string()); + assert_eq!(request_session_id, session_id_string.as_str()); assert_eq!(request_thread_id, thread_id_string.as_str()); assert_eq!(request_originator, originator().value); assert_eq!(request_authorization, "Bearer Test API Key"); @@ -792,9 +846,11 @@ async fn includes_session_id_thread_id_and_model_headers_in_request() { request_body["prompt_cache_key"].as_str(), Some(thread_id_string.as_str()) ); - assert_eq!( - request_body["client_metadata"]["x-codex-installation-id"].as_str(), - Some(installation_id.as_str()) + assert_codex_client_metadata( + &request_body, + installation_id.as_str(), + session_id_string.as_str(), + thread_id_string.as_str(), ); } @@ -900,18 +956,16 @@ async fn send_provider_auth_request(server: &MockServer, auth: ModelProviderAuth Some(AuthManager::from_auth_for_testing(CodexAuth::from_api_key( "unused-api-key", ))), - thread_id.into(), thread_id, - /*installation_id*/ "11111111-1111-4111-8111-111111111111".to_string(), provider, SessionSource::Exec, - /*parent_thread_id*/ None, config.model_verbosity, /*enable_request_compression*/ false, /*include_timing_metrics*/ false, /*beta_features_header*/ None, /*attestation_provider*/ None, ); + let responses_metadata = test_turn_responses_metadata(&client, thread_id); let mut client_session = client.new_session(); let mut prompt = Prompt::default(); prompt.input.push(ResponseItem::Message { @@ -925,14 +979,13 @@ async fn send_provider_auth_request(server: &MockServer, auth: ModelProviderAuth let mut stream = client_session .stream( - TEST_WINDOW_ID, &prompt, &model_info, &session_telemetry, effort, summary.unwrap_or(ReasoningSummary::Auto), /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -1056,15 +1109,19 @@ async fn chatgpt_auth_sends_correct_request() { let installation_id = std::fs::read_to_string(test.codex_home_path().join(INSTALLATION_ID_FILENAME)) .expect("read installation id"); - assert_eq!(request_session_id, expected_session_id.to_string()); - assert_eq!(request_thread_id, expected_thread_id.to_string()); + let session_id_string = expected_session_id.to_string(); + let thread_id_string = expected_thread_id.to_string(); + assert_eq!(request_session_id, session_id_string.as_str()); + assert_eq!(request_thread_id, thread_id_string.as_str()); assert_eq!(request_originator, originator().value); assert_eq!(request_authorization, "Bearer Access Token"); assert_eq!(request_chatgpt_account_id, "account_id"); - assert_eq!( - request_body["client_metadata"]["x-codex-installation-id"].as_str(), - Some(installation_id.as_str()) + assert_codex_client_metadata( + &request_body, + installation_id.as_str(), + session_id_string.as_str(), + thread_id_string.as_str(), ); assert!(request_body["stream"].as_bool().unwrap()); assert_eq!( @@ -1137,6 +1194,7 @@ async fn prefers_apikey_when_config_prefers_apikey_even_with_chatgpt_tokens() { SessionSource::Exec, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), empty_extension_registry(), + Arc::new(codex_core::test_support::EmptyUserInstructionsProvider), /*analytics_events_client*/ None, thread_store_from_config(&config, /*state_db*/ None), /*state_db*/ None, @@ -1178,8 +1236,8 @@ async fn includes_user_instructions_message_in_request() { let mut builder = test_codex() .with_auth(CodexAuth::from_api_key("Test API Key")) - .with_config(|config| { - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing("be nice")); + .with_pre_build_hook(|home| { + std::fs::write(home.join("AGENTS.md"), "be nice").expect("write global instructions"); }); let codex = builder .build(&server) @@ -2246,8 +2304,10 @@ async fn includes_developer_instructions_message_in_request() { .await; let mut builder = test_codex() .with_auth(CodexAuth::from_api_key("Test API Key")) + .with_pre_build_hook(|home| { + std::fs::write(home.join("AGENTS.md"), "be nice").expect("write global instructions"); + }) .with_config(|config| { - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing("be nice")); config.developer_instructions = Some("be useful".to_string()); }); let codex = builder @@ -2390,18 +2450,16 @@ async fn azure_responses_request_includes_store_and_reasoning_ids() { let client = ModelClient::new( /*auth_manager*/ None, - thread_id.into(), thread_id, - /*installation_id*/ "11111111-1111-4111-8111-111111111111".to_string(), provider.clone(), SessionSource::Exec, - /*parent_thread_id*/ None, config.model_verbosity, /*enable_request_compression*/ false, /*include_timing_metrics*/ false, /*beta_features_header*/ None, /*attestation_provider*/ None, ); + let responses_metadata = test_turn_responses_metadata(&client, thread_id); let mut client_session = client.new_session(); let mut prompt = Prompt::default(); @@ -2469,14 +2527,13 @@ async fn azure_responses_request_includes_store_and_reasoning_ids() { let mut stream = client_session .stream( - TEST_WINDOW_ID, &prompt, &model_info, &session_telemetry, effort, summary.unwrap_or(ReasoningSummary::Auto), /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await diff --git a/codex-rs/core/tests/suite/client_websockets.rs b/codex-rs/core/tests/suite/client_websockets.rs index 8b135b6b75..4aac3dcbd8 100755 --- a/codex-rs/core/tests/suite/client_websockets.rs +++ b/codex-rs/core/tests/suite/client_websockets.rs @@ -1,6 +1,7 @@ #![allow(clippy::expect_used, clippy::unwrap_used)] use codex_api::WS_REQUEST_HEADER_TRACEPARENT_CLIENT_METADATA_KEY; use codex_api::WS_REQUEST_HEADER_TRACESTATE_CLIENT_METADATA_KEY; +use codex_core::CodexResponsesMetadata; use codex_core::ModelClient; use codex_core::ModelClientSession; use codex_core::Prompt; @@ -35,6 +36,7 @@ use codex_rollout_trace::InferenceTraceContext; use codex_rollout_trace::RawTraceEventPayload; use codex_rollout_trace::TraceWriter; use codex_rollout_trace::replay_bundle; +use core_test_support::TestCodexResponsesRequestKind; use core_test_support::load_default_config_for_test; use core_test_support::responses::WebSocketConnectionConfig; use core_test_support::responses::WebSocketTestServer; @@ -43,6 +45,7 @@ use core_test_support::responses::ev_completed; use core_test_support::responses::ev_response_created; use core_test_support::responses::start_websocket_server; use core_test_support::responses::start_websocket_server_with_headers; +use core_test_support::responses_metadata as test_responses_metadata; use core_test_support::skip_if_no_network; use core_test_support::test_codex::test_codex; use core_test_support::tracing::install_test_tracing; @@ -106,6 +109,42 @@ struct WebsocketTestHarness { session_telemetry: SessionTelemetry, } +fn responses_metadata( + harness: &WebsocketTestHarness, + turn_id: Option<&str>, + request_kind: TestCodexResponsesRequestKind, +) -> CodexResponsesMetadata { + test_responses_metadata( + TEST_INSTALLATION_ID, + &harness.session_id.to_string(), + &harness.thread_id.to_string(), + turn_id, + TEST_WINDOW_ID.to_string(), + &SessionSource::Exec, + /*parent_thread_id*/ None, + request_kind, + ) +} + +fn turn_metadata(harness: &WebsocketTestHarness, turn_id: Option<&str>) -> CodexResponsesMetadata { + responses_metadata(harness, turn_id, TestCodexResponsesRequestKind::Turn) +} + +fn prewarm_metadata( + harness: &WebsocketTestHarness, + turn_id: Option<&str>, +) -> CodexResponsesMetadata { + responses_metadata(harness, turn_id, TestCodexResponsesRequestKind::Prewarm) +} + +fn websocket_connection_metadata(harness: &WebsocketTestHarness) -> CodexResponsesMetadata { + responses_metadata( + harness, + /*turn_id*/ None, + TestCodexResponsesRequestKind::WebsocketConnection, + ) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn responses_websocket_streams_request() { skip_if_no_network!(); @@ -274,8 +313,9 @@ async fn responses_websocket_preconnect_does_not_replace_turn_trace_payload() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); + let responses_metadata = websocket_connection_metadata(&harness); client_session - .preconnect_websocket(&harness.session_telemetry) + .preconnect_websocket(&harness.session_telemetry, &responses_metadata) .await .expect("websocket preconnect failed"); let prompt = prompt_with_input(vec![message_item("hello")]); @@ -310,8 +350,9 @@ async fn responses_websocket_preconnect_reuses_connection() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); + let responses_metadata = websocket_connection_metadata(&harness); client_session - .preconnect_websocket(&harness.session_telemetry) + .preconnect_websocket(&harness.session_telemetry, &responses_metadata) .await .expect("websocket preconnect failed"); let prompt = prompt_with_input(vec![message_item("hello")]); @@ -322,7 +363,10 @@ async fn responses_websocket_preconnect_reuses_connection() { server.single_handshake().header(USER_AGENT_HEADER), Some(codex_login::default_client::get_codex_user_agent()) ); - assert_eq!(server.single_handshake().header("x-codex-window-id"), None); + assert_eq!( + server.single_handshake().header("x-codex-window-id"), + Some(TEST_WINDOW_ID.to_string()) + ); let connection = server.single_connection(); assert_eq!(connection.len(), 1); @@ -342,16 +386,16 @@ async fn responses_websocket_request_prewarm_reuses_connection() { let harness = websocket_harness_with_options(&server, /*runtime_metrics_enabled*/ true).await; let mut client_session = harness.client.new_session(); let prompt = prompt_with_input(vec![message_item("hello")]); + let responses_metadata = prewarm_metadata(&harness, /*turn_id*/ None); client_session .prewarm_websocket( - TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, ) .await .expect("websocket prewarm failed"); @@ -376,6 +420,16 @@ async fn responses_websocket_request_prewarm_reuses_connection() { assert_eq!(warmup["type"].as_str(), Some("response.create")); assert_eq!(warmup["generate"].as_bool(), Some(false)); assert_eq!(warmup["tools"], serde_json::json!([])); + let warmup_turn_metadata: serde_json::Value = serde_json::from_str( + warmup["client_metadata"]["x-codex-turn-metadata"] + .as_str() + .expect("warmup turn metadata"), + ) + .expect("valid warmup turn metadata"); + assert_eq!( + warmup_turn_metadata["request_kind"].as_str(), + Some("prewarm") + ); assert_eq!(follow_up["type"].as_str(), Some("response.create")); assert_eq!(follow_up["previous_response_id"].as_str(), Some("warm-1")); assert_eq!(follow_up["input"], serde_json::json!([])); @@ -383,6 +437,49 @@ async fn responses_websocket_request_prewarm_reuses_connection() { server.shutdown().await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn responses_websocket_request_prewarm_uses_caller_supplied_metadata() { + skip_if_no_network!(); + + let server = start_websocket_server(vec![vec![vec![ + ev_response_created("warm-1"), + ev_completed("warm-1"), + ]]]) + .await; + + let harness = websocket_harness_with_options(&server, /*runtime_metrics_enabled*/ true).await; + let mut client_session = harness.client.new_session(); + let prompt = prompt_with_input(vec![message_item("hello")]); + let responses_metadata = turn_metadata(&harness, /*turn_id*/ None); + client_session + .prewarm_websocket( + &prompt, + &harness.model_info, + &harness.session_telemetry, + harness.effort.clone(), + harness.summary, + /*service_tier*/ None, + &responses_metadata, + ) + .await + .expect("websocket prewarm failed"); + + let warmup = server + .single_connection() + .first() + .expect("missing warmup request") + .body_json(); + let warmup_turn_metadata: serde_json::Value = serde_json::from_str( + warmup["client_metadata"]["x-codex-turn-metadata"] + .as_str() + .expect("warmup turn metadata"), + ) + .expect("valid warmup turn metadata"); + assert_eq!(warmup_turn_metadata["request_kind"].as_str(), Some("turn")); + + server.shutdown().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn responses_websocket_request_prewarm_traces_logical_request() { skip_if_no_network!(); @@ -396,17 +493,17 @@ async fn responses_websocket_request_prewarm_traces_logical_request() { let harness = websocket_harness_with_options(&server, /*runtime_metrics_enabled*/ true).await; let mut client_session = harness.client.new_session(); let prompt = prompt_with_input(vec![message_item("hello")]); + let prewarm_responses_metadata = prewarm_metadata(&harness, /*turn_id*/ None); client_session .prewarm_websocket( - TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &prewarm_responses_metadata, ) .await .expect("websocket prewarm failed"); @@ -443,16 +540,16 @@ async fn responses_websocket_request_prewarm_traces_logical_request() { "test-provider".to_string(), ); + let responses_metadata = turn_metadata(&harness, /*turn_id*/ None); let mut stream = client_session .stream( - TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &inference_trace, ) .await @@ -610,21 +707,22 @@ async fn responses_websocket_preconnect_is_reused_even_with_header_changes() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); + let preconnect_metadata = websocket_connection_metadata(&harness); client_session - .preconnect_websocket(&harness.session_telemetry) + .preconnect_websocket(&harness.session_telemetry, &preconnect_metadata) .await .expect("websocket preconnect failed"); let prompt = prompt_with_input(vec![message_item("hello")]); + let responses_metadata = turn_metadata(&harness, /*turn_id*/ None); let mut stream = client_session .stream( - TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -655,29 +753,29 @@ async fn responses_websocket_request_prewarm_is_reused_even_with_header_changes( let harness = websocket_harness_with_options(&server, /*runtime_metrics_enabled*/ true).await; let mut client_session = harness.client.new_session(); let prompt = prompt_with_input(vec![message_item("hello")]); + let prewarm_responses_metadata = prewarm_metadata(&harness, /*turn_id*/ None); client_session .prewarm_websocket( - TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &prewarm_responses_metadata, ) .await .expect("websocket prewarm failed"); + let responses_metadata = turn_metadata(&harness, /*turn_id*/ None); let mut stream = client_session .stream( - TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -723,16 +821,16 @@ async fn responses_websocket_prewarm_uses_v2_when_provider_supports_websockets() let harness = websocket_harness_with_options(&server, /*runtime_metrics_enabled*/ false).await; let mut client_session = harness.client.new_session(); let prompt = prompt_with_input(vec![message_item("hello")]); + let responses_metadata = prewarm_metadata(&harness, /*turn_id*/ None); client_session .prewarm_websocket( - TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, ) .await .expect("websocket prewarm failed"); @@ -780,13 +878,18 @@ async fn responses_websocket_preconnect_runs_when_only_v2_feature_enabled() { let harness = websocket_harness_with_options(&server, /*runtime_metrics_enabled*/ true).await; let mut client_session = harness.client.new_session(); + let responses_metadata = websocket_connection_metadata(&harness); client_session - .preconnect_websocket(&harness.session_telemetry) + .preconnect_websocket(&harness.session_telemetry, &responses_metadata) .await .expect("websocket preconnect failed"); assert_eq!(server.handshakes().len(), 1); assert_eq!(server.single_connection().len(), 0); + assert_eq!( + server.single_handshake().header("x-codex-turn-metadata"), + None + ); let prompt = prompt_with_input(vec![message_item("hello")]); stream_until_complete(&mut client_session, &harness, &prompt).await; @@ -1072,17 +1175,17 @@ async fn responses_websocket_emits_reasoning_included_event() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); let prompt = prompt_with_input(vec![message_item("hello")]); + let responses_metadata = turn_metadata(&harness, /*turn_id*/ None); let mut stream = client_session .stream( - TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -1147,17 +1250,17 @@ async fn responses_websocket_emits_rate_limit_events() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); let prompt = prompt_with_input(vec![message_item("hello")]); + let responses_metadata = turn_metadata(&harness, /*turn_id*/ None); let mut stream = client_session .stream( - TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -1471,30 +1574,29 @@ async fn responses_websocket_forwards_turn_metadata_on_initial_and_incremental_c let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); - let first_turn_metadata = - r#"{"turn_id":"turn-123","thread_source":"user","sandbox":"workspace-write"}"#; - let enriched_turn_metadata = r#"{"turn_id":"turn-123","thread_source":"user","sandbox":"workspace-write","workspaces":[{"root_path":"/tmp/repo","latest_git_commit_hash":"abc123","associated_remote_urls":["git@github.com:openai/codex.git"],"has_changes":true}]}"#; let prompt_one = prompt_with_input(vec![message_item("hello")]); let prompt_two = prompt_with_input(vec![ message_item("hello"), assistant_message_item("msg-1", "assistant output"), message_item("second"), ]); + let first_responses_metadata = turn_metadata(&harness, Some("turn-123")); + let second_responses_metadata = turn_metadata(&harness, Some("turn-456")); - stream_until_complete_with_turn_metadata( + stream_until_complete_with_metadata( &mut client_session, &harness, &prompt_one, /*service_tier*/ None, - Some(first_turn_metadata), + &first_responses_metadata, ) .await; - stream_until_complete_with_turn_metadata( + stream_until_complete_with_metadata( &mut client_session, &harness, &prompt_two, /*service_tier*/ None, - Some(enriched_turn_metadata), + &second_responses_metadata, ) .await; @@ -1504,36 +1606,37 @@ async fn responses_websocket_forwards_turn_metadata_on_initial_and_incremental_c let second = connection.get(1).expect("missing request").body_json(); assert_eq!(first["type"].as_str(), Some("response.create")); - assert_eq!( - first["client_metadata"]["x-codex-turn-metadata"].as_str(), - Some(first_turn_metadata) - ); assert_eq!(second["type"].as_str(), Some("response.create")); assert_eq!(second["previous_response_id"].as_str(), Some("resp-1")); - assert_eq!( - second["client_metadata"]["x-codex-turn-metadata"].as_str(), - Some(enriched_turn_metadata) - ); - - let first_metadata: serde_json::Value = - serde_json::from_str(first_turn_metadata).expect("first metadata should be valid json"); - let second_metadata: serde_json::Value = serde_json::from_str(enriched_turn_metadata) - .expect("enriched metadata should be valid json"); + let first_metadata: serde_json::Value = serde_json::from_str( + first["client_metadata"]["x-codex-turn-metadata"] + .as_str() + .expect("first turn metadata"), + ) + .expect("first metadata should be valid json"); + let second_metadata: serde_json::Value = serde_json::from_str( + second["client_metadata"]["x-codex-turn-metadata"] + .as_str() + .expect("second turn metadata"), + ) + .expect("second metadata should be valid json"); assert_eq!(first_metadata["turn_id"].as_str(), Some("turn-123")); - assert_eq!(second_metadata["turn_id"].as_str(), Some("turn-123")); - assert_eq!(first_metadata["thread_source"].as_str(), Some("user")); - assert_eq!(second_metadata["thread_source"].as_str(), Some("user")); + assert_eq!(second_metadata["turn_id"].as_str(), Some("turn-456")); assert_eq!( - second_metadata["workspaces"][0]["has_changes"].as_bool(), - Some(true) + first["client_metadata"]["turn_id"].as_str(), + first_metadata["turn_id"].as_str() + ); + assert_eq!( + second["client_metadata"]["turn_id"].as_str(), + second_metadata["turn_id"].as_str() ); server.shutdown().await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn responses_websocket_preserves_custom_turn_metadata_fields() { +async fn responses_websocket_sends_canonical_turn_metadata() { skip_if_no_network!(); let server = start_websocket_server(vec![vec![vec![ @@ -1545,19 +1648,14 @@ async fn responses_websocket_preserves_custom_turn_metadata_fields() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); let prompt = prompt_with_input(vec![message_item("hello")]); - let turn_metadata = json!({ - "turn_id": "turn-123", - "fiber_run_id": "fiber-123", - "origin": "app-server", - }) - .to_string(); + let responses_metadata = turn_metadata(&harness, Some("turn-123")); - stream_until_complete_with_turn_metadata( + stream_until_complete_with_metadata( &mut client_session, &harness, &prompt, /*service_tier*/ None, - Some(&turn_metadata), + &responses_metadata, ) .await; @@ -1568,15 +1666,16 @@ async fn responses_websocket_preserves_custom_turn_metadata_fields() { .body_json(); assert_eq!(body["type"].as_str(), Some("response.create")); - assert_eq!( + let turn_metadata: serde_json::Value = serde_json::from_str( body["client_metadata"]["x-codex-turn-metadata"] .as_str() - .map(|value| serde_json::from_str::(value).expect("valid json")), - Some(json!({ - "turn_id": "turn-123", - "fiber_run_id": "fiber-123", - "origin": "app-server", - })) + .expect("turn metadata"), + ) + .expect("valid turn metadata"); + assert_eq!(turn_metadata["turn_id"].as_str(), Some("turn-123")); + assert_eq!( + body["client_metadata"]["turn_id"].as_str(), + turn_metadata["turn_id"].as_str() ); server.shutdown().await; @@ -1803,16 +1902,16 @@ async fn responses_websocket_v2_after_error_uses_full_create_without_previous_re stream_until_complete(&mut session, &harness, &prompt_one).await; + let responses_metadata = turn_metadata(&harness, /*turn_id*/ None); let mut second_stream = session .stream( - TEST_WINDOW_ID, &prompt_two, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -1892,16 +1991,16 @@ async fn responses_websocket_v2_surfaces_terminal_error_without_close_handshake( stream_until_complete(&mut session, &harness, &prompt_one).await; + let responses_metadata = turn_metadata(&harness, /*turn_id*/ None); let mut second_stream = session .stream( - TEST_WINDOW_ID, &prompt_two, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -2081,12 +2180,9 @@ async fn websocket_harness_with_provider_options( let summary = ReasoningSummary::Auto; let client = ModelClient::new( /*auth_manager*/ None, - session_id, thread_id, - /*installation_id*/ TEST_INSTALLATION_ID.to_string(), provider.clone(), SessionSource::Exec, - /*parent_thread_id*/ None, config.model_verbosity, /*enable_request_compression*/ false, runtime_metrics_enabled, @@ -2127,16 +2223,16 @@ async fn stream_until_complete_with_model_info( model_info: &ModelInfo, expected_response_id: &str, ) { + let responses_metadata = turn_metadata(harness, /*turn_id*/ None); let mut stream = client_session .stream( - TEST_WINDOW_ID, prompt, model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, /*service_tier*/ None, - /*turn_metadata_header*/ None, + &responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await @@ -2161,50 +2257,33 @@ async fn stream_until_complete_with_service_tier( prompt: &Prompt, service_tier: Option, ) { - stream_until_complete_with_turn_metadata( + let responses_metadata = turn_metadata(harness, /*turn_id*/ None); + stream_until_complete_with_metadata( client_session, harness, prompt, service_tier, - /*turn_metadata_header*/ None, + &responses_metadata, ) .await; } -async fn stream_until_complete_with_turn_metadata( +async fn stream_until_complete_with_metadata( client_session: &mut ModelClientSession, harness: &WebsocketTestHarness, prompt: &Prompt, service_tier: Option, - turn_metadata_header: Option<&str>, -) { - stream_until_complete_with_request_metadata( - client_session, - harness, - prompt, - service_tier, - turn_metadata_header, - ) - .await; -} - -async fn stream_until_complete_with_request_metadata( - client_session: &mut ModelClientSession, - harness: &WebsocketTestHarness, - prompt: &Prompt, - service_tier: Option, - turn_metadata_header: Option<&str>, + responses_metadata: &CodexResponsesMetadata, ) { let mut stream = client_session .stream( - TEST_WINDOW_ID, prompt, &harness.model_info, &harness.session_telemetry, harness.effort.clone(), harness.summary, service_tier.map(|service_tier| service_tier.request_value().to_string()), - turn_metadata_header, + responses_metadata, &codex_rollout_trace::InferenceTraceContext::disabled(), ) .await diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index e1b02292ef..fdd87274a7 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -42,11 +42,16 @@ use core_test_support::test_codex::turn_permission_fields; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; use core_test_support::wait_for_mcp_server; +use image::DynamicImage; +use image::GenericImageView; +use image::ImageBuffer; +use image::Rgba; use pretty_assertions::assert_eq; use serde_json::Value; use std::collections::HashMap; use std::collections::HashSet; use std::fs; +use std::io::Cursor; use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -168,12 +173,21 @@ async fn run_code_mode_turn_with_config( code: &str, configure: impl FnOnce(&mut Config) + Send + 'static, ) -> Result<(TestCodex, ResponseMock)> { - let mut builder = test_codex() - .with_model("test-gpt-5.1-codex") - .with_config(move |config| { - let _ = config.features.enable(Feature::CodeMode); - configure(config); - }); + run_code_mode_turn_with_model_and_config(server, prompt, code, "test-gpt-5.1-codex", configure) + .await +} + +async fn run_code_mode_turn_with_model_and_config( + server: &MockServer, + prompt: &str, + code: &str, + model: &'static str, + configure: impl FnOnce(&mut Config) + Send + 'static, +) -> Result<(TestCodex, ResponseMock)> { + let mut builder = test_codex().with_model(model).with_config(move |config| { + let _ = config.features.enable(Feature::CodeMode); + configure(config); + }); let test = builder.build(server).await?; responses::mount_sse_once( @@ -2536,6 +2550,103 @@ image("data:image/png;base64,AAA"); Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resize_all_images_replaces_malformed_code_mode_image_only() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let (_test, second_mock) = run_code_mode_turn_with_config( + &server, + "use exec to return images", + r#" +image("https://example.com/image.jpg"); +image("data:image/png;base64,AAA"); +"#, + |config| { + let _ = config.features.enable(Feature::ResizeAllImages); + }, + ) + .await?; + + let req = second_mock.single_request(); + let items = custom_tool_output_items(&req, "call-1"); + let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); + assert_ne!(success, Some(false)); + assert_eq!(items.len(), 3); + assert_eq!( + items[1], + serde_json::json!({ + "type": "input_image", + "image_url": "https://example.com/image.jpg", + "detail": "high" + }) + ); + assert_eq!( + items[2], + serde_json::json!({ + "type": "input_text", + "text": "image content omitted because it could not be processed" + }) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resize_all_images_resizes_explicit_original_code_mode_image() -> Result<()> { + skip_if_no_network!(Ok(())); + + let original_dimensions = (6401, 100); + let image = ImageBuffer::from_pixel( + original_dimensions.0, + original_dimensions.1, + Rgba([20, 40, 60, 255]), + ); + let mut encoded = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?; + let image_data_url = format!( + "data:image/png;base64,{}", + BASE64_STANDARD.encode(encoded.into_inner()) + ); + let code = format!( + "image({}, \"original\");", + serde_json::to_string(&image_data_url)? + ); + + let server = responses::start_mock_server().await; + let (_test, second_mock) = run_code_mode_turn_with_model_and_config( + &server, + "use exec to return a large original-detail image", + &code, + "gpt-5.3-codex", + |config| { + config + .features + .enable(Feature::ResizeAllImages) + .expect("resize_all_images should be enabled"); + }, + ) + .await?; + + let req = second_mock.single_request(); + let items = custom_tool_output_items(&req, "call-1"); + let (_, success) = custom_tool_output_body_and_success(&req, "call-1"); + assert_ne!(success, Some(false)); + let resized_url = items[1]["image_url"] + .as_str() + .expect("code mode image output should contain a data URL"); + assert_eq!(items[1]["detail"], "original"); + let (_, resized_base64) = resized_url + .split_once(',') + .expect("resized image should contain a data URL prefix"); + let resized_bytes = BASE64_STANDARD.decode(resized_base64)?; + let resized = image::load_from_memory(&resized_bytes)?; + let resized_dimensions = resized.dimensions(); + assert_eq!(resized_dimensions, (6000, 94)); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 339cadda4e..73bc05df43 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -376,6 +376,12 @@ fn model_info_with_context_window(slug: &str, context_window: i64) -> ModelInfo model_info } +fn model_info_with_optional_comp_hash(slug: &str, comp_hash: Option<&str>) -> ModelInfo { + let mut model_info = model_info_with_context_window(slug, /*context_window*/ 273_000); + model_info.comp_hash = comp_hash.map(str::to_string); + model_info +} + fn assert_pre_sampling_switch_compaction_requests( first: &serde_json::Value, compact: &serde_json::Value, @@ -2009,6 +2015,7 @@ async fn auto_compact_runs_after_resume_when_token_usage_is_over_limit() { let mut builder = test_codex().with_config(move |config| { set_test_compact_prompt(config); config.model_auto_compact_token_limit = Some(limit); + let _ = config.features.disable(Feature::RemoteCompactionV2); }); let initial = builder.build(&server).await.unwrap(); let home = initial.home.clone(); @@ -2037,6 +2044,7 @@ async fn auto_compact_runs_after_resume_when_token_usage_is_over_limit() { let mut resume_builder = test_codex().with_config(move |config| { set_test_compact_prompt(config); config.model_auto_compact_token_limit = Some(limit); + let _ = config.features.disable(Feature::RemoteCompactionV2); }); let resumed = resume_builder .resume(&server, home, rollout_path) @@ -2189,6 +2197,203 @@ async fn pre_sampling_compact_runs_on_switch_to_smaller_context_model() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn pre_sampling_compact_runs_when_comp_hash_changes() { + skip_if_no_network!(); + + let server = MockServer::start().await; + let previous_model = "gpt-5.3-codex"; + let next_model = "gpt-5.2"; + + let models_mock = mount_models_once( + &server, + ModelsResponse { + models: vec![ + model_info_with_optional_comp_hash(previous_model, Some("hash-a")), + model_info_with_optional_comp_hash(next_model, Some("hash-b")), + ], + }, + ) + .await; + + let request_log = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_assistant_message("m1", "before switch"), + ev_completed_with_tokens("r1", /*total_tokens*/ 100), + ]), + sse(vec![ + ev_assistant_message("m2", "COMP_HASH_SUMMARY"), + ev_completed_with_tokens("r2", /*total_tokens*/ 10), + ]), + sse(vec![ + ev_assistant_message("m3", "after switch"), + ev_completed_with_tokens("r3", /*total_tokens*/ 100), + ]), + ], + ) + .await; + + let model_provider = non_openai_model_provider(&server); + let mut builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_model(previous_model) + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + }); + let test = builder.build(&server).await.expect("build test codex"); + + test.codex + .submit(disabled_permission_user_turn( + "before switch", + test.cwd.path().to_path_buf(), + previous_model.to_string(), + )) + .await + .expect("submit first user turn"); + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + test.codex + .submit(disabled_permission_user_turn( + "after switch", + test.cwd.path().to_path_buf(), + next_model.to_string(), + )) + .await + .expect("submit second user turn"); + assert_compaction_uses_turn_lifecycle_id(&test.codex).await; + + let requests = request_log.requests(); + assert_eq!(models_mock.requests().len(), 1); + assert_eq!( + requests.len(), + 3, + "a comp-hash change should compact before sampling the next turn" + ); + assert_pre_sampling_switch_compaction_requests( + &requests[0].body_json(), + &requests[1].body_json(), + &requests[2].body_json(), + previous_model, + next_model, + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn pre_sampling_compact_skips_when_either_comp_hash_is_missing() { + skip_if_no_network!(); + + let server = MockServer::start().await; + let model_without_hash = "gpt-5.4"; + let model_with_hash = "gpt-5.3-codex"; + let next_model_without_hash = "gpt-5.2"; + + let models_mock = mount_models_once( + &server, + ModelsResponse { + models: vec![ + model_info_with_optional_comp_hash(model_without_hash, /*comp_hash*/ None), + model_info_with_optional_comp_hash(model_with_hash, Some("hash-a")), + model_info_with_optional_comp_hash( + next_model_without_hash, + /*comp_hash*/ None, + ), + ], + }, + ) + .await; + + let request_log = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_assistant_message("m1", "before hash"), + ev_completed_with_tokens("r1", /*total_tokens*/ 100), + ]), + sse(vec![ + ev_assistant_message("m2", "hash introduced"), + ev_completed_with_tokens("r2", /*total_tokens*/ 100), + ]), + sse(vec![ + ev_assistant_message("m3", "hash removed"), + ev_completed_with_tokens("r3", /*total_tokens*/ 100), + ]), + ], + ) + .await; + + let model_provider = non_openai_model_provider(&server); + let mut builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_model(model_without_hash) + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + }); + let test = builder.build(&server).await.expect("build test codex"); + + test.codex + .submit(disabled_permission_user_turn( + "before hash", + test.cwd.path().to_path_buf(), + model_without_hash.to_string(), + )) + .await + .expect("submit first user turn"); + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + test.codex + .submit(disabled_permission_user_turn( + "hash introduced", + test.cwd.path().to_path_buf(), + model_with_hash.to_string(), + )) + .await + .expect("submit second user turn"); + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + test.codex + .submit(disabled_permission_user_turn( + "hash removed", + test.cwd.path().to_path_buf(), + next_model_without_hash.to_string(), + )) + .await + .expect("submit third user turn"); + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let requests = request_log.requests(); + assert_eq!(models_mock.requests().len(), 1); + assert_eq!( + requests + .iter() + .map(|request| request.body_json()["model"].as_str().map(str::to_string)) + .collect::>(), + vec![ + Some(model_without_hash.to_string()), + Some(model_with_hash.to_string()), + Some(next_model_without_hash.to_string()), + ] + ); + assert!(requests.iter().all(|request| { + !body_contains_text(&request.body_json().to_string(), SUMMARIZATION_PROMPT) + })); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn body_after_prefix_model_switch_budget_compacts_with_next_model() { skip_if_no_network!(); @@ -2404,6 +2609,266 @@ async fn pre_sampling_compact_runs_after_resume_and_switch_to_smaller_model() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn pre_sampling_compact_recovers_comp_hash_after_resume() { + skip_if_no_network!(); + + let server = MockServer::start().await; + let previous_model = "gpt-5.3-codex"; + let next_model = "gpt-5.2"; + + let models_mock = mount_models_once( + &server, + ModelsResponse { + models: vec![ + model_info_with_optional_comp_hash(previous_model, Some("hash-a")), + model_info_with_optional_comp_hash(next_model, Some("hash-b")), + ], + }, + ) + .await; + + let request_log = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_assistant_message("m1", "before resume"), + ev_completed_with_tokens("r1", /*total_tokens*/ 100), + ]), + sse(vec![ + ev_assistant_message("m2", "RESUMED_COMP_HASH_SUMMARY"), + ev_completed_with_tokens("r2", /*total_tokens*/ 10), + ]), + sse(vec![ + ev_assistant_message("m3", "after resume"), + ev_completed_with_tokens("r3", /*total_tokens*/ 100), + ]), + ], + ) + .await; + + let model_provider = non_openai_model_provider(&server); + let mut initial_builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_model(previous_model) + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + }); + let initial = initial_builder + .build(&server) + .await + .expect("build initial test codex"); + let home = initial.home.clone(); + let rollout_path = initial + .session_configured + .rollout_path + .clone() + .expect("rollout path"); + + initial + .codex + .submit(disabled_permission_user_turn( + "before resume", + initial.cwd.path().to_path_buf(), + previous_model.to_string(), + )) + .await + .expect("submit pre-resume turn"); + wait_for_event(&initial.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + initial + .codex + .submit(Op::Shutdown) + .await + .expect("shutdown initial session"); + wait_for_event(&initial.codex, |event| { + matches!(event, EventMsg::ShutdownComplete) + }) + .await; + + let rollout = fs::read_to_string(&rollout_path).expect("read rollout"); + let persisted_comp_hash = rollout + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .find_map(|line| match line.item { + RolloutItem::TurnContext(context) => context.comp_hash, + _ => None, + }); + assert_eq!(persisted_comp_hash.as_deref(), Some("hash-a")); + + let model_provider = non_openai_model_provider(&server); + let mut resumed_builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_model(previous_model) + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + }); + let resumed = resumed_builder + .resume(&server, home, rollout_path) + .await + .expect("resume codex"); + + resumed + .codex + .submit(disabled_permission_user_turn( + "after resume", + resumed.cwd.path().to_path_buf(), + next_model.to_string(), + )) + .await + .expect("submit resumed user turn"); + assert_compaction_uses_turn_lifecycle_id(&resumed.codex).await; + + let requests = request_log.requests(); + assert_eq!(models_mock.requests().len(), 1); + assert_eq!( + requests.len(), + 3, + "the resumed turn should compact using the comp hash recovered from rollout" + ); + assert_pre_sampling_switch_compaction_requests( + &requests[0].body_json(), + &requests[1].body_json(), + &requests[2].body_json(), + previous_model, + next_model, + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn pre_sampling_compact_skips_missing_comp_hash_after_resume() { + skip_if_no_network!(); + + let server = MockServer::start().await; + let previous_model = "gpt-5.3-codex"; + let next_model = "gpt-5.2"; + + let models_mock = mount_models_once( + &server, + ModelsResponse { + models: vec![ + model_info_with_optional_comp_hash(previous_model, /*comp_hash*/ None), + model_info_with_optional_comp_hash(next_model, Some("hash-b")), + ], + }, + ) + .await; + + let request_log = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_assistant_message("m1", "before resume"), + ev_completed_with_tokens("r1", /*total_tokens*/ 100), + ]), + sse(vec![ + ev_assistant_message("m2", "after resume"), + ev_completed_with_tokens("r2", /*total_tokens*/ 100), + ]), + ], + ) + .await; + + let model_provider = non_openai_model_provider(&server); + let mut initial_builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_model(previous_model) + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + }); + let initial = initial_builder + .build(&server) + .await + .expect("build initial test codex"); + let home = initial.home.clone(); + let rollout_path = initial + .session_configured + .rollout_path + .clone() + .expect("rollout path"); + + initial + .codex + .submit(disabled_permission_user_turn( + "before resume", + initial.cwd.path().to_path_buf(), + previous_model.to_string(), + )) + .await + .expect("submit pre-resume turn"); + wait_for_event(&initial.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + initial + .codex + .submit(Op::Shutdown) + .await + .expect("shutdown initial session"); + wait_for_event(&initial.codex, |event| { + matches!(event, EventMsg::ShutdownComplete) + }) + .await; + + let rollout = fs::read_to_string(&rollout_path).expect("read rollout"); + let persisted_turn_context = rollout + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .find(|line| line["type"] == "turn_context") + .expect("persisted turn context"); + assert!(persisted_turn_context["payload"].get("comp_hash").is_none()); + + let model_provider = non_openai_model_provider(&server); + let mut resumed_builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_model(previous_model) + .with_config(move |config| { + config.model_provider = model_provider; + set_test_compact_prompt(config); + }); + let resumed = resumed_builder + .resume(&server, home, rollout_path) + .await + .expect("resume codex"); + + resumed + .codex + .submit(disabled_permission_user_turn( + "after resume", + resumed.cwd.path().to_path_buf(), + next_model.to_string(), + )) + .await + .expect("submit resumed user turn"); + wait_for_event(&resumed.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let requests = request_log.requests(); + assert_eq!(models_mock.requests().len(), 1); + assert_eq!( + requests + .iter() + .map(|request| request.body_json()["model"].as_str().map(str::to_string)) + .collect::>(), + vec![ + Some(previous_model.to_string()), + Some(next_model.to_string()), + ] + ); + assert!(requests.iter().all(|request| { + !body_contains_text(&request.body_json().to_string(), SUMMARIZATION_PROMPT) + })); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn auto_compact_persists_rollout_entries() { skip_if_no_network!(); @@ -3560,6 +4025,7 @@ async fn auto_compact_counts_encrypted_reasoning_before_last_user() { config.chatgpt_base_url = chatgpt_base_url; set_test_compact_prompt(config); config.model_auto_compact_token_limit = Some(300); + let _ = config.features.disable(Feature::RemoteCompactionV2); }) .build(&server) .await @@ -3682,6 +4148,7 @@ async fn auto_compact_runs_when_reasoning_header_clears_between_turns() { .with_config(|config| { set_test_compact_prompt(config); config.model_auto_compact_token_limit = Some(300); + let _ = config.features.disable(Feature::RemoteCompactionV2); }) .build(&server) .await diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index eee445ea9f..a63397c559 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -36,7 +36,7 @@ use core_test_support::responses::start_websocket_server; use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodexBuilder; use core_test_support::test_codex::TestCodexHarness; -use core_test_support::test_codex::test_codex; +use core_test_support::test_codex::test_codex as base_test_codex; use core_test_support::test_path_buf; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; @@ -160,6 +160,12 @@ fn compacted_summary_only_output(summary: &str) -> Vec { }] } +fn test_codex() -> TestCodexBuilder { + base_test_codex().with_config(|config| { + let _ = config.features.disable(Feature::RemoteCompactionV2); + }) +} + fn remote_realtime_test_codex_builder( realtime_server: &responses::WebSocketTestServer, ) -> TestCodexBuilder { @@ -380,6 +386,10 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { .expect("remote compact request should include turn metadata"), ) .expect("remote compact turn metadata should be valid json"); + assert_eq!( + compact_request.header("x-codex-installation-id").as_deref(), + compact_metadata["installation_id"].as_str() + ); assert!( compact_metadata["turn_id"] .as_str() diff --git a/codex-rs/core/tests/suite/compact_remote_parity.rs b/codex-rs/core/tests/suite/compact_remote_parity.rs index 1b7c50936c..87f771b743 100644 --- a/codex-rs/core/tests/suite/compact_remote_parity.rs +++ b/codex-rs/core/tests/suite/compact_remote_parity.rs @@ -5,7 +5,6 @@ use std::path::Path; use std::path::PathBuf; use anyhow::Result; -use codex_core::LoadedAgentsMd; use codex_features::Feature; use codex_login::CodexAuth; use codex_protocol::config_types::ServiceTier; @@ -29,6 +28,7 @@ const FIXED_CWD: &str = "/tmp/codex_remote_compaction_parity_workspace"; const IMAGE_URL: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII="; const SUMMARY: &str = "REMOTE_COMPACTION_PARITY_ENCRYPTED_SUMMARY"; const DUMMY_FUNCTION_NAME: &str = "test_tool"; +const USER_INSTRUCTIONS: &str = "PARITY_USER_INSTRUCTIONS"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum Mode { @@ -509,7 +509,12 @@ async fn build_harness_inner( auto_compact_limit: Option, ) -> Result { fs::create_dir_all(FIXED_CWD)?; - let mut builder = test_codex().with_auth(settings.auth.build()); + let mut builder = test_codex() + .with_auth(settings.auth.build()) + .with_pre_build_hook(|home| { + fs::write(home.join("AGENTS.md"), USER_INSTRUCTIONS) + .expect("write global instructions"); + }); if hooks { builder = builder.with_pre_build_hook(write_manual_compact_hooks); } @@ -518,9 +523,6 @@ async fn build_harness_inner( FIXED_CWD, )) .expect("fixed cwd should be absolute"); - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "PARITY_USER_INSTRUCTIONS", - )); config.developer_instructions = Some("PARITY_DEVELOPER_INSTRUCTIONS".to_string()); if settings.service_tier_fast { config.service_tier = Some(ServiceTier::Fast.request_value().to_string()); @@ -529,8 +531,8 @@ async fn build_harness_inner( if hooks { trust_discovered_hooks(config); } - if mode == Mode::V2 { - let _ = config.features.enable(Feature::RemoteCompactionV2); + if mode == Mode::Legacy { + let _ = config.features.disable(Feature::RemoteCompactionV2); } })) .await diff --git a/codex-rs/core/tests/suite/hierarchical_agents.rs b/codex-rs/core/tests/suite/hierarchical_agents.rs index 751807623b..276fd235f9 100644 --- a/codex-rs/core/tests/suite/hierarchical_agents.rs +++ b/codex-rs/core/tests/suite/hierarchical_agents.rs @@ -1,4 +1,5 @@ use codex_features::Feature; +use codex_utils_path_uri::PathUri; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_response_created; use core_test_support::responses::mount_sse_once; @@ -27,7 +28,8 @@ async fn hierarchical_agents_appends_to_project_doc_in_user_instructions() { }) .with_workspace_setup(|cwd, fs| async move { let agents_md = cwd.join("AGENTS.md"); - fs.write_file(&agents_md, b"be nice".to_vec(), /*sandbox*/ None) + let agents_md_uri = PathUri::from_path(&agents_md)?; + fs.write_file(&agents_md_uri, b"be nice".to_vec(), /*sandbox*/ None) .await?; Ok::<(), anyhow::Error>(()) }); diff --git a/codex-rs/core/tests/suite/hooks.rs b/codex-rs/core/tests/suite/hooks.rs index 16412c2d27..31e54c5e7b 100644 --- a/codex-rs/core/tests/suite/hooks.rs +++ b/codex-rs/core/tests/suite/hooks.rs @@ -34,7 +34,6 @@ use core_test_support::responses::ev_function_call; use core_test_support::responses::ev_message_item_added; use core_test_support::responses::ev_output_text_delta; use core_test_support::responses::ev_response_created; -use core_test_support::responses::mount_compact_json_once; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; @@ -1441,21 +1440,32 @@ async fn resumed_thread_runs_resume_then_compact_session_start_hooks() -> Result let remote_summary = "remote compact summary"; let resume_context = "remember the resumed reef"; let compact_context = "remember the compacted reef"; - let compacted_history = vec![ - ResponseItem::Message { - id: None, - role: "assistant".to_string(), - content: vec![ContentItem::OutputText { - text: remote_summary.to_string(), - }], - phase: None, - }, - ResponseItem::Compaction { - encrypted_content: "encrypted compact summary".to_string(), - }, - ]; - let compact_mock = - mount_compact_json_once(&server, serde_json::json!({ "output": compacted_history })).await; + let responses_mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_assistant_message("msg-1", "hello before resume"), + ev_completed_with_tokens("resp-1", over_limit_tokens), + ]), + sse(vec![ + serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "compaction", + "encrypted_content": remote_summary, + } + }), + ev_completed("resp-compact"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-2", "hello after resume"), + ev_completed("resp-2"), + ]), + ], + ) + .await; let mut builder = test_codex() .with_pre_build_hook(move |home| { @@ -1479,37 +1489,19 @@ async fn resumed_thread_runs_resume_then_compact_session_start_hooks() -> Result .clone() .context("rollout path")?; - mount_sse_once( - &server, - sse(vec![ - ev_response_created("resp-1"), - ev_assistant_message("msg-1", "hello before resume"), - ev_completed_with_tokens("resp-1", over_limit_tokens), - ]), - ) - .await; initial.submit_turn("hello before resume").await?; - assert!(compact_mock.requests().is_empty()); + assert_eq!(responses_mock.requests().len(), 1); let mut resume_builder = test_codex().with_config(move |config| { config.model_auto_compact_token_limit = Some(limit); trust_discovered_hooks(config); }); let resumed = resume_builder.resume(&server, home, rollout_path).await?; - let follow_up = mount_sse_once( - &server, - sse(vec![ - ev_response_created("resp-2"), - ev_assistant_message("msg-2", "hello after resume"), - ev_completed("resp-2"), - ]), - ) - .await; - resumed.submit_turn("hello after resume").await?; - assert_eq!(compact_mock.requests().len(), 1); - let developer_messages = follow_up.single_request().message_input_texts("developer"); + let requests = responses_mock.requests(); + assert_eq!(requests.len(), 3); + let developer_messages = requests[2].message_input_texts("developer"); assert!( developer_messages .iter() diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 234fb8a2f0..5764068bd9 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -110,6 +110,7 @@ mod sqlite_state; mod stream_error_allows_next_turn; mod stream_no_completed; mod subagent_notifications; +mod token_budget; mod tool_harness; mod tool_parallelism; mod tools; diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index d176b5bc24..22363eabea 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -136,6 +136,7 @@ fn test_model_info( context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), } @@ -957,6 +958,7 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result< context_window: Some(large_context_window), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent, experimental_supported_tools: Vec::new(), }; diff --git a/codex-rs/core/tests/suite/models_cache_ttl.rs b/codex-rs/core/tests/suite/models_cache_ttl.rs index b343c4bbaf..5103f82123 100644 --- a/codex-rs/core/tests/suite/models_cache_ttl.rs +++ b/codex-rs/core/tests/suite/models_cache_ttl.rs @@ -365,6 +365,7 @@ fn test_remote_model(slug: &str, priority: i32) -> ModelInfo { context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), diff --git a/codex-rs/core/tests/suite/personality.rs b/codex-rs/core/tests/suite/personality.rs index 06c2fc41f9..a58e5f20fe 100644 --- a/codex-rs/core/tests/suite/personality.rs +++ b/codex-rs/core/tests/suite/personality.rs @@ -587,6 +587,7 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow context_window: Some(128_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), @@ -701,6 +702,7 @@ async fn user_turn_personality_remote_model_template_includes_update_message() - context_window: Some(128_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), diff --git a/codex-rs/core/tests/suite/prompt_caching.rs b/codex-rs/core/tests/suite/prompt_caching.rs index 80dee582ad..809cb0ea32 100644 --- a/codex-rs/core/tests/suite/prompt_caching.rs +++ b/codex-rs/core/tests/suite/prompt_caching.rs @@ -1,6 +1,8 @@ #![allow(clippy::unwrap_used)] -use codex_core::LoadedAgentsMd; +use std::fs; +use std::path::Path; + use codex_core::shell::default_user_shell; use codex_features::Feature; use codex_prompts::APPLY_PATCH_TOOL_INSTRUCTIONS; @@ -32,6 +34,11 @@ use core_test_support::wait_for_event; use pretty_assertions::assert_eq; use tempfile::TempDir; +fn write_global_instructions(home: &Path) { + fs::write(home.join("AGENTS.md"), "be consistent and helpful") + .unwrap_or_else(|err| panic!("write global instructions: {err}")); +} + fn text_user_input(text: String) -> serde_json::Value { text_user_input_parts(vec![text]) } @@ -123,10 +130,8 @@ async fn prompt_tools_are_consistent_across_requests() -> anyhow::Result<()> { thread_manager, .. } = test_codex() + .with_pre_build_hook(write_global_instructions) .with_config(|config| { - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "be consistent and helpful", - )); config.model = Some("gpt-5.2".to_string()); // Keep tool expectations stable when the default web_search mode changes. config @@ -235,10 +240,8 @@ async fn gpt_5_tools_without_apply_patch_append_apply_patch_instructions() -> an .await; let TestCodex { codex, .. } = test_codex() + .with_pre_build_hook(write_global_instructions) .with_config(|config| { - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "be consistent and helpful", - )); config .features .enable(Feature::CollaborationModes) @@ -317,10 +320,8 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests .await; let TestCodex { codex, config, .. } = test_codex() + .with_pre_build_hook(write_global_instructions) .with_config(|config| { - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "be consistent and helpful", - )); config .features .enable(Feature::CollaborationModes) @@ -415,10 +416,8 @@ async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() -> an .await; let TestCodex { codex, config, .. } = test_codex() + .with_pre_build_hook(write_global_instructions) .with_config(|config| { - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "be consistent and helpful", - )); config .features .enable(Feature::CollaborationModes) @@ -706,10 +705,8 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res .await; let TestCodex { codex, .. } = test_codex() + .with_pre_build_hook(write_global_instructions) .with_config(|config| { - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "be consistent and helpful", - )); config .features .enable(Feature::CollaborationModes) @@ -841,10 +838,8 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a session_configured, .. } = test_codex() + .with_pre_build_hook(write_global_instructions) .with_config(|config| { - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "be consistent and helpful", - )); config .features .enable(Feature::CollaborationModes) @@ -982,10 +977,8 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu session_configured, .. } = test_codex() + .with_pre_build_hook(write_global_instructions) .with_config(|config| { - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "be consistent and helpful", - )); config .features .enable(Feature::CollaborationModes) diff --git a/codex-rs/core/tests/suite/prompt_debug_tests.rs b/codex-rs/core/tests/suite/prompt_debug_tests.rs index bad7e1ab5a..c2ebe3999b 100644 --- a/codex-rs/core/tests/suite/prompt_debug_tests.rs +++ b/codex-rs/core/tests/suite/prompt_debug_tests.rs @@ -1,19 +1,24 @@ +use std::sync::Arc; + use anyhow::Result; -use codex_core::LoadedAgentsMd; use codex_core::build_prompt_input; use codex_core::config::ConfigBuilder; use codex_core::config::ConfigOverrides; +use codex_home::CodexHomeUserInstructionsProvider; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::user_input::UserInput; use pretty_assertions::assert_eq; use tempfile::TempDir; +const TEST_INSTRUCTIONS: &str = "Global test instructions"; + #[tokio::test] async fn build_prompt_input_includes_context_and_user_message() -> Result<()> { let codex_home = TempDir::new()?; let cwd = TempDir::new()?; - let mut config = ConfigBuilder::default() + std::fs::write(codex_home.path().join("AGENTS.md"), TEST_INSTRUCTIONS)?; + let config = ConfigBuilder::default() .codex_home(codex_home.path().to_path_buf()) .harness_overrides(ConfigOverrides { cwd: Some(cwd.path().to_path_buf()), @@ -22,8 +27,8 @@ async fn build_prompt_input_includes_context_and_user_message() -> Result<()> { }) .build() .await?; - config.user_instructions = Some(LoadedAgentsMd::from_text_for_testing( - "Project-specific test instructions", + let user_instructions_provider = Arc::new(CodexHomeUserInstructionsProvider::new( + config.codex_home.clone(), )); let input = build_prompt_input( @@ -33,6 +38,7 @@ async fn build_prompt_input_includes_context_and_user_message() -> Result<()> { text_elements: Vec::new(), }], /*state_db*/ None, + user_instructions_provider, ) .await?; @@ -55,7 +61,7 @@ async fn build_prompt_input_includes_context_and_user_message() -> Result<()> { else { return false; }; - text.contains("Project-specific test instructions") + text.contains(TEST_INSTRUCTIONS) }) })); diff --git a/codex-rs/core/tests/suite/remote_env.rs b/codex-rs/core/tests/suite/remote_env.rs index 00f5a0c514..64d5c7081c 100644 --- a/codex-rs/core/tests/suite/remote_env.rs +++ b/codex-rs/core/tests/suite/remote_env.rs @@ -28,6 +28,7 @@ use codex_protocol::request_permissions::RequestPermissionProfile; use codex_protocol::request_permissions::RequestPermissionsResponse; use codex_protocol::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use core_test_support::PathBufExt; use core_test_support::PathExt; use core_test_support::get_remote_test_env; @@ -155,19 +156,20 @@ async fn remote_test_env_can_connect_and_use_filesystem() -> Result<()> { let file_system = test_env.environment().get_filesystem(); let file_path_abs = remote_test_file_path().abs(); + let file_path_uri = PathUri::from_path(&file_path_abs)?; let payload = b"remote-test-env-ok".to_vec(); file_system - .write_file(&file_path_abs, payload.clone(), /*sandbox*/ None) + .write_file(&file_path_uri, payload.clone(), /*sandbox*/ None) .await?; let actual = file_system - .read_file(&file_path_abs, /*sandbox*/ None) + .read_file(&file_path_uri, /*sandbox*/ None) .await?; assert_eq!(actual, payload); file_system .remove( - &file_path_abs, + &file_path_uri, RemoveOptions { recursive: false, force: true, @@ -295,16 +297,18 @@ async fn exec_command_routes_to_selected_remote_environment() -> Result<()> { )) .abs(); let remote_marker_name = "marker.txt"; + let remote_cwd_uri = PathUri::from_path(&remote_cwd)?; + let remote_marker_uri = PathUri::from_path(remote_cwd.join(remote_marker_name))?; test.fs() .create_directory( - &remote_cwd, + &remote_cwd_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) .await?; test.fs() .write_file( - &remote_cwd.join(remote_marker_name), + &remote_marker_uri, b"remote-routing".to_vec(), /*sandbox*/ None, ) @@ -338,7 +342,7 @@ async fn exec_command_routes_to_selected_remote_environment() -> Result<()> { test.fs() .remove( - &remote_cwd, + &remote_cwd_uri, RemoveOptions { recursive: true, force: true, @@ -390,9 +394,10 @@ async fn remote_request_permissions_grant_unblocks_later_remote_exec() -> Result let local_write_root = local_cwd.path().join(relative_write_root); let local_target_path = local_cwd.path().join(relative_target_path); fs::create_dir(&local_write_root)?; + let remote_write_root_uri = PathUri::from_path(&remote_write_root)?; test.fs() .create_directory( - &remote_write_root, + &remote_write_root_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) @@ -527,7 +532,10 @@ async fn remote_request_permissions_grant_unblocks_later_remote_exec() -> Result ); assert_eq!( test.fs() - .read_file_text(&remote_target_path, /*sandbox*/ None) + .read_file_text( + &PathUri::from_path(&remote_target_path)?, + /*sandbox*/ None, + ) .await?, "remote-request-permissions-ok" ); @@ -538,7 +546,7 @@ async fn remote_request_permissions_grant_unblocks_later_remote_exec() -> Result test.fs() .remove( - &remote_cwd, + &PathUri::from_abs_path(&remote_cwd)?, RemoveOptions { recursive: true, force: true, @@ -567,9 +575,10 @@ async fn apply_patch_freeform_routes_to_selected_remote_environment() -> Result< SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() )) .abs(); + let remote_cwd_uri = PathUri::from_path(&remote_cwd)?; test.fs() .create_directory( - &remote_cwd, + &remote_cwd_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) @@ -610,7 +619,10 @@ async fn apply_patch_freeform_routes_to_selected_remote_environment() -> Result< let remote_contents = test .fs() - .read_file_text(&remote_cwd.join(file_name), /*sandbox*/ None) + .read_file_text( + &PathUri::from_path(remote_cwd.join(file_name))?, + /*sandbox*/ None, + ) .await?; assert_eq!(remote_contents, "patched remote freeform\n"); assert!( @@ -620,7 +632,7 @@ async fn apply_patch_freeform_routes_to_selected_remote_environment() -> Result< test.fs() .remove( - &remote_cwd, + &remote_cwd_uri, RemoveOptions { recursive: true, force: true, @@ -651,9 +663,10 @@ async fn apply_patch_approvals_are_remembered_per_environment() -> Result<()> { SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() )) .abs(); + let remote_cwd_uri = PathUri::from_path(&remote_cwd)?; test.fs() .create_directory( - &remote_cwd, + &remote_cwd_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) @@ -664,10 +677,11 @@ async fn apply_patch_approvals_are_remembered_per_environment() -> Result<()> { SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() )) .abs(); + let target_path_uri = PathUri::from_path(&target_path)?; let _ = fs::remove_file(&target_path); test.fs() .remove( - &target_path, + &target_path_uri, RemoveOptions { recursive: false, force: true, @@ -771,7 +785,7 @@ async fn apply_patch_approvals_are_remembered_per_environment() -> Result<()> { .await; assert_eq!( test.fs() - .read_file_text(&target_path, /*sandbox*/ None) + .read_file_text(&target_path_uri, /*sandbox*/ None) .await?, "remote\n" ); @@ -785,7 +799,7 @@ async fn apply_patch_approvals_are_remembered_per_environment() -> Result<()> { wait_for_completion_without_patch_approval(&test).await; assert_eq!( test.fs() - .read_file_text(&target_path, /*sandbox*/ None) + .read_file_text(&target_path_uri, /*sandbox*/ None) .await?, "remote updated\n" ); @@ -793,7 +807,7 @@ async fn apply_patch_approvals_are_remembered_per_environment() -> Result<()> { let _ = fs::remove_file(&target_path); test.fs() .remove( - &target_path, + &target_path_uri, RemoveOptions { recursive: false, force: true, @@ -803,7 +817,7 @@ async fn apply_patch_approvals_are_remembered_per_environment() -> Result<()> { .await?; test.fs() .remove( - &remote_cwd, + &remote_cwd_uri, RemoveOptions { recursive: true, force: true, @@ -832,9 +846,10 @@ async fn apply_patch_intercepted_exec_command_routes_to_selected_remote_environm SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis() )) .abs(); + let remote_cwd_uri = PathUri::from_path(&remote_cwd)?; test.fs() .create_directory( - &remote_cwd, + &remote_cwd_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) @@ -885,7 +900,10 @@ async fn apply_patch_intercepted_exec_command_routes_to_selected_remote_environm let remote_contents = test .fs() - .read_file_text(&remote_cwd.join(file_name), /*sandbox*/ None) + .read_file_text( + &PathUri::from_path(remote_cwd.join(file_name))?, + /*sandbox*/ None, + ) .await?; assert_eq!(remote_contents, "patched remote exec\n"); assert!( @@ -895,7 +913,7 @@ async fn apply_patch_intercepted_exec_command_routes_to_selected_remote_environm test.fs() .remove( - &remote_cwd, + &remote_cwd_uri, RemoveOptions { recursive: true, force: true, @@ -919,16 +937,18 @@ async fn remote_test_env_sandboxed_read_allows_readable_root() -> Result<()> { let allowed_dir = PathBuf::from(format!("/tmp/codex-remote-readable-{}", std::process::id())); let file_path = allowed_dir.join("note.txt"); + let allowed_dir_uri = PathUri::from_path(&allowed_dir)?; + let file_path_uri = PathUri::from_path(&file_path)?; file_system .create_directory( - &absolute_path(allowed_dir.clone()), + &allowed_dir_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) .await?; file_system .write_file( - &absolute_path(file_path.clone()), + &file_path_uri, b"sandboxed hello".to_vec(), /*sandbox*/ None, ) @@ -936,13 +956,13 @@ async fn remote_test_env_sandboxed_read_allows_readable_root() -> Result<()> { let sandbox = read_only_sandbox(allowed_dir.clone()); let contents = file_system - .read_file(&absolute_path(file_path.clone()), Some(&sandbox)) + .read_file(&file_path_uri, Some(&sandbox)) .await?; assert_eq!(contents, b"sandboxed hello"); file_system .remove( - &absolute_path(allowed_dir), + &allowed_dir_uri, RemoveOptions { recursive: true, force: true, @@ -976,7 +996,8 @@ async fn remote_test_env_sandboxed_read_rejects_symlink_parent_dotdot_escape() - secret = secret_path.display(), ))?; - let requested_path = absolute_path(allowed_dir.join("link").join("..").join("secret.txt")); + let requested_path = + PathUri::from_path(allowed_dir.join("link").join("..").join("secret.txt"))?; let sandbox = read_only_sandbox(allowed_dir.clone()); let error = match file_system.read_file(&requested_path, Some(&sandbox)).await { Ok(_) => anyhow::bail!("read should fail after path normalization"), @@ -1023,7 +1044,7 @@ async fn remote_test_env_remove_removes_symlink_not_target() -> Result<()> { let sandbox = workspace_write_sandbox(allowed_dir.clone()); file_system .remove( - &absolute_path(symlink_path.clone()), + &PathUri::from_path(&symlink_path)?, RemoveOptions { recursive: false, force: false, @@ -1033,18 +1054,21 @@ async fn remote_test_env_remove_removes_symlink_not_target() -> Result<()> { .await?; let symlink_exists = file_system - .get_metadata(&absolute_path(symlink_path), /*sandbox*/ None) + .get_metadata( + &PathUri::from_abs_path(&absolute_path(symlink_path))?, + /*sandbox*/ None, + ) .await .is_ok(); assert!(!symlink_exists); let outside = file_system - .read_file_text(&absolute_path(outside_file.clone()), /*sandbox*/ None) + .read_file_text(&PathUri::from_path(&outside_file)?, /*sandbox*/ None) .await?; assert_eq!(outside, "outside"); file_system .remove( - &absolute_path(root), + &PathUri::from_path(&root)?, RemoveOptions { recursive: true, force: true, @@ -1085,8 +1109,8 @@ async fn remote_test_env_copy_preserves_symlink_source() -> Result<()> { let sandbox = workspace_write_sandbox(allowed_dir.clone()); file_system .copy( - &absolute_path(source_symlink), - &absolute_path(copied_symlink.clone()), + &PathUri::from_path(&source_symlink)?, + &PathUri::from_path(&copied_symlink)?, CopyOptions { recursive: false }, Some(&sandbox), ) @@ -1117,7 +1141,7 @@ async fn remote_test_env_copy_preserves_symlink_source() -> Result<()> { file_system .remove( - &absolute_path(root), + &PathUri::from_path(&root)?, RemoveOptions { recursive: true, force: true, diff --git a/codex-rs/core/tests/suite/remote_models.rs b/codex-rs/core/tests/suite/remote_models.rs index dc8df93c9b..b3fc9b9db7 100644 --- a/codex-rs/core/tests/suite/remote_models.rs +++ b/codex-rs/core/tests/suite/remote_models.rs @@ -499,6 +499,7 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), }; @@ -751,6 +752,7 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> { context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), }; @@ -1237,6 +1239,7 @@ fn test_remote_model_with_policy( context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), } diff --git a/codex-rs/core/tests/suite/responses_lite.rs b/codex-rs/core/tests/suite/responses_lite.rs index 60db4c064d..b137d5bc73 100644 --- a/codex-rs/core/tests/suite/responses_lite.rs +++ b/codex-rs/core/tests/suite/responses_lite.rs @@ -9,9 +9,11 @@ use codex_features::Feature; use codex_image_generation_extension::install as install_image_generation_extension; use codex_login::CodexAuth; use codex_protocol::config_types::WebSearchMode; +use codex_protocol::models::ImageDetail; use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; +use codex_protocol::user_input::UserInput; use codex_web_search_extension::install as install_web_search_extension; use core_test_support::responses; use core_test_support::skip_if_no_network; @@ -52,6 +54,62 @@ fn has_hosted_tool(tools: &[Value], tool_type: &str) -> bool { .any(|tool| tool.get("type").and_then(Value::as_str) == Some(tool_type)) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn responses_lite_strips_data_image_detail_without_resize_all_images() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; + let mut builder = test_codex().with_model_info_override("gpt-5.4", |model_info| { + model_info.use_responses_lite = true; + configure_image_capable_model(model_info); + }); + let test = builder.build(&server).await?; + + test.codex + .submit(Op::UserInput { + items: vec![UserInput::Image { + image_url: image_url.to_string(), + detail: Some(ImageDetail::Original), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let request = response_mock.single_request(); + let input = request.input(); + let image = input + .iter() + .filter_map(|item| item.get("content").and_then(Value::as_array)) + .flatten() + .find(|item| item.get("type").and_then(Value::as_str) == Some("input_image")) + .context("request should contain an image")?; + assert_eq!( + image, + &serde_json::json!({ + "type": "input_image", + "image_url": image_url + }) + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn responses_lite_uses_standalone_web_search_and_image_generation() -> Result<()> { skip_if_no_network!(Ok(())); @@ -119,10 +177,14 @@ async fn responses_lite_compact_request_uses_lite_transport_contract() -> Result let compact_mock = responses::mount_compact_json_once(&server, serde_json::json!({ "output": [] })).await; - let mut builder = test_codex().with_model_info_override("gpt-5.4", |model_info| { - model_info.use_responses_lite = true; - model_info.supports_parallel_tool_calls = true; - }); + let mut builder = test_codex() + .with_model_info_override("gpt-5.4", |model_info| { + model_info.use_responses_lite = true; + model_info.supports_parallel_tool_calls = true; + }) + .with_config(|config| { + let _ = config.features.disable(Feature::RemoteCompactionV2); + }); let test = builder.build(&server).await?; test.submit_turn("Compact this conversation").await?; diff --git a/codex-rs/core/tests/suite/resume_warning.rs b/codex-rs/core/tests/suite/resume_warning.rs index 422489a86a..2120559751 100644 --- a/codex-rs/core/tests/suite/resume_warning.rs +++ b/codex-rs/core/tests/suite/resume_warning.rs @@ -37,6 +37,7 @@ fn resume_history( network: None, file_system_sandbox_policy: None, model: previous_model.to_string(), + comp_hash: None, personality: None, collaboration_mode: None, multi_agent_version: None, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index c47aa0970a..d9397cfc55 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -2,6 +2,8 @@ use anyhow::Context as _; use anyhow::ensure; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use core_test_support::test_codex::local_selections; use std::collections::HashMap; use std::ffi::OsStr; @@ -25,6 +27,7 @@ use codex_core::config::Config; use codex_exec_server::CreateDirectoryOptions; use codex_exec_server::Environment; use codex_exec_server::HttpRequestParams; +use codex_features::Feature; use codex_login::CodexAuth; use codex_mcp::MCP_SANDBOX_STATE_META_CAPABILITY; use codex_models_manager::manager::RefreshStrategy; @@ -45,6 +48,7 @@ use codex_protocol::protocol::McpToolCallBeginEvent; use codex_protocol::protocol::Op; use codex_protocol::user_input::UserInput; use codex_utils_cargo_bin::cargo_bin; +use codex_utils_path_uri::PathUri; use core_test_support::assert_regex_match; use core_test_support::remote_env_env_var; use core_test_support::responses; @@ -57,11 +61,16 @@ use core_test_support::test_codex::test_codex; use core_test_support::test_codex::turn_permission_fields; use core_test_support::wait_for_event; use core_test_support::wait_for_mcp_server; +use image::DynamicImage; +use image::GenericImageView; +use image::ImageBuffer; +use image::Rgba; use reqwest::Client; use reqwest::StatusCode; use serde_json::Value; use serde_json::json; use serial_test::serial; +use std::io::Cursor; use tempfile::tempdir; use tokio::process::Child; use tokio::process::Command; @@ -617,8 +626,10 @@ async fn stdio_server_uses_configured_cwd_before_runtime_fallback() -> anyhow::R let fixture = test_codex() .with_workspace_setup(|cwd, fs| async move { + let configured_cwd = cwd.join("mcp-configured-cwd"); + let configured_cwd_uri = PathUri::from_path(&configured_cwd)?; fs.create_directory( - &cwd.join("mcp-configured-cwd"), + &configured_cwd_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) @@ -1257,6 +1268,107 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[serial(mcp_test_value)] +async fn stdio_image_responses_resize_large_image() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let call_id = "img-resize-1"; + let server_name = "rmcp"; + let namespace = format!("mcp__{server_name}"); + + let original_dimensions = (3000, 2000); + let image = ImageBuffer::from_pixel( + original_dimensions.0, + original_dimensions.1, + Rgba([20, 40, 60, 255]), + ); + let mut encoded = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?; + let image_data_url = format!( + "data:image/png;base64,{}", + BASE64_STANDARD.encode(encoded.into_inner()) + ); + let tool_arguments = serde_json::to_string(&json!({ + "scenario": "image_only", + "data_url": image_data_url, + }))?; + + mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace( + call_id, + &namespace, + "image_scenario", + &tool_arguments, + ), + responses::ev_completed("resp-1"), + ]), + ) + .await; + let final_mock = mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "done"), + responses::ev_completed("resp-2"), + ]), + ) + .await; + + let rmcp_test_server_bin = remote_aware_stdio_server_bin()?; + let fixture = test_codex() + .with_config(move |config| { + config + .features + .enable(Feature::ResizeAllImages) + .expect("resize_all_images should be enabled"); + insert_mcp_server( + config, + server_name, + stdio_transport(rmcp_test_server_bin, /*env*/ None, Vec::new()), + TestMcpServerOptions { + environment_id: remote_aware_environment_id(), + ..Default::default() + }, + ); + }) + .build_with_remote_env(&server) + .await?; + wait_for_mcp_server(&fixture.codex, server_name).await?; + + fixture + .codex + .submit(read_only_user_turn( + &fixture, + "call the rmcp image_scenario tool", + )) + .await?; + wait_for_event(&fixture.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let output_item = final_mock.single_request().function_call_output(call_id); + assert_eq!(output_item["call_id"], call_id); + let output = output_item["output"] + .as_array() + .expect("image MCP output should be content items"); + let resized_url = output[1]["image_url"] + .as_str() + .expect("MCP image output should contain a data URL"); + assert_eq!(output[1]["detail"], "high"); + let (_, resized_base64) = resized_url + .split_once(',') + .expect("resized image should contain a data URL prefix"); + let resized_bytes = BASE64_STANDARD.decode(resized_base64)?; + let resized = image::load_from_memory(&resized_bytes)?; + let resized_dimensions = resized.dimensions(); + assert_eq!(resized_dimensions, (1920, 1280)); + + server.verify().await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[serial(mcp_test_value)] async fn stdio_image_responses_preserve_original_detail_metadata() -> anyhow::Result<()> { @@ -1390,6 +1502,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), input_modalities: vec![InputModality::Text], diff --git a/codex-rs/core/tests/suite/skills.rs b/codex-rs/core/tests/suite/skills.rs index 45b52c6ace..55c75016d9 100644 --- a/codex-rs/core/tests/suite/skills.rs +++ b/codex-rs/core/tests/suite/skills.rs @@ -9,6 +9,7 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::Op; use codex_protocol::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_response_created; @@ -29,15 +30,17 @@ async fn write_repo_skill( body: &str, ) -> Result<()> { let skill_dir = cwd.join(".agents").join("skills").join(name); + let skill_dir_uri = PathUri::from_path(&skill_dir)?; fs.create_directory( - &skill_dir, + &skill_dir_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) .await?; let contents = format!("---\nname: {name}\ndescription: {description}\n---\n\n{body}\n"); let path = skill_dir.join("SKILL.md"); - fs.write_file(&path, contents.into_bytes(), /*sandbox*/ None) + let path_uri = PathUri::from_path(&path)?; + fs.write_file(&path_uri, contents.into_bytes(), /*sandbox*/ None) .await?; Ok(()) } diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_api_auth_prompt_cache_key_request_diff.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_api_auth_prompt_cache_key_request_diff.snap index 89796cb918..0df96a7c06 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_api_auth_prompt_cache_key_request_diff.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_api_auth_prompt_cache_key_request_diff.snap @@ -7,7 +7,11 @@ Scenario: After five varied API-key-auth turns, remote manual compaction omits s --- Last Normal /responses Request +++ Remote /responses/compact Request - "client_metadata": { +- "session_id": "", +- "thread_id": "", +- "turn_id": "", - "x-codex-installation-id": "", +- "x-codex-turn-metadata": "{\"installation_id\":\"\",\"session_id\":\"\",\"thread_id\":\"\",\"turn_id\":\"\",\"window_id\":\":0\",\"request_kind\":\"turn\",\"sandbox\":\"\",\"turn_started_at_unix_ms\":}", - "x-codex-window-id": ":0" - }, - "include": [ diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_chatgpt_auth_service_tier_prompt_cache_key_request_diff.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_chatgpt_auth_service_tier_prompt_cache_key_request_diff.snap index ab6c9d1b37..d959d526cc 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_chatgpt_auth_service_tier_prompt_cache_key_request_diff.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_chatgpt_auth_service_tier_prompt_cache_key_request_diff.snap @@ -7,7 +7,11 @@ Scenario: After five varied ChatGPT-auth turns, remote manual compaction reuses --- Last Normal /responses Request +++ Remote /responses/compact Request - "client_metadata": { +- "session_id": "", +- "thread_id": "", +- "turn_id": "", - "x-codex-installation-id": "", +- "x-codex-turn-metadata": "{\"installation_id\":\"\",\"session_id\":\"\",\"thread_id\":\"\",\"turn_id\":\"\",\"window_id\":\":0\",\"request_kind\":\"turn\",\"sandbox\":\"\",\"turn_started_at_unix_ms\":}", - "x-codex-window-id": ":0" - }, - "include": [ diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap b/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap new file mode 100644 index 0000000000..d042616afc --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap @@ -0,0 +1,15 @@ +--- +source: core/tests/suite/token_budget.rs +assertion_line: 300 +expression: "context_snapshot::format_labeled_requests_snapshot(\"New context window tool installs fresh full context before the next follow-up request.\",\n&[(\"Final Follow-Up Request\", &requests[2])],\n&ContextSnapshotOptions::default(),)" +--- +Scenario: New context window tool installs fresh full context before the next follow-up request. + +## Final Follow-Up Request +00:message/developer[3]: + [01] + [02] + [03] \nCurrent context window 1.\nYou have 121600 tokens left in this context window.\n +01:message/user:> +02:function_call/update_plan +03:function_call_output:Plan updated diff --git a/codex-rs/core/tests/suite/spawn_agent_description.rs b/codex-rs/core/tests/suite/spawn_agent_description.rs index ba29b4f00c..30aab8926e 100644 --- a/codex-rs/core/tests/suite/spawn_agent_description.rs +++ b/codex-rs/core/tests/suite/spawn_agent_description.rs @@ -84,6 +84,7 @@ fn test_model_info( context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), } diff --git a/codex-rs/core/tests/suite/token_budget.rs b/codex-rs/core/tests/suite/token_budget.rs new file mode 100644 index 0000000000..f50ec09894 --- /dev/null +++ b/codex-rs/core/tests/suite/token_budget.rs @@ -0,0 +1,449 @@ +use anyhow::Result; +use codex_features::Feature; +use codex_model_provider_info::built_in_model_providers; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use core_test_support::PathBufExt; +use core_test_support::context_snapshot; +use core_test_support::context_snapshot::ContextSnapshotOptions; +use core_test_support::responses::ResponsesRequest; +use core_test_support::responses::ev_assistant_message; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_completed_with_tokens; +use core_test_support::responses::ev_function_call; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_sequence; +use core_test_support::responses::sse; +use core_test_support::responses::start_mock_server; +use core_test_support::skip_if_no_network; +use core_test_support::test_codex::local; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; +use pretty_assertions::assert_eq; +use serde_json::Value; +use serde_json::json; + +const CONFIGURED_CONTEXT_WINDOW: i64 = 128_000; +const EFFECTIVE_CONTEXT_WINDOW: i64 = CONFIGURED_CONTEXT_WINDOW * 95 / 100; + +fn token_budget_texts(request: &ResponsesRequest) -> Vec { + request + .message_input_texts("developer") + .into_iter() + .filter(|text| text.starts_with("")) + .collect() +} + +fn tool_names(request: &ResponsesRequest) -> Vec { + request + .body_json() + .get("tools") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|tool| tool.get("name").and_then(Value::as_str).map(str::to_string)) + .collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn token_budget_context_is_only_emitted_with_full_context() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]), + ], + ) + .await; + let test = test_codex() + .with_config(|config| { + config.model_context_window = Some(CONFIGURED_CONTEXT_WINDOW); + config + .features + .enable(Feature::TokenBudget) + .expect("test config should allow token budget"); + }) + .build(&server) + .await?; + + test.submit_turn("first turn").await?; + + let second_cwd = test.workspace_path("second-cwd"); + std::fs::create_dir_all(&second_cwd)?; + test.submit_turn_with_environments("second turn", Some(vec![local(second_cwd.abs())])) + .await?; + + let requests = responses.requests(); + assert_eq!(requests.len(), 2); + + let expected = vec![format!( + "\nCurrent context window 0.\nYou have {EFFECTIVE_CONTEXT_WINDOW} tokens left in this context window.\n" + )]; + assert_eq!( + token_budget_texts(&requests[0]), + expected, + "initial full context should report context window 0" + ); + assert_eq!( + token_budget_texts(&requests[1]), + expected, + "steady-state context update should not advance the context window" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn token_budget_remaining_context_emits_on_first_threshold_crossing() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_completed_with_tokens("resp-1", /*total_tokens*/ 2_500), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_completed_with_tokens("resp-2", /*total_tokens*/ 3_000), + ]), + sse(vec![ + ev_response_created("resp-3"), + ev_completed_with_tokens("resp-3", /*total_tokens*/ 5_000), + ]), + sse(vec![ + ev_response_created("resp-4"), + ev_completed_with_tokens("resp-4", /*total_tokens*/ 8_000), + ]), + sse(vec![ev_response_created("resp-5"), ev_completed("resp-5")]), + ], + ) + .await; + let test = test_codex() + .with_config(|config| { + config.model_context_window = Some(10_000); + config + .features + .enable(Feature::TokenBudget) + .expect("test config should allow token budget"); + }) + .build(&server) + .await?; + + for turn in 1..=5 { + test.submit_turn(&format!("turn {turn}")).await?; + } + + let requests = responses.requests(); + assert_eq!(requests.len(), 5); + + let full_context = "\nCurrent context window 0.\nYou have 9500 tokens left in this context window.\n" + .to_string(); + let threshold_25 = + "\nYou have 7000 tokens left in this context window.\n" + .to_string(); + let threshold_50 = + "\nYou have 4500 tokens left in this context window.\n" + .to_string(); + let threshold_75 = + "\nYou have 1500 tokens left in this context window.\n" + .to_string(); + + assert_eq!(token_budget_texts(&requests[0]), vec![full_context.clone()]); + assert_eq!( + token_budget_texts(&requests[1]), + vec![full_context.clone(), threshold_25.clone()] + ); + assert_eq!( + token_budget_texts(&requests[2]), + vec![full_context.clone(), threshold_25.clone()] + ); + assert_eq!( + token_budget_texts(&requests[3]), + vec![ + full_context.clone(), + threshold_25.clone(), + threshold_50.clone() + ] + ); + assert_eq!( + token_budget_texts(&requests[4]), + vec![full_context, threshold_25, threshold_50, threshold_75] + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_context_remaining_returns_token_budget_remaining_fragment() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let call_id = "remaining-call"; + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_assistant_message("msg-1", "noted"), + ev_completed_with_tokens("resp-1", /*total_tokens*/ 2_500), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_function_call(call_id, "get_context_remaining", "{}"), + ev_completed_with_tokens("resp-2", /*total_tokens*/ 2_500), + ]), + sse(vec![ + ev_response_created("resp-3"), + ev_assistant_message("msg-3", "done"), + ev_completed("resp-3"), + ]), + ], + ) + .await; + let test = test_codex() + .with_config(|config| { + config.model_context_window = Some(10_000); + config + .features + .enable(Feature::TokenBudget) + .expect("test config should allow token budget"); + }) + .build(&server) + .await?; + + test.submit_turn("spend some tokens").await?; + test.submit_turn("check remaining context").await?; + + let requests = responses.requests(); + assert_eq!(requests.len(), 3); + assert!( + tool_names(&requests[1]) + .iter() + .any(|name| name == "get_context_remaining"), + "get_context_remaining should be exposed when token budget is enabled" + ); + + let full_context = "\nCurrent context window 0.\nYou have 9500 tokens left in this context window.\n" + .to_string(); + let remaining_context = + "\nYou have 7000 tokens left in this context window.\n" + .to_string(); + assert_eq!( + token_budget_texts(&requests[1]), + vec![full_context, remaining_context.clone()] + ); + assert_eq!( + requests[2].function_call_output_content_and_success(call_id), + Some((Some(remaining_context), None)) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_context_remaining_returns_unknown_when_window_is_unavailable() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let call_id = "remaining-call"; + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "get_context_remaining", "{}"), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_assistant_message("msg-2", "done"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + let test = test_codex() + .with_model_info_override("gpt-5.2", |model_info| { + model_info.context_window = None; + model_info.max_context_window = None; + }) + .with_config(|config| { + config.model_context_window = None; + config + .features + .enable(Feature::TokenBudget) + .expect("test config should allow token budget"); + }) + .build(&server) + .await?; + + test.submit_turn("check remaining context").await?; + + let requests = responses.requests(); + assert_eq!(requests.len(), 2); + assert!( + tool_names(&requests[0]) + .iter() + .any(|name| name == "get_context_remaining"), + "get_context_remaining should be exposed when token budget is enabled" + ); + + assert_eq!(token_budget_texts(&requests[0]), Vec::::new()); + assert_eq!( + requests[1].function_call_output_content_and_success(call_id), + Some(( + Some( + "\nYou have unknown tokens left in this context window.\n" + .to_string() + ), + None, + )) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn token_budget_context_uses_new_window_after_compaction() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + sse(vec![ + ev_response_created("resp-compact"), + ev_assistant_message("msg-compact", "compact summary"), + ev_completed("resp-compact"), + ]), + sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]), + ], + ) + .await; + + let mut model_provider = built_in_model_providers(/*openai_base_url*/ None)["openai"].clone(); + model_provider.name = "OpenAI-compatible test provider".to_string(); + model_provider.base_url = Some(format!("{}/v1", server.uri())); + model_provider.supports_websockets = false; + + let test = test_codex() + .with_config(move |config| { + config.model_provider = model_provider; + config.model_context_window = Some(CONFIGURED_CONTEXT_WINDOW); + config + .features + .enable(Feature::TokenBudget) + .expect("test config should allow token budget"); + }) + .build(&server) + .await?; + + test.submit_turn("before compact").await?; + test.codex.submit(Op::Compact).await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + test.submit_turn("after compact").await?; + + let requests = responses.requests(); + assert_eq!(requests.len(), 3); + + assert_eq!( + token_budget_texts(&requests[2]), + vec![format!( + "\nCurrent context window 1.\nYou have {EFFECTIVE_CONTEXT_WINDOW} tokens left in this context window.\n" + )], + "post-compaction full context should report context window 1" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn new_context_tool_starts_new_window_before_follow_up() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let call_id = "new-window-call"; + let continue_call_id = "continue-call"; + let continue_args = json!({ + "plan": [ + {"step": "Continue in the new context window", "status": "in_progress"} + ], + }) + .to_string(); + let responses = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "new_context", "{}"), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_function_call(continue_call_id, "update_plan", &continue_args), + ev_completed("resp-2"), + ]), + sse(vec![ + ev_response_created("resp-3"), + ev_assistant_message("msg-3", "done"), + ev_completed("resp-3"), + ]), + ], + ) + .await; + let test = test_codex() + .with_config(|config| { + config.model_context_window = Some(CONFIGURED_CONTEXT_WINDOW); + config + .features + .enable(Feature::TokenBudget) + .expect("test config should allow token budget"); + }) + .build(&server) + .await?; + + test.submit_turn("request new context window").await?; + + let requests = responses.requests(); + assert_eq!(requests.len(), 3); + assert!( + tool_names(&requests[0]) + .iter() + .any(|name| name == "new_context"), + "new_context should be exposed when token budget is enabled" + ); + assert_eq!( + token_budget_texts(&requests[2]), + vec![format!( + "\nCurrent context window 1.\nYou have {EFFECTIVE_CONTEXT_WINDOW} tokens left in this context window.\n" + )] + ); + assert!( + !requests[2].body_contains_text("request new context window"), + "new_context should drop the prior window history before continuing the turn" + ); + assert_eq!( + requests[2].function_call_output_text(continue_call_id), + Some("Plan updated".to_string()) + ); + insta::assert_snapshot!( + "token_budget_new_context_window_tool_full_context", + context_snapshot::format_labeled_requests_snapshot( + "New context window tool installs fresh full context before the next follow-up request.", + &[("Final Follow-Up Request", &requests[2])], + &ContextSnapshotOptions::default(), + ) + ); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index eb8bf29551..06b5288cdc 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -17,6 +17,7 @@ use codex_protocol::protocol::ExecCommandSource; use codex_protocol::protocol::ExecCommandStatus; use codex_protocol::protocol::Op; use codex_protocol::user_input::UserInput; +use codex_utils_path_uri::PathUri; use core_test_support::TempDirExt; use core_test_support::assert_regex_match; use core_test_support::managed_network_requirements_loader; @@ -228,9 +229,10 @@ async fn create_workspace_directory( rel_path: impl AsRef, ) -> Result { let abs_path = test.config.cwd.join(rel_path.as_ref()); + let abs_path_uri = PathUri::from_path(&abs_path)?; test.fs() .create_directory( - &abs_path, + &abs_path_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index 5820788566..3790eb19b3 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -7,6 +7,7 @@ use codex_exec_server::CreateDirectoryOptions; use codex_exec_server::LOCAL_ENVIRONMENT_ID; use codex_exec_server::REMOTE_ENVIRONMENT_ID; use codex_exec_server::RemoveOptions; +use codex_features::Feature; use codex_login::CodexAuth; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::PermissionProfile; @@ -28,6 +29,7 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use codex_protocol::protocol::TurnEnvironmentSelection; use codex_protocol::user_input::UserInput; +use codex_utils_path_uri::PathUri; use core_test_support::PathBufExt; use core_test_support::PathExt; use core_test_support::get_remote_test_env; @@ -133,9 +135,10 @@ fn png_bytes(width: u32, height: u32, rgba: [u8; 4]) -> anyhow::Result> async fn create_workspace_directory(test: &TestCodex, rel_path: &str) -> anyhow::Result { let abs_path = test.config.cwd.join(rel_path); + let abs_path_uri = PathUri::from_path(&abs_path)?; test.fs() .create_directory( - &abs_path, + &abs_path_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) @@ -150,16 +153,18 @@ async fn write_workspace_file( ) -> anyhow::Result { let abs_path = test.config.cwd.join(rel_path); if let Some(parent) = abs_path.parent() { + let parent_uri = PathUri::from_path(&parent)?; test.fs() .create_directory( - &parent, + &parent_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) .await?; } + let abs_path_uri = PathUri::from_path(&abs_path)?; test.fs() - .write_file(&abs_path, contents, /*sandbox*/ None) + .write_file(&abs_path_uri, contents, /*sandbox*/ None) .await?; Ok(abs_path.into_path_buf()) } @@ -177,10 +182,15 @@ async fn write_workspace_png( async fn assert_user_turn_local_image_resizes_to( original_dimensions: (u32, u32), expected_dimensions: (u32, u32), + resize_policy: TestImageResizePolicy, ) -> anyhow::Result<()> { let server = start_mock_server().await; - let mut builder = test_codex(); + let mut builder = test_codex().with_config(move |config| { + if resize_policy == TestImageResizePolicy::AllImages { + let _ = config.features.enable(Feature::ResizeAllImages); + } + }); let test = builder.build_with_remote_env(&server).await?; let TestCodex { codex, @@ -254,18 +264,42 @@ async fn assert_user_turn_local_image_resizes_to( Ok(()) } +#[derive(Clone, Copy, Eq, PartialEq)] +enum TestImageResizePolicy { + Legacy, + AllImages, +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); - assert_user_turn_local_image_resizes_to((2304, 864), (2048, 768)).await + assert_user_turn_local_image_resizes_to((2304, 864), (2048, 768), TestImageResizePolicy::Legacy) + .await } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn user_turn_with_vertical_local_image_resizes_to_square_bounds() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); - assert_user_turn_local_image_resizes_to((1024, 4096), (512, 2048)).await + assert_user_turn_local_image_resizes_to( + (1024, 4096), + (512, 2048), + TestImageResizePolicy::Legacy, + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resize_all_images_applies_patch_budget_to_local_user_image() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + assert_user_turn_local_image_resizes_to( + (2048, 2048), + (1600, 1600), + TestImageResizePolicy::AllImages, + ) + .await } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -577,16 +611,18 @@ async fn view_image_routes_to_selected_remote_environment() -> anyhow::Result<() )) .abs(); let image_path = remote_cwd.join("remote.png"); + let remote_cwd_uri = PathUri::from_path(&remote_cwd)?; test.fs() .create_directory( - &remote_cwd, + &remote_cwd_uri, CreateDirectoryOptions { recursive: true }, /*sandbox*/ None, ) .await?; let png = png_bytes(/*width*/ 1, /*height*/ 1, [0, 255, 0, 255])?; + let image_path_uri = PathUri::from_path(&image_path)?; test.fs() - .write_file(&image_path, png, /*sandbox*/ None) + .write_file(&image_path_uri, png, /*sandbox*/ None) .await?; let remote_selection = TurnEnvironmentSelection { environment_id: REMOTE_ENVIRONMENT_ID.to_string(), @@ -645,7 +681,7 @@ async fn view_image_routes_to_selected_remote_environment() -> anyhow::Result<() test.fs() .remove( - &remote_cwd, + &remote_cwd_uri, RemoveOptions { recursive: true, force: true, @@ -1248,6 +1284,72 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resize_all_images_turns_invalid_view_image_into_placeholder() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let mut builder = test_codex().with_config(|config| { + let _ = config.features.enable(Feature::ResizeAllImages); + }); + let test = builder.build_with_remote_env(&server).await?; + let TestCodex { + codex, + session_configured, + .. + } = &test; + + let rel_path = "assets/invalid-image.json"; + write_workspace_file(&test, rel_path, br#"{ "message": "hello" }"#.to_vec()).await?; + let call_id = "view-image-invalid-placeholder"; + let arguments = serde_json::json!({ "path": rel_path }).to_string(); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "view_image", &arguments), + ev_completed("resp-1"), + ]), + ) + .await; + let second_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + codex + .submit(disabled_user_turn( + &test, + vec![UserInput::Text { + text: "please inspect the image".into(), + text_elements: Vec::new(), + }], + session_configured.model.clone(), + )) + .await?; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; + + let request = second_mock.single_request(); + assert_eq!( + request.function_call_output(call_id).get("output"), + Some(&serde_json::json!([{ + "type": "input_text", + "text": "image content omitted because it could not be processed" + }])) + ); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -1374,6 +1476,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an context_window: Some(272_000), max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), }; diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml index eadab83fe7..3173ad1bdd 100644 --- a/codex-rs/exec-server/Cargo.toml +++ b/codex-rs/exec-server/Cargo.toml @@ -24,6 +24,7 @@ codex-protocol = { workspace = true } codex-sandboxing = { workspace = true } codex-shell-command = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-path-uri = { workspace = true } codex-utils-pty = { workspace = true } codex-utils-rustls-provider = { workspace = true } futures = { workspace = true } diff --git a/codex-rs/exec-server/src/environment.rs b/codex-rs/exec-server/src/environment.rs index 1dfd49d3fd..bc100b9422 100644 --- a/codex-rs/exec-server/src/environment.rs +++ b/codex-rs/exec-server/src/environment.rs @@ -882,6 +882,7 @@ mod tests { std::env::current_exe().expect("current exe").as_path(), ) .expect("absolute current exe"); + let path = codex_utils_path_uri::PathUri::from_abs_path(&path).expect("path URI"); let sandbox = crate::FileSystemSandboxContext::from_permission_profile( codex_protocol::models::PermissionProfile::from_runtime_permissions( &codex_protocol::permissions::FileSystemSandboxPolicy::restricted(Vec::new()), diff --git a/codex-rs/exec-server/src/environment_path.rs b/codex-rs/exec-server/src/environment_path.rs deleted file mode 100644 index e9e07abef8..0000000000 --- a/codex-rs/exec-server/src/environment_path.rs +++ /dev/null @@ -1,527 +0,0 @@ -use std::fmt; -use std::hash::Hash; -use std::hash::Hasher; -use std::io; -use std::path::Path; -use std::sync::Arc; - -use codex_utils_absolute_path::AbsolutePathBuf; - -use crate::ExecutorFileSystem; -use crate::FileMetadata; -use crate::FileSystemSandboxContext; -use crate::LOCAL_FS; -use crate::ReadDirectoryEntry; - -/// Binds an absolute path to the executor filesystem that owns it. -#[derive(Clone)] -pub struct EnvironmentPathRef { - file_system: Arc, - path: AbsolutePathBuf, -} - -impl EnvironmentPathRef { - /// Creates a path ref bound to the filesystem that owns `path`. - pub fn new(file_system: Arc, path: AbsolutePathBuf) -> Self { - Self { file_system, path } - } - - /// Creates a path ref bound to the shared unsandboxed local filesystem. - pub fn local(path: AbsolutePathBuf) -> Self { - Self::new(Arc::clone(&LOCAL_FS), path) - } - - /// Returns the absolute path held by this ref. - pub fn path(&self) -> &AbsolutePathBuf { - &self.path - } - - /// Returns the filesystem that owns this path. - pub fn file_system(&self) -> Arc { - Arc::clone(&self.file_system) - } - - /// Reads this path as UTF-8 text through its bound filesystem. - pub async fn read_to_string( - &self, - sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result { - self.file_system.read_file_text(&self.path, sandbox).await - } - - /// Reads metadata for this path through its bound filesystem. - pub async fn metadata( - &self, - sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result { - self.file_system.get_metadata(&self.path, sandbox).await - } - - /// Reads directory entries for this path through its bound filesystem. - pub async fn read_directory( - &self, - sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result> { - self.file_system.read_directory(&self.path, sandbox).await - } - - /// Returns a ref with the same filesystem and a replacement path. - pub fn with_path(&self, path: AbsolutePathBuf) -> Self { - Self::new(Arc::clone(&self.file_system), path) - } - - /// Lexically joins `path` onto this path through its bound filesystem. - pub async fn join>(&self, path: P) -> io::Result { - self.file_system - .join(&self.path, path.as_ref()) - .await - .map(|path| self.with_path(path)) - } - - /// Returns the parent of this path through its bound filesystem. - pub async fn parent(&self) -> io::Result> { - self.file_system - .parent(&self.path) - .await - .map(|path| path.map(|path| self.with_path(path))) - } - - /// Canonicalizes this path through its bound filesystem. - pub async fn canonicalize( - &self, - sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result { - self.file_system - .canonicalize(&self.path, sandbox) - .await - .map(|path| self.with_path(path)) - } -} - -impl PartialEq for EnvironmentPathRef { - fn eq(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.file_system, &other.file_system) && self.path == other.path - } -} - -impl Eq for EnvironmentPathRef {} - -impl Hash for EnvironmentPathRef { - fn hash(&self, state: &mut H) { - (Arc::as_ptr(&self.file_system) as *const () as usize).hash(state); - self.path.hash(state); - } -} - -impl fmt::Debug for EnvironmentPathRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("EnvironmentPathRef") - .field("path", &self.path) - .finish() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - use async_trait::async_trait; - use codex_protocol::models::PermissionProfile; - use codex_protocol::permissions::FileSystemSandboxPolicy; - use codex_protocol::permissions::NetworkSandboxPolicy; - use codex_utils_absolute_path::test_support::PathBufExt; - use pretty_assertions::assert_eq; - use std::collections::HashSet; - use std::path::Path; - use std::sync::Mutex; - - use crate::LOCAL_FS; - - #[derive(Clone, Debug, Eq, PartialEq)] - enum RecordedMethod { - Canonicalize, - Join, - Parent, - ReadFileText, - Metadata, - ReadDirectory, - } - - #[derive(Clone, Debug, Eq, PartialEq)] - struct RecordedCall { - method: RecordedMethod, - path: AbsolutePathBuf, - sandbox: Option, - } - - struct RecordingFileSystem { - calls: Mutex>, - } - - impl Default for RecordingFileSystem { - fn default() -> Self { - Self { - calls: Mutex::new(Vec::new()), - } - } - } - - impl RecordingFileSystem { - fn recorded_calls(&self) -> Vec { - match self.calls.lock() { - Ok(calls) => calls.clone(), - Err(err) => err.into_inner().clone(), - } - } - - fn push_call(&self, call: RecordedCall) { - match self.calls.lock() { - Ok(mut calls) => calls.push(call), - Err(err) => err.into_inner().push(call), - } - } - } - - fn local_path_ref(path: AbsolutePathBuf) -> EnvironmentPathRef { - EnvironmentPathRef::new(Arc::clone(&LOCAL_FS), path) - } - - #[async_trait] - impl ExecutorFileSystem for RecordingFileSystem { - async fn canonicalize( - &self, - path: &AbsolutePathBuf, - _sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result { - self.push_call(RecordedCall { - method: RecordedMethod::Canonicalize, - path: path.clone(), - sandbox: None, - }); - Ok(path.parent().unwrap()) - } - - async fn join( - &self, - base_path: &AbsolutePathBuf, - path: &Path, - ) -> io::Result { - self.push_call(RecordedCall { - method: RecordedMethod::Join, - path: base_path.clone(), - sandbox: None, - }); - AbsolutePathBuf::from_absolute_path_checked(base_path.as_path().join(path)) - } - - async fn parent(&self, path: &AbsolutePathBuf) -> io::Result> { - self.push_call(RecordedCall { - method: RecordedMethod::Parent, - path: path.clone(), - sandbox: None, - }); - Ok(path.parent()) - } - - async fn read_file( - &self, - path: &AbsolutePathBuf, - sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result> { - self.push_call(RecordedCall { - method: RecordedMethod::ReadFileText, - path: path.clone(), - sandbox: sandbox.cloned(), - }); - Ok(b"skill contents".to_vec()) - } - - async fn write_file( - &self, - _path: &AbsolutePathBuf, - _contents: Vec, - _sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result<()> { - unreachable!("write_file should not be called") - } - - async fn create_directory( - &self, - _path: &AbsolutePathBuf, - _create_directory_options: crate::CreateDirectoryOptions, - _sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result<()> { - unreachable!("create_directory should not be called") - } - - async fn get_metadata( - &self, - path: &AbsolutePathBuf, - sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result { - self.push_call(RecordedCall { - method: RecordedMethod::Metadata, - path: path.clone(), - sandbox: sandbox.cloned(), - }); - Ok(FileMetadata { - is_directory: true, - is_file: false, - is_symlink: false, - created_at_ms: 1, - modified_at_ms: 2, - }) - } - - async fn read_directory( - &self, - path: &AbsolutePathBuf, - sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result> { - self.push_call(RecordedCall { - method: RecordedMethod::ReadDirectory, - path: path.clone(), - sandbox: sandbox.cloned(), - }); - Ok(vec![ReadDirectoryEntry { - file_name: "SKILL.md".to_string(), - is_directory: false, - is_file: true, - }]) - } - - async fn remove( - &self, - _path: &AbsolutePathBuf, - _remove_options: crate::RemoveOptions, - _sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result<()> { - unreachable!("remove should not be called") - } - - async fn copy( - &self, - _source_path: &AbsolutePathBuf, - _destination_path: &AbsolutePathBuf, - _copy_options: crate::CopyOptions, - _sandbox: Option<&FileSystemSandboxContext>, - ) -> io::Result<()> { - unreachable!("copy should not be called") - } - } - - fn restricted_sandbox() -> FileSystemSandboxContext { - FileSystemSandboxContext::from_permission_profile( - PermissionProfile::from_runtime_permissions( - &FileSystemSandboxPolicy::restricted(Vec::new()), - NetworkSandboxPolicy::Restricted, - ), - ) - } - - #[tokio::test] - async fn environment_path_ref_forwards_sandbox_to_file_system_methods() { - let path = std::env::temp_dir().join("skills/demo").abs(); - let file_system = Arc::new(RecordingFileSystem::default()); - let path_ref = EnvironmentPathRef::new(file_system.clone(), path.clone()); - let sandbox = restricted_sandbox(); - - assert_eq!( - path_ref - .read_to_string(Some(&sandbox)) - .await - .expect("read skill contents"), - "skill contents".to_string() - ); - assert_eq!( - path_ref - .metadata(Some(&sandbox)) - .await - .expect("read metadata"), - FileMetadata { - is_directory: true, - is_file: false, - is_symlink: false, - created_at_ms: 1, - modified_at_ms: 2, - } - ); - assert_eq!( - path_ref - .read_directory(Some(&sandbox)) - .await - .expect("read directory"), - vec![ReadDirectoryEntry { - file_name: "SKILL.md".to_string(), - is_directory: false, - is_file: true, - }] - ); - assert_eq!( - file_system.recorded_calls(), - vec![ - RecordedCall { - method: RecordedMethod::ReadFileText, - path: path.clone(), - sandbox: Some(sandbox.clone()), - }, - RecordedCall { - method: RecordedMethod::Metadata, - path: path.clone(), - sandbox: Some(sandbox.clone()), - }, - RecordedCall { - method: RecordedMethod::ReadDirectory, - path, - sandbox: Some(sandbox), - }, - ] - ); - } - - #[test] - fn environment_path_ref_equality_and_hash_include_file_system_identity() { - let path = std::env::temp_dir().join("skills/demo").abs(); - let file_system = Arc::new(RecordingFileSystem::default()); - let same_file_system: Arc = file_system.clone(); - let different_file_system: Arc = - Arc::new(RecordingFileSystem::default()); - - let left = EnvironmentPathRef::new(same_file_system.clone(), path.clone()); - let same = EnvironmentPathRef::new(same_file_system, path.clone()); - let different_path = EnvironmentPathRef::new(file_system, path.parent().unwrap()); - let different_fs = EnvironmentPathRef::new(different_file_system, path); - - assert_eq!(left, same); - assert_ne!(left, different_path); - assert_ne!(left, different_fs); - - let set = HashSet::from([left, same, different_path, different_fs]); - assert_eq!(set.len(), 3); - } - #[tokio::test] - async fn canonicalize_keeps_bound_file_system_identity() { - let path = std::env::temp_dir().join("skills/demo").abs(); - let file_system = Arc::new(RecordingFileSystem::default()); - let path_ref = EnvironmentPathRef::new(file_system.clone(), path.clone()); - - let canonicalized = path_ref - .canonicalize(/*sandbox*/ None) - .await - .expect("canonicalize"); - - assert_eq!(canonicalized.path(), &path.parent().unwrap()); - assert_eq!( - canonicalized, - EnvironmentPathRef::new(file_system.clone(), path.parent().unwrap()) - ); - assert_eq!( - file_system.recorded_calls(), - vec![RecordedCall { - method: RecordedMethod::Canonicalize, - path, - sandbox: None, - }] - ); - } - - #[tokio::test] - async fn join_keeps_bound_file_system_identity() { - let path = std::env::temp_dir().join("skills").abs(); - let file_system = Arc::new(RecordingFileSystem::default()); - let path_ref = EnvironmentPathRef::new(file_system.clone(), path.clone()); - - assert_eq!( - path_ref.join(Path::new("demo")).await.ok(), - Some(EnvironmentPathRef::new( - file_system.clone(), - std::env::temp_dir().join("skills/demo").abs(), - )) - ); - assert_eq!( - file_system.recorded_calls(), - vec![RecordedCall { - method: RecordedMethod::Join, - path, - sandbox: None, - }] - ); - } - - #[tokio::test] - async fn join_matches_absolute_path_buf_for_tilde_paths() { - let path_ref = local_path_ref(std::env::temp_dir().join("skills").abs()); - - assert_eq!( - path_ref - .join(Path::new("~")) - .await - .ok() - .map(|path_ref| path_ref.path().clone()), - Some(path_ref.path().join(Path::new("~"))) - ); - } - - #[tokio::test] - async fn join_matches_absolute_path_buf_for_parent_dirs() { - let path_ref = local_path_ref(std::env::temp_dir().join("skills").abs()); - - assert_eq!( - path_ref - .join(Path::new("../outside")) - .await - .expect("join") - .path() - .clone(), - path_ref.path().join(Path::new("../outside")) - ); - } - - #[tokio::test] - async fn parent_keeps_bound_file_system_identity() { - let path = std::env::temp_dir().join("skills/demo").abs(); - let file_system = Arc::new(RecordingFileSystem::default()); - let path_ref = EnvironmentPathRef::new(file_system.clone(), path.clone()); - - assert_eq!( - path_ref.parent().await.expect("parent"), - Some(EnvironmentPathRef::new( - file_system.clone(), - std::env::temp_dir().join("skills").abs(), - )) - ); - assert_eq!( - file_system.recorded_calls(), - vec![RecordedCall { - method: RecordedMethod::Parent, - path, - sandbox: None, - }] - ); - } - - #[cfg(windows)] - #[tokio::test] - async fn join_matches_absolute_path_buf_for_windows_prefixed_and_rooted_paths() { - let path_ref = local_path_ref(std::env::temp_dir().join("skills").abs()); - - assert_eq!( - path_ref - .join(Path::new(r"C:temp")) - .await - .expect("join") - .path() - .clone(), - path_ref.path().join(Path::new(r"C:temp")) - ); - assert_eq!( - path_ref - .join(Path::new(r"\temp")) - .await - .expect("join") - .path() - .clone(), - path_ref.path().join(Path::new(r"\temp")) - ); - } -} diff --git a/codex-rs/exec-server/src/fs_helper.rs b/codex-rs/exec-server/src/fs_helper.rs index 0a210175b2..526e682976 100644 --- a/codex-rs/exec-server/src/fs_helper.rs +++ b/codex-rs/exec-server/src/fs_helper.rs @@ -189,8 +189,10 @@ pub(crate) async fn run_direct_request( let file_system = DirectFileSystem; match request { FsHelperRequest::ReadFile(params) => { + let path = + codex_utils_path_uri::PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let data = file_system - .read_file(¶ms.path, /*sandbox*/ None) + .read_file(&path, /*sandbox*/ None) .await .map_err(map_fs_error)?; Ok(FsHelperPayload::ReadFile(FsReadFileResponse { @@ -198,21 +200,25 @@ pub(crate) async fn run_direct_request( })) } FsHelperRequest::WriteFile(params) => { + let path = + codex_utils_path_uri::PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let bytes = STANDARD.decode(params.data_base64).map_err(|err| { invalid_request(format!( "{FS_WRITE_FILE_METHOD} requires valid base64 dataBase64: {err}" )) })?; file_system - .write_file(¶ms.path, bytes, /*sandbox*/ None) + .write_file(&path, bytes, /*sandbox*/ None) .await .map_err(map_fs_error)?; Ok(FsHelperPayload::WriteFile(FsWriteFileResponse {})) } FsHelperRequest::CreateDirectory(params) => { + let path = + codex_utils_path_uri::PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; file_system .create_directory( - ¶ms.path, + &path, CreateDirectoryOptions { recursive: params.recursive.unwrap_or(true), }, @@ -225,8 +231,10 @@ pub(crate) async fn run_direct_request( )) } FsHelperRequest::GetMetadata(params) => { + let path = + codex_utils_path_uri::PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let metadata = file_system - .get_metadata(¶ms.path, /*sandbox*/ None) + .get_metadata(&path, /*sandbox*/ None) .await .map_err(map_fs_error)?; Ok(FsHelperPayload::GetMetadata(FsGetMetadataResponse { @@ -238,17 +246,22 @@ pub(crate) async fn run_direct_request( })) } FsHelperRequest::Canonicalize(params) => { + let path = + codex_utils_path_uri::PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let path = file_system - .canonicalize(¶ms.path, /*sandbox*/ None) + .canonicalize(&path, /*sandbox*/ None) .await .map_err(map_fs_error)?; + let path = path.to_abs_path().map_err(map_fs_error)?; Ok(FsHelperPayload::Canonicalize(FsCanonicalizeResponse { path, })) } FsHelperRequest::ReadDirectory(params) => { + let path = + codex_utils_path_uri::PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let entries = file_system - .read_directory(¶ms.path, /*sandbox*/ None) + .read_directory(&path, /*sandbox*/ None) .await .map_err(map_fs_error)? .into_iter() @@ -263,9 +276,11 @@ pub(crate) async fn run_direct_request( })) } FsHelperRequest::Remove(params) => { + let path = + codex_utils_path_uri::PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; file_system .remove( - ¶ms.path, + &path, RemoveOptions { recursive: params.recursive.unwrap_or(true), force: params.force.unwrap_or(true), @@ -277,10 +292,15 @@ pub(crate) async fn run_direct_request( Ok(FsHelperPayload::Remove(FsRemoveResponse {})) } FsHelperRequest::Copy(params) => { + let source_path = codex_utils_path_uri::PathUri::from_abs_path(¶ms.source_path) + .map_err(map_fs_error)?; + let destination_path = + codex_utils_path_uri::PathUri::from_abs_path(¶ms.destination_path) + .map_err(map_fs_error)?; file_system .copy( - ¶ms.source_path, - ¶ms.destination_path, + &source_path, + &destination_path, CopyOptions { recursive: params.recursive, }, @@ -305,23 +325,72 @@ fn map_fs_error(err: io::Error) -> JSONRPCErrorError { #[cfg(test)] mod tests { + use codex_utils_absolute_path::AbsolutePathBuf; + use pretty_assertions::assert_eq; + use serde_json::json; + use super::*; #[test] - fn helper_requests_use_fs_method_names() -> serde_json::Result<()> { - assert_eq!( - serde_json::to_value(FsHelperRequest::WriteFile(FsWriteFileParams { - path: std::env::current_dir() - .expect("cwd") - .join("file") - .as_path() - .try_into() - .expect("absolute path"), + fn helper_protocol_keeps_native_absolute_paths() -> serde_json::Result<()> { + let local_path = + AbsolutePathBuf::from_absolute_path(std::env::current_dir().expect("cwd").join("file")) + .expect("absolute path"); + #[cfg(not(windows))] + let paths = [local_path]; + #[cfg(windows)] + let paths = [ + local_path, + AbsolutePathBuf::from_absolute_path(r"\\server\share\file").expect("absolute UNC path"), + ]; + + for path in paths { + let expected_path = path.to_string_lossy().into_owned(); + + let request = serde_json::to_value(FsHelperRequest::WriteFile(FsWriteFileParams { + path: path.clone(), data_base64: String::new(), sandbox: None, - }))?["operation"], - FS_WRITE_FILE_METHOD, - ); + }))?; + assert_eq!( + request, + json!({ + "operation": FS_WRITE_FILE_METHOD, + "params": { + "path": expected_path.as_str(), + "dataBase64": "", + "sandbox": null, + }, + }), + ); + let request_path = request["params"]["path"] + .as_str() + .expect("request path should be a string"); + assert_eq!(request_path, expected_path); + assert!(!request_path.starts_with("file:")); + + let response = serde_json::to_value(FsHelperResponse::Ok( + FsHelperPayload::Canonicalize(FsCanonicalizeResponse { path }), + ))?; + assert_eq!( + response, + json!({ + "status": "ok", + "payload": { + "operation": FS_CANONICALIZE_METHOD, + "response": { + "path": expected_path.as_str(), + }, + }, + }), + ); + let response_path = response["payload"]["response"]["path"] + .as_str() + .expect("canonicalize response path should be a string"); + assert_eq!(response_path, expected_path); + assert!(!response_path.starts_with("file:")); + } + Ok(()) } } diff --git a/codex-rs/exec-server/src/lib.rs b/codex-rs/exec-server/src/lib.rs index 378d12e632..2d198d921d 100644 --- a/codex-rs/exec-server/src/lib.rs +++ b/codex-rs/exec-server/src/lib.rs @@ -3,7 +3,6 @@ mod client_api; mod client_transport; mod connection; mod environment; -mod environment_path; mod environment_provider; mod environment_toml; mod fs_helper; @@ -44,7 +43,6 @@ pub use environment::Environment; pub use environment::EnvironmentManager; pub use environment::LOCAL_ENVIRONMENT_ID; pub use environment::REMOTE_ENVIRONMENT_ID; -pub use environment_path::EnvironmentPathRef; pub use environment_provider::DefaultEnvironmentProvider; pub use environment_provider::EnvironmentProvider; pub use fs_helper::CODEX_FS_HELPER_ARG1; diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs index 3f410f926d..ee38d7ff94 100644 --- a/codex-rs/exec-server/src/local_file_system.rs +++ b/codex-rs/exec-server/src/local_file_system.rs @@ -1,5 +1,6 @@ use async_trait::async_trait; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -81,28 +82,16 @@ impl LocalFileSystem { impl ExecutorFileSystem for LocalFileSystem { async fn canonicalize( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, - ) -> FileSystemResult { + ) -> FileSystemResult { let (file_system, sandbox) = self.file_system_for(sandbox)?; file_system.canonicalize(path, sandbox).await } - async fn join( - &self, - base_path: &AbsolutePathBuf, - path: &Path, - ) -> FileSystemResult { - self.unsandboxed.join(base_path, path).await - } - - async fn parent(&self, path: &AbsolutePathBuf) -> FileSystemResult> { - self.unsandboxed.parent(path).await - } - async fn read_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { let (file_system, sandbox) = self.file_system_for(sandbox)?; @@ -111,7 +100,7 @@ impl ExecutorFileSystem for LocalFileSystem { async fn write_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, contents: Vec, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -121,7 +110,7 @@ impl ExecutorFileSystem for LocalFileSystem { async fn create_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, options: CreateDirectoryOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -131,7 +120,7 @@ impl ExecutorFileSystem for LocalFileSystem { async fn get_metadata( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult { let (file_system, sandbox) = self.file_system_for(sandbox)?; @@ -140,7 +129,7 @@ impl ExecutorFileSystem for LocalFileSystem { async fn read_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { let (file_system, sandbox) = self.file_system_for(sandbox)?; @@ -149,7 +138,7 @@ impl ExecutorFileSystem for LocalFileSystem { async fn remove( &self, - path: &AbsolutePathBuf, + path: &PathUri, options: RemoveOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -159,8 +148,8 @@ impl ExecutorFileSystem for LocalFileSystem { async fn copy( &self, - source_path: &AbsolutePathBuf, - destination_path: &AbsolutePathBuf, + source_path: &PathUri, + destination_path: &PathUri, options: CopyOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -175,28 +164,16 @@ impl ExecutorFileSystem for LocalFileSystem { impl ExecutorFileSystem for UnsandboxedFileSystem { async fn canonicalize( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, - ) -> FileSystemResult { + ) -> FileSystemResult { reject_platform_sandbox_context(sandbox)?; self.file_system.canonicalize(path, /*sandbox*/ None).await } - async fn join( - &self, - base_path: &AbsolutePathBuf, - path: &Path, - ) -> FileSystemResult { - self.file_system.join(base_path, path).await - } - - async fn parent(&self, path: &AbsolutePathBuf) -> FileSystemResult> { - self.file_system.parent(path).await - } - async fn read_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { reject_platform_sandbox_context(sandbox)?; @@ -205,7 +182,7 @@ impl ExecutorFileSystem for UnsandboxedFileSystem { async fn write_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, contents: Vec, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -217,7 +194,7 @@ impl ExecutorFileSystem for UnsandboxedFileSystem { async fn create_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, options: CreateDirectoryOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -229,7 +206,7 @@ impl ExecutorFileSystem for UnsandboxedFileSystem { async fn get_metadata( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult { reject_platform_sandbox_context(sandbox)?; @@ -238,7 +215,7 @@ impl ExecutorFileSystem for UnsandboxedFileSystem { async fn read_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { reject_platform_sandbox_context(sandbox)?; @@ -249,7 +226,7 @@ impl ExecutorFileSystem for UnsandboxedFileSystem { async fn remove( &self, - path: &AbsolutePathBuf, + path: &PathUri, options: RemoveOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -261,8 +238,8 @@ impl ExecutorFileSystem for UnsandboxedFileSystem { async fn copy( &self, - source_path: &AbsolutePathBuf, - destination_path: &AbsolutePathBuf, + source_path: &PathUri, + destination_path: &PathUri, options: CopyOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -282,31 +259,23 @@ impl ExecutorFileSystem for UnsandboxedFileSystem { impl ExecutorFileSystem for DirectFileSystem { async fn canonicalize( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, - ) -> FileSystemResult { + ) -> FileSystemResult { reject_sandbox_context(sandbox)?; - AbsolutePathBuf::from_absolute_path(tokio::fs::canonicalize(path.as_path()).await?) - } - - async fn join( - &self, - base_path: &AbsolutePathBuf, - path: &Path, - ) -> FileSystemResult { - Ok(base_path.join(path)) - } - - async fn parent(&self, path: &AbsolutePathBuf) -> FileSystemResult> { - Ok(path.parent()) + let path = path.to_abs_path()?; + let canonicalized = + AbsolutePathBuf::from_absolute_path(tokio::fs::canonicalize(path.as_path()).await?)?; + PathUri::from_abs_path(&canonicalized) } async fn read_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { reject_sandbox_context(sandbox)?; + let path = path.to_abs_path()?; let metadata = tokio::fs::metadata(path.as_path()).await?; if metadata.len() > MAX_READ_FILE_BYTES { return Err(io::Error::new( @@ -319,21 +288,23 @@ impl ExecutorFileSystem for DirectFileSystem { async fn write_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, contents: Vec, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { reject_sandbox_context(sandbox)?; + let path = path.to_abs_path()?; tokio::fs::write(path.as_path(), contents).await } async fn create_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, options: CreateDirectoryOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { reject_sandbox_context(sandbox)?; + let path = path.to_abs_path()?; if options.recursive { tokio::fs::create_dir_all(path.as_path()).await?; } else { @@ -344,10 +315,11 @@ impl ExecutorFileSystem for DirectFileSystem { async fn get_metadata( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult { reject_sandbox_context(sandbox)?; + let path = path.to_abs_path()?; let metadata = tokio::fs::metadata(path.as_path()).await?; let symlink_metadata = tokio::fs::symlink_metadata(path.as_path()).await?; Ok(FileMetadata { @@ -361,10 +333,11 @@ impl ExecutorFileSystem for DirectFileSystem { async fn read_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { reject_sandbox_context(sandbox)?; + let path = path.to_abs_path()?; let mut entries = Vec::new(); let mut read_dir = tokio::fs::read_dir(path.as_path()).await?; while let Some(entry) = read_dir.next_entry().await? { @@ -382,11 +355,12 @@ impl ExecutorFileSystem for DirectFileSystem { async fn remove( &self, - path: &AbsolutePathBuf, + path: &PathUri, options: RemoveOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { reject_sandbox_context(sandbox)?; + let path = path.to_abs_path()?; match tokio::fs::symlink_metadata(path.as_path()).await { Ok(metadata) => { let file_type = metadata.file_type(); @@ -408,14 +382,14 @@ impl ExecutorFileSystem for DirectFileSystem { async fn copy( &self, - source_path: &AbsolutePathBuf, - destination_path: &AbsolutePathBuf, + source_path: &PathUri, + destination_path: &PathUri, options: CopyOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { reject_sandbox_context(sandbox)?; - let source_path = source_path.to_path_buf(); - let destination_path = destination_path.to_path_buf(); + let source_path = source_path.to_abs_path()?.into_path_buf(); + let destination_path = destination_path.to_abs_path()?.into_path_buf(); tokio::task::spawn_blocking(move || -> FileSystemResult<()> { let metadata = std::fs::symlink_metadata(source_path.as_path())?; let file_type = metadata.file_type(); @@ -576,6 +550,10 @@ fn system_time_to_unix_ms(time: SystemTime) -> i64 { .unwrap_or(0) } +#[cfg(all(test, any(unix, windows)))] +#[path = "local_file_system_path_uri_tests.rs"] +mod path_uri_tests; + #[cfg(all(test, unix))] mod tests { use super::*; diff --git a/codex-rs/exec-server/src/local_file_system_path_uri_tests.rs b/codex-rs/exec-server/src/local_file_system_path_uri_tests.rs new file mode 100644 index 0000000000..dcddd98e08 --- /dev/null +++ b/codex-rs/exec-server/src/local_file_system_path_uri_tests.rs @@ -0,0 +1,27 @@ +use codex_utils_path_uri::PathUri; +use pretty_assertions::assert_eq; +use tokio::io; + +use super::*; + +#[tokio::test] +async fn direct_file_system_rejects_non_native_uri_as_invalid_input() { + let error = DirectFileSystem + .read_file(&non_native_uri(), /*sandbox*/ None) + .await + .expect_err("non-native URI should be rejected"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); +} + +fn non_native_uri() -> PathUri { + #[cfg(unix)] + let uri = "file://server/share/file.txt"; + #[cfg(windows)] + let uri = "file:///usr/local/file.txt"; + + match PathUri::parse(uri) { + Ok(uri) => uri, + Err(err) => panic!("valid non-native URI should parse: {err}"), + } +} diff --git a/codex-rs/exec-server/src/remote_file_system.rs b/codex-rs/exec-server/src/remote_file_system.rs index 54198c93ba..9bcdcc5b09 100644 --- a/codex-rs/exec-server/src/remote_file_system.rs +++ b/codex-rs/exec-server/src/remote_file_system.rs @@ -1,8 +1,7 @@ use async_trait::async_trait; use base64::Engine as _; use base64::engine::general_purpose::STANDARD; -use codex_utils_absolute_path::AbsolutePathBuf; -use std::path::Path; +use codex_utils_path_uri::PathUri; use tokio::io; use tracing::trace; @@ -20,8 +19,6 @@ use crate::protocol::FsCanonicalizeParams; use crate::protocol::FsCopyParams; use crate::protocol::FsCreateDirectoryParams; use crate::protocol::FsGetMetadataParams; -use crate::protocol::FsJoinParams; -use crate::protocol::FsParentParams; use crate::protocol::FsReadDirectoryParams; use crate::protocol::FsReadFileParams; use crate::protocol::FsRemoveParams; @@ -45,58 +42,33 @@ impl RemoteFileSystem { impl ExecutorFileSystem for RemoteFileSystem { async fn canonicalize( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, - ) -> FileSystemResult { + ) -> FileSystemResult { trace!("remote fs canonicalize"); + let path = path.to_abs_path()?; let client = self.client.get().await.map_err(map_remote_error)?; let response = client .fs_canonicalize(FsCanonicalizeParams { - path: path.clone(), + path, sandbox: remote_sandbox_context(sandbox), }) .await .map_err(map_remote_error)?; - Ok(response.path) - } - - async fn join( - &self, - base_path: &AbsolutePathBuf, - path: &Path, - ) -> FileSystemResult { - trace!("remote fs join"); - let client = self.client.get().await.map_err(map_remote_error)?; - let response = client - .fs_join(FsJoinParams { - base_path: base_path.clone(), - path: path.to_path_buf(), - }) - .await - .map_err(map_remote_error)?; - Ok(response.path) - } - - async fn parent(&self, path: &AbsolutePathBuf) -> FileSystemResult> { - trace!("remote fs parent"); - let client = self.client.get().await.map_err(map_remote_error)?; - let response = client - .fs_parent(FsParentParams { path: path.clone() }) - .await - .map_err(map_remote_error)?; - Ok(response.path) + PathUri::from_abs_path(&response.path) } async fn read_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { trace!("remote fs read_file"); + let path = path.to_abs_path()?; let client = self.client.get().await.map_err(map_remote_error)?; let response = client .fs_read_file(FsReadFileParams { - path: path.clone(), + path, sandbox: remote_sandbox_context(sandbox), }) .await @@ -111,15 +83,16 @@ impl ExecutorFileSystem for RemoteFileSystem { async fn write_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, contents: Vec, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { trace!("remote fs write_file"); + let path = path.to_abs_path()?; let client = self.client.get().await.map_err(map_remote_error)?; client .fs_write_file(FsWriteFileParams { - path: path.clone(), + path, data_base64: STANDARD.encode(contents), sandbox: remote_sandbox_context(sandbox), }) @@ -130,15 +103,16 @@ impl ExecutorFileSystem for RemoteFileSystem { async fn create_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, options: CreateDirectoryOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { trace!("remote fs create_directory"); + let path = path.to_abs_path()?; let client = self.client.get().await.map_err(map_remote_error)?; client .fs_create_directory(FsCreateDirectoryParams { - path: path.clone(), + path, recursive: Some(options.recursive), sandbox: remote_sandbox_context(sandbox), }) @@ -149,14 +123,15 @@ impl ExecutorFileSystem for RemoteFileSystem { async fn get_metadata( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult { trace!("remote fs get_metadata"); + let path = path.to_abs_path()?; let client = self.client.get().await.map_err(map_remote_error)?; let response = client .fs_get_metadata(FsGetMetadataParams { - path: path.clone(), + path, sandbox: remote_sandbox_context(sandbox), }) .await @@ -172,14 +147,15 @@ impl ExecutorFileSystem for RemoteFileSystem { async fn read_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { trace!("remote fs read_directory"); + let path = path.to_abs_path()?; let client = self.client.get().await.map_err(map_remote_error)?; let response = client .fs_read_directory(FsReadDirectoryParams { - path: path.clone(), + path, sandbox: remote_sandbox_context(sandbox), }) .await @@ -197,15 +173,16 @@ impl ExecutorFileSystem for RemoteFileSystem { async fn remove( &self, - path: &AbsolutePathBuf, + path: &PathUri, options: RemoveOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { trace!("remote fs remove"); + let path = path.to_abs_path()?; let client = self.client.get().await.map_err(map_remote_error)?; client .fs_remove(FsRemoveParams { - path: path.clone(), + path, recursive: Some(options.recursive), force: Some(options.force), sandbox: remote_sandbox_context(sandbox), @@ -217,17 +194,19 @@ impl ExecutorFileSystem for RemoteFileSystem { async fn copy( &self, - source_path: &AbsolutePathBuf, - destination_path: &AbsolutePathBuf, + source_path: &PathUri, + destination_path: &PathUri, options: CopyOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { trace!("remote fs copy"); + let source_path = source_path.to_abs_path()?; + let destination_path = destination_path.to_abs_path()?; let client = self.client.get().await.map_err(map_remote_error)?; client .fs_copy(FsCopyParams { - source_path: source_path.clone(), - destination_path: destination_path.clone(), + source_path, + destination_path, recursive: options.recursive, sandbox: remote_sandbox_context(sandbox), }) @@ -261,6 +240,10 @@ fn map_remote_error(error: ExecServerError) -> io::Error { } } +#[cfg(all(test, any(unix, windows)))] +#[path = "remote_file_system_path_uri_tests.rs"] +mod path_uri_tests; + #[cfg(test)] mod tests { use codex_protocol::models::PermissionProfile; @@ -270,6 +253,7 @@ mod tests { use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::permissions::FileSystemSpecialPath; use codex_protocol::permissions::NetworkSandboxPolicy; + use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; use super::*; diff --git a/codex-rs/exec-server/src/remote_file_system_path_uri_tests.rs b/codex-rs/exec-server/src/remote_file_system_path_uri_tests.rs new file mode 100644 index 0000000000..29d0edaffb --- /dev/null +++ b/codex-rs/exec-server/src/remote_file_system_path_uri_tests.rs @@ -0,0 +1,225 @@ +#![allow(clippy::expect_used)] + +#[cfg(windows)] +use codex_app_server_protocol::JSONRPCMessage; +#[cfg(windows)] +use codex_app_server_protocol::JSONRPCResponse; +#[cfg(windows)] +use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; +#[cfg(windows)] +use futures::SinkExt; +#[cfg(windows)] +use futures::StreamExt; +use pretty_assertions::assert_eq; +use tokio::io; +#[cfg(windows)] +use tokio::net::TcpListener; +#[cfg(windows)] +use tokio::net::TcpStream; +#[cfg(windows)] +use tokio::sync::oneshot; +#[cfg(windows)] +use tokio::time::Duration; +#[cfg(windows)] +use tokio::time::timeout; +#[cfg(windows)] +use tokio_tungstenite::WebSocketStream; +#[cfg(windows)] +use tokio_tungstenite::accept_async; +#[cfg(windows)] +use tokio_tungstenite::tungstenite::Message; + +use super::*; +use crate::client_api::ExecServerTransportParams; +#[cfg(windows)] +use crate::protocol::FS_READ_FILE_METHOD; +#[cfg(windows)] +use crate::protocol::FsReadFileParams; +#[cfg(windows)] +use crate::protocol::FsReadFileResponse; +#[cfg(windows)] +use crate::protocol::INITIALIZE_METHOD; +#[cfg(windows)] +use crate::protocol::INITIALIZED_METHOD; +#[cfg(windows)] +use crate::protocol::InitializeResponse; + +#[tokio::test] +async fn non_native_uri_is_rejected_before_connecting() { + let file_system = RemoteFileSystem::new(LazyRemoteExecServerClient::new( + ExecServerTransportParams::websocket_url("not a websocket URL".to_string()), + )); + + let error = file_system + .read_file(&non_native_uri(), /*sandbox*/ None) + .await + .expect_err("non-native URI should be rejected"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); +} + +#[cfg(windows)] +#[tokio::test] +async fn remote_file_system_sends_explicit_windows_native_paths() { + let (websocket_url, captured_paths, server) = record_read_file_paths(2).await; + let file_system = RemoteFileSystem::new(LazyRemoteExecServerClient::new( + ExecServerTransportParams::websocket_url(websocket_url), + )); + let paths = vec![ + ( + PathUri::parse("file:///C:/Users/Alice/src/main.rs").expect("valid drive URI"), + absolute_windows_path(r"C:\Users\Alice\src\main.rs"), + ), + ( + PathUri::parse("file://server/share/src/main.rs").expect("valid UNC URI"), + absolute_windows_path(r"\\server\share\src\main.rs"), + ), + ]; + let expected_paths = paths + .iter() + .map(|(_, expected_path)| expected_path.clone()) + .collect::>(); + + for (path, _) in paths { + assert_eq!( + file_system + .read_file(&path, /*sandbox*/ None) + .await + .expect("remote read should succeed"), + Vec::::new() + ); + } + + assert_eq!( + captured_paths.await.expect("captured paths"), + expected_paths + ); + server.await.expect("recording server should succeed"); +} + +fn non_native_uri() -> PathUri { + #[cfg(unix)] + let uri = "file://server/share/file.txt"; + #[cfg(windows)] + let uri = "file:///usr/local/file.txt"; + + PathUri::parse(uri).expect("valid non-native URI") +} + +#[cfg(windows)] +fn absolute_windows_path(path: &str) -> AbsolutePathBuf { + AbsolutePathBuf::from_absolute_path_checked(path).expect("absolute Windows path") +} + +#[cfg(windows)] +async fn record_read_file_paths( + expected_requests: usize, +) -> ( + String, + oneshot::Receiver>, + tokio::task::JoinHandle<()>, +) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let websocket_url = format!("ws://{}", listener.local_addr().expect("listener address")); + let (captured_paths_tx, captured_paths_rx) = oneshot::channel(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("listener should accept"); + let mut websocket = accept_async(stream) + .await + .expect("websocket handshake should succeed"); + complete_websocket_initialize(&mut websocket).await; + + let mut captured_paths = Vec::with_capacity(expected_requests); + for _ in 0..expected_requests { + let request = match read_jsonrpc_websocket(&mut websocket).await { + JSONRPCMessage::Request(request) if request.method == FS_READ_FILE_METHOD => { + request + } + other => panic!("expected fs/readFile request, got {other:?}"), + }; + let params: FsReadFileParams = + serde_json::from_value(request.params.expect("fs/readFile params should exist")) + .expect("fs/readFile params should deserialize"); + captured_paths.push(params.path); + write_jsonrpc_websocket( + &mut websocket, + JSONRPCMessage::Response(JSONRPCResponse { + id: request.id, + result: serde_json::to_value(FsReadFileResponse { + data_base64: String::new(), + }) + .expect("fs/readFile response should serialize"), + }), + ) + .await; + } + captured_paths_tx + .send(captured_paths) + .expect("captured paths receiver should stay open"); + }); + + (websocket_url, captured_paths_rx, server) +} + +#[cfg(windows)] +async fn complete_websocket_initialize(websocket: &mut WebSocketStream) { + let request = match read_jsonrpc_websocket(websocket).await { + JSONRPCMessage::Request(request) if request.method == INITIALIZE_METHOD => request, + other => panic!("expected initialize request, got {other:?}"), + }; + write_jsonrpc_websocket( + websocket, + JSONRPCMessage::Response(JSONRPCResponse { + id: request.id, + result: serde_json::to_value(InitializeResponse { + session_id: "session-1".to_string(), + }) + .expect("initialize response should serialize"), + }), + ) + .await; + + match read_jsonrpc_websocket(websocket).await { + JSONRPCMessage::Notification(notification) if notification.method == INITIALIZED_METHOD => { + } + other => panic!("expected initialized notification, got {other:?}"), + } +} + +#[cfg(windows)] +async fn read_jsonrpc_websocket(websocket: &mut WebSocketStream) -> JSONRPCMessage { + loop { + match timeout(Duration::from_secs(1), websocket.next()) + .await + .expect("json-rpc websocket read should not time out") + .expect("websocket should stay open") + .expect("websocket frame should read") + { + Message::Text(text) => { + return serde_json::from_str(text.as_ref()) + .expect("json-rpc text frame should parse"); + } + Message::Binary(bytes) => { + return serde_json::from_slice(bytes.as_ref()) + .expect("json-rpc binary frame should parse"); + } + Message::Ping(_) | Message::Pong(_) => {} + other => panic!("expected json-rpc websocket frame, got {other:?}"), + } + } +} + +#[cfg(windows)] +async fn write_jsonrpc_websocket( + websocket: &mut WebSocketStream, + message: JSONRPCMessage, +) { + let encoded = serde_json::to_string(&message).expect("json-rpc should serialize"); + websocket + .send(Message::Text(encoded.into())) + .await + .expect("json-rpc websocket frame should write"); +} diff --git a/codex-rs/exec-server/src/sandboxed_file_system.rs b/codex-rs/exec-server/src/sandboxed_file_system.rs index 00b6966c3b..14f26e60b9 100644 --- a/codex-rs/exec-server/src/sandboxed_file_system.rs +++ b/codex-rs/exec-server/src/sandboxed_file_system.rs @@ -2,8 +2,7 @@ use async_trait::async_trait; use base64::Engine as _; use base64::engine::general_purpose::STANDARD; use codex_app_server_protocol::JSONRPCErrorError; -use codex_utils_absolute_path::AbsolutePathBuf; -use std::path::Path; +use codex_utils_path_uri::PathUri; use tokio::io; use crate::CopyOptions; @@ -55,39 +54,27 @@ impl SandboxedFileSystem { impl ExecutorFileSystem for SandboxedFileSystem { async fn canonicalize( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, - ) -> FileSystemResult { + ) -> FileSystemResult { let sandbox = require_platform_sandbox(sandbox)?; let response = self .run_sandboxed( sandbox, FsHelperRequest::Canonicalize(FsCanonicalizeParams { - path: path.clone(), + path: path.to_abs_path()?, sandbox: None, }), ) .await? .expect_canonicalize() .map_err(map_sandbox_error)?; - Ok(response.path) - } - - async fn join( - &self, - base_path: &AbsolutePathBuf, - path: &Path, - ) -> FileSystemResult { - Ok(base_path.join(path)) - } - - async fn parent(&self, path: &AbsolutePathBuf) -> FileSystemResult> { - Ok(path.parent()) + PathUri::from_abs_path(&response.path) } async fn read_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { let sandbox = require_platform_sandbox(sandbox)?; @@ -95,7 +82,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { .run_sandboxed( sandbox, FsHelperRequest::ReadFile(FsReadFileParams { - path: path.clone(), + path: path.to_abs_path()?, sandbox: None, }), ) @@ -112,7 +99,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { async fn write_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, contents: Vec, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -120,7 +107,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { self.run_sandboxed( sandbox, FsHelperRequest::WriteFile(FsWriteFileParams { - path: path.clone(), + path: path.to_abs_path()?, data_base64: STANDARD.encode(contents), sandbox: None, }), @@ -133,7 +120,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { async fn create_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, options: CreateDirectoryOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -141,7 +128,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { self.run_sandboxed( sandbox, FsHelperRequest::CreateDirectory(FsCreateDirectoryParams { - path: path.clone(), + path: path.to_abs_path()?, recursive: Some(options.recursive), sandbox: None, }), @@ -154,7 +141,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { async fn get_metadata( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult { let sandbox = require_platform_sandbox(sandbox)?; @@ -162,7 +149,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { .run_sandboxed( sandbox, FsHelperRequest::GetMetadata(FsGetMetadataParams { - path: path.clone(), + path: path.to_abs_path()?, sandbox: None, }), ) @@ -180,7 +167,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { async fn read_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { let sandbox = require_platform_sandbox(sandbox)?; @@ -188,7 +175,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { .run_sandboxed( sandbox, FsHelperRequest::ReadDirectory(FsReadDirectoryParams { - path: path.clone(), + path: path.to_abs_path()?, sandbox: None, }), ) @@ -208,7 +195,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { async fn remove( &self, - path: &AbsolutePathBuf, + path: &PathUri, remove_options: RemoveOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -216,7 +203,7 @@ impl ExecutorFileSystem for SandboxedFileSystem { self.run_sandboxed( sandbox, FsHelperRequest::Remove(FsRemoveParams { - path: path.clone(), + path: path.to_abs_path()?, recursive: Some(remove_options.recursive), force: Some(remove_options.force), sandbox: None, @@ -230,8 +217,8 @@ impl ExecutorFileSystem for SandboxedFileSystem { async fn copy( &self, - source_path: &AbsolutePathBuf, - destination_path: &AbsolutePathBuf, + source_path: &PathUri, + destination_path: &PathUri, options: CopyOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -239,8 +226,8 @@ impl ExecutorFileSystem for SandboxedFileSystem { self.run_sandboxed( sandbox, FsHelperRequest::Copy(FsCopyParams { - source_path: source_path.clone(), - destination_path: destination_path.clone(), + source_path: source_path.to_abs_path()?, + destination_path: destination_path.to_abs_path()?, recursive: options.recursive, sandbox: None, }), @@ -272,3 +259,7 @@ fn map_sandbox_error(error: JSONRPCErrorError) -> io::Error { _ => io::Error::other(error.message), } } + +#[cfg(all(test, any(unix, windows)))] +#[path = "sandboxed_file_system_path_uri_tests.rs"] +mod path_uri_tests; diff --git a/codex-rs/exec-server/src/sandboxed_file_system_path_uri_tests.rs b/codex-rs/exec-server/src/sandboxed_file_system_path_uri_tests.rs new file mode 100644 index 0000000000..eb1a0d5dc4 --- /dev/null +++ b/codex-rs/exec-server/src/sandboxed_file_system_path_uri_tests.rs @@ -0,0 +1,43 @@ +use codex_protocol::models::PermissionProfile; +use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::NetworkSandboxPolicy; +use codex_utils_path_uri::PathUri; +use pretty_assertions::assert_eq; +use tokio::io; + +use super::*; + +#[tokio::test] +async fn sandboxed_file_system_rejects_non_native_uri_as_invalid_input() { + let runtime_paths = ExecServerRuntimePaths::new( + std::env::current_exe().expect("current exe"), + /*codex_linux_sandbox_exe*/ None, + ) + .expect("runtime paths"); + let file_system = SandboxedFileSystem::new(runtime_paths); + let sandbox = FileSystemSandboxContext::from_permission_profile( + PermissionProfile::from_runtime_permissions( + &FileSystemSandboxPolicy::restricted(Vec::new()), + NetworkSandboxPolicy::Restricted, + ), + ); + + let error = file_system + .read_file(&non_native_uri(), Some(&sandbox)) + .await + .expect_err("non-native URI should be rejected"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); +} + +fn non_native_uri() -> PathUri { + #[cfg(unix)] + let uri = "file://server/share/file.txt"; + #[cfg(windows)] + let uri = "file:///usr/local/file.txt"; + + match PathUri::parse(uri) { + Ok(uri) => uri, + Err(err) => panic!("valid non-native URI should parse: {err}"), + } +} diff --git a/codex-rs/exec-server/src/server/file_system_handler.rs b/codex-rs/exec-server/src/server/file_system_handler.rs index b60589d020..97d028f89b 100644 --- a/codex-rs/exec-server/src/server/file_system_handler.rs +++ b/codex-rs/exec-server/src/server/file_system_handler.rs @@ -3,6 +3,7 @@ use std::io; use base64::Engine as _; use base64::engine::general_purpose::STANDARD; use codex_app_server_protocol::JSONRPCErrorError; +use codex_utils_path_uri::PathUri; use crate::CopyOptions; use crate::CreateDirectoryOptions; @@ -52,9 +53,10 @@ impl FileSystemHandler { &self, params: FsReadFileParams, ) -> Result { + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let bytes = self .file_system - .read_file(¶ms.path, params.sandbox.as_ref()) + .read_file(&path, params.sandbox.as_ref()) .await .map_err(map_fs_error)?; Ok(FsReadFileResponse { @@ -66,13 +68,14 @@ impl FileSystemHandler { &self, params: FsWriteFileParams, ) -> Result { + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let bytes = STANDARD.decode(params.data_base64).map_err(|err| { invalid_request(format!( "{FS_WRITE_FILE_METHOD} requires valid base64 dataBase64: {err}" )) })?; self.file_system - .write_file(¶ms.path, bytes, params.sandbox.as_ref()) + .write_file(&path, bytes, params.sandbox.as_ref()) .await .map_err(map_fs_error)?; Ok(FsWriteFileResponse {}) @@ -83,9 +86,10 @@ impl FileSystemHandler { params: FsCreateDirectoryParams, ) -> Result { let recursive = params.recursive.unwrap_or(true); + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; self.file_system .create_directory( - ¶ms.path, + &path, CreateDirectoryOptions { recursive }, params.sandbox.as_ref(), ) @@ -98,9 +102,10 @@ impl FileSystemHandler { &self, params: FsGetMetadataParams, ) -> Result { + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let metadata = self .file_system - .get_metadata(¶ms.path, params.sandbox.as_ref()) + .get_metadata(&path, params.sandbox.as_ref()) .await .map_err(map_fs_error)?; Ok(FsGetMetadataResponse { @@ -116,11 +121,13 @@ impl FileSystemHandler { &self, params: FsCanonicalizeParams, ) -> Result { + let requested_path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let path = self .file_system - .canonicalize(¶ms.path, params.sandbox.as_ref()) + .canonicalize(&requested_path, params.sandbox.as_ref()) .await .map_err(map_fs_error)?; + let path = path.to_abs_path().map_err(map_fs_error)?; Ok(FsCanonicalizeResponse { path }) } @@ -128,11 +135,8 @@ impl FileSystemHandler { &self, params: FsJoinParams, ) -> Result { - let path = self - .file_system - .join(¶ms.base_path, ¶ms.path) - .await - .map_err(map_fs_error)?; + // TODO(anp): remove and migrate callers to PathUri. + let path = params.base_path.join(params.path); Ok(FsJoinResponse { path }) } @@ -140,11 +144,8 @@ impl FileSystemHandler { &self, params: FsParentParams, ) -> Result { - let path = self - .file_system - .parent(¶ms.path) - .await - .map_err(map_fs_error)?; + // TODO(anp): remove and migrate callers to PathUri. + let path = params.path.parent(); Ok(FsParentResponse { path }) } @@ -152,9 +153,10 @@ impl FileSystemHandler { &self, params: FsReadDirectoryParams, ) -> Result { + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; let entries = self .file_system - .read_directory(¶ms.path, params.sandbox.as_ref()) + .read_directory(&path, params.sandbox.as_ref()) .await .map_err(map_fs_error)? .into_iter() @@ -173,9 +175,10 @@ impl FileSystemHandler { ) -> Result { let recursive = params.recursive.unwrap_or(true); let force = params.force.unwrap_or(true); + let path = PathUri::from_abs_path(¶ms.path).map_err(map_fs_error)?; self.file_system .remove( - ¶ms.path, + &path, RemoveOptions { recursive, force }, params.sandbox.as_ref(), ) @@ -188,10 +191,13 @@ impl FileSystemHandler { &self, params: FsCopyParams, ) -> Result { + let source_path = PathUri::from_abs_path(¶ms.source_path).map_err(map_fs_error)?; + let destination_path = + PathUri::from_abs_path(¶ms.destination_path).map_err(map_fs_error)?; self.file_system .copy( - ¶ms.source_path, - ¶ms.destination_path, + &source_path, + &destination_path, CopyOptions { recursive: params.recursive, }, @@ -262,6 +268,24 @@ mod tests { .await .expect("write file"); + let canonicalized = handler + .canonicalize(FsCanonicalizeParams { + path: path.clone(), + sandbox: Some(FileSystemSandboxContext::from_legacy_sandbox_policy( + sandbox_policy.clone(), + sandbox_cwd.clone(), + )), + }) + .await + .expect("canonicalize file"); + assert_eq!( + canonicalized.path, + AbsolutePathBuf::from_absolute_path( + std::fs::canonicalize(path.as_path()).expect("canonical path"), + ) + .expect("absolute canonical path"), + ); + let response = handler .read_file(FsReadFileParams { path, @@ -276,4 +300,34 @@ mod tests { assert_eq!(response.data_base64, STANDARD.encode("ok")); } } + + #[tokio::test] + async fn protocol_join_and_parent_remain_native_path_operations() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let runtime_paths = ExecServerRuntimePaths::new( + std::env::current_exe().expect("current exe"), + /*codex_linux_sandbox_exe*/ None, + ) + .expect("runtime paths"); + let handler = FileSystemHandler::new(runtime_paths); + let base_path = + AbsolutePathBuf::from_absolute_path(temp_dir.path()).expect("absolute tempdir"); + + let joined = handler + .join(FsJoinParams { + base_path: base_path.clone(), + path: "nested/file.txt".into(), + }) + .await + .expect("join path"); + assert_eq!(joined.path, base_path.join("nested/file.txt")); + + let parent = handler + .parent(FsParentParams { + path: joined.path.clone(), + }) + .await + .expect("parent path"); + assert_eq!(parent.path, joined.path.parent()); + } } diff --git a/codex-rs/exec-server/tests/file_system.rs b/codex-rs/exec-server/tests/file_system.rs deleted file mode 100644 index b8d1c21894..0000000000 --- a/codex-rs/exec-server/tests/file_system.rs +++ /dev/null @@ -1,1247 +0,0 @@ -#![cfg(unix)] - -mod common; - -use std::os::unix::fs::MetadataExt; -#[cfg(target_os = "linux")] -use std::os::unix::fs::PermissionsExt; -use std::os::unix::fs::symlink; -use std::path::Path; -use std::path::PathBuf; -use std::process::Command; -use std::sync::Arc; - -use anyhow::Context; -use anyhow::Result; -use codex_exec_server::CopyOptions; -use codex_exec_server::CreateDirectoryOptions; -use codex_exec_server::Environment; -use codex_exec_server::ExecServerRuntimePaths; -use codex_exec_server::ExecutorFileSystem; -use codex_exec_server::FileSystemSandboxContext; -use codex_exec_server::LocalFileSystem; -use codex_exec_server::ReadDirectoryEntry; -use codex_exec_server::RemoveOptions; -use codex_protocol::models::AdditionalPermissionProfile; -use codex_protocol::models::FileSystemPermissions; -use codex_protocol::models::PermissionProfile; -use codex_protocol::permissions::FileSystemAccessMode; -use codex_protocol::permissions::FileSystemPath; -use codex_protocol::permissions::FileSystemSandboxEntry; -use codex_protocol::permissions::FileSystemSandboxPolicy; -use codex_protocol::permissions::NetworkSandboxPolicy; -use codex_sandboxing::policy_transforms::effective_file_system_sandbox_policy; -use codex_sandboxing::policy_transforms::effective_network_sandbox_policy; -use codex_utils_absolute_path::AbsolutePathBuf; -use pretty_assertions::assert_eq; -use tempfile::TempDir; -use test_case::test_case; - -use common::exec_server::ExecServerHarness; -use common::exec_server::TestCodexHelperPaths; -use common::exec_server::exec_server; -#[cfg(target_os = "linux")] -use common::exec_server::exec_server_with_env; -use common::exec_server::test_codex_helper_paths; - -struct FileSystemContext { - file_system: Arc, - _helper_paths: Option, - _server: Option, -} - -async fn create_file_system_context(use_remote: bool) -> Result { - if use_remote { - let server = exec_server().await?; - let environment = Environment::create_for_tests(Some(server.websocket_url().to_string()))?; - Ok(FileSystemContext { - file_system: environment.get_filesystem(), - _helper_paths: None, - _server: Some(server), - }) - } else { - let helper_paths = test_codex_helper_paths()?; - let runtime_paths = ExecServerRuntimePaths::new( - helper_paths.codex_exe.clone(), - helper_paths.codex_linux_sandbox_exe.clone(), - )?; - Ok(FileSystemContext { - file_system: Arc::new(LocalFileSystem::with_runtime_paths(runtime_paths)), - _helper_paths: Some(helper_paths), - _server: None, - }) - } -} - -fn absolute_path(path: std::path::PathBuf) -> AbsolutePathBuf { - assert!( - path.is_absolute(), - "path must be absolute: {}", - path.display() - ); - match AbsolutePathBuf::try_from(path) { - Ok(path) => path, - Err(err) => panic!("path should be absolute: {err}"), - } -} - -fn read_only_sandbox(readable_root: std::path::PathBuf) -> FileSystemSandboxContext { - let readable_root = absolute_path(readable_root); - sandbox_context(vec![FileSystemSandboxEntry { - path: FileSystemPath::Path { - path: readable_root, - }, - access: FileSystemAccessMode::Read, - }]) -} - -fn workspace_write_sandbox(writable_root: std::path::PathBuf) -> FileSystemSandboxContext { - let writable_root = absolute_path(writable_root); - sandbox_context(vec![FileSystemSandboxEntry { - path: FileSystemPath::Path { - path: writable_root, - }, - access: FileSystemAccessMode::Write, - }]) -} - -fn sandbox_context(entries: Vec) -> FileSystemSandboxContext { - FileSystemSandboxContext::from_permission_profile(PermissionProfile::from_runtime_permissions( - &FileSystemSandboxPolicy::restricted(entries), - NetworkSandboxPolicy::Restricted, - )) -} - -#[test] -fn sandbox_context_from_profile_preserves_workspace_write_read_only_subpaths() -> Result<()> { - let tmp = TempDir::new()?; - let writable_dir = tmp.path().join("writable"); - let git_dir = writable_dir.join(".git"); - std::fs::create_dir_all(&git_dir)?; - - let sandbox = workspace_write_sandbox(writable_dir.clone()); - let policy = sandbox.permissions.file_system_sandbox_policy(); - let cwd = absolute_path(writable_dir.clone()); - let writable_roots = policy.get_writable_roots_with_cwd(cwd.as_path()); - let writable_dir = absolute_path(std::fs::canonicalize(writable_dir)?); - let git_dir = absolute_path(std::fs::canonicalize(git_dir)?); - let Some(writable_root) = writable_roots - .iter() - .find(|writable_root| writable_root.root == writable_dir) - else { - panic!("writable root should be preserved"); - }; - - assert!(writable_root.read_only_subpaths.contains(&git_dir)); - - Ok(()) -} - -fn assert_sandbox_denied(error: &std::io::Error) { - match error.kind() { - std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied => { - let message = error.to_string(); - assert!( - message.contains("is not permitted") - || message.contains("Operation not permitted") - || message.contains("Permission denied"), - "unexpected sandbox error message: {message}", - ); - } - std::io::ErrorKind::NotFound => assert!( - error.to_string().contains("No such file or directory"), - "unexpected sandbox not-found message: {error}", - ), - std::io::ErrorKind::Other => assert!( - error.to_string().contains("Read-only file system"), - "unexpected sandbox other error message: {error}", - ), - other => panic!("unexpected sandbox error kind: {other:?}: {error:?}"), - } -} - -fn assert_normalized_path_rejected(error: &std::io::Error) { - match error.kind() { - std::io::ErrorKind::NotFound => assert!( - error.to_string().contains("No such file or directory"), - "unexpected not-found message: {error}", - ), - std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied => { - let message = error.to_string(); - assert!( - message.contains("is not permitted") - || message.contains("Operation not permitted") - || message.contains("Permission denied"), - "unexpected rejection message: {message}", - ); - } - other => panic!("unexpected normalized-path error kind: {other:?}: {error:?}"), - } -} - -fn alias_root_candidate() -> Result> { - for root in [Path::new("/tmp").to_path_buf(), std::env::temp_dir()] { - if root.is_dir() && root.canonicalize().is_ok_and(|canonical| canonical != root) { - return Ok(Some(root)); - } - } - Ok(None) -} - -#[cfg(target_os = "linux")] -fn write_fake_bwrap(bin_dir: &Path) -> Result { - std::fs::create_dir_all(bin_dir)?; - let fake_bwrap = bin_dir.join("bwrap"); - std::fs::write( - &fake_bwrap, - r#"#!/bin/bash -set -euo pipefail - -for arg in "$@"; do - if [[ "${arg}" == "--help" ]]; then - echo "Usage: bwrap --argv0 --perms" - exit 0 - fi -done - -printf '%s\n' "$*" >> "${0}.log" - -args=("$@") -argv0="" -command_start=-1 -for i in "${!args[@]}"; do - if [[ "${args[$i]}" == "--argv0" && $((i + 1)) -lt ${#args[@]} ]]; then - argv0="${args[$((i + 1))]}" - fi - if [[ "${args[$i]}" == "--" ]]; then - command_start=$((i + 1)) - break - fi -done - -if [[ "${command_start}" -lt 0 || "${command_start}" -ge "${#args[@]}" ]]; then - echo "fake bwrap did not find an inner command" >&2 - exit 125 -fi - -cmd=("${args[@]:$command_start}") -if [[ -n "${argv0}" ]]; then - exec -a "${argv0}" "${cmd[@]}" -fi -exec "${cmd[@]}" -"#, - )?; - let mut permissions = std::fs::metadata(&fake_bwrap)?.permissions(); - permissions.set_mode(0o755); - std::fs::set_permissions(&fake_bwrap, permissions)?; - Ok(fake_bwrap) -} - -#[cfg(target_os = "linux")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn sandboxed_file_system_helper_finds_bwrap_on_preserved_path() -> Result<()> { - let tmp = TempDir::new()?; - let fake_bin_dir = tmp.path().join("bin"); - let fake_bwrap = write_fake_bwrap(&fake_bin_dir)?; - let mut path_entries = vec![fake_bin_dir]; - if let Some(path) = std::env::var_os("PATH") { - path_entries.extend(std::env::split_paths(&path)); - } - let helper_path = std::env::join_paths(path_entries)?; - - let server = exec_server_with_env([("PATH", helper_path.as_os_str())]).await?; - let environment = Environment::create_for_tests(Some(server.websocket_url().to_string()))?; - let file_system = environment.get_filesystem(); - let workspace = tmp.path().join("workspace"); - std::fs::create_dir_all(&workspace)?; - let file_path = workspace.join("created.txt"); - let sandbox = workspace_write_sandbox(workspace); - - file_system - .write_file( - &absolute_path(file_path.clone()), - b"written through fs helper".to_vec(), - Some(&sandbox), - ) - .await?; - - assert_eq!(std::fs::read(&file_path)?, b"written through fs helper"); - - let bwrap_log = fake_bwrap.with_file_name("bwrap.log"); - let log = std::fs::read_to_string(&bwrap_log) - .with_context(|| format!("expected fake bwrap log at {}", bwrap_log.display()))?; - assert!( - log.contains("--argv0"), - "expected fs helper sandbox path to invoke PATH bwrap with --argv0, got: {log}" - ); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_get_metadata_returns_expected_fields(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let file_path = tmp.path().join("note.txt"); - std::fs::write(&file_path, "hello")?; - - let metadata = file_system - .get_metadata(&absolute_path(file_path.clone()), /*sandbox*/ None) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!(metadata.is_directory, false); - assert_eq!(metadata.is_file, true); - assert_eq!(metadata.is_symlink, false); - assert!(metadata.modified_at_ms > 0); - - let symlink_path = tmp.path().join("note-link.txt"); - symlink(&file_path, &symlink_path)?; - let symlink_metadata = file_system - .get_metadata(&absolute_path(symlink_path.clone()), /*sandbox*/ None) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!(symlink_metadata.is_directory, false); - assert_eq!(symlink_metadata.is_file, true); - assert_eq!(symlink_metadata.is_symlink, true); - assert!(symlink_metadata.modified_at_ms > 0); - - let dir_path = tmp.path().join("notes"); - std::fs::create_dir(&dir_path)?; - let dir_symlink_path = tmp.path().join("notes-link"); - symlink(&dir_path, &dir_symlink_path)?; - let dir_symlink_metadata = file_system - .get_metadata(&absolute_path(dir_symlink_path), /*sandbox*/ None) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!(dir_symlink_metadata.is_directory, true); - assert_eq!(dir_symlink_metadata.is_file, false); - assert_eq!(dir_symlink_metadata.is_symlink, true); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_methods_cover_surface_area(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let source_dir = tmp.path().join("source"); - let nested_dir = source_dir.join("nested"); - let source_file = source_dir.join("root.txt"); - let nested_file = nested_dir.join("note.txt"); - let copied_dir = tmp.path().join("copied"); - let copied_file = tmp.path().join("copy.txt"); - - file_system - .create_directory( - &absolute_path(nested_dir.clone()), - CreateDirectoryOptions { recursive: true }, - /*sandbox*/ None, - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - - file_system - .write_file( - &absolute_path(nested_file.clone()), - b"hello from trait".to_vec(), - /*sandbox*/ None, - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - file_system - .write_file( - &absolute_path(source_file.clone()), - b"hello from source root".to_vec(), - /*sandbox*/ None, - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - - let source_link = tmp.path().join("source-link"); - symlink(&source_dir, &source_link)?; - let joined_nested = file_system - .join( - &absolute_path(source_link.clone()), - Path::new("nested/note.txt"), - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!( - joined_nested, - absolute_path(source_link.join("nested").join("note.txt")) - ); - let joined_parent = file_system - .parent(&joined_nested) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!( - joined_parent, - Some(absolute_path(source_link.join("nested"))) - ); - let joined_parent_traversal = file_system - .join(&absolute_path(source_dir.clone()), Path::new("../outside")) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!( - joined_parent_traversal, - absolute_path(source_dir.join("../outside")) - ); - let canonical_nested = file_system - .canonicalize( - &absolute_path(source_link.join("nested").join("note.txt")), - /*sandbox*/ None, - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!( - canonical_nested, - absolute_path(std::fs::canonicalize( - source_dir.join("nested").join("note.txt") - )?) - ); - - let nested_file_contents = file_system - .read_file(&absolute_path(nested_file.clone()), /*sandbox*/ None) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!(nested_file_contents, b"hello from trait"); - - let nested_file_text = file_system - .read_file_text(&absolute_path(nested_file.clone()), /*sandbox*/ None) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!(nested_file_text, "hello from trait"); - - file_system - .copy( - &absolute_path(nested_file), - &absolute_path(copied_file.clone()), - CopyOptions { recursive: false }, - /*sandbox*/ None, - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!(std::fs::read_to_string(copied_file)?, "hello from trait"); - - file_system - .copy( - &absolute_path(source_dir.clone()), - &absolute_path(copied_dir.clone()), - CopyOptions { recursive: true }, - /*sandbox*/ None, - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!( - std::fs::read_to_string(copied_dir.join("nested").join("note.txt"))?, - "hello from trait" - ); - - symlink( - source_dir.join("missing-target"), - source_dir.join("broken-link"), - )?; - - let mut entries = file_system - .read_directory(&absolute_path(source_dir), /*sandbox*/ None) - .await - .with_context(|| format!("mode={use_remote}"))?; - entries.sort_by(|left, right| left.file_name.cmp(&right.file_name)); - assert_eq!( - entries, - vec![ - ReadDirectoryEntry { - file_name: "nested".to_string(), - is_directory: true, - is_file: false, - }, - ReadDirectoryEntry { - file_name: "root.txt".to_string(), - is_directory: false, - is_file: true, - }, - ] - ); - - file_system - .remove( - &absolute_path(copied_dir.clone()), - RemoveOptions { - recursive: true, - force: true, - }, - /*sandbox*/ None, - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert!(!copied_dir.exists()); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_write_file_reports_missing_parent(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let missing_parent_path = tmp.path().join("missing").join("note.txt"); - - let error = match file_system - .write_file( - &absolute_path(missing_parent_path.clone()), - b"hello from trait".to_vec(), - /*sandbox*/ None, - ) - .await - { - Ok(()) => anyhow::bail!("write should fail when parent directory is absent"), - Err(error) => error, - }; - assert_eq!( - error.kind(), - std::io::ErrorKind::NotFound, - "mode={use_remote}" - ); - assert!(!missing_parent_path.exists(), "mode={use_remote}"); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_rejects_directory_without_recursive(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let source_dir = tmp.path().join("source"); - std::fs::create_dir_all(&source_dir)?; - - let error = file_system - .copy( - &absolute_path(source_dir), - &absolute_path(tmp.path().join("dest")), - CopyOptions { recursive: false }, - /*sandbox*/ None, - ) - .await; - let error = match error { - Ok(()) => panic!("copy should fail"), - Err(error) => error, - }; - assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); - assert_eq!( - error.to_string(), - "fs/copy requires recursive: true when sourcePath is a directory" - ); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_read_allows_readable_root(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let file_path = allowed_dir.join("note.txt"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::write(&file_path, "sandboxed hello")?; - let sandbox = read_only_sandbox(allowed_dir); - - let contents = file_system - .read_file(&absolute_path(file_path), Some(&sandbox)) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!(contents, b"sandboxed hello"); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_canonicalize_allows_readable_root(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let file_path = allowed_dir.join("note.txt"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::write(&file_path, "sandboxed hello")?; - let sandbox = read_only_sandbox(allowed_dir); - - let canonical_path = file_system - .canonicalize(&absolute_path(file_path.clone()), Some(&sandbox)) - .await - .with_context(|| format!("mode={use_remote}"))?; - assert_eq!( - canonical_path, - absolute_path(std::fs::canonicalize(file_path)?) - ); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_write_rejects_unwritable_path(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let blocked_path = tmp.path().join("blocked.txt"); - - let sandbox = read_only_sandbox(tmp.path().to_path_buf()); - let error = match file_system - .write_file( - &absolute_path(blocked_path.clone()), - b"nope".to_vec(), - Some(&sandbox), - ) - .await - { - Ok(()) => anyhow::bail!("write should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - assert!(!blocked_path.exists()); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_write_allows_explicit_alias_roots(use_remote: bool) -> Result<()> { - let Some(alias_root) = alias_root_candidate()? else { - return Ok(()); - }; - - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = tempfile::Builder::new() - .prefix("codex-fs-sandbox-alias-") - .tempdir_in(&alias_root)?; - let file_path = tmp.path().join("note.txt"); - let sandbox = workspace_write_sandbox(alias_root.clone()); - - file_system - .write_file( - &absolute_path(file_path.clone()), - b"created".to_vec(), - Some(&sandbox), - ) - .await - .with_context(|| format!("write file through alias root mode={use_remote}"))?; - assert_eq!(std::fs::read(&file_path)?, b"created"); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_write_allows_additional_write_root(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let readable_dir = tmp.path().join("readable"); - let writable_dir = tmp.path().join("writable"); - let file_path = writable_dir.join("note.txt"); - std::fs::create_dir_all(&readable_dir)?; - std::fs::create_dir_all(&writable_dir)?; - - let mut sandbox = read_only_sandbox(readable_dir); - let additional_permissions = AdditionalPermissionProfile { - network: None, - file_system: Some(FileSystemPermissions::from_read_write_roots( - /*read*/ None, - Some(vec![absolute_path(writable_dir)]), - )), - }; - let file_system_policy = effective_file_system_sandbox_policy( - &sandbox.permissions.file_system_sandbox_policy(), - Some(&additional_permissions), - ); - let network_policy = effective_network_sandbox_policy( - sandbox.permissions.network_sandbox_policy(), - Some(&additional_permissions), - ); - sandbox.permissions = PermissionProfile::from_runtime_permissions_with_enforcement( - sandbox.permissions.enforcement(), - &file_system_policy, - network_policy, - ); - - file_system - .write_file( - &absolute_path(file_path.clone()), - b"created".to_vec(), - Some(&sandbox), - ) - .await - .with_context(|| format!("write file through additional root mode={use_remote}"))?; - assert_eq!(std::fs::read(&file_path)?, b"created"); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_read_rejects_symlink_escape(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(outside_dir.join("secret.txt"), "nope")?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_path = allowed_dir.join("link").join("secret.txt"); - let sandbox = read_only_sandbox(allowed_dir); - let error = match file_system - .read_file(&absolute_path(requested_path.clone()), Some(&sandbox)) - .await - { - Ok(_) => anyhow::bail!("read should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_read_rejects_symlink_parent_dotdot_escape( - use_remote: bool, -) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - let secret_path = tmp.path().join("secret.txt"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(&secret_path, "nope")?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_path = absolute_path(allowed_dir.join("link").join("..").join("secret.txt")); - let sandbox = read_only_sandbox(allowed_dir); - let error = match file_system.read_file(&requested_path, Some(&sandbox)).await { - Ok(_) => anyhow::bail!("read should fail after path normalization"), - Err(error) => error, - }; - // AbsolutePathBuf normalizes `link/../secret.txt` to `allowed/secret.txt` - // before the request reaches the filesystem layer. Depending on whether - // the platform/runtime resolves that normalized path through a top-level - // symlink alias, the request can surface as either "missing file" or an - // upfront sandbox rejection. - assert_normalized_path_rejected(&error); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_write_rejects_symlink_escape(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_path = allowed_dir.join("link").join("blocked.txt"); - let sandbox = workspace_write_sandbox(allowed_dir); - let error = match file_system - .write_file( - &absolute_path(requested_path.clone()), - b"nope".to_vec(), - Some(&sandbox), - ) - .await - { - Ok(()) => anyhow::bail!("write should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - assert!(!outside_dir.join("blocked.txt").exists()); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_sandboxed_write_preserves_existing_hard_link(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - - let outside_file = outside_dir.join("outside.txt"); - let hard_link = allowed_dir.join("hard-link.txt"); - std::fs::write(&outside_file, "outside\n")?; - std::fs::hard_link(&outside_file, &hard_link)?; - - let sandbox = workspace_write_sandbox(allowed_dir); - file_system - .write_file( - &absolute_path(hard_link.clone()), - b"updated through existing hard link\n".to_vec(), - Some(&sandbox), - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - - assert_eq!( - std::fs::read_to_string(&outside_file)?, - "updated through existing hard link\n" - ); - assert_eq!( - std::fs::read_to_string(&hard_link)?, - "updated through existing hard link\n" - ); - - let outside_metadata = std::fs::metadata(&outside_file)?; - let link_metadata = std::fs::metadata(&hard_link)?; - assert_eq!( - (link_metadata.dev(), link_metadata.ino()), - (outside_metadata.dev(), outside_metadata.ino()) - ); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_create_directory_rejects_symlink_escape(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_path = allowed_dir.join("link").join("created"); - let sandbox = workspace_write_sandbox(allowed_dir); - let error = match file_system - .create_directory( - &absolute_path(requested_path.clone()), - CreateDirectoryOptions { recursive: false }, - Some(&sandbox), - ) - .await - { - Ok(()) => anyhow::bail!("create_directory should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - assert!(!outside_dir.join("created").exists()); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_read_directory_rejects_symlink_escape(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(outside_dir.join("secret.txt"), "nope")?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_path = allowed_dir.join("link"); - let sandbox = read_only_sandbox(allowed_dir); - let error = match file_system - .read_directory(&absolute_path(requested_path.clone()), Some(&sandbox)) - .await - { - Ok(_) => anyhow::bail!("read_directory should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_rejects_symlink_escape_destination(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(allowed_dir.join("source.txt"), "hello")?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_destination = allowed_dir.join("link").join("copied.txt"); - let sandbox = workspace_write_sandbox(allowed_dir.clone()); - let error = match file_system - .copy( - &absolute_path(allowed_dir.join("source.txt")), - &absolute_path(requested_destination.clone()), - CopyOptions { recursive: false }, - Some(&sandbox), - ) - .await - { - Ok(()) => anyhow::bail!("copy should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - assert!(!outside_dir.join("copied.txt").exists()); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_remove_removes_symlink_not_target(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - let outside_file = outside_dir.join("keep.txt"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(&outside_file, "outside")?; - let symlink_path = allowed_dir.join("link"); - symlink(&outside_file, &symlink_path)?; - - let sandbox = workspace_write_sandbox(allowed_dir); - file_system - .remove( - &absolute_path(symlink_path.clone()), - RemoveOptions { - recursive: false, - force: false, - }, - Some(&sandbox), - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - - assert!(!symlink_path.exists()); - assert!(outside_file.exists()); - assert_eq!(std::fs::read_to_string(outside_file)?, "outside"); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_preserves_symlink_source(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - let outside_file = outside_dir.join("outside.txt"); - let source_symlink = allowed_dir.join("link"); - let copied_symlink = allowed_dir.join("copied-link"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(&outside_file, "outside")?; - symlink(&outside_file, &source_symlink)?; - - let sandbox = workspace_write_sandbox(allowed_dir.clone()); - file_system - .copy( - &absolute_path(source_symlink), - &absolute_path(copied_symlink.clone()), - CopyOptions { recursive: false }, - Some(&sandbox), - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - - let copied_metadata = std::fs::symlink_metadata(&copied_symlink)?; - assert!(copied_metadata.file_type().is_symlink()); - assert_eq!(std::fs::read_link(copied_symlink)?, outside_file); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_remove_rejects_symlink_escape(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - let outside_file = outside_dir.join("secret.txt"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(&outside_file, "outside")?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_path = allowed_dir.join("link").join("secret.txt"); - let sandbox = workspace_write_sandbox(allowed_dir); - let error = match file_system - .remove( - &absolute_path(requested_path.clone()), - RemoveOptions { - recursive: false, - force: false, - }, - Some(&sandbox), - ) - .await - { - Ok(()) => anyhow::bail!("remove should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - assert_eq!(std::fs::read_to_string(outside_file)?, "outside"); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_rejects_symlink_escape_source(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let allowed_dir = tmp.path().join("allowed"); - let outside_dir = tmp.path().join("outside"); - let outside_file = outside_dir.join("secret.txt"); - let requested_destination = allowed_dir.join("copied.txt"); - std::fs::create_dir_all(&allowed_dir)?; - std::fs::create_dir_all(&outside_dir)?; - std::fs::write(&outside_file, "outside")?; - symlink(&outside_dir, allowed_dir.join("link"))?; - - let requested_source = allowed_dir.join("link").join("secret.txt"); - let sandbox = workspace_write_sandbox(allowed_dir); - let error = match file_system - .copy( - &absolute_path(requested_source.clone()), - &absolute_path(requested_destination.clone()), - CopyOptions { recursive: false }, - Some(&sandbox), - ) - .await - { - Ok(()) => anyhow::bail!("copy should be blocked"), - Err(error) => error, - }; - assert_sandbox_denied(&error); - assert!(!requested_destination.exists()); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_rejects_copying_directory_into_descendant( - use_remote: bool, -) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let source_dir = tmp.path().join("source"); - std::fs::create_dir_all(source_dir.join("nested"))?; - - let error = file_system - .copy( - &absolute_path(source_dir.clone()), - &absolute_path(source_dir.join("nested").join("copy")), - CopyOptions { recursive: true }, - /*sandbox*/ None, - ) - .await; - let error = match error { - Ok(()) => panic!("copy should fail"), - Err(error) => error, - }; - assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); - assert_eq!( - error.to_string(), - "fs/copy cannot copy a directory to itself or one of its descendants" - ); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_preserves_symlinks_in_recursive_copy(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let source_dir = tmp.path().join("source"); - let nested_dir = source_dir.join("nested"); - let copied_dir = tmp.path().join("copied"); - std::fs::create_dir_all(&nested_dir)?; - symlink("nested", source_dir.join("nested-link"))?; - - file_system - .copy( - &absolute_path(source_dir), - &absolute_path(copied_dir.clone()), - CopyOptions { recursive: true }, - /*sandbox*/ None, - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - - let copied_link = copied_dir.join("nested-link"); - let metadata = std::fs::symlink_metadata(&copied_link)?; - assert!(metadata.file_type().is_symlink()); - assert_eq!( - std::fs::read_link(copied_link)?, - std::path::PathBuf::from("nested") - ); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_ignores_unknown_special_files_in_recursive_copy( - use_remote: bool, -) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let source_dir = tmp.path().join("source"); - let copied_dir = tmp.path().join("copied"); - std::fs::create_dir_all(&source_dir)?; - std::fs::write(source_dir.join("note.txt"), "hello")?; - - let fifo_path = source_dir.join("named-pipe"); - let output = Command::new("mkfifo").arg(&fifo_path).output()?; - if !output.status.success() { - anyhow::bail!( - "mkfifo failed: stdout={} stderr={}", - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim() - ); - } - - file_system - .copy( - &absolute_path(source_dir), - &absolute_path(copied_dir.clone()), - CopyOptions { recursive: true }, - /*sandbox*/ None, - ) - .await - .with_context(|| format!("mode={use_remote}"))?; - - assert_eq!( - std::fs::read_to_string(copied_dir.join("note.txt"))?, - "hello" - ); - assert!(!copied_dir.join("named-pipe").exists()); - - Ok(()) -} - -#[test_case(false ; "local")] -#[test_case(true ; "remote")] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn file_system_copy_rejects_standalone_fifo_source(use_remote: bool) -> Result<()> { - let context = create_file_system_context(use_remote).await?; - let file_system = context.file_system; - - let tmp = TempDir::new()?; - let fifo_path = tmp.path().join("named-pipe"); - let output = Command::new("mkfifo").arg(&fifo_path).output()?; - if !output.status.success() { - anyhow::bail!( - "mkfifo failed: stdout={} stderr={}", - String::from_utf8_lossy(&output.stdout).trim(), - String::from_utf8_lossy(&output.stderr).trim() - ); - } - - let error = file_system - .copy( - &absolute_path(fifo_path), - &absolute_path(tmp.path().join("copied")), - CopyOptions { recursive: false }, - /*sandbox*/ None, - ) - .await; - let error = match error { - Ok(()) => panic!("copy should fail"), - Err(error) => error, - }; - assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); - assert_eq!( - error.to_string(), - "fs/copy only supports regular files, directories, and symlinks" - ); - - Ok(()) -} diff --git a/codex-rs/exec-server/tests/file_system/shared.rs b/codex-rs/exec-server/tests/file_system/shared.rs new file mode 100644 index 0000000000..0bef4aa288 --- /dev/null +++ b/codex-rs/exec-server/tests/file_system/shared.rs @@ -0,0 +1,563 @@ +use anyhow::Context; +use anyhow::Result; +use codex_exec_server::CopyOptions; +use codex_exec_server::CreateDirectoryOptions; +use codex_exec_server::ReadDirectoryEntry; +use codex_exec_server::RemoveOptions; +use codex_protocol::models::AdditionalPermissionProfile; +use codex_protocol::models::FileSystemPermissions; +use codex_protocol::models::PermissionProfile; +use codex_sandboxing::policy_transforms::effective_file_system_sandbox_policy; +use codex_sandboxing::policy_transforms::effective_network_sandbox_policy; +use codex_utils_path_uri::PathUri; +use pretty_assertions::assert_eq; +use std::path::Path; +use tempfile::TempDir; +use test_case::test_case; + +use super::support::FileSystemImplementation; +use super::support::absolute_path; +use super::support::create_file_system_context; +use super::support::read_only_sandbox; +use super::support::workspace_write_sandbox; + +#[test] +fn sandbox_context_from_profile_preserves_workspace_write_read_only_subpaths() -> Result<()> { + let tmp = TempDir::new()?; + let writable_dir = tmp.path().join("writable"); + let git_dir = writable_dir.join(".git"); + std::fs::create_dir_all(&git_dir)?; + + let sandbox = workspace_write_sandbox(writable_dir.clone()); + let policy = sandbox.permissions.file_system_sandbox_policy(); + let cwd = absolute_path(writable_dir.clone()); + let writable_roots = policy.get_writable_roots_with_cwd(cwd.as_path()); + let writable_dir = absolute_path(std::fs::canonicalize(writable_dir)?); + let git_dir = absolute_path(std::fs::canonicalize(git_dir)?); + let Some(writable_root) = writable_roots + .iter() + .find(|writable_root| writable_root.root == writable_dir) + else { + panic!("writable root should be preserved"); + }; + + assert!(writable_root.read_only_subpaths.contains(&git_dir)); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_get_metadata_reports_files_and_directories( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let file_path = tmp.path().join("note.txt"); + let directory_path = tmp.path().join("notes"); + std::fs::write(&file_path, "hello")?; + std::fs::create_dir(&directory_path)?; + + let file_metadata = file_system + .get_metadata(&PathUri::from_path(&file_path)?, /*sandbox*/ None) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(file_metadata.is_directory, false); + assert_eq!(file_metadata.is_file, true); + assert_eq!(file_metadata.is_symlink, false); + assert!(file_metadata.modified_at_ms > 0); + + let directory_metadata = file_system + .get_metadata(&PathUri::from_path(&directory_path)?, /*sandbox*/ None) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(directory_metadata.is_directory, true); + assert_eq!(directory_metadata.is_file, false); + assert_eq!(directory_metadata.is_symlink, false); + assert!(directory_metadata.modified_at_ms > 0); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_create_directory_creates_nested_directories( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let nested_dir = tmp.path().join("source").join("nested"); + + file_system + .create_directory( + &PathUri::from_path(&nested_dir)?, + CreateDirectoryOptions { recursive: true }, + /*sandbox*/ None, + ) + .await + .with_context(|| format!("mode={implementation}"))?; + assert!(nested_dir.is_dir()); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_write_file_writes_bytes( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let file_path = tmp.path().join("note.txt"); + file_system + .write_file( + &PathUri::from_path(&file_path)?, + b"hello from trait".to_vec(), + /*sandbox*/ None, + ) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(std::fs::read(file_path)?, b"hello from trait"); + + Ok(()) +} + +#[test] +fn path_uri_join_and_parent_preserve_lexical_paths() -> Result<()> { + let tmp = TempDir::new()?; + let source_dir = tmp.path().join("source"); + let source_dir_uri = PathUri::from_path(&source_dir)?; + let joined_nested = source_dir_uri.join("nested/note.txt")?; + assert_eq!( + joined_nested, + PathUri::from_path(source_dir.join("nested").join("note.txt"))? + ); + let joined_parent = joined_nested.parent(); + assert_eq!( + joined_parent, + Some(PathUri::from_path(source_dir.join("nested"))?) + ); + let joined_parent_traversal = source_dir_uri.join("../outside")?; + assert_eq!( + joined_parent_traversal, + PathUri::from_path(source_dir.join("../outside"))? + ); + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_read_file_returns_bytes( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let file_path = tmp.path().join("note.txt"); + std::fs::write(&file_path, "hello from trait")?; + + let contents = file_system + .read_file(&PathUri::from_path(&file_path)?, /*sandbox*/ None) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(contents, b"hello from trait"); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_read_file_text_returns_string( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let file_path = tmp.path().join("note.txt"); + std::fs::write(&file_path, "hello from trait")?; + + let contents = file_system + .read_file_text(&PathUri::from_path(&file_path)?, /*sandbox*/ None) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(contents, "hello from trait"); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_copies_file(implementation: FileSystemImplementation) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let source_file = tmp.path().join("source.txt"); + let copied_file = tmp.path().join("copy.txt"); + std::fs::write(&source_file, "hello from trait")?; + + file_system + .copy( + &PathUri::from_path(&source_file)?, + &PathUri::from_path(&copied_file)?, + CopyOptions { recursive: false }, + /*sandbox*/ None, + ) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(std::fs::read_to_string(copied_file)?, "hello from trait"); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_copies_directory_recursively( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let source_dir = tmp.path().join("source"); + let nested_dir = source_dir.join("nested"); + let nested_file = nested_dir.join("note.txt"); + let copied_dir = tmp.path().join("copied"); + std::fs::create_dir_all(&nested_dir)?; + std::fs::write(&nested_file, "hello from trait")?; + + file_system + .copy( + &PathUri::from_path(&source_dir)?, + &PathUri::from_path(&copied_dir)?, + CopyOptions { recursive: true }, + /*sandbox*/ None, + ) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!( + std::fs::read_to_string(copied_dir.join("nested").join("note.txt"))?, + "hello from trait" + ); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_read_directory_lists_entries( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let source_dir = tmp.path().join("source"); + std::fs::create_dir_all(source_dir.join("nested"))?; + std::fs::write(source_dir.join("root.txt"), "hello")?; + + let mut entries = file_system + .read_directory(&PathUri::from_path(&source_dir)?, /*sandbox*/ None) + .await + .with_context(|| format!("mode={implementation}"))?; + entries.sort_by(|left, right| left.file_name.cmp(&right.file_name)); + assert_eq!( + entries, + vec![ + ReadDirectoryEntry { + file_name: "nested".to_string(), + is_directory: true, + is_file: false, + }, + ReadDirectoryEntry { + file_name: "root.txt".to_string(), + is_directory: false, + is_file: true, + }, + ] + ); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_remove_removes_directory( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let directory_path = tmp.path().join("remove-me"); + std::fs::create_dir_all(directory_path.join("nested"))?; + + file_system + .remove( + &PathUri::from_path(&directory_path)?, + RemoveOptions { + recursive: true, + force: true, + }, + /*sandbox*/ None, + ) + .await + .with_context(|| format!("mode={implementation}"))?; + assert!(!directory_path.exists()); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_write_file_reports_missing_parent( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let missing_parent_path = tmp.path().join("missing").join("note.txt"); + + let error = match file_system + .write_file( + &PathUri::from_path(&missing_parent_path)?, + b"hello from trait".to_vec(), + /*sandbox*/ None, + ) + .await + { + Ok(()) => anyhow::bail!("write should fail when parent directory is absent"), + Err(error) => error, + }; + assert_eq!( + error.kind(), + std::io::ErrorKind::NotFound, + "mode={implementation}" + ); + assert!(!missing_parent_path.exists(), "mode={implementation}"); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_rejects_directory_without_recursive( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let source_dir = tmp.path().join("source"); + std::fs::create_dir_all(&source_dir)?; + + let error = file_system + .copy( + &PathUri::from_path(&source_dir)?, + &PathUri::from_path(tmp.path().join("dest"))?, + CopyOptions { recursive: false }, + /*sandbox*/ None, + ) + .await; + let error = match error { + Ok(()) => panic!("copy should fail"), + Err(error) => error, + }; + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!( + error.to_string(), + "fs/copy requires recursive: true when sourcePath is a directory" + ); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_read_allows_readable_root( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let file_path = allowed_dir.join("note.txt"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::write(&file_path, "sandboxed hello")?; + let sandbox = read_only_sandbox(allowed_dir); + + let contents = file_system + .read_file(&PathUri::from_path(&file_path)?, Some(&sandbox)) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(contents, b"sandboxed hello"); + + Ok(()) +} + +pub(crate) async fn assert_canonicalize_resolves_directory_alias( + implementation: FileSystemImplementation, + create_directory_alias: impl FnOnce(&Path, &Path) -> Result<()>, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let source_dir = tmp.path().join("source"); + let nested_dir = source_dir.join("nested"); + let file_path = nested_dir.join("note.txt"); + let alias_dir = tmp.path().join("source-alias"); + std::fs::create_dir_all(&nested_dir)?; + std::fs::write(&file_path, "canonical hello")?; + create_directory_alias(&source_dir, &alias_dir)?; + + let requested_path = PathUri::from_path(alias_dir.join("nested").join("note.txt"))?; + let expected_path = PathUri::from_path(std::fs::canonicalize(&file_path)?)?; + assert_ne!(requested_path, expected_path); + + let canonical_path = file_system + .canonicalize(&requested_path, /*sandbox*/ None) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(canonical_path, expected_path); + + Ok(()) +} + +pub(crate) async fn assert_sandboxed_canonicalize_resolves_directory_alias( + implementation: FileSystemImplementation, + create_directory_alias: impl FnOnce(&Path, &Path) -> Result<()>, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let source_dir = tmp.path().join("source"); + let nested_dir = source_dir.join("nested"); + let file_path = nested_dir.join("note.txt"); + let alias_dir = tmp.path().join("source-alias"); + std::fs::create_dir_all(&nested_dir)?; + std::fs::write(&file_path, "sandboxed canonical hello")?; + create_directory_alias(&source_dir, &alias_dir)?; + let sandbox = read_only_sandbox(tmp.path().to_path_buf()); + + let requested_path = PathUri::from_path(alias_dir.join("nested").join("note.txt"))?; + let expected_path = PathUri::from_path(std::fs::canonicalize(&file_path)?)?; + assert_ne!(requested_path, expected_path); + + let canonical_path = file_system + .canonicalize(&requested_path, Some(&sandbox)) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(canonical_path, expected_path); + + Ok(()) +} + +/// Verifies that effective additional permissions extend a read-only sandbox with a writable root. +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_allows_additional_write_root( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let readable_dir = tmp.path().join("readable"); + let writable_dir = tmp.path().join("writable"); + let file_path = writable_dir.join("note.txt"); + std::fs::create_dir_all(&readable_dir)?; + std::fs::create_dir_all(&writable_dir)?; + + let mut sandbox = read_only_sandbox(readable_dir); + let additional_permissions = AdditionalPermissionProfile { + network: None, + file_system: Some(FileSystemPermissions::from_read_write_roots( + /*read*/ None, + Some(vec![absolute_path(writable_dir)]), + )), + }; + let file_system_policy = effective_file_system_sandbox_policy( + &sandbox.permissions.file_system_sandbox_policy(), + Some(&additional_permissions), + ); + let network_policy = effective_network_sandbox_policy( + sandbox.permissions.network_sandbox_policy(), + Some(&additional_permissions), + ); + sandbox.permissions = PermissionProfile::from_runtime_permissions_with_enforcement( + sandbox.permissions.enforcement(), + &file_system_policy, + network_policy, + ); + + file_system + .write_file( + &PathUri::from_path(&file_path)?, + b"created".to_vec(), + Some(&sandbox), + ) + .await + .with_context(|| format!("write file through additional root mode={implementation}"))?; + assert_eq!(std::fs::read(&file_path)?, b"created"); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_rejects_copying_directory_into_descendant( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let source_dir = tmp.path().join("source"); + std::fs::create_dir_all(source_dir.join("nested"))?; + + let error = file_system + .copy( + &PathUri::from_path(&source_dir)?, + &PathUri::from_path(source_dir.join("nested").join("copy"))?, + CopyOptions { recursive: true }, + /*sandbox*/ None, + ) + .await; + let error = match error { + Ok(()) => panic!("copy should fail"), + Err(error) => error, + }; + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!( + error.to_string(), + "fs/copy cannot copy a directory to itself or one of its descendants" + ); + + Ok(()) +} diff --git a/codex-rs/exec-server/tests/file_system/support.rs b/codex-rs/exec-server/tests/file_system/support.rs new file mode 100644 index 0000000000..715faa9965 --- /dev/null +++ b/codex-rs/exec-server/tests/file_system/support.rs @@ -0,0 +1,112 @@ +use std::fmt; +use std::sync::Arc; + +use anyhow::Result; +use codex_exec_server::Environment; +use codex_exec_server::ExecServerRuntimePaths; +use codex_exec_server::ExecutorFileSystem; +use codex_exec_server::FileSystemSandboxContext; +use codex_exec_server::LocalFileSystem; +use codex_protocol::models::PermissionProfile; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::NetworkSandboxPolicy; +use codex_utils_absolute_path::AbsolutePathBuf; + +use crate::common::exec_server::ExecServerHarness; +use crate::common::exec_server::TestCodexHelperPaths; +use crate::common::exec_server::exec_server; +use crate::common::exec_server::test_codex_helper_paths; + +pub(crate) struct FileSystemContext { + pub(crate) file_system: Arc, + _helper_paths: Option, + _server: Option, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum FileSystemImplementation { + Local, + Remote, +} + +impl fmt::Display for FileSystemImplementation { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Local => formatter.write_str("local"), + Self::Remote => formatter.write_str("remote"), + } + } +} + +pub(crate) async fn create_file_system_context( + implementation: FileSystemImplementation, +) -> Result { + match implementation { + FileSystemImplementation::Local => { + let helper_paths = test_codex_helper_paths()?; + let runtime_paths = ExecServerRuntimePaths::new( + helper_paths.codex_exe.clone(), + helper_paths.codex_linux_sandbox_exe.clone(), + )?; + Ok(FileSystemContext { + file_system: Arc::new(LocalFileSystem::with_runtime_paths(runtime_paths)), + _helper_paths: Some(helper_paths), + _server: None, + }) + } + FileSystemImplementation::Remote => { + let server = exec_server().await?; + let environment = + Environment::create_for_tests(Some(server.websocket_url().to_string()))?; + Ok(FileSystemContext { + file_system: environment.get_filesystem(), + _helper_paths: None, + _server: Some(server), + }) + } + } +} + +pub(crate) fn absolute_path(path: std::path::PathBuf) -> AbsolutePathBuf { + assert!( + path.is_absolute(), + "path must be absolute: {}", + path.display() + ); + match AbsolutePathBuf::try_from(path) { + Ok(path) => path, + Err(err) => panic!("path should be absolute: {err}"), + } +} + +pub(crate) fn read_only_sandbox(readable_root: std::path::PathBuf) -> FileSystemSandboxContext { + let readable_root = absolute_path(readable_root); + sandbox_context(vec![FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: readable_root, + }, + access: FileSystemAccessMode::Read, + }]) +} + +pub(crate) fn workspace_write_sandbox( + writable_root: std::path::PathBuf, +) -> FileSystemSandboxContext { + let writable_root = absolute_path(writable_root); + sandbox_context(vec![FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: writable_root, + }, + access: FileSystemAccessMode::Write, + }]) +} + +fn sandbox_context(entries: Vec) -> FileSystemSandboxContext { + FileSystemSandboxContext::from_permission_profile(PermissionProfile::from_runtime_permissions( + &FileSystemSandboxPolicy::restricted(entries), + NetworkSandboxPolicy::Restricted, + )) +} diff --git a/codex-rs/exec-server/tests/file_system_unix.rs b/codex-rs/exec-server/tests/file_system_unix.rs new file mode 100644 index 0000000000..3538f32366 --- /dev/null +++ b/codex-rs/exec-server/tests/file_system_unix.rs @@ -0,0 +1,839 @@ +#![cfg(unix)] + +mod common; + +#[path = "file_system/shared.rs"] +mod shared; +#[path = "file_system/support.rs"] +mod support; + +use std::os::unix::fs::MetadataExt; +#[cfg(target_os = "linux")] +use std::os::unix::fs::PermissionsExt; +use std::os::unix::fs::symlink; +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; + +use anyhow::Context; +use anyhow::Result; +use codex_exec_server::CopyOptions; +use codex_exec_server::CreateDirectoryOptions; +#[cfg(target_os = "linux")] +use codex_exec_server::Environment; +use codex_exec_server::RemoveOptions; +use codex_utils_path_uri::PathUri; +use pretty_assertions::assert_eq; +use tempfile::TempDir; +use test_case::test_case; + +#[cfg(target_os = "linux")] +use crate::common::exec_server::exec_server_with_env; + +use crate::support::FileSystemImplementation; +use crate::support::create_file_system_context; +use crate::support::read_only_sandbox; +use crate::support::workspace_write_sandbox; + +fn assert_sandbox_denied(error: &std::io::Error) { + match error.kind() { + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied => { + let message = error.to_string(); + assert!( + message.contains("is not permitted") + || message.contains("Operation not permitted") + || message.contains("Permission denied"), + "unexpected sandbox error message: {message}", + ); + } + std::io::ErrorKind::NotFound => assert!( + error.to_string().contains("No such file or directory"), + "unexpected sandbox not-found message: {error}", + ), + std::io::ErrorKind::Other => assert!( + error.to_string().contains("Read-only file system"), + "unexpected sandbox other error message: {error}", + ), + other => panic!("unexpected sandbox error kind: {other:?}: {error:?}"), + } +} + +fn assert_normalized_path_rejected(error: &std::io::Error) { + match error.kind() { + std::io::ErrorKind::NotFound => assert!( + error.to_string().contains("No such file or directory"), + "unexpected not-found message: {error}", + ), + std::io::ErrorKind::InvalidInput | std::io::ErrorKind::PermissionDenied => { + let message = error.to_string(); + assert!( + message.contains("is not permitted") + || message.contains("Operation not permitted") + || message.contains("Permission denied"), + "unexpected rejection message: {message}", + ); + } + other => panic!("unexpected normalized-path error kind: {other:?}: {error:?}"), + } +} + +fn alias_root_candidate() -> Result> { + for root in [Path::new("/tmp").to_path_buf(), std::env::temp_dir()] { + if root.is_dir() && root.canonicalize().is_ok_and(|canonical| canonical != root) { + return Ok(Some(root)); + } + } + Ok(None) +} + +fn create_directory_symlink(target: &Path, alias: &Path) -> Result<()> { + symlink(target, alias)?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn write_fake_bwrap(bin_dir: &Path) -> Result { + std::fs::create_dir_all(bin_dir)?; + let fake_bwrap = bin_dir.join("bwrap"); + std::fs::write( + &fake_bwrap, + r#"#!/bin/bash +set -euo pipefail + +for arg in "$@"; do + if [[ "${arg}" == "--help" ]]; then + echo "Usage: bwrap --argv0 --perms" + exit 0 + fi +done + +printf '%s\n' "$*" >> "${0}.log" + +args=("$@") +argv0="" +command_start=-1 +for i in "${!args[@]}"; do + if [[ "${args[$i]}" == "--argv0" && $((i + 1)) -lt ${#args[@]} ]]; then + argv0="${args[$((i + 1))]}" + fi + if [[ "${args[$i]}" == "--" ]]; then + command_start=$((i + 1)) + break + fi +done + +if [[ "${command_start}" -lt 0 || "${command_start}" -ge "${#args[@]}" ]]; then + echo "fake bwrap did not find an inner command" >&2 + exit 125 +fi + +cmd=("${args[@]:$command_start}") +if [[ -n "${argv0}" ]]; then + exec -a "${argv0}" "${cmd[@]}" +fi +exec "${cmd[@]}" +"#, + )?; + let mut permissions = std::fs::metadata(&fake_bwrap)?.permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&fake_bwrap, permissions)?; + Ok(fake_bwrap) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_canonicalize_resolves_directory_symlink( + implementation: FileSystemImplementation, +) -> Result<()> { + shared::assert_canonicalize_resolves_directory_alias(implementation, create_directory_symlink) + .await +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_canonicalize_resolves_directory_symlink( + implementation: FileSystemImplementation, +) -> Result<()> { + shared::assert_sandboxed_canonicalize_resolves_directory_alias( + implementation, + create_directory_symlink, + ) + .await +} + +#[cfg(target_os = "linux")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sandboxed_file_system_helper_finds_bwrap_on_preserved_path() -> Result<()> { + let tmp = TempDir::new()?; + let fake_bin_dir = tmp.path().join("bin"); + let fake_bwrap = write_fake_bwrap(&fake_bin_dir)?; + let mut path_entries = vec![fake_bin_dir]; + if let Some(path) = std::env::var_os("PATH") { + path_entries.extend(std::env::split_paths(&path)); + } + let helper_path = std::env::join_paths(path_entries)?; + + let server = exec_server_with_env([("PATH", helper_path.as_os_str())]).await?; + let environment = Environment::create_for_tests(Some(server.websocket_url().to_string()))?; + let file_system = environment.get_filesystem(); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&workspace)?; + let file_path = workspace.join("created.txt"); + let sandbox = workspace_write_sandbox(workspace); + + file_system + .write_file( + &PathUri::from_path(&file_path)?, + b"written through fs helper".to_vec(), + Some(&sandbox), + ) + .await?; + + assert_eq!(std::fs::read(&file_path)?, b"written through fs helper"); + + let bwrap_log = fake_bwrap.with_file_name("bwrap.log"); + let log = std::fs::read_to_string(&bwrap_log) + .with_context(|| format!("expected fake bwrap log at {}", bwrap_log.display()))?; + assert!( + log.contains("--argv0"), + "expected fs helper sandbox path to invoke PATH bwrap with --argv0, got: {log}" + ); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_get_metadata_reports_symlink_targets( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let file_path = tmp.path().join("note.txt"); + std::fs::write(&file_path, "hello")?; + let symlink_path = tmp.path().join("note-link.txt"); + symlink(&file_path, &symlink_path)?; + let symlink_metadata = file_system + .get_metadata(&PathUri::from_path(&symlink_path)?, /*sandbox*/ None) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(symlink_metadata.is_directory, false); + assert_eq!(symlink_metadata.is_file, true); + assert_eq!(symlink_metadata.is_symlink, true); + assert!(symlink_metadata.modified_at_ms > 0); + + let dir_path = tmp.path().join("notes"); + std::fs::create_dir(&dir_path)?; + let dir_symlink_path = tmp.path().join("notes-link"); + symlink(&dir_path, &dir_symlink_path)?; + let dir_symlink_metadata = file_system + .get_metadata( + &PathUri::from_path(&dir_symlink_path)?, + /*sandbox*/ None, + ) + .await + .with_context(|| format!("mode={implementation}"))?; + assert_eq!(dir_symlink_metadata.is_directory, true); + assert_eq!(dir_symlink_metadata.is_file, false); + assert_eq!(dir_symlink_metadata.is_symlink, true); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_rejects_unwritable_path( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let blocked_path = tmp.path().join("blocked.txt"); + + let sandbox = read_only_sandbox(tmp.path().to_path_buf()); + let error = match file_system + .write_file( + &PathUri::from_path(&blocked_path)?, + b"nope".to_vec(), + Some(&sandbox), + ) + .await + { + Ok(()) => anyhow::bail!("write should be blocked"), + Err(error) => error, + }; + assert_sandbox_denied(&error); + assert!(!blocked_path.exists()); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_allows_explicit_alias_roots( + implementation: FileSystemImplementation, +) -> Result<()> { + let Some(alias_root) = alias_root_candidate()? else { + return Ok(()); + }; + + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = tempfile::Builder::new() + .prefix("codex-fs-sandbox-alias-") + .tempdir_in(&alias_root)?; + let file_path = tmp.path().join("note.txt"); + let sandbox = workspace_write_sandbox(alias_root.clone()); + + file_system + .write_file( + &PathUri::from_path(&file_path)?, + b"created".to_vec(), + Some(&sandbox), + ) + .await + .with_context(|| format!("write file through alias root mode={implementation}"))?; + assert_eq!(std::fs::read(&file_path)?, b"created"); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_read_rejects_symlink_escape( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + std::fs::write(outside_dir.join("secret.txt"), "nope")?; + symlink(&outside_dir, allowed_dir.join("link"))?; + + let requested_path = allowed_dir.join("link").join("secret.txt"); + let sandbox = read_only_sandbox(allowed_dir); + let error = match file_system + .read_file(&PathUri::from_path(&requested_path)?, Some(&sandbox)) + .await + { + Ok(_) => anyhow::bail!("read should be blocked"), + Err(error) => error, + }; + assert_sandbox_denied(&error); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_read_rejects_symlink_parent_dotdot_escape( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + let secret_path = tmp.path().join("secret.txt"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + std::fs::write(&secret_path, "nope")?; + symlink(&outside_dir, allowed_dir.join("link"))?; + + let requested_path = + PathUri::from_path(allowed_dir.join("link").join("..").join("secret.txt"))?; + let sandbox = read_only_sandbox(allowed_dir); + let error = match file_system.read_file(&requested_path, Some(&sandbox)).await { + Ok(_) => anyhow::bail!("read should fail after path normalization"), + Err(error) => error, + }; + // PathUri's native path constructor normalizes `link/../secret.txt` to + // `allowed/secret.txt` before the request reaches the filesystem layer. + // Depending on whether the platform/runtime resolves that normalized path + // through a top-level symlink alias, the request can surface as either + // "missing file" or an upfront sandbox rejection. + assert_normalized_path_rejected(&error); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_rejects_symlink_escape( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + symlink(&outside_dir, allowed_dir.join("link"))?; + + let requested_path = allowed_dir.join("link").join("blocked.txt"); + let sandbox = workspace_write_sandbox(allowed_dir); + let error = match file_system + .write_file( + &PathUri::from_path(&requested_path)?, + b"nope".to_vec(), + Some(&sandbox), + ) + .await + { + Ok(()) => anyhow::bail!("write should be blocked"), + Err(error) => error, + }; + assert_sandbox_denied(&error); + assert!(!outside_dir.join("blocked.txt").exists()); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_write_preserves_existing_hard_link( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + + let outside_file = outside_dir.join("outside.txt"); + let hard_link = allowed_dir.join("hard-link.txt"); + std::fs::write(&outside_file, "outside\n")?; + std::fs::hard_link(&outside_file, &hard_link)?; + + let sandbox = workspace_write_sandbox(allowed_dir); + file_system + .write_file( + &PathUri::from_path(&hard_link)?, + b"updated through existing hard link\n".to_vec(), + Some(&sandbox), + ) + .await + .with_context(|| format!("mode={implementation}"))?; + + assert_eq!( + std::fs::read_to_string(&outside_file)?, + "updated through existing hard link\n" + ); + assert_eq!( + std::fs::read_to_string(&hard_link)?, + "updated through existing hard link\n" + ); + + let outside_metadata = std::fs::metadata(&outside_file)?; + let link_metadata = std::fs::metadata(&hard_link)?; + assert_eq!( + (link_metadata.dev(), link_metadata.ino()), + (outside_metadata.dev(), outside_metadata.ino()) + ); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_create_directory_rejects_symlink_escape( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + symlink(&outside_dir, allowed_dir.join("link"))?; + + let requested_path = allowed_dir.join("link").join("created"); + let sandbox = workspace_write_sandbox(allowed_dir); + let error = match file_system + .create_directory( + &PathUri::from_path(&requested_path)?, + CreateDirectoryOptions { recursive: false }, + Some(&sandbox), + ) + .await + { + Ok(()) => anyhow::bail!("create_directory should be blocked"), + Err(error) => error, + }; + assert_sandbox_denied(&error); + assert!(!outside_dir.join("created").exists()); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_read_directory_rejects_symlink_escape( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + std::fs::write(outside_dir.join("secret.txt"), "nope")?; + symlink(&outside_dir, allowed_dir.join("link"))?; + + let requested_path = allowed_dir.join("link"); + let sandbox = read_only_sandbox(allowed_dir); + let error = match file_system + .read_directory(&PathUri::from_path(&requested_path)?, Some(&sandbox)) + .await + { + Ok(_) => anyhow::bail!("read_directory should be blocked"), + Err(error) => error, + }; + assert_sandbox_denied(&error); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_rejects_symlink_escape_destination( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + std::fs::write(allowed_dir.join("source.txt"), "hello")?; + symlink(&outside_dir, allowed_dir.join("link"))?; + + let requested_destination = allowed_dir.join("link").join("copied.txt"); + let sandbox = workspace_write_sandbox(allowed_dir.clone()); + let error = match file_system + .copy( + &PathUri::from_path(allowed_dir.join("source.txt"))?, + &PathUri::from_path(&requested_destination)?, + CopyOptions { recursive: false }, + Some(&sandbox), + ) + .await + { + Ok(()) => anyhow::bail!("copy should be blocked"), + Err(error) => error, + }; + assert_sandbox_denied(&error); + assert!(!outside_dir.join("copied.txt").exists()); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_remove_removes_symlink_not_target( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + let outside_file = outside_dir.join("keep.txt"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + std::fs::write(&outside_file, "outside")?; + let symlink_path = allowed_dir.join("link"); + symlink(&outside_file, &symlink_path)?; + + let sandbox = workspace_write_sandbox(allowed_dir); + file_system + .remove( + &PathUri::from_path(&symlink_path)?, + RemoveOptions { + recursive: false, + force: false, + }, + Some(&sandbox), + ) + .await + .with_context(|| format!("mode={implementation}"))?; + + assert!(!symlink_path.exists()); + assert!(outside_file.exists()); + assert_eq!(std::fs::read_to_string(outside_file)?, "outside"); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_preserves_symlink_source( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + let outside_file = outside_dir.join("outside.txt"); + let source_symlink = allowed_dir.join("link"); + let copied_symlink = allowed_dir.join("copied-link"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + std::fs::write(&outside_file, "outside")?; + symlink(&outside_file, &source_symlink)?; + + let sandbox = workspace_write_sandbox(allowed_dir.clone()); + file_system + .copy( + &PathUri::from_path(&source_symlink)?, + &PathUri::from_path(&copied_symlink)?, + CopyOptions { recursive: false }, + Some(&sandbox), + ) + .await + .with_context(|| format!("mode={implementation}"))?; + + let copied_metadata = std::fs::symlink_metadata(&copied_symlink)?; + assert!(copied_metadata.file_type().is_symlink()); + assert_eq!(std::fs::read_link(copied_symlink)?, outside_file); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_remove_rejects_symlink_escape( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + let outside_file = outside_dir.join("secret.txt"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + std::fs::write(&outside_file, "outside")?; + symlink(&outside_dir, allowed_dir.join("link"))?; + + let requested_path = allowed_dir.join("link").join("secret.txt"); + let sandbox = workspace_write_sandbox(allowed_dir); + let error = match file_system + .remove( + &PathUri::from_path(&requested_path)?, + RemoveOptions { + recursive: false, + force: false, + }, + Some(&sandbox), + ) + .await + { + Ok(()) => anyhow::bail!("remove should be blocked"), + Err(error) => error, + }; + assert_sandbox_denied(&error); + assert_eq!(std::fs::read_to_string(outside_file)?, "outside"); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_rejects_symlink_escape_source( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let allowed_dir = tmp.path().join("allowed"); + let outside_dir = tmp.path().join("outside"); + let outside_file = outside_dir.join("secret.txt"); + let requested_destination = allowed_dir.join("copied.txt"); + std::fs::create_dir_all(&allowed_dir)?; + std::fs::create_dir_all(&outside_dir)?; + std::fs::write(&outside_file, "outside")?; + symlink(&outside_dir, allowed_dir.join("link"))?; + + let requested_source = allowed_dir.join("link").join("secret.txt"); + let sandbox = workspace_write_sandbox(allowed_dir); + let error = match file_system + .copy( + &PathUri::from_path(&requested_source)?, + &PathUri::from_path(&requested_destination)?, + CopyOptions { recursive: false }, + Some(&sandbox), + ) + .await + { + Ok(()) => anyhow::bail!("copy should be blocked"), + Err(error) => error, + }; + assert_sandbox_denied(&error); + assert!(!requested_destination.exists()); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_preserves_symlinks_in_recursive_copy( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let source_dir = tmp.path().join("source"); + let nested_dir = source_dir.join("nested"); + let copied_dir = tmp.path().join("copied"); + std::fs::create_dir_all(&nested_dir)?; + symlink("nested", source_dir.join("nested-link"))?; + + file_system + .copy( + &PathUri::from_path(&source_dir)?, + &PathUri::from_path(&copied_dir)?, + CopyOptions { recursive: true }, + /*sandbox*/ None, + ) + .await + .with_context(|| format!("mode={implementation}"))?; + + let copied_link = copied_dir.join("nested-link"); + let metadata = std::fs::symlink_metadata(&copied_link)?; + assert!(metadata.file_type().is_symlink()); + assert_eq!( + std::fs::read_link(copied_link)?, + std::path::PathBuf::from("nested") + ); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_ignores_unknown_special_files_in_recursive_copy( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let source_dir = tmp.path().join("source"); + let copied_dir = tmp.path().join("copied"); + std::fs::create_dir_all(&source_dir)?; + std::fs::write(source_dir.join("note.txt"), "hello")?; + + let fifo_path = source_dir.join("named-pipe"); + let output = Command::new("mkfifo").arg(&fifo_path).output()?; + if !output.status.success() { + anyhow::bail!( + "mkfifo failed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout).trim(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + file_system + .copy( + &PathUri::from_path(&source_dir)?, + &PathUri::from_path(&copied_dir)?, + CopyOptions { recursive: true }, + /*sandbox*/ None, + ) + .await + .with_context(|| format!("mode={implementation}"))?; + + assert_eq!( + std::fs::read_to_string(copied_dir.join("note.txt"))?, + "hello" + ); + assert!(!copied_dir.join("named-pipe").exists()); + + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_copy_rejects_standalone_fifo_source( + implementation: FileSystemImplementation, +) -> Result<()> { + let context = create_file_system_context(implementation).await?; + let file_system = context.file_system; + + let tmp = TempDir::new()?; + let fifo_path = tmp.path().join("named-pipe"); + let output = Command::new("mkfifo").arg(&fifo_path).output()?; + if !output.status.success() { + anyhow::bail!( + "mkfifo failed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout).trim(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + let error = file_system + .copy( + &PathUri::from_path(&fifo_path)?, + &PathUri::from_path(tmp.path().join("copied"))?, + CopyOptions { recursive: false }, + /*sandbox*/ None, + ) + .await; + let error = match error { + Ok(()) => panic!("copy should fail"), + Err(error) => error, + }; + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!( + error.to_string(), + "fs/copy only supports regular files, directories, and symlinks" + ); + + Ok(()) +} diff --git a/codex-rs/exec-server/tests/file_system_windows.rs b/codex-rs/exec-server/tests/file_system_windows.rs new file mode 100644 index 0000000000..68d826ced3 --- /dev/null +++ b/codex-rs/exec-server/tests/file_system_windows.rs @@ -0,0 +1,55 @@ +#![cfg(windows)] + +mod common; + +#[path = "file_system/shared.rs"] +mod shared; +#[path = "file_system/support.rs"] +mod support; + +use std::path::Path; +use std::process::Command; + +use anyhow::Result; +use test_case::test_case; + +use crate::support::FileSystemImplementation; + +fn create_directory_junction(target: &Path, alias: &Path) -> Result<()> { + let output = Command::new("cmd") + .args(["/C", "mklink", "/J"]) + .arg(alias) + .arg(target) + .output()?; + if !output.status.success() { + anyhow::bail!( + "mklink /J failed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout).trim(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_canonicalize_resolves_directory_junction( + implementation: FileSystemImplementation, +) -> Result<()> { + shared::assert_canonicalize_resolves_directory_alias(implementation, create_directory_junction) + .await +} + +#[test_case(FileSystemImplementation::Local ; "local")] +#[test_case(FileSystemImplementation::Remote ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn file_system_sandboxed_canonicalize_resolves_directory_junction( + implementation: FileSystemImplementation, +) -> Result<()> { + shared::assert_sandboxed_canonicalize_resolves_directory_alias( + implementation, + create_directory_junction, + ) + .await +} diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 00a9f957f1..60667bb5df 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -239,6 +239,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { ); CodexStatus::Running } + ServerNotification::Warning(notification) => self.process_warning(notification.message), ServerNotification::Error(notification) => { eprintln!( "{} {}", diff --git a/codex-rs/exec/src/event_processor_with_jsonl_output.rs b/codex-rs/exec/src/event_processor_with_jsonl_output.rs index 89e41eacb1..79bf88eaaf 100644 --- a/codex-rs/exec/src/event_processor_with_jsonl_output.rs +++ b/codex-rs/exec/src/event_processor_with_jsonl_output.rs @@ -430,6 +430,11 @@ impl EventProcessorWithJsonOutput { })); CodexStatus::Running } + ServerNotification::Warning(notification) => { + let warning = self.collect_warning(notification.message); + events.extend(warning.events); + warning.status + } ServerNotification::Error(notification) => { let message = match notification.error.additional_details { Some(details) if !details.is_empty() => { diff --git a/codex-rs/exec/src/event_processor_with_jsonl_output_tests.rs b/codex-rs/exec/src/event_processor_with_jsonl_output_tests.rs index b8b0267fc4..f8521071e3 100644 --- a/codex-rs/exec/src/event_processor_with_jsonl_output_tests.rs +++ b/codex-rs/exec/src/event_processor_with_jsonl_output_tests.rs @@ -59,6 +59,33 @@ fn failed_turn_does_not_overwrite_output_last_message_file() { ); } +#[test] +fn runtime_warning_emits_a_non_fatal_error_item() { + let mut processor = EventProcessorWithJsonOutput::new(/*last_message_path*/ None); + + let collected = processor.collect_thread_events(ServerNotification::Warning( + codex_app_server_protocol::WarningNotification { + thread_id: Some("thread-1".to_string()), + message: "invalid global instructions".to_string(), + }, + )); + + assert_eq!( + collected, + CollectedThreadEvents { + events: vec![ThreadEvent::ItemCompleted(ItemCompletedEvent { + item: ExecThreadItem { + id: "item_0".to_string(), + details: ThreadItemDetails::Error(ErrorItem { + message: "invalid global instructions".to_string(), + }), + }, + })], + status: CodexStatus::Running, + } + ); +} + #[test] fn mcp_tool_call_result_preserves_meta_in_jsonl_event() { let mut processor = EventProcessorWithJsonOutput::new(/*last_message_path*/ None); diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index b3dd5f3277..e1e0907432 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -1277,6 +1277,11 @@ fn should_process_notification( ) -> bool { match notification { ServerNotification::ConfigWarning(_) | ServerNotification::DeprecationNotice(_) => true, + // TODO(anp) resolve duplicate startup warnings + ServerNotification::Warning(notification) => notification + .thread_id + .as_deref() + .is_none_or(|candidate| candidate == thread_id), ServerNotification::Error(notification) => { notification.thread_id == thread_id && notification.turn_id == turn_id } diff --git a/codex-rs/exec/src/lib_tests.rs b/codex-rs/exec/src/lib_tests.rs index 8b2f214987..de81f60d70 100644 --- a/codex-rs/exec/src/lib_tests.rs +++ b/codex-rs/exec/src/lib_tests.rs @@ -267,6 +267,35 @@ fn lagged_event_warning_message_is_explicit() { ); } +#[test] +fn runtime_warnings_are_filtered_to_the_primary_thread() { + let primary_thread_id = "thread-1"; + let turn_id = "turn-1"; + let outcomes = [ + codex_app_server_protocol::WarningNotification { + thread_id: None, + message: "global warning".to_string(), + }, + codex_app_server_protocol::WarningNotification { + thread_id: Some(primary_thread_id.to_string()), + message: "primary warning".to_string(), + }, + codex_app_server_protocol::WarningNotification { + thread_id: Some("thread-2".to_string()), + message: "other warning".to_string(), + }, + ] + .map(|warning| { + should_process_notification( + &ServerNotification::Warning(warning), + primary_thread_id, + turn_id, + ) + }); + + assert_eq!(outcomes, [true, true, false]); +} + #[tokio::test] async fn resume_lookup_model_providers_filters_only_last_lookup() { let codex_home = tempdir().expect("create temp codex home"); diff --git a/codex-rs/exec/tests/suite/agents_md.rs b/codex-rs/exec/tests/suite/agents_md.rs index 2891721ae6..d4a765f5c6 100644 --- a/codex-rs/exec/tests/suite/agents_md.rs +++ b/codex-rs/exec/tests/suite/agents_md.rs @@ -2,6 +2,8 @@ use core_test_support::responses; use core_test_support::test_codex_exec::test_codex_exec; +use predicates::prelude::PredicateBooleanExt; +use predicates::str::contains; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn exec_includes_workspace_agents_md_in_request() -> anyhow::Result<()> { @@ -72,3 +74,103 @@ async fn exec_prefers_workspace_agents_override_md() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_surfaces_project_instruction_loading_warnings() -> anyhow::Result<()> { + let test = test_codex_exec(); + let project_agents_path = test.cwd_path().join("AGENTS.md"); + std::fs::write(&project_agents_path, b"project\xFFinstructions")?; + + let server = responses::start_mock_server().await; + let body = responses::sse(vec![ + responses::ev_response_created("resp1"), + responses::ev_assistant_message("m1", "fixture hello"), + responses::ev_completed("resp1"), + ]); + responses::mount_sse_once(&server, body).await; + + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("tell me something") + .assert() + .success() + .stderr(contains("invalid UTF-8").and(contains(project_agents_path.display().to_string()))); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_json_surfaces_project_instruction_loading_warnings() -> anyhow::Result<()> { + let test = test_codex_exec(); + let project_agents_path = test.cwd_path().join("AGENTS.md"); + std::fs::write(&project_agents_path, b"project\xFFinstructions")?; + + let server = responses::start_mock_server().await; + let body = responses::sse(vec![ + responses::ev_response_created("resp1"), + responses::ev_assistant_message("m1", "fixture hello"), + responses::ev_completed("resp1"), + ]); + responses::mount_sse_once(&server, body).await; + + let output = test + .cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("--json") + .arg("tell me something") + .assert() + .success() + .get_output() + .stdout + .clone(); + let events = String::from_utf8(output)? + .lines() + .map(serde_json::from_str::) + .collect::, _>>()?; + + assert!( + events.iter().any(|event| { + event["type"] == "item.completed" + && event["item"]["type"] == "error" + && event["item"]["message"].as_str().is_some_and(|message| { + message.contains("invalid UTF-8") + && message.contains(project_agents_path.display().to_string().as_str()) + }) + }), + "expected a JSONL warning event; observed: {events:?}" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_surfaces_global_instruction_loading_warnings() -> anyhow::Result<()> { + let test = test_codex_exec(); + let global_agents_path = test.home_path().join("AGENTS.md"); + let global_agents_source_suffix = format!( + "{}{}AGENTS.md", + test.home_path() + .file_name() + .expect("temporary Codex home should have a file name") + .to_string_lossy(), + std::path::MAIN_SEPARATOR, + ); + std::fs::write(&global_agents_path, b"global\xFFinstructions")?; + + let server = responses::start_mock_server().await; + let body = responses::sse(vec![ + responses::ev_response_created("resp1"), + responses::ev_assistant_message("m1", "fixture hello"), + responses::ev_completed("resp1"), + ]); + responses::mount_sse_once(&server, body).await; + + test.cmd_with_server(&server) + .arg("--skip-git-repo-check") + .arg("tell me something") + .assert() + .success() + .stderr(contains("invalid UTF-8").and(contains(global_agents_source_suffix))); + + Ok(()) +} diff --git a/codex-rs/ext/extension-api/Cargo.toml b/codex-rs/ext/extension-api/Cargo.toml index 25ed8d0d71..1a5a9742fb 100644 --- a/codex-rs/ext/extension-api/Cargo.toml +++ b/codex-rs/ext/extension-api/Cargo.toml @@ -18,6 +18,7 @@ codex-config = { workspace = true } codex-context-fragments = { workspace = true } codex-protocol = { workspace = true } codex-tools = { workspace = true } +codex-utils-absolute-path = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index e93088b5d0..efc22f5fd9 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -48,6 +48,9 @@ pub type ExtensionFuture<'a, T> = Pin + Send + 'a>>; /// Plugin-owned servers and their provenance continue to be resolved by the /// plugin manager until that ownership moves into an extension explicitly. pub trait McpServerContributor: Send + Sync { + /// Stable identity used for registration provenance and conflict diagnostics. + fn id(&self) -> &'static str; + fn contribute<'a>(&'a self, config: &'a C) -> ExtensionFuture<'a, Vec>; } diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index 52f15c894c..7004af2633 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -2,6 +2,7 @@ mod capabilities; mod contributors; mod registry; mod state; +mod user_instructions; pub use capabilities::AgentSpawnFuture; pub use capabilities::AgentSpawner; @@ -64,3 +65,7 @@ pub use registry::ExtensionRegistryBuilder; pub use registry::empty_extension_registry; pub use state::ExtensionData; pub use state::ExtensionDataInit; +pub use user_instructions::LoadUserInstructionsFuture; +pub use user_instructions::LoadedUserInstructions; +pub use user_instructions::UserInstructions; +pub use user_instructions::UserInstructionsProvider; diff --git a/codex-rs/ext/extension-api/src/user_instructions.rs b/codex-rs/ext/extension-api/src/user_instructions.rs new file mode 100644 index 0000000000..d5fd63c197 --- /dev/null +++ b/codex-rs/ext/extension-api/src/user_instructions.rs @@ -0,0 +1,41 @@ +use std::future::Future; +use std::pin::Pin; + +use codex_utils_absolute_path::AbsolutePathBuf; + +/// User instructions supplied by the host. +/// +/// `source` must be an absolute filesystem path because the app-server +/// `instructionSources` API currently exposes instruction sources as +/// `AbsolutePathBuf` values. +// TODO(anp): Replace the absolute path with a more general instruction-source +// abstraction when non-filesystem providers need first-class attribution. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct UserInstructions { + /// Model-visible user instruction text. + pub text: String, + /// Absolute filesystem path reported through `instructionSources`. + pub source: AbsolutePathBuf, +} + +/// Result of loading host-provided user instructions. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct LoadedUserInstructions { + /// Loaded instructions, or `None` when the provider has no applicable text. + pub instructions: Option, + /// Recoverable loading problems that should be surfaced during startup. + pub warnings: Vec, +} + +/// Future returned by a [`UserInstructionsProvider`]. +pub type LoadUserInstructionsFuture<'a> = + Pin + Send + 'a>>; + +/// Loads the user instructions that apply when a root thread runtime starts. +/// +/// Implementations should return any recoverable loading problems as warnings +/// while still returning usable fallback instructions when available. +pub trait UserInstructionsProvider: Send + Sync { + /// Loads the snapshot to use for a newly created root runtime. + fn load_user_instructions(&self) -> LoadUserInstructionsFuture<'_>; +} diff --git a/codex-rs/ext/mcp/src/lib.rs b/codex-rs/ext/mcp/src/lib.rs index 8568d58150..3843e06840 100644 --- a/codex-rs/ext/mcp/src/lib.rs +++ b/codex-rs/ext/mcp/src/lib.rs @@ -9,6 +9,10 @@ use codex_mcp::hosted_plugin_runtime_mcp_server_config; struct HostedPluginRuntimeExtension; impl McpServerContributor for HostedPluginRuntimeExtension { + fn id(&self) -> &'static str { + "hosted_plugin_runtime" + } + fn contribute<'a>( &'a self, config: &'a Config, diff --git a/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs b/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs index bb401578f4..4a3902630b 100644 --- a/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs +++ b/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs @@ -42,6 +42,34 @@ async fn contributes_hosted_plugin_runtime_without_an_executor() -> TestResult { Ok(()) } +#[tokio::test] +async fn runtime_overlay_preserves_disabled_server() -> TestResult { + let codex_home = tempfile::tempdir()?; + let config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .fallback_cwd(Some(codex_home.path().to_path_buf())) + .cli_overrides(vec![ + ("features.apps".to_string(), true.into()), + ( + "mcp_servers.codex_apps.url".to_string(), + "https://example.com/mcp".into(), + ), + ("mcp_servers.codex_apps.enabled".to_string(), false.into()), + ]) + .build() + .await?; + let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); + let manager = installed_manager(&config); + + let servers = manager.effective_servers(&config, Some(&auth)).await; + let server = servers + .get(CODEX_APPS_MCP_SERVER_NAME) + .ok_or("hosted plugin runtime should remain configured")?; + + assert!(!server.enabled()); + Ok(()) +} + #[tokio::test] async fn legacy_fallback_overwrites_reserved_config_without_an_extension() -> TestResult { let codex_home = tempfile::tempdir()?; @@ -76,7 +104,7 @@ async fn legacy_fallback_overwrites_reserved_config_without_an_extension() -> Te } #[tokio::test] -async fn extension_can_remove_legacy_fallback_while_apps_are_enabled() -> TestResult { +async fn later_extension_can_remove_same_name_registration() -> TestResult { let codex_home = tempfile::tempdir()?; let config = ConfigBuilder::default() .codex_home(codex_home.path().to_path_buf()) @@ -86,6 +114,7 @@ async fn extension_can_remove_legacy_fallback_while_apps_are_enabled() -> TestRe .await?; let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let mut builder = ExtensionRegistryBuilder::new(); + codex_mcp_extension::install(&mut builder); builder.mcp_server_contributor(Arc::new(RemoveCodexApps)); let manager = McpManager::new_with_extensions( Arc::new(PluginsManager::new(config.codex_home.to_path_buf())), @@ -117,7 +146,7 @@ async fn hosted_apps_mcp_requires_chatgpt_auth() -> TestResult { } #[tokio::test] -async fn disabled_apps_remove_reserved_server_config() -> TestResult { +async fn disabled_apps_remove_reserved_server_config_for_all_hosts() -> TestResult { let codex_home = tempfile::tempdir()?; let config = ConfigBuilder::default() .codex_home(codex_home.path().to_path_buf()) @@ -131,11 +160,16 @@ async fn disabled_apps_remove_reserved_server_config() -> TestResult { ]) .build() .await?; - let manager = installed_manager(&config); - - let servers = manager.runtime_servers(&config).await; - - assert!(!servers.contains_key(CODEX_APPS_MCP_SERVER_NAME)); + let managers = [ + installed_manager(&config), + McpManager::new(Arc::new(PluginsManager::new( + config.codex_home.to_path_buf(), + ))), + ]; + for manager in managers { + let servers = manager.runtime_servers(&config).await; + assert!(!servers.contains_key(CODEX_APPS_MCP_SERVER_NAME)); + } Ok(()) } @@ -151,6 +185,10 @@ fn installed_manager(config: &Config) -> McpManager { struct RemoveCodexApps; impl McpServerContributor for RemoveCodexApps { + fn id(&self) -> &'static str { + "remove_codex_apps" + } + fn contribute<'a>( &'a self, _config: &'a Config, diff --git a/codex-rs/ext/skills/Cargo.toml b/codex-rs/ext/skills/Cargo.toml index c67d4b0e1b..cc8dcb1149 100644 --- a/codex-rs/ext/skills/Cargo.toml +++ b/codex-rs/ext/skills/Cargo.toml @@ -14,13 +14,21 @@ doctest = false workspace = true [dependencies] -codex-core = { workspace = true } codex-core-skills = { workspace = true } codex-exec-server = { workspace = true } codex-extension-api = { workspace = true } +codex-mcp = { workspace = true } codex-protocol = { workspace = true } +codex-tools = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-path-uri = { workspace = true } codex-utils-string = { workspace = true } +schemars = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["sync", "time"] } +tracing = { workspace = true } +url = { workspace = true } [dev-dependencies] async-trait = { workspace = true } diff --git a/codex-rs/ext/skills/src/catalog.rs b/codex-rs/ext/skills/src/catalog.rs index c966cc09e9..96b8fec10f 100644 --- a/codex-rs/ext/skills/src/catalog.rs +++ b/codex-rs/ext/skills/src/catalog.rs @@ -1,5 +1,5 @@ use codex_core_skills::model::SkillDependencies; -use codex_exec_server::EnvironmentPathRef; +use codex_utils_absolute_path::AbsolutePathBuf; /// Source authority that owns a skill package and must be used to read it. #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -9,8 +9,8 @@ pub enum SkillSourceKind { Host, /// Skills owned by an execution environment. Executor, - /// Skills read through an authenticated remote catalog/API. - Remote, + /// Skills owned by the orchestrator rather than an execution environment. + Orchestrator, /// Extension-private source kind for future providers that do not fit an /// existing transport category. Custom(String), @@ -25,7 +25,7 @@ impl SkillSourceKind { match self { Self::Host => "host", Self::Executor => "executor", - Self::Remote => "remote", + Self::Orchestrator => "orchestrator", Self::Custom(kind) => kind, } } @@ -62,7 +62,7 @@ pub struct SkillPackageId(pub String); #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct SkillResourceId { id: String, - environment_path: Option, + environment_path: Option, } impl SkillResourceId { @@ -73,10 +73,17 @@ impl SkillResourceId { } } - pub fn environment(id: impl Into, path: EnvironmentPathRef) -> Self { + pub fn environment( + id: impl Into, + environment_id: impl Into, + path: AbsolutePathBuf, + ) -> Self { Self { id: id.into(), - environment_path: Some(path), + environment_path: Some(EnvironmentSkillResource { + environment_id: environment_id.into(), + path, + }), } } @@ -84,11 +91,19 @@ impl SkillResourceId { &self.id } - pub(crate) fn environment_path(&self) -> Option<&EnvironmentPathRef> { - self.environment_path.as_ref() + pub(crate) fn environment_path(&self) -> Option<(&str, &AbsolutePathBuf)> { + self.environment_path + .as_ref() + .map(|resource| (resource.environment_id.as_str(), &resource.path)) } } +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct EnvironmentSkillResource { + environment_id: String, + path: AbsolutePathBuf, +} + /// Metadata shown in the always-visible skills catalog. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SkillCatalogEntry { diff --git a/codex-rs/ext/skills/src/config.rs b/codex-rs/ext/skills/src/config.rs new file mode 100644 index 0000000000..e883f0c5f5 --- /dev/null +++ b/codex-rs/ext/skills/src/config.rs @@ -0,0 +1,8 @@ +/// Host-supplied configuration used by the skills extension. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SkillsExtensionConfig { + /// Whether the available-skills catalog is included in model context. + pub include_instructions: bool, + /// Whether bundled skills are eligible for discovery. + pub bundled_skills_enabled: bool, +} diff --git a/codex-rs/ext/skills/src/extension.rs b/codex-rs/ext/skills/src/extension.rs index e876c66b29..23dee27ebb 100644 --- a/codex-rs/ext/skills/src/extension.rs +++ b/codex-rs/ext/skills/src/extension.rs @@ -1,10 +1,7 @@ use std::sync::Arc; -use codex_core::config::Config; use codex_core_skills::HostLoadedSkills; -use codex_core_skills::SkillInstructions; use codex_core_skills::injection::InjectedHostSkillPrompts; -use codex_core_skills::injection::SkillInjection; use codex_extension_api::ConfigContributor; use codex_extension_api::ContextContributor; use codex_extension_api::ContextualUserFragment; @@ -15,16 +12,23 @@ use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::PromptFragment; use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadStartInput; +use codex_extension_api::ToolCall; +use codex_extension_api::ToolContributor; +use codex_extension_api::ToolExecutor; use codex_extension_api::TurnInputContext; use codex_extension_api::TurnInputContributor; +use codex_mcp::McpResourceClient; use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::WarningEvent; +use crate::SkillsExtensionConfig; +use crate::catalog::SkillCatalog; use crate::catalog::SkillCatalogEntry; use crate::catalog::SkillReadResult; use crate::catalog::SkillSourceKind; +use crate::fragments::SkillInstructions; use crate::provider::HostSkillProvider; use crate::provider::SkillListQuery; use crate::provider::SkillReadRequest; @@ -35,21 +39,21 @@ use crate::render::truncate_main_prompt_contents; use crate::render::truncate_utf8_to_bytes; use crate::selection::collect_explicit_skill_mentions; use crate::sources::SkillProviders; -use crate::state::SkillsExtensionConfig; use crate::state::SkillsThreadState; use crate::state::SkillsTurnState; +use crate::tools::skill_tools; -#[derive(Clone)] -struct SkillsExtension { +struct SkillsExtension { providers: SkillProviders, event_sink: Arc, + config_from_host: Arc SkillsExtensionConfig + Send + Sync>, } -impl ThreadLifecycleContributor for SkillsExtension { - fn on_thread_start<'a>( - &'a self, - input: ThreadStartInput<'a, Config>, - ) -> ExtensionFuture<'a, ()> { +impl ThreadLifecycleContributor for SkillsExtension +where + C: Send + Sync + 'static, +{ + fn on_thread_start<'a>(&'a self, input: ThreadStartInput<'a, C>) -> ExtensionFuture<'a, ()> { Box::pin(async move { let selected_roots = input .thread_store @@ -57,22 +61,25 @@ impl ThreadLifecycleContributor for SkillsExtension { .map(|selected_roots| selected_roots.as_ref().clone()) .unwrap_or_default(); input.thread_store.insert(SkillsThreadState::new( - SkillsExtensionConfig::from_config(input.config), + (self.config_from_host)(input.config), selected_roots, )); }) } } -impl ConfigContributor for SkillsExtension { +impl ConfigContributor for SkillsExtension +where + C: Send + Sync + 'static, +{ fn on_config_changed( &self, _session_store: &ExtensionData, thread_store: &ExtensionData, - _previous_config: &Config, - new_config: &Config, + _previous_config: &C, + new_config: &C, ) { - let next_config = SkillsExtensionConfig::from_config(new_config); + let next_config = (self.config_from_host)(new_config); if let Some(state) = thread_store.get::() { state.set_config(next_config); } else { @@ -81,10 +88,13 @@ impl ConfigContributor for SkillsExtension { } } -impl ContextContributor for SkillsExtension { +impl ContextContributor for SkillsExtension +where + C: Send + Sync + 'static, +{ fn contribute<'a>( &'a self, - _session_store: &'a ExtensionData, + session_store: &'a ExtensionData, thread_store: &'a ExtensionData, ) -> std::pin::Pin> + Send + 'a>> { Box::pin(async move { @@ -92,19 +102,22 @@ impl ContextContributor for SkillsExtension { return Vec::new(); }; let config = thread_state.config(); - if !config.include_instructions || thread_state.selected_roots().is_empty() { + if !config.include_instructions { return Vec::new(); } let catalog = self - .providers - .list_for_turn(SkillListQuery { - turn_id: thread_store.level_id().to_string(), - executor_roots: thread_state.selected_roots().to_vec(), - host: None, - include_host_skills: false, - include_bundled_skills: config.bundled_skills_enabled, - include_remote_skills: false, - }) + .list_skills( + SkillListQuery { + turn_id: thread_store.level_id().to_string(), + executor_roots: thread_state.selected_roots().to_vec(), + host: None, + include_host_skills: false, + include_bundled_skills: config.bundled_skills_enabled, + include_orchestrator_skills: true, + mcp_resources: session_store.get::(), + }, + &thread_state, + ) .await; for warning in &catalog.warnings { self.emit_warning(thread_store.level_id(), warning.clone()); @@ -117,11 +130,34 @@ impl ContextContributor for SkillsExtension { } } -impl TurnInputContributor for SkillsExtension { +impl ToolContributor for SkillsExtension +where + C: Send + Sync + 'static, +{ + fn tools( + &self, + session_store: &ExtensionData, + _thread_store: &ExtensionData, + ) -> Vec>> { + if !self.providers.has_orchestrator_provider() { + return Vec::new(); + } + + skill_tools( + self.providers.clone(), + session_store.get::(), + ) + } +} + +impl TurnInputContributor for SkillsExtension +where + C: Send + Sync + 'static, +{ fn contribute<'a>( &'a self, input: TurnInputContext, - _session_store: &'a ExtensionData, + session_store: &'a ExtensionData, thread_store: &'a ExtensionData, turn_store: &'a ExtensionData, ) -> ExtensionFuture<'a, Vec>> { @@ -138,9 +174,10 @@ impl TurnInputContributor for SkillsExtension { host: host_loaded_skills.clone(), include_host_skills: true, include_bundled_skills: config.bundled_skills_enabled, - include_remote_skills: true, + include_orchestrator_skills: true, + mcp_resources: session_store.get::(), }; - let catalog = self.providers.list_for_turn(query).await; + let catalog = self.list_skills(query, &thread_state).await; for warning in &catalog.warnings { self.emit_warning(&input.turn_id, warning.clone()); } @@ -149,9 +186,10 @@ impl TurnInputContributor for SkillsExtension { let mut fragments: Vec> = Vec::new(); if config.include_instructions { let mut turn_catalog = catalog.clone(); - turn_catalog - .entries - .retain(|entry| entry.authority.kind != SkillSourceKind::Executor); + turn_catalog.entries.retain(|entry| { + entry.authority.kind != SkillSourceKind::Executor + && entry.authority.kind != SkillSourceKind::Orchestrator + }); if let Some(fragment) = available_skills_fragment(&turn_catalog) { fragments.push(Box::new(fragment)); } @@ -162,7 +200,7 @@ impl TurnInputContributor for SkillsExtension { let mut injected_host_skill_prompts = InjectedHostSkillPrompts::default(); for entry in &selected_entries { match self - .read_main_prompt(entry, host_loaded_skills.clone()) + .read_main_prompt(entry, host_loaded_skills.clone(), session_store) .await { Ok(read_result) => { @@ -176,7 +214,7 @@ impl TurnInputContributor for SkillsExtension { self.emit_warning(&input.turn_id, warning.clone()); warnings.push(warning); } - let injection = SkillInjection { + let fragment = SkillInstructions { name: truncate_utf8_to_bytes(&entry.name, MAX_SKILL_NAME_BYTES).0, path: truncate_utf8_to_bytes( entry.rendered_path(), @@ -185,7 +223,7 @@ impl TurnInputContributor for SkillsExtension { .0, contents, }; - fragments.push(Box::new(SkillInstructions::from(&injection))); + fragments.push(Box::new(fragment)); main_prompts_injected = true; if entry.authority.kind == SkillSourceKind::Host { injected_host_skill_prompts.insert_path(entry.main_prompt.as_str()); @@ -231,11 +269,34 @@ impl TurnInputContributor for SkillsExtension { } } -impl SkillsExtension { +impl SkillsExtension { + async fn list_skills( + &self, + mut query: SkillListQuery, + thread_state: &SkillsThreadState, + ) -> SkillCatalog { + let include_orchestrator_skills = query.include_orchestrator_skills; + let orchestrator_query = query.clone(); + query.include_orchestrator_skills = false; + + let mut catalog = self.providers.list_for_turn(query).await; + if include_orchestrator_skills { + let orchestrator_catalog = thread_state + .orchestrator_catalog_snapshot( + self.providers + .list_orchestrator_for_turn(orchestrator_query), + ) + .await; + catalog.extend(orchestrator_catalog); + } + catalog + } + async fn read_main_prompt( &self, entry: &SkillCatalogEntry, host_loaded_skills: Option>, + session_store: &ExtensionData, ) -> Result { self.providers .read(SkillReadRequest { @@ -243,6 +304,7 @@ impl SkillsExtension { package: entry.id.clone(), resource: entry.main_prompt.clone(), host: host_loaded_skills, + mcp_resources: session_store.get::(), }) .await .map_err(|err| err.message) @@ -256,23 +318,34 @@ impl SkillsExtension { } } -pub fn install(registry: &mut ExtensionRegistryBuilder) { +pub fn install( + registry: &mut ExtensionRegistryBuilder, + config_from_host: impl Fn(&C) -> SkillsExtensionConfig + Send + Sync + 'static, +) where + C: Send + Sync + 'static, +{ install_with_providers( registry, SkillProviders::new().with_host_provider(Arc::new(HostSkillProvider::new())), + config_from_host, ); } -pub fn install_with_providers( - registry: &mut ExtensionRegistryBuilder, +pub fn install_with_providers( + registry: &mut ExtensionRegistryBuilder, providers: SkillProviders, -) { + config_from_host: impl Fn(&C) -> SkillsExtensionConfig + Send + Sync + 'static, +) where + C: Send + Sync + 'static, +{ let extension = Arc::new(SkillsExtension { providers, event_sink: registry.event_sink(), + config_from_host: Arc::new(config_from_host), }); registry.thread_lifecycle_contributor(extension.clone()); registry.config_contributor(extension.clone()); registry.prompt_contributor(extension.clone()); - registry.turn_input_contributor(extension); + registry.turn_input_contributor(extension.clone()); + registry.tool_contributor(extension); } diff --git a/codex-rs/ext/skills/src/fragments.rs b/codex-rs/ext/skills/src/fragments.rs new file mode 100644 index 0000000000..0a24b48b5a --- /dev/null +++ b/codex-rs/ext/skills/src/fragments.rs @@ -0,0 +1,61 @@ +use codex_core_skills::render_available_skills_body; +use codex_extension_api::ContextualUserFragment; +use codex_protocol::protocol::SKILLS_INSTRUCTIONS_CLOSE_TAG; +use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AvailableSkillsInstructions { + skill_lines: Vec, +} + +impl AvailableSkillsInstructions { + pub(crate) fn from_skill_lines(skill_lines: Vec) -> Self { + Self { skill_lines } + } +} + +impl ContextualUserFragment for AvailableSkillsInstructions { + fn role(&self) -> &'static str { + "developer" + } + + fn markers(&self) -> (&'static str, &'static str) { + Self::type_markers() + } + + fn type_markers() -> (&'static str, &'static str) { + (SKILLS_INSTRUCTIONS_OPEN_TAG, SKILLS_INSTRUCTIONS_CLOSE_TAG) + } + + fn body(&self) -> String { + render_available_skills_body(&[], &self.skill_lines) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SkillInstructions { + pub(crate) name: String, + pub(crate) path: String, + pub(crate) contents: String, +} + +impl ContextualUserFragment for SkillInstructions { + fn role(&self) -> &'static str { + "user" + } + + fn markers(&self) -> (&'static str, &'static str) { + Self::type_markers() + } + + fn type_markers() -> (&'static str, &'static str) { + ("", "") + } + + fn body(&self) -> String { + let name = &self.name; + let path = &self.path; + let contents = &self.contents; + format!("\n{name}\n{path}\n{contents}\n") + } +} diff --git a/codex-rs/ext/skills/src/lib.rs b/codex-rs/ext/skills/src/lib.rs index 5ab03ddd9e..8abdf6a93e 100644 --- a/codex-rs/ext/skills/src/lib.rs +++ b/codex-rs/ext/skills/src/lib.rs @@ -1,15 +1,20 @@ pub mod catalog; +mod config; mod extension; +mod fragments; pub mod provider; mod render; mod selection; mod sources; mod state; +mod tools; +pub use config::SkillsExtensionConfig; pub use extension::install; pub use extension::install_with_providers; pub use provider::ExecutorSkillProvider; pub use provider::HostSkillProvider; +pub use provider::OrchestratorSkillProvider; pub use provider::SkillProvider; pub use sources::SkillProviderSource; pub use sources::SkillProviders; diff --git a/codex-rs/ext/skills/src/provider.rs b/codex-rs/ext/skills/src/provider.rs index 8d13a2f981..329c0c5813 100644 --- a/codex-rs/ext/skills/src/provider.rs +++ b/codex-rs/ext/skills/src/provider.rs @@ -4,8 +4,10 @@ use std::sync::Arc; mod executor; mod host; +mod orchestrator; use codex_core_skills::HostLoadedSkills; +use codex_mcp::McpResourceClient; use codex_protocol::capabilities::SelectedCapabilityRoot; use crate::catalog::SkillAuthority; @@ -18,6 +20,7 @@ use crate::catalog::SkillSearchResult; pub use executor::ExecutorSkillProvider; pub use host::HostSkillProvider; +pub use orchestrator::OrchestratorSkillProvider; #[derive(Clone, Debug)] pub struct SkillListQuery { @@ -26,7 +29,8 @@ pub struct SkillListQuery { pub host: Option>, pub include_host_skills: bool, pub include_bundled_skills: bool, - pub include_remote_skills: bool, + pub include_orchestrator_skills: bool, + pub mcp_resources: Option>, } #[derive(Clone, Debug)] @@ -35,6 +39,7 @@ pub struct SkillReadRequest { pub package: SkillPackageId, pub resource: SkillResourceId, pub host: Option>, + pub mcp_resources: Option>, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/codex-rs/ext/skills/src/provider/executor.rs b/codex-rs/ext/skills/src/provider/executor.rs index e23e5a95bd..1ed1d11fa0 100644 --- a/codex-rs/ext/skills/src/provider/executor.rs +++ b/codex-rs/ext/skills/src/provider/executor.rs @@ -6,11 +6,11 @@ use codex_core_skills::filter_skill_load_outcome_for_product; use codex_core_skills::loader::SkillRoot; use codex_core_skills::loader::load_skills_from_roots; use codex_exec_server::EnvironmentManager; -use codex_exec_server::EnvironmentPathRef; use codex_protocol::capabilities::CapabilityRootLocation; use codex_protocol::protocol::Product; use codex_protocol::protocol::SkillScope; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use crate::catalog::SkillAuthority; use crate::catalog::SkillCatalog; @@ -99,7 +99,7 @@ impl SkillProvider for ExecutorSkillProvider { enabled, authority.clone(), &selected_root_id, - Arc::clone(&file_system), + &environment_id, )); } } @@ -121,13 +121,25 @@ impl SkillProvider for ExecutorSkillProvider { "executor skill resource does not match its package", )); } - let Some(resource_path) = request.resource.environment_path() else { + let Some((environment_id, resource_path)) = request.resource.environment_path() else { return Err(SkillProviderError::new( "executor skill resource is not bound to an environment", )); }; - let contents = resource_path - .read_to_string(/*sandbox*/ None) + let Some(environment) = self.environment_manager.get_environment(environment_id) else { + return Err(SkillProviderError::new(format!( + "executor skill resource references unavailable environment `{environment_id}`" + ))); + }; + let resource_path = PathUri::from_abs_path(resource_path).map_err(|err| { + SkillProviderError::new(format!( + "failed to read executor skill resource {}: {err}", + request.resource.as_str() + )) + })?; + let contents = environment + .get_filesystem() + .read_file_text(&resource_path, /*sandbox*/ None) .await .map_err(|err| { SkillProviderError::new(format!( @@ -153,7 +165,7 @@ fn catalog_entry_from_skill( enabled: bool, authority: SkillAuthority, selected_root_id: &str, - file_system: Arc, + environment_id: &str, ) -> SkillCatalogEntry { let skill_path = skill.path_to_skills_md.to_string_lossy().into_owned(); let normalized_path = skill_path.replace('\\', "/"); @@ -168,7 +180,8 @@ fn catalog_entry_from_skill( skill.description.clone(), SkillResourceId::environment( display_path.clone(), - EnvironmentPathRef::new(file_system, skill.path_to_skills_md.clone()), + environment_id, + skill.path_to_skills_md.clone(), ), ) .with_short_description(skill.short_description.clone()) diff --git a/codex-rs/ext/skills/src/provider/orchestrator.rs b/codex-rs/ext/skills/src/provider/orchestrator.rs new file mode 100644 index 0000000000..045b67de62 --- /dev/null +++ b/codex-rs/ext/skills/src/provider/orchestrator.rs @@ -0,0 +1,327 @@ +use std::collections::HashSet; +use std::time::Duration; + +use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use codex_protocol::mcp::Resource; +use codex_protocol::mcp::ResourceContent; +use url::Url; + +use crate::catalog::SkillAuthority; +use crate::catalog::SkillCatalog; +use crate::catalog::SkillCatalogEntry; +use crate::catalog::SkillPackageId; +use crate::catalog::SkillProviderError; +use crate::catalog::SkillReadResult; +use crate::catalog::SkillResourceId; +use crate::catalog::SkillSearchResult; +use crate::catalog::SkillSourceKind; +use crate::provider::SkillListQuery; +use crate::provider::SkillProvider; +use crate::provider::SkillProviderFuture; +use crate::provider::SkillReadRequest; +use crate::provider::SkillSearchRequest; + +const ORCHESTRATOR_SKILL_MIME_TYPE: &str = "mcp/skill"; +const ORCHESTRATOR_SKILL_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(10); +const ORCHESTRATOR_SKILL_READ_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_RESOURCE_PAGES: usize = 10; +const MAX_ORCHESTRATOR_SKILLS: usize = 100; +const MAX_SKILL_NAME_CHARS: usize = 64; +const MAX_QUALIFIED_SKILL_NAME_CHARS: usize = 128; +const MAX_SKILL_DESCRIPTION_CHARS: usize = 1_024; +const MAX_SKILL_PACKAGE_URI_CHARS: usize = 1_024; +const MAX_SKILL_RESOURCE_URI_CHARS: usize = 2_048; +const MAX_SKILL_RESOURCE_CONTENT_BYTES: usize = 1024 * 1024; + +/// Discovers and reads skills owned by the orchestrator. +/// +/// The provider uses session-scoped resources without exposing the transport or +/// resource server to callers that configure the skills extension. +#[derive(Clone, Debug, Default)] +pub struct OrchestratorSkillProvider; + +impl OrchestratorSkillProvider { + pub fn new() -> Self { + Self + } +} + +impl SkillProvider for OrchestratorSkillProvider { + fn list(&self, query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> { + Box::pin(async move { + let Some(client) = query.mcp_resources else { + return Ok(SkillCatalog::default()); + }; + if !client.has_server(CODEX_APPS_MCP_SERVER_NAME).await { + return Ok(SkillCatalog::default()); + } + + let discovery_deadline = + tokio::time::Instant::now() + ORCHESTRATOR_SKILL_DISCOVERY_TIMEOUT; + let mut catalog = SkillCatalog::default(); + let mut cursor = None; + let mut seen_cursors = HashSet::new(); + let mut skill_resources_seen = 0usize; + let mut skipped_resources = 0usize; + let mut truncated = false; + let mut completed_pages = 0usize; + + for _ in 0..MAX_RESOURCE_PAGES { + let page = match tokio::time::timeout_at( + discovery_deadline, + client.list_resources(CODEX_APPS_MCP_SERVER_NAME, cursor.clone()), + ) + .await + { + Ok(result) => result.map_err(|err| { + SkillProviderError::new(format!( + "failed to list orchestrator skill resources: {err:#}" + )) + }), + Err(_) => Err(SkillProviderError::new(format!( + "orchestrator skill discovery timed out after {ORCHESTRATOR_SKILL_DISCOVERY_TIMEOUT:?}" + ))), + }; + let result = match page { + Ok(result) => result, + Err(err) if completed_pages == 0 => return Err(err), + Err(err) => { + let page_word = if completed_pages == 1 { + "page" + } else { + "pages" + }; + catalog.warnings.push(format!( + "Orchestrator skill discovery stopped after {completed_pages} resource {page_word}: {}", + err.message + )); + cursor = None; + break; + } + }; + completed_pages = completed_pages.saturating_add(1); + + for resource in &result.resources { + if resource.mime_type.as_deref() != Some(ORCHESTRATOR_SKILL_MIME_TYPE) { + continue; + } + if skill_resources_seen >= MAX_ORCHESTRATOR_SKILLS { + truncated = true; + break; + } + skill_resources_seen = skill_resources_seen.saturating_add(1); + match catalog_entry_from_resource(resource) { + Some(entry) => catalog.push_entry(entry), + None => skipped_resources = skipped_resources.saturating_add(1), + } + } + + if truncated { + break; + } + let Some(next_cursor) = result.next_cursor else { + cursor = None; + break; + }; + if !seen_cursors.insert(next_cursor.clone()) { + catalog.warnings.push( + "Orchestrator skill resource pagination returned a duplicate cursor." + .to_string(), + ); + cursor = None; + break; + } + cursor = Some(next_cursor); + } + + if cursor.is_some() || truncated { + catalog.warnings.push(format!( + "Orchestrator skill discovery was truncated at {MAX_ORCHESTRATOR_SKILLS} skills or {MAX_RESOURCE_PAGES} resource pages." + )); + } + if skipped_resources > 0 { + catalog.warnings.push(format!( + "Skipped {skipped_resources} malformed orchestrator skill resources." + )); + } + + Ok(catalog) + }) + } + + fn read(&self, request: SkillReadRequest) -> SkillProviderFuture<'_, SkillReadResult> { + Box::pin(async move { + if request.authority + != SkillAuthority::new(SkillSourceKind::Orchestrator, CODEX_APPS_MCP_SERVER_NAME) + { + return Err(SkillProviderError::new(format!( + "orchestrator skill provider cannot read authority {}", + request.authority.id + ))); + } + if !resource_belongs_to_package(&request.package.0, request.resource.as_str()) { + return Err(SkillProviderError::new( + "orchestrator skill resource does not match its package", + )); + } + + let Some(client) = request.mcp_resources.as_ref() else { + return Err(SkillProviderError::new( + "session MCP resource client is not configured", + )); + }; + let result = tokio::time::timeout( + ORCHESTRATOR_SKILL_READ_TIMEOUT, + client.read_resource(CODEX_APPS_MCP_SERVER_NAME, request.resource.as_str()), + ) + .await + .map_err(|_| { + SkillProviderError::new(format!( + "orchestrator skill read timed out after {ORCHESTRATOR_SKILL_READ_TIMEOUT:?}" + )) + })? + .map_err(|err| { + SkillProviderError::new(format!( + "failed to read orchestrator skill resource {}: {err:#}", + request.resource.as_str() + )) + })?; + let contents = result + .contents + .into_iter() + .find_map(|contents| match contents { + ResourceContent::Text { uri, text, .. } if uri == request.resource.as_str() => { + Some(text) + } + ResourceContent::Text { .. } | ResourceContent::Blob { .. } => None, + }); + let Some(contents) = contents else { + return Err(SkillProviderError::new(format!( + "orchestrator skill resource {} did not return matching text contents", + request.resource.as_str() + ))); + }; + if contents.len() > MAX_SKILL_RESOURCE_CONTENT_BYTES { + return Err(SkillProviderError::new(format!( + "orchestrator skill resource {} exceeds the {MAX_SKILL_RESOURCE_CONTENT_BYTES}-byte read limit", + request.resource.as_str() + ))); + } + + Ok(SkillReadResult { + resource: request.resource, + contents, + }) + }) + } + + fn search(&self, _request: SkillSearchRequest) -> SkillProviderFuture<'_, SkillSearchResult> { + Box::pin(async { Ok(SkillSearchResult::default()) }) + } +} + +fn catalog_entry_from_resource(resource: &Resource) -> Option { + let uri = validated_skill_uri(resource.uri.as_str(), MAX_SKILL_PACKAGE_URI_CHARS)?; + let meta = resource.meta.as_ref()?.as_object()?; + let skill_name = normalized_label(meta.get("skill_name")?.as_str()?, MAX_SKILL_NAME_CHARS)?; + let name = if meta.get("source").and_then(|value| value.as_str()) == Some("user") { + skill_name + } else { + let plugin_name = + normalized_label(meta.get("plugin_name")?.as_str()?, MAX_SKILL_NAME_CHARS)?; + let qualified_name = format!("{plugin_name}:{skill_name}"); + (qualified_name.chars().count() <= MAX_QUALIFIED_SKILL_NAME_CHARS) + .then_some(qualified_name)? + }; + let description = normalized_description(resource.description.as_deref().unwrap_or_default())?; + let main_prompt = main_prompt_uri(uri); + + Some( + SkillCatalogEntry::new( + SkillPackageId(uri.to_string()), + SkillAuthority::new(SkillSourceKind::Orchestrator, CODEX_APPS_MCP_SERVER_NAME), + name, + description, + SkillResourceId::new(main_prompt), + ) + .with_display_path(uri), + ) +} + +fn validated_skill_uri(uri: &str, max_chars: usize) -> Option<&str> { + validated_skill_url(uri, max_chars).map(|_| uri) +} + +fn validated_skill_url(uri: &str, max_chars: usize) -> Option { + if uri.chars().count() > max_chars + || uri + .chars() + .any(|ch| ch.is_control() || ch.is_whitespace() || matches!(ch, '<' | '>')) + { + return None; + } + + let url = Url::parse(uri).ok()?; + let path_is_valid = url.path_segments().is_some_and(|segments| { + let segments = segments.collect::>(); + !segments.is_empty() && segments.iter().all(|segment| !segment.is_empty()) + }); + (url.scheme() == "skill" + && url.as_str() == uri + && url.host_str().is_some_and(|host| !host.is_empty()) + && url.username().is_empty() + && url.password().is_none() + && url.port().is_none() + && url.query().is_none() + && url.fragment().is_none() + && path_is_valid) + .then_some(url) +} + +fn resource_belongs_to_package(package: &str, resource: &str) -> bool { + let Some(package) = validated_skill_url(package, MAX_SKILL_PACKAGE_URI_CHARS) else { + return false; + }; + let Some(resource) = validated_skill_url(resource, MAX_SKILL_RESOURCE_URI_CHARS) else { + return false; + }; + + let Some(package_segments) = package.path_segments() else { + return false; + }; + let Some(resource_segments) = resource.path_segments() else { + return false; + }; + let package_segments = package_segments.collect::>(); + let resource_segments = resource_segments.collect::>(); + + package.scheme() == resource.scheme() + && package.host_str() == resource.host_str() + && resource_segments.len() > package_segments.len() + && resource_segments.starts_with(&package_segments) +} + +fn normalized_label(value: &str, max_chars: usize) -> Option { + let value = normalized_single_line(value, max_chars)?; + let invalid = value.is_empty() || value.chars().any(|ch| matches!(ch, '&' | '<' | '>')); + (!invalid).then_some(value) +} + +fn normalized_description(value: &str) -> Option { + normalized_single_line(value, MAX_SKILL_DESCRIPTION_CHARS).map(|value| { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + }) +} + +fn normalized_single_line(value: &str, max_chars: usize) -> Option { + let value = value.split_whitespace().collect::>().join(" "); + let valid = value.chars().count() <= max_chars && !value.chars().any(char::is_control); + valid.then_some(value) +} + +fn main_prompt_uri(package_uri: &str) -> String { + format!("{}/SKILL.md", package_uri.trim_end_matches('/')) +} diff --git a/codex-rs/ext/skills/src/render.rs b/codex-rs/ext/skills/src/render.rs index 6b77d02140..d913e4161c 100644 --- a/codex-rs/ext/skills/src/render.rs +++ b/codex-rs/ext/skills/src/render.rs @@ -1,40 +1,18 @@ -use codex_core_skills::render_available_skills_body; -use codex_extension_api::ContextualUserFragment; -use codex_protocol::protocol::SKILLS_INSTRUCTIONS_CLOSE_TAG; -use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; use codex_utils_string::take_bytes_at_char_boundary; use crate::catalog::SkillCatalog; +use crate::catalog::SkillCatalogEntry; +use crate::catalog::SkillSourceKind; +use crate::fragments::AvailableSkillsInstructions; const MAX_AVAILABLE_SKILLS_BYTES: usize = 8_000; const MAX_MAIN_PROMPT_BYTES: usize = 8_000; pub(crate) const MAX_SKILL_NAME_BYTES: usize = 256; pub(crate) const MAX_SKILL_PATH_BYTES: usize = 1_024; -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct AvailableSkillsFragment { - body: String, -} - -impl ContextualUserFragment for AvailableSkillsFragment { - fn role(&self) -> &'static str { - "developer" - } - - fn markers(&self) -> (&'static str, &'static str) { - Self::type_markers() - } - - fn body(&self) -> String { - self.body.clone() - } - - fn type_markers() -> (&'static str, &'static str) { - (SKILLS_INSTRUCTIONS_OPEN_TAG, SKILLS_INSTRUCTIONS_CLOSE_TAG) - } -} - -pub(crate) fn available_skills_fragment(catalog: &SkillCatalog) -> Option { +pub(crate) fn available_skills_fragment( + catalog: &SkillCatalog, +) -> Option { let mut total_bytes = 0usize; let mut omitted = 0usize; let mut skill_lines = Vec::new(); @@ -48,7 +26,7 @@ pub(crate) fn available_skills_fragment(catalog: &SkillCatalog) -> Option MAX_AVAILABLE_SKILLS_BYTES { omitted = omitted.saturating_add(1); @@ -68,16 +46,22 @@ pub(crate) fn available_skills_fragment(catalog: &SkillCatalog) -> Option String { +fn render_skill_line(entry: &SkillCatalogEntry, description: &str) -> String { + let locator_kind = match &entry.authority.kind { + SkillSourceKind::Host => "file", + SkillSourceKind::Executor => "environment resource", + SkillSourceKind::Orchestrator => "orchestrator resource", + SkillSourceKind::Custom(_) => "custom resource", + }; + let name = entry.name.as_str(); + let path = entry.rendered_path(); if description.is_empty() { - format!("- {name}: (file: {path})") + format!("- {name}: ({locator_kind}: {path})") } else { - format!("- {name}: {description} (file: {path})") + format!("- {name}: {description} ({locator_kind}: {path})") } } diff --git a/codex-rs/ext/skills/src/sources.rs b/codex-rs/ext/skills/src/sources.rs index 5ed51b878e..d558c13b3b 100644 --- a/codex-rs/ext/skills/src/sources.rs +++ b/codex-rs/ext/skills/src/sources.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use crate::catalog::SkillCatalog; use crate::catalog::SkillProviderError; +use crate::catalog::SkillProviderResult; use crate::catalog::SkillReadResult; use crate::catalog::SkillSearchResult; use crate::catalog::SkillSourceKind; @@ -39,15 +40,15 @@ impl SkillProviderSource { Self::new(SkillSourceKind::Executor, label, provider) } - pub fn remote(label: impl Into, provider: Arc) -> Self { - Self::new(SkillSourceKind::Remote, label, provider) + pub fn orchestrator(label: impl Into, provider: Arc) -> Self { + Self::new(SkillSourceKind::Orchestrator, label, provider) } fn should_list(&self, query: &SkillListQuery) -> bool { match &self.kind { SkillSourceKind::Host => query.include_host_skills, SkillSourceKind::Executor => !query.executor_roots.is_empty(), - SkillSourceKind::Remote => query.include_remote_skills, + SkillSourceKind::Orchestrator => query.include_orchestrator_skills, SkillSourceKind::Custom(_) => true, } } @@ -94,20 +95,54 @@ impl SkillProviders { self } - pub fn with_remote_provider(mut self, provider: Arc) -> Self { + pub fn with_orchestrator_provider(mut self, provider: Arc) -> Self { self.sources - .push(SkillProviderSource::remote("remote", provider)); + .push(SkillProviderSource::orchestrator("orchestrator", provider)); self } + pub(crate) fn has_orchestrator_provider(&self) -> bool { + self.sources + .iter() + .any(|source| source.kind == SkillSourceKind::Orchestrator) + } + pub(crate) async fn list_for_turn(&self, query: SkillListQuery) -> SkillCatalog { + self.list_matching(&query, |source| source.should_list(&query)) + .await + } + + pub(crate) async fn list_orchestrator_for_turn( + &self, + query: SkillListQuery, + ) -> SkillProviderResult { let mut catalog = SkillCatalog::default(); for source in self .sources .iter() - .filter(|source| source.should_list(&query)) + .filter(|source| source.kind == SkillSourceKind::Orchestrator) { + let source_catalog = source.provider.list(query.clone()).await.map_err(|err| { + SkillProviderError::new(format!( + "{} skills unavailable: {}", + source.label, err.message + )) + })?; + catalog.extend(source_catalog); + } + + Ok(catalog) + } + + async fn list_matching( + &self, + query: &SkillListQuery, + should_list: impl Fn(&SkillProviderSource) -> bool, + ) -> SkillCatalog { + let mut catalog = SkillCatalog::default(); + + for source in self.sources.iter().filter(|source| should_list(source)) { extend_catalog( &mut catalog, source.provider.list(query.clone()).await, diff --git a/codex-rs/ext/skills/src/state.rs b/codex-rs/ext/skills/src/state.rs index aa77cce822..0616ae4d71 100644 --- a/codex-rs/ext/skills/src/state.rs +++ b/codex-rs/ext/skills/src/state.rs @@ -1,29 +1,18 @@ -use codex_core::config::Config; use codex_protocol::capabilities::SelectedCapabilityRoot; +use std::future::Future; use std::sync::Mutex; +use tokio::sync::OnceCell; +use crate::SkillsExtensionConfig; use crate::catalog::SkillCatalog; use crate::catalog::SkillCatalogEntry; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct SkillsExtensionConfig { - pub(crate) include_instructions: bool, - pub(crate) bundled_skills_enabled: bool, -} - -impl SkillsExtensionConfig { - pub(crate) fn from_config(config: &Config) -> Self { - Self { - include_instructions: config.include_skill_instructions, - bundled_skills_enabled: config.bundled_skills_enabled(), - } - } -} +use crate::catalog::SkillProviderError; #[derive(Debug)] pub(crate) struct SkillsThreadState { config: Mutex, selected_roots: Vec, + orchestrator_catalog: OnceCell, } impl SkillsThreadState { @@ -34,6 +23,7 @@ impl SkillsThreadState { Self { config: Mutex::new(config), selected_roots, + orchestrator_catalog: OnceCell::new(), } } @@ -54,6 +44,21 @@ impl SkillsThreadState { pub(crate) fn selected_roots(&self) -> &[SelectedCapabilityRoot] { &self.selected_roots } + + pub(crate) async fn orchestrator_catalog_snapshot( + &self, + initialize: impl Future> + Send, + ) -> SkillCatalog { + self.orchestrator_catalog + .get_or_init(|| async { + initialize.await.unwrap_or_else(|err| SkillCatalog { + warnings: vec![err.message], + ..Default::default() + }) + }) + .await + .clone() + } } #[derive(Clone, Debug, Default, PartialEq, Eq)] diff --git a/codex-rs/ext/skills/src/tools/list.rs b/codex-rs/ext/skills/src/tools/list.rs new file mode 100644 index 0000000000..31cb68003d --- /dev/null +++ b/codex-rs/ext/skills/src/tools/list.rs @@ -0,0 +1,112 @@ +use codex_extension_api::ToolCall; +use codex_extension_api::ToolExecutor; +use codex_extension_api::ToolExecutorFuture; +use codex_extension_api::ToolName; +use codex_extension_api::ToolSpec; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; + +use crate::catalog::SkillCatalogEntry; +use crate::render::truncate_utf8_to_bytes; + +use super::MAX_HANDLE_BYTES; +use super::SkillToolAuthority; +use super::SkillToolContext; +use super::external_json_output; +use super::is_bounded_handle; +use super::parse_args; +use super::skill_function_tool; +use super::skill_tool_name; + +const TOOL_NAME: &str = "list"; +const MAX_WARNINGS: usize = 4; +const MAX_WARNING_BYTES: usize = 256; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct ListArgs { + authority: SkillToolAuthority, +} + +#[derive(Debug, Eq, JsonSchema, PartialEq, Serialize)] +#[schemars(deny_unknown_fields)] +struct ListedSkill { + authority: SkillToolAuthority, + package: String, + name: String, + description: String, + main_resource: String, +} + +#[derive(Debug, Eq, JsonSchema, PartialEq, Serialize)] +#[schemars(deny_unknown_fields)] +struct ListResponse { + skills: Vec, + warnings: Vec, +} + +#[derive(Clone)] +pub(super) struct ListTool { + pub(super) context: SkillToolContext, +} + +impl ToolExecutor for ListTool { + fn tool_name(&self) -> ToolName { + skill_tool_name(TOOL_NAME) + } + + fn spec(&self) -> ToolSpec { + skill_function_tool::( + TOOL_NAME, + "List enabled skills owned by the requested authority. Only orchestrator-owned skills are currently supported. Returns the opaque package and main-resource handles required by skills.read.", + ) + } + + fn handle(&self, call: ToolCall) -> ToolExecutorFuture<'_> { + Box::pin(async move { + let args: ListArgs = parse_args(&call)?; + let authority = args.authority.into_authority(); + let catalog = self.context.catalog(&call.turn_id, args.authority).await; + let response = ListResponse { + skills: catalog + .entries + .into_iter() + .filter(|entry| entry.enabled && entry.authority == authority) + .filter_map(listed_skill) + .collect(), + warnings: bounded_warnings(catalog.warnings), + }; + + external_json_output(&response) + }) + } +} + +fn listed_skill(entry: SkillCatalogEntry) -> Option { + let authority = SkillToolAuthority::from_authority(&entry.authority)?; + if !is_bounded_handle(&entry.id.0, MAX_HANDLE_BYTES) + || !is_bounded_handle(entry.main_prompt.as_str(), MAX_HANDLE_BYTES) + { + return None; + } + + Some(ListedSkill { + authority, + package: entry.id.0, + name: entry.name, + description: entry.description, + main_resource: entry.main_prompt.as_str().to_string(), + }) +} + +fn bounded_warnings(warnings: Vec) -> Vec { + warnings + .into_iter() + .take(MAX_WARNINGS) + .map(|warning| { + let (warning, _) = truncate_utf8_to_bytes(&warning, MAX_WARNING_BYTES); + warning + }) + .collect() +} diff --git a/codex-rs/ext/skills/src/tools/mod.rs b/codex-rs/ext/skills/src/tools/mod.rs new file mode 100644 index 0000000000..12b050338e --- /dev/null +++ b/codex-rs/ext/skills/src/tools/mod.rs @@ -0,0 +1,160 @@ +use std::sync::Arc; + +use codex_extension_api::FunctionCallError; +use codex_extension_api::JsonToolOutput; +use codex_extension_api::ResponsesApiTool; +use codex_extension_api::ToolCall; +use codex_extension_api::ToolExecutor; +use codex_extension_api::ToolName; +use codex_extension_api::ToolOutput; +use codex_extension_api::ToolSpec; +use codex_extension_api::parse_tool_input_schema; +use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use codex_mcp::McpResourceClient; +use codex_tools::ResponsesApiNamespace; +use codex_tools::ResponsesApiNamespaceTool; +use codex_tools::default_namespace_description; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use serde_json::Value; + +use crate::catalog::SkillAuthority; +use crate::catalog::SkillCatalog; +use crate::catalog::SkillSourceKind; +use crate::provider::SkillListQuery; +use crate::sources::SkillProviders; + +mod list; +mod read; +mod schema; + +const SKILLS_NAMESPACE: &str = "skills"; +const MAX_HANDLE_BYTES: usize = 2_048; + +pub(crate) fn skill_tools( + providers: SkillProviders, + mcp_resources: Option>, +) -> Vec>> { + let context = SkillToolContext { + providers, + mcp_resources, + }; + vec![ + Arc::new(list::ListTool { + context: context.clone(), + }), + Arc::new(read::ReadTool { context }), + ] +} + +#[derive(Clone)] +struct SkillToolContext { + providers: SkillProviders, + mcp_resources: Option>, +} + +impl SkillToolContext { + async fn catalog(&self, turn_id: &str, authority: SkillToolAuthority) -> SkillCatalog { + match authority { + SkillToolAuthority::Orchestrator => match self + .providers + .list_orchestrator_for_turn(SkillListQuery { + turn_id: turn_id.to_string(), + executor_roots: Vec::new(), + host: None, + include_host_skills: false, + include_bundled_skills: false, + include_orchestrator_skills: true, + mcp_resources: self.mcp_resources.clone(), + }) + .await + { + Ok(catalog) => catalog, + Err(err) => SkillCatalog { + warnings: vec![err.message], + ..Default::default() + }, + }, + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum SkillToolAuthority { + Orchestrator, +} + +impl SkillToolAuthority { + fn from_authority(authority: &SkillAuthority) -> Option { + if authority + != &SkillAuthority::new(SkillSourceKind::Orchestrator, CODEX_APPS_MCP_SERVER_NAME) + { + return None; + } + Some(Self::Orchestrator) + } + + fn into_authority(self) -> SkillAuthority { + match self { + Self::Orchestrator => { + SkillAuthority::new(SkillSourceKind::Orchestrator, CODEX_APPS_MCP_SERVER_NAME) + } + } + } +} + +fn skill_tool_name(name: &str) -> ToolName { + ToolName::namespaced(SKILLS_NAMESPACE, name) +} + +fn skill_function_tool(name: &str, description: &str) -> ToolSpec { + let tool = ResponsesApiTool { + name: name.to_string(), + description: description.to_string(), + strict: false, + defer_loading: None, + parameters: parse_tool_input_schema(&schema::input_schema_for::()) + .unwrap_or_else(|err| panic!("generated input schema for {name} should parse: {err}")), + output_schema: Some(schema::output_schema_for::()), + }; + + ToolSpec::Namespace(ResponsesApiNamespace { + name: SKILLS_NAMESPACE.to_string(), + description: default_namespace_description(SKILLS_NAMESPACE), + tools: vec![ResponsesApiNamespaceTool::Function(tool)], + }) +} + +fn parse_args Deserialize<'de>>(call: &ToolCall) -> Result { + let arguments = call.function_arguments()?; + let value = if arguments.trim().is_empty() { + Value::Object(serde_json::Map::new()) + } else { + serde_json::from_str(arguments) + .map_err(|err| FunctionCallError::RespondToModel(err.to_string()))? + }; + serde_json::from_value(value).map_err(|err| FunctionCallError::RespondToModel(err.to_string())) +} + +fn validate_handle(name: &str, value: &str, max_bytes: usize) -> Result<(), FunctionCallError> { + if is_bounded_handle(value, max_bytes) { + return Ok(()); + } + + Err(FunctionCallError::RespondToModel(format!( + "{name} must be non-empty, contain no control characters, and be at most {max_bytes} bytes" + ))) +} + +fn is_bounded_handle(value: &str, max_bytes: usize) -> bool { + !value.is_empty() && value.len() <= max_bytes && !value.chars().any(char::is_control) +} + +fn external_json_output(value: &T) -> Result, FunctionCallError> { + let value = serde_json::to_value(value).map_err(|err| { + FunctionCallError::Fatal(format!("failed to serialize tool output: {err}")) + })?; + Ok(Box::new(JsonToolOutput::new(value).with_external_context())) +} diff --git a/codex-rs/ext/skills/src/tools/read.rs b/codex-rs/ext/skills/src/tools/read.rs new file mode 100644 index 0000000000..e8b29905b2 --- /dev/null +++ b/codex-rs/ext/skills/src/tools/read.rs @@ -0,0 +1,109 @@ +use codex_extension_api::FunctionCallError; +use codex_extension_api::ToolCall; +use codex_extension_api::ToolExecutor; +use codex_extension_api::ToolExecutorFuture; +use codex_extension_api::ToolName; +use codex_extension_api::ToolSpec; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; + +use crate::catalog::SkillPackageId; +use crate::catalog::SkillResourceId; +use crate::provider::SkillReadRequest; + +use super::MAX_HANDLE_BYTES; +use super::SkillToolAuthority; +use super::SkillToolContext; +use super::external_json_output; +use super::parse_args; +use super::skill_function_tool; +use super::skill_tool_name; +use super::validate_handle; + +const TOOL_NAME: &str = "read"; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct ReadArgs { + authority: SkillToolAuthority, + package: String, + resource: String, +} + +#[derive(Debug, Eq, JsonSchema, PartialEq, Serialize)] +#[schemars(deny_unknown_fields)] +struct ReadResponse { + resource: String, + contents: String, +} + +#[derive(Clone)] +pub(super) struct ReadTool { + pub(super) context: SkillToolContext, +} + +impl ToolExecutor for ReadTool { + fn tool_name(&self) -> ToolName { + skill_tool_name(TOOL_NAME) + } + + fn spec(&self) -> ToolSpec { + skill_function_tool::( + TOOL_NAME, + "Read one complete resource from an enabled skill. Pass the exact authority and package returned by skills.list; resource identifiers remain opaque and are routed to that authority.", + ) + } + + fn handle(&self, call: ToolCall) -> ToolExecutorFuture<'_> { + Box::pin(async move { + let args: ReadArgs = parse_args(&call)?; + let authority = args.authority.into_authority(); + validate_handle("package", &args.package, MAX_HANDLE_BYTES)?; + validate_handle("resource", &args.resource, MAX_HANDLE_BYTES)?; + + let catalog = self.context.catalog(&call.turn_id, args.authority).await; + let package_is_available = catalog.entries.iter().any(|entry| { + entry.enabled && entry.authority == authority && entry.id.0 == args.package + }); + if !package_is_available { + return Err(FunctionCallError::RespondToModel( + "skill package is not available from the requested authority".to_string(), + )); + } + + let requested_resource = SkillResourceId::new(args.resource); + let result = self + .context + .providers + .read(SkillReadRequest { + authority, + package: SkillPackageId(args.package), + resource: requested_resource.clone(), + host: None, + mcp_resources: self.context.mcp_resources.clone(), + }) + .await + .map_err(|err| { + tracing::warn!( + error = %err, + turn_id = %call.turn_id, + call_id = %call.call_id, + resource = requested_resource.as_str(), + "skills.read provider request failed" + ); + FunctionCallError::RespondToModel("failed to read skill resource".to_string()) + })?; + if result.resource != requested_resource { + return Err(FunctionCallError::Fatal( + "skill provider returned a different resource".to_string(), + )); + } + + external_json_output(&ReadResponse { + resource: result.resource.as_str().to_string(), + contents: result.contents, + }) + }) + } +} diff --git a/codex-rs/ext/skills/src/tools/schema.rs b/codex-rs/ext/skills/src/tools/schema.rs new file mode 100644 index 0000000000..d0e9425e47 --- /dev/null +++ b/codex-rs/ext/skills/src/tools/schema.rs @@ -0,0 +1,42 @@ +use schemars::JsonSchema; +use schemars::r#gen::SchemaSettings; +use serde_json::Map; +use serde_json::Value; + +pub(super) fn input_schema_for() -> Value { + schema_for::(/*option_add_null_type*/ false) +} + +pub(super) fn output_schema_for() -> Value { + schema_for::(/*option_add_null_type*/ true) +} + +fn schema_for(option_add_null_type: bool) -> Value { + let schema = SchemaSettings::draft2019_09() + .with(|settings| { + settings.inline_subschemas = true; + settings.option_add_null_type = option_add_null_type; + }) + .into_generator() + .into_root_schema_for::(); + let schema_value = serde_json::to_value(schema) + .unwrap_or_else(|err| panic!("generated skill tool schema should serialize: {err}")); + let Value::Object(mut schema_object) = schema_value else { + unreachable!("root tool schema must be an object"); + }; + + let mut tool_schema = Map::new(); + for key in [ + "properties", + "required", + "type", + "additionalProperties", + "$defs", + "definitions", + ] { + if let Some(value) = schema_object.remove(key) { + tool_schema.insert(key.to_string(), value); + } + } + Value::Object(tool_schema) +} diff --git a/codex-rs/ext/skills/tests/executor_file_system_authority.rs b/codex-rs/ext/skills/tests/executor_file_system_authority.rs index 026c3f3348..5b9a440d87 100644 --- a/codex-rs/ext/skills/tests/executor_file_system_authority.rs +++ b/codex-rs/ext/skills/tests/executor_file_system_authority.rs @@ -1,5 +1,4 @@ use std::io; -use std::path::Path; use std::sync::Arc; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; @@ -21,11 +20,10 @@ use codex_protocol::capabilities::CapabilityRootLocation; use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::protocol::SkillScope; use codex_skills_extension::ExecutorSkillProvider; -use codex_skills_extension::catalog::SkillReadResult; use codex_skills_extension::provider::SkillListQuery; use codex_skills_extension::provider::SkillProvider; -use codex_skills_extension::provider::SkillReadRequest; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use pretty_assertions::assert_eq; const SKILL_CONTENTS: &str = @@ -62,34 +60,23 @@ impl SyntheticFileSystem { impl ExecutorFileSystem for SyntheticFileSystem { async fn canonicalize( &self, - path: &AbsolutePathBuf, + path: &PathUri, _sandbox: Option<&FileSystemSandboxContext>, - ) -> FileSystemResult { - if path == &self.alias_root { - return Ok(self.canonical_root.clone()); + ) -> FileSystemResult { + let path = path.to_abs_path()?; + if path == self.alias_root { + return PathUri::from_abs_path(&self.canonical_root); } - self.metadata(path)?; - Ok(path.clone()) - } - - async fn join( - &self, - base_path: &AbsolutePathBuf, - path: &Path, - ) -> FileSystemResult { - Ok(base_path.join(path)) - } - - async fn parent(&self, path: &AbsolutePathBuf) -> FileSystemResult> { - Ok(path.parent()) + self.metadata(&path)?; + PathUri::from_abs_path(&path) } async fn read_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { - if path == &self.canonical_root.join("skill/SKILL.md") { + if path.to_abs_path()? == self.canonical_root.join("skill/SKILL.md") { Ok(SKILL_CONTENTS.as_bytes().to_vec()) } else { Err(io::Error::new(io::ErrorKind::NotFound, "not found")) @@ -98,7 +85,7 @@ impl ExecutorFileSystem for SyntheticFileSystem { async fn write_file( &self, - _path: &AbsolutePathBuf, + _path: &PathUri, _contents: Vec, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -107,7 +94,7 @@ impl ExecutorFileSystem for SyntheticFileSystem { async fn create_directory( &self, - _path: &AbsolutePathBuf, + _path: &PathUri, _options: CreateDirectoryOptions, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -116,24 +103,25 @@ impl ExecutorFileSystem for SyntheticFileSystem { async fn get_metadata( &self, - path: &AbsolutePathBuf, + path: &PathUri, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult { - self.metadata(path) + self.metadata(&path.to_abs_path()?) } async fn read_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult> { - if path == &self.canonical_root { + let path = path.to_abs_path()?; + if path == self.canonical_root { Ok(vec![ReadDirectoryEntry { file_name: "skill".to_string(), is_directory: true, is_file: false, }]) - } else if path == &self.canonical_root.join("skill") { + } else if path == self.canonical_root.join("skill") { Ok(vec![ReadDirectoryEntry { file_name: "SKILL.md".to_string(), is_directory: false, @@ -146,7 +134,7 @@ impl ExecutorFileSystem for SyntheticFileSystem { async fn remove( &self, - _path: &AbsolutePathBuf, + _path: &PathUri, _options: RemoveOptions, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -155,8 +143,8 @@ impl ExecutorFileSystem for SyntheticFileSystem { async fn copy( &self, - _source_path: &AbsolutePathBuf, - _destination_path: &AbsolutePathBuf, + _source_path: &PathUri, + _destination_path: &PathUri, _options: CopyOptions, _sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()> { @@ -202,62 +190,6 @@ async fn skill_loading_and_reads_use_the_supplied_executor_file_system() { ); } -#[tokio::test] -async fn executor_provider_reads_from_the_environment_instance_used_for_listing() { - let test_root = create_local_skill_root("bound-instance").expect("create local skill root"); - let root_path = test_root.to_string_lossy().into_owned(); - let environment_manager = Arc::new(EnvironmentManager::default_for_tests()); - let provider = ExecutorSkillProvider::new_with_restriction_product( - Arc::clone(&environment_manager), - /*restriction_product*/ None, - ); - let catalog = provider - .list(SkillListQuery { - turn_id: "turn-1".to_string(), - executor_roots: vec![SelectedCapabilityRoot { - id: "root-a".to_string(), - location: CapabilityRootLocation::Environment { - environment_id: "local".to_string(), - path: root_path, - }, - }], - host: None, - include_host_skills: false, - include_bundled_skills: true, - include_remote_skills: false, - }) - .await - .expect("list executor skills"); - let entry = catalog - .entries - .into_iter() - .next() - .expect("listed executor skill"); - let resource = entry.main_prompt.clone(); - - environment_manager - .upsert_environment("local".to_string(), "http://127.0.0.1:1".to_string()) - .expect("replace environment"); - - assert_eq!( - provider - .read(SkillReadRequest { - authority: entry.authority, - package: entry.id, - resource: resource.clone(), - host: None, - }) - .await - .expect("read bound executor skill"), - SkillReadResult { - resource, - contents: SKILL_CONTENTS.to_string(), - } - ); - - std::fs::remove_dir_all(test_root).expect("remove skill directory"); -} - #[tokio::test] async fn selected_root_id_distinguishes_identical_executor_paths() { let test_root = create_local_skill_root("root-identity").expect("create local skill root"); @@ -272,7 +204,6 @@ async fn selected_root_id_distinguishes_identical_executor_paths() { Arc::new(EnvironmentManager::default_for_tests()), /*restriction_product*/ None, ); - let catalog = provider .list(SkillListQuery { turn_id: "turn-1".to_string(), @@ -289,7 +220,8 @@ async fn selected_root_id_distinguishes_identical_executor_paths() { host: None, include_host_skills: false, include_bundled_skills: true, - include_remote_skills: false, + include_orchestrator_skills: false, + mcp_resources: None, }) .await .expect("list executor skills"); diff --git a/codex-rs/ext/skills/tests/skills_extension.rs b/codex-rs/ext/skills/tests/skills_extension.rs index dc1aa01407..dcbd828fce 100644 --- a/codex-rs/ext/skills/tests/skills_extension.rs +++ b/codex-rs/ext/skills/tests/skills_extension.rs @@ -4,26 +4,33 @@ use std::sync::Mutex; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; -use codex_core::config::Config; -use codex_core::config::ConfigBuilder; use codex_core_skills::HostLoadedSkills; -use codex_core_skills::SkillsLoadInput; -use codex_core_skills::SkillsManager; +use codex_core_skills::SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS; +use codex_core_skills::SKILLS_INTRO_WITH_ABSOLUTE_PATHS; +use codex_core_skills::SkillLoadOutcome; +use codex_core_skills::SkillMetadata; use codex_core_skills::injection::InjectedHostSkillPrompts; use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionEventSink; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ThreadStartInput; use codex_extension_api::TurnInputContext; use codex_protocol::capabilities::CapabilityRootLocation; use codex_protocol::capabilities::SelectedCapabilityRoot; +use codex_protocol::protocol::Event; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::SKILLS_INSTRUCTIONS_CLOSE_TAG; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SkillScope; use codex_protocol::user_input::UserInput; use codex_skills_extension::SkillProviders; +use codex_skills_extension::SkillsExtensionConfig; use codex_skills_extension::catalog::SkillAuthority; use codex_skills_extension::catalog::SkillCatalog; use codex_skills_extension::catalog::SkillCatalogEntry; use codex_skills_extension::catalog::SkillPackageId; +use codex_skills_extension::catalog::SkillProviderError; use codex_skills_extension::catalog::SkillReadResult; use codex_skills_extension::catalog::SkillResourceId; use codex_skills_extension::catalog::SkillSearchResult; @@ -35,14 +42,17 @@ use codex_skills_extension::provider::SkillProvider; use codex_skills_extension::provider::SkillProviderFuture; use codex_skills_extension::provider::SkillReadRequest; use codex_skills_extension::provider::SkillSearchRequest; +use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; type TestResult = Result<(), Box>; static NEXT_CODEX_HOME_ID: AtomicUsize = AtomicUsize::new(0); +const DEMO_SKILL_CONTENTS: &str = + "---\nname: demo\ndescription: Demo skill.\n---\n# Demo\n\nUse the demo skill.\n"; #[tokio::test] -async fn installed_extension_loads_host_skills_from_legacy_roots() -> TestResult { +async fn installed_extension_uses_host_loaded_skills() -> TestResult { let codex_home = test_codex_home(); let skill_path = codex_home.join("skills").join("demo").join("SKILL.md"); std::fs::create_dir_all( @@ -50,18 +60,11 @@ async fn installed_extension_loads_host_skills_from_legacy_roots() -> TestResult .parent() .ok_or("skill path should have a parent")?, )?; - std::fs::write( - &skill_path, - "---\nname: demo\ndescription: Demo skill.\n---\n# Demo\n\nUse the demo skill.\n", - )?; - let config = ConfigBuilder::default() - .codex_home(codex_home.clone()) - .fallback_cwd(Some(codex_home.clone())) - .build() - .await?; + std::fs::write(&skill_path, DEMO_SKILL_CONTENTS)?; + let config = default_config(); let mut builder = ExtensionRegistryBuilder::new(); - install(&mut builder); + install(&mut builder, skills_extension_config); let registry = builder.build(); let session_store = ExtensionData::new("session"); let thread_store = ExtensionData::new("thread"); @@ -76,22 +79,21 @@ async fn installed_extension_loads_host_skills_from_legacy_roots() -> TestResult }) .await; - let manager = SkillsManager::new(config.codex_home.clone(), config.bundled_skills_enabled()); - let input = SkillsLoadInput::new( - config.cwd.clone(), - Vec::new(), - config.config_layer_stack.clone(), - config.bundled_skills_enabled(), - ); - let loaded_skills = Arc::new(manager.skills_for_config(&input, /*fs*/ None).await); - let skill_path_string = loaded_skills - .skills - .iter() - .find(|skill| skill.name == "demo") - .ok_or("demo skill should load")? - .path_to_skills_md - .to_string_lossy() - .into_owned(); + let skill_path = AbsolutePathBuf::try_from(skill_path)?; + let skill_path_string = skill_path.to_string_lossy().into_owned(); + let mut outcome = SkillLoadOutcome::default(); + outcome.skills.push(SkillMetadata { + name: "demo".to_string(), + description: "Demo skill.".to_string(), + short_description: None, + interface: None, + dependencies: None, + policy: None, + path_to_skills_md: skill_path, + scope: SkillScope::User, + plugin_id: None, + }); + let loaded_skills = Arc::new(outcome); let skill_prompt_path = skill_path_string.replace('\\', "/"); let turn_store = ExtensionData::new("turn-1"); turn_store.insert(HostLoadedSkills::new(Arc::clone(&loaded_skills))); @@ -112,14 +114,19 @@ async fn installed_extension_loads_host_skills_from_legacy_roots() -> TestResult ) .await; - assert_eq!(2, fragments.len()); - assert_eq!("developer", fragments[0].role()); - assert!(fragments[0].render().contains("demo")); - assert!(fragments[0].render().contains(&skill_prompt_path)); - assert_eq!("user", fragments[1].role()); - assert!(fragments[1].render().contains("demo")); - assert!(fragments[1].render().contains("# Demo")); - assert!(fragments[1].render().contains(&skill_prompt_path)); + let expected_catalog = format!( + "{SKILLS_INSTRUCTIONS_OPEN_TAG}\n## Skills\n{SKILLS_INTRO_WITH_ABSOLUTE_PATHS}\n### Available skills\n- demo: Demo skill. (file: {skill_prompt_path})\n### How to use skills\n{SKILLS_HOW_TO_USE_WITH_ABSOLUTE_PATHS}\n{SKILLS_INSTRUCTIONS_CLOSE_TAG}" + ); + let expected_skill = format!( + "\ndemo\n{skill_prompt_path}\n{DEMO_SKILL_CONTENTS}\n" + ); + assert_eq!( + vec![("developer", expected_catalog), ("user", expected_skill),], + fragments + .iter() + .map(|fragment| (fragment.role(), fragment.render())) + .collect::>() + ); let injected_host_skill_prompts = turn_store .get::() .ok_or("host skill prompt marker should be set")?; @@ -144,10 +151,12 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in warnings: Vec::new(), }, read_requests: Arc::clone(&read_requests), + list_calls: None, + fail_first_list: false, }); let providers = SkillProviders::new().with_executor_provider(executor_provider); let mut builder = ExtensionRegistryBuilder::new(); - install_with_providers(&mut builder, providers); + install_with_providers(&mut builder, providers, skills_extension_config); let registry = builder.build(); let session_store = ExtensionData::new("session"); @@ -160,7 +169,7 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in }, }]); let session_source = SessionSource::Cli; - let config = default_config().await?; + let config = default_config(); registry.thread_lifecycle_contributors()[0] .on_thread_start(ThreadStartInput { config: &config, @@ -181,6 +190,11 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in .starts_with(SKILLS_INSTRUCTIONS_OPEN_TAG) ); assert!(prompt_fragments[0].text().contains("lint-fix")); + assert!( + prompt_fragments[0] + .text() + .contains("(environment resource: skill://executor/lint-fix/SKILL.md)") + ); let turn_store = ExtensionData::new("turn-1"); let fragments = registry.turn_input_contributors()[0] @@ -239,6 +253,78 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in Ok(()) } +#[tokio::test] +async fn orchestrator_catalog_snapshot_caches_failure() -> TestResult { + let list_calls = Arc::new(AtomicUsize::new(0)); + let providers = + SkillProviders::new().with_orchestrator_provider(Arc::new(StaticSkillProvider { + catalog: SkillCatalog { + entries: vec![test_entry( + SkillSourceKind::Orchestrator, + "codex_apps", + "orchestrator/first", + "skill://orchestrator/first/SKILL.md", + )], + warnings: Vec::new(), + }, + read_requests: Arc::new(Mutex::new(Vec::new())), + list_calls: Some(Arc::clone(&list_calls)), + fail_first_list: true, + })); + let (event_tx, event_rx) = std::sync::mpsc::channel(); + let mut builder = + ExtensionRegistryBuilder::with_event_sink(Arc::new(ChannelEventSink(event_tx))); + install_with_providers(&mut builder, providers, skills_extension_config); + let registry = builder.build(); + let session_store = ExtensionData::new("session"); + let thread_store = ExtensionData::new("thread"); + let session_source = SessionSource::Cli; + let config = default_config(); + registry.thread_lifecycle_contributors()[0] + .on_thread_start(ThreadStartInput { + config: &config, + session_source: &session_source, + persistent_thread_state_available: true, + session_store: &session_store, + thread_store: &thread_store, + }) + .await; + + let initial_fragments = registry.context_contributors()[0] + .contribute(&session_store, &thread_store) + .await; + assert!(initial_fragments.is_empty()); + let EventMsg::Warning(warning) = event_rx.try_recv()?.msg else { + panic!("expected warning event"); + }; + assert_eq!( + warning.message, + "orchestrator skills unavailable: temporary orchestrator failure" + ); + + for turn_id in ["turn-1", "turn-2"] { + let fragments = registry.turn_input_contributors()[0] + .contribute( + TurnInputContext { + turn_id: turn_id.to_string(), + user_input: vec![UserInput::Text { + text: "$first".to_string(), + text_elements: Vec::new(), + }], + environments: Vec::new(), + }, + &session_store, + &thread_store, + &ExtensionData::new(turn_id), + ) + .await; + assert!(fragments.is_empty()); + } + assert_eq!(1, list_calls.load(Ordering::Relaxed)); + + Ok(()) +} + #[tokio::test] async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> TestResult { let read_requests = Arc::new(Mutex::new(Vec::new())); @@ -262,10 +348,12 @@ async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> Te warnings: Vec::new(), }, read_requests: Arc::clone(&read_requests), + list_calls: None, + fail_first_list: false, }); let providers = SkillProviders::new().with_executor_provider(executor_provider); let mut builder = ExtensionRegistryBuilder::new(); - install_with_providers(&mut builder, providers); + install_with_providers(&mut builder, providers, skills_extension_config); let registry = builder.build(); let session_store = ExtensionData::new("session"); let thread_store = ExtensionData::new("thread"); @@ -282,7 +370,7 @@ async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> Te .collect::>(), ); let session_source = SessionSource::Cli; - let config = default_config().await?; + let config = default_config(); registry.thread_lifecycle_contributors()[0] .on_thread_start(ThreadStartInput { config: &config, @@ -346,15 +434,17 @@ async fn prompt_hidden_skill_can_still_be_invoked() -> TestResult { warnings: Vec::new(), }, read_requests: Arc::clone(&read_requests), + list_calls: None, + fail_first_list: false, }); let providers = SkillProviders::new().with_host_provider(provider); let mut builder = ExtensionRegistryBuilder::new(); - install_with_providers(&mut builder, providers); + install_with_providers(&mut builder, providers, skills_extension_config); let registry = builder.build(); let session_store = ExtensionData::new("session"); let thread_store = ExtensionData::new("thread"); let session_source = SessionSource::Cli; - let config = default_config().await?; + let config = default_config(); registry.thread_lifecycle_contributors()[0] .on_thread_start(ThreadStartInput { config: &config, @@ -401,12 +491,33 @@ async fn prompt_hidden_skill_can_still_be_invoked() -> TestResult { struct StaticSkillProvider { catalog: SkillCatalog, read_requests: Arc>>, + list_calls: Option>, + fail_first_list: bool, +} + +struct ChannelEventSink(std::sync::mpsc::Sender); + +impl ExtensionEventSink for ChannelEventSink { + fn emit(&self, event: Event) { + let _ = self.0.send(event); + } } impl SkillProvider for StaticSkillProvider { fn list(&self, _query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> { + let list_call = self + .list_calls + .as_ref() + .map(|list_calls| list_calls.fetch_add(1, Ordering::Relaxed)); + let fail = self.fail_first_list && list_call == Some(0); let catalog = self.catalog.clone(); - Box::pin(async move { Ok(catalog) }) + Box::pin(async move { + if fail { + Err(SkillProviderError::new("temporary orchestrator failure")) + } else { + Ok(catalog) + } + }) } fn read(&self, request: SkillReadRequest) -> SkillProviderFuture<'_, SkillReadResult> { @@ -445,13 +556,24 @@ fn test_entry( .with_display_path(format!("skill://{package_id}/SKILL.md")) } -async fn default_config() -> std::io::Result { - let codex_home = test_codex_home(); - std::fs::create_dir_all(&codex_home)?; - let config = - Config::load_default_with_cli_overrides_for_codex_home(codex_home.clone(), vec![]).await?; - std::fs::remove_dir_all(codex_home)?; - Ok(config) +#[derive(Clone, Debug, Eq, PartialEq)] +struct TestConfig { + include_instructions: bool, + bundled_skills_enabled: bool, +} + +fn default_config() -> TestConfig { + TestConfig { + include_instructions: true, + bundled_skills_enabled: true, + } +} + +fn skills_extension_config(config: &TestConfig) -> SkillsExtensionConfig { + SkillsExtensionConfig { + include_instructions: config.include_instructions, + bundled_skills_enabled: config.bundled_skills_enabled, + } } fn test_codex_home() -> PathBuf { diff --git a/codex-rs/ext/web-search/web_run_description.md b/codex-rs/ext/web-search/web_run_description.md index 0f22f6df3c..77be9a0a03 100644 --- a/codex-rs/ext/web-search/web_run_description.md +++ b/codex-rs/ext/web-search/web_run_description.md @@ -48,6 +48,35 @@ Below is a list of scenarios where browsing the internet MUST be used. PAY CLOSE --- +## Citations + +Results from `web.run` include internal reference IDs such as `turn2search5`. Use +those reference IDs only in calls to `web.run`; do not expose them in the final +response. + +Cite sources in the final response using Markdown links: + +- Cite a single source as `[descriptive source title](https://example.com/page)`. +- Cite multiple sources with separate Markdown links, for example + `[first source](https://example.com/one), [second source](https://example.com/two)`. +- Link directly to the page that supports the claim. Do not link to search result + pages or use bare URLs. + +Formatting of citations: + +- Place each citation as near as possible to the claim it supports, normally at + the end of the sentence or paragraph and after punctuation. +- Do not place citations inside code fences. +- Do not put citations on a line by themselves or collect all citations at the + end of the response. + +If you browse the internet, cite statements supported by web sources. Each cited +source must directly support the associated claim. Prefer primary and +authoritative sources, and use sources from different domains when the response +benefits from multiple perspectives. + +--- + ## Special cases If these conflict with any other instructions, these should take precedence. @@ -74,7 +103,3 @@ Responses may not excessively quote or draw on a specific source. There are seve - You must avoid providing full articles, long verbatim passages, or extensive direct quotes due to copyright concerns. - If the user asked for a verbatim quote, the response should provide a short compliant excerpt and then answer with paraphrases and summaries. - Again, this limit does not apply to reddit content, as long as it's appropriately indicated that those are direct quotes and you link to the source. - ---- - -Cite your sources by using markdown links to the sites you used in your response. Do NOT use citations in the `turnX` style. diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index b30a9259eb..6284c0726c 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -181,12 +181,14 @@ pub enum Feature { RemotePlugin, /// Enable remote plugin sharing flows. PluginSharing, - /// Show the startup prompt for migrating external agent config into Codex. + /// Removed compatibility flag retained as a no-op. ExternalMigration, /// Allow the model to invoke the built-in image generation tool. ImageGeneration, /// Replace hosted image generation with the standalone image-generation extension. ImageGenExt, + /// Resize all inline data-URL images before recording them in history. + ResizeAllImages, /// Allow prompting and installing missing MCP dependencies. SkillMcpDependencyInstall, /// Removed compatibility flag for deleted skill env var dependency prompting. @@ -199,6 +201,8 @@ pub enum Feature { GuardianApproval, /// Enable persisted thread goals and automatic goal continuation. Goals, + /// Add current context-window metadata to model-visible context. + TokenBudget, /// Route MCP tool approval prompts through the MCP elicitation request path. ToolCallMcpElicitation, /// Prompt Codex Apps connector auth failures through MCP URL elicitations. @@ -1090,11 +1094,7 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::ExternalMigration, key: "external_migration", - stage: Stage::Experimental { - name: "External migration", - menu_description: "Show a startup prompt when Codex detects migratable external agent config for this machine or project.", - announcement: "", - }, + stage: Stage::Removed, default_enabled: false, }, FeatureSpec { @@ -1109,6 +1109,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::ResizeAllImages, + key: "resize_all_images", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::SkillMcpDependencyInstall, key: "skill_mcp_dependency_install", @@ -1157,6 +1163,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Stable, default_enabled: true, }, + FeatureSpec { + id: Feature::TokenBudget, + key: "token_budget", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::CollaborationModes, key: "collaboration_modes", @@ -1256,8 +1268,8 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::RemoteCompactionV2, key: "remote_compaction_v2", - stage: Stage::UnderDevelopment, - default_enabled: false, + stage: Stage::Stable, + default_enabled: true, }, FeatureSpec { id: Feature::WorkspaceDependencies, diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 8f08752450..19383c97ad 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -83,6 +83,16 @@ fn plugin_hooks_is_removed_and_disabled_by_default() { assert_eq!(feature_for_key("plugin_hooks"), Some(Feature::PluginHooks)); } +#[test] +fn external_migration_is_removed_and_disabled_by_default() { + assert_eq!(Feature::ExternalMigration.stage(), Stage::Removed); + assert_eq!(Feature::ExternalMigration.default_enabled(), false); + assert_eq!( + feature_for_key("external_migration"), + Some(Feature::ExternalMigration) + ); +} + #[test] fn removed_apps_mcp_path_override_shapes_are_ignored() { let features = [ @@ -122,23 +132,6 @@ fn guardian_approval_is_stable_and_enabled_by_default() { assert_eq!(Feature::GuardianApproval.default_enabled(), true); } -#[test] -fn external_migration_is_experimental_and_disabled_by_default() { - let spec = Feature::ExternalMigration.info(); - let stage = spec.stage; - - assert!(matches!(stage, Stage::Experimental { .. })); - assert_eq!(stage.experimental_menu_name(), Some("External migration")); - assert_eq!( - stage.experimental_menu_description(), - Some( - "Show a startup prompt when Codex detects migratable external agent config for this machine or project." - ) - ); - assert_eq!(stage.experimental_announcement(), None); - assert_eq!(Feature::ExternalMigration.default_enabled(), false); -} - #[test] fn request_permissions_is_under_development() { assert_eq!( @@ -157,16 +150,6 @@ fn request_permissions_tool_is_under_development() { assert_eq!(Feature::RequestPermissionsTool.default_enabled(), false); } -#[test] -fn remote_compaction_v2_is_under_development() { - assert_eq!(Feature::RemoteCompactionV2.stage(), Stage::UnderDevelopment); - assert_eq!(Feature::RemoteCompactionV2.default_enabled(), false); - assert_eq!( - feature_for_key("remote_compaction_v2"), - Some(Feature::RemoteCompactionV2) - ); -} - #[test] fn terminal_resize_reflow_is_experimental_and_enabled_by_default() { assert_eq!( diff --git a/codex-rs/file-system/Cargo.toml b/codex-rs/file-system/Cargo.toml index 85e083567b..bae7527576 100644 --- a/codex-rs/file-system/Cargo.toml +++ b/codex-rs/file-system/Cargo.toml @@ -11,6 +11,7 @@ workspace = true async-trait = { workspace = true } codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-path-uri = { workspace = true } serde = { workspace = true, features = ["derive"] } [lib] diff --git a/codex-rs/file-system/src/lib.rs b/codex-rs/file-system/src/lib.rs index 8fad1f5b62..95537402af 100644 --- a/codex-rs/file-system/src/lib.rs +++ b/codex-rs/file-system/src/lib.rs @@ -9,6 +9,7 @@ use codex_protocol::permissions::FileSystemSpecialPath; use codex_protocol::permissions::NetworkSandboxPolicy; use codex_protocol::protocol::SandboxPolicy; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use std::io; use std::path::Path; @@ -136,30 +137,20 @@ pub trait ExecutorFileSystem: Send + Sync { /// Resolves a path within this filesystem. async fn canonicalize( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, - ) -> FileSystemResult; - - /// Lexically joins a path onto an existing bound path. - async fn join( - &self, - base_path: &AbsolutePathBuf, - path: &Path, - ) -> FileSystemResult; - - /// Returns the parent directory of a bound path. - async fn parent(&self, path: &AbsolutePathBuf) -> FileSystemResult>; + ) -> FileSystemResult; async fn read_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult>; /// Reads a file and decodes it as UTF-8 text. async fn read_file_text( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult { let bytes = self.read_file(path, sandbox).await?; @@ -168,41 +159,41 @@ pub trait ExecutorFileSystem: Send + Sync { async fn write_file( &self, - path: &AbsolutePathBuf, + path: &PathUri, contents: Vec, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()>; async fn create_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, create_directory_options: CreateDirectoryOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()>; async fn get_metadata( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult; async fn read_directory( &self, - path: &AbsolutePathBuf, + path: &PathUri, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult>; async fn remove( &self, - path: &AbsolutePathBuf, + path: &PathUri, remove_options: RemoveOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()>; async fn copy( &self, - source_path: &AbsolutePathBuf, - destination_path: &AbsolutePathBuf, + source_path: &PathUri, + destination_path: &PathUri, copy_options: CopyOptions, sandbox: Option<&FileSystemSandboxContext>, ) -> FileSystemResult<()>; diff --git a/codex-rs/git-utils/Cargo.toml b/codex-rs/git-utils/Cargo.toml index 9f9c6d0c1d..18ba488252 100644 --- a/codex-rs/git-utils/Cargo.toml +++ b/codex-rs/git-utils/Cargo.toml @@ -14,6 +14,7 @@ chrono = { workspace = true } codex-file-system = { workspace = true } codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-path-uri = { workspace = true } futures = { workspace = true, features = ["alloc"] } gix = { workspace = true } once_cell = { workspace = true } diff --git a/codex-rs/git-utils/src/info.rs b/codex-rs/git-utils/src/info.rs index 7c0d59462c..beff18b519 100644 --- a/codex-rs/git-utils/src/info.rs +++ b/codex-rs/git-utils/src/info.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use codex_file_system::ExecutorFileSystem; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use futures::future::join_all; use schemars::JsonSchema; use serde::Deserialize; @@ -46,7 +47,8 @@ pub async fn get_git_repo_root_with_fs( fs: &dyn ExecutorFileSystem, cwd: &AbsolutePathBuf, ) -> Option { - let base = match fs.get_metadata(cwd, /*sandbox*/ None).await { + let cwd_uri = PathUri::from_abs_path(cwd).ok()?; + let base = match fs.get_metadata(&cwd_uri, /*sandbox*/ None).await { Ok(metadata) if metadata.is_directory => cwd.clone(), _ => cwd.parent()?, }; @@ -803,8 +805,9 @@ pub async fn resolve_root_git_project_for_trust( ) -> Option { let repo_root = get_git_repo_root_with_fs(fs, cwd).await?; let dot_git = repo_root.join(".git"); + let dot_git_uri = PathUri::from_abs_path(&dot_git).ok()?; if fs - .get_metadata(&dot_git, /*sandbox*/ None) + .get_metadata(&dot_git_uri, /*sandbox*/ None) .await .ok()? .is_directory @@ -812,7 +815,10 @@ pub async fn resolve_root_git_project_for_trust( return Some(repo_root); } - let git_dir_s = fs.read_file_text(&dot_git, /*sandbox*/ None).await.ok()?; + let git_dir_s = fs + .read_file_text(&dot_git_uri, /*sandbox*/ None) + .await + .ok()?; let git_dir_rel = git_dir_s.trim().strip_prefix("gitdir:")?.trim(); if git_dir_rel.is_empty() { return None; @@ -853,7 +859,12 @@ async fn find_ancestor_git_entry_with_fs( ) -> Option<(AbsolutePathBuf, AbsolutePathBuf)> { for dir in base_dir.ancestors() { let dot_git = dir.join(".git"); - if fs.get_metadata(&dot_git, /*sandbox*/ None).await.is_ok() { + let dot_git_uri = PathUri::from_abs_path(&dot_git).ok()?; + if fs + .get_metadata(&dot_git_uri, /*sandbox*/ None) + .await + .is_ok() + { return Some((dir, dot_git)); } } diff --git a/codex-rs/login/src/auth/auth_tests.rs b/codex-rs/login/src/auth/auth_tests.rs index 2a8380c086..c509de6bac 100644 --- a/codex-rs/login/src/auth/auth_tests.rs +++ b/codex-rs/login/src/auth/auth_tests.rs @@ -167,6 +167,7 @@ async fn login_with_access_token_writes_only_personal_access_token() { last_refresh: None, agent_identity: None, personal_access_token: Some("at-login-test".to_string()), + bedrock_api_key: None, } ); assert_eq!(auth.resolved_mode(), AuthMode::PersonalAccessToken); @@ -327,6 +328,7 @@ async fn pro_account_with_no_api_key_uses_chatgpt_auth() { last_refresh: Some(last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }, auth_dot_json ); @@ -370,6 +372,7 @@ fn logout_removes_auth_file() -> Result<(), std::io::Error> { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; super::save_auth(dir.path(), &auth_dot_json, AuthCredentialsStoreMode::File)?; let auth_file = get_auth_file(dir.path()); @@ -1120,6 +1123,7 @@ async fn enforce_login_restrictions_logs_out_for_agent_identity_workspace_mismat last_refresh: None, agent_identity: Some(agent_identity), personal_access_token: None, + bedrock_api_key: None, }, AuthCredentialsStoreMode::File, ) diff --git a/codex-rs/login/src/auth/bedrock_api_key.rs b/codex-rs/login/src/auth/bedrock_api_key.rs new file mode 100644 index 0000000000..803b7a01ec --- /dev/null +++ b/codex-rs/login/src/auth/bedrock_api_key.rs @@ -0,0 +1,42 @@ +use std::path::Path; + +use codex_config::types::AuthCredentialsStoreMode; +use serde::Deserialize; +use serde::Serialize; + +use super::manager::save_auth; +use super::storage::AuthDotJson; +use codex_app_server_protocol::AuthMode; + +/// Managed Amazon Bedrock API key persisted in `auth.json`. +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] +pub struct BedrockApiKeyAuth { + pub api_key: String, + pub region: String, +} + +/// Writes an `auth.json` that contains only the Amazon Bedrock API key auth. +pub fn login_with_bedrock_api_key( + codex_home: &Path, + api_key: &str, + region: &str, + auth_credentials_store_mode: AuthCredentialsStoreMode, +) -> std::io::Result<()> { + let auth_dot_json = AuthDotJson { + auth_mode: Some(AuthMode::BedrockApiKey), + openai_api_key: None, + tokens: None, + last_refresh: None, + agent_identity: None, + personal_access_token: None, + bedrock_api_key: Some(BedrockApiKeyAuth { + api_key: api_key.to_string(), + region: region.to_string(), + }), + }; + save_auth(codex_home, &auth_dot_json, auth_credentials_store_mode) +} + +#[cfg(test)] +#[path = "bedrock_api_key_tests.rs"] +mod tests; diff --git a/codex-rs/login/src/auth/bedrock_api_key_tests.rs b/codex-rs/login/src/auth/bedrock_api_key_tests.rs new file mode 100644 index 0000000000..372baadec7 --- /dev/null +++ b/codex-rs/login/src/auth/bedrock_api_key_tests.rs @@ -0,0 +1,162 @@ +use codex_app_server_protocol::AuthMode; +use codex_config::types::AuthCredentialsStoreMode; +use pretty_assertions::assert_eq; +use tempfile::tempdir; + +use super::*; +use crate::auth::AuthManager; +use crate::auth::CodexAuth; +use crate::auth::storage::AuthStorageBackend; +use crate::auth::storage::FileAuthStorage; + +fn api_key_auth() -> AuthDotJson { + AuthDotJson { + auth_mode: Some(AuthMode::ApiKey), + openai_api_key: Some("sk-test-key".to_string()), + tokens: None, + last_refresh: None, + agent_identity: None, + personal_access_token: None, + bedrock_api_key: None, + } +} + +fn bedrock_only_auth() -> AuthDotJson { + AuthDotJson { + auth_mode: None, + openai_api_key: None, + tokens: None, + last_refresh: None, + agent_identity: None, + personal_access_token: None, + bedrock_api_key: Some(bedrock_auth()), + } +} + +fn bedrock_auth() -> BedrockApiKeyAuth { + BedrockApiKeyAuth { + api_key: "bedrock-api-key-test".to_string(), + region: "us-east-1".to_string(), + } +} + +#[tokio::test] +async fn login_with_bedrock_api_key_replaces_openai_auth() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + storage.save(&api_key_auth())?; + login_with_bedrock_api_key( + codex_home.path(), + "bedrock-api-key-test", + "us-east-1", + AuthCredentialsStoreMode::File, + )?; + + let auth_manager = AuthManager::new( + codex_home.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + /*chatgpt_base_url*/ None, + ) + .await; + + let loaded = storage.load()?.expect("auth should be stored"); + let expected = AuthDotJson { + auth_mode: Some(AuthMode::BedrockApiKey), + openai_api_key: None, + tokens: None, + last_refresh: None, + agent_identity: None, + personal_access_token: None, + bedrock_api_key: Some(bedrock_auth()), + }; + assert_eq!(loaded, expected); + assert_eq!(auth_manager.auth_mode(), Some(AuthMode::BedrockApiKey)); + assert_eq!( + auth_manager.auth_cached().and_then(|auth| match auth { + CodexAuth::BedrockApiKey(auth) => Some(auth), + CodexAuth::ApiKey(_) + | CodexAuth::Chatgpt(_) + | CodexAuth::ChatgptAuthTokens(_) + | CodexAuth::AgentIdentity(_) + | CodexAuth::PersonalAccessToken(_) => None, + }), + Some(bedrock_auth()) + ); + Ok(()) +} + +#[tokio::test] +async fn logout_removes_bedrock_auth() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + login_with_bedrock_api_key( + codex_home.path(), + "bedrock-api-key-test", + "us-east-1", + AuthCredentialsStoreMode::File, + )?; + let auth_manager = AuthManager::new( + codex_home.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + /*chatgpt_base_url*/ None, + ) + .await; + + assert!(auth_manager.logout().await?); + + assert_eq!(storage.load()?, None); + assert_eq!(auth_manager.auth_cached(), None); + Ok(()) +} + +#[tokio::test] +async fn bedrock_only_auth_storage_creates_primary_auth() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + storage.save(&bedrock_only_auth())?; + + let auth_manager = AuthManager::new( + codex_home.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + /*chatgpt_base_url*/ None, + ) + .await; + + assert_eq!(auth_manager.auth_mode(), Some(AuthMode::BedrockApiKey)); + assert_eq!( + auth_manager.auth_cached().and_then(|auth| match auth { + CodexAuth::BedrockApiKey(auth) => Some(auth), + CodexAuth::ApiKey(_) + | CodexAuth::Chatgpt(_) + | CodexAuth::ChatgptAuthTokens(_) + | CodexAuth::AgentIdentity(_) + | CodexAuth::PersonalAccessToken(_) => None, + }), + Some(bedrock_auth()) + ); + Ok(()) +} + +#[tokio::test] +async fn login_with_api_key_clears_bedrock_api_key() -> anyhow::Result<()> { + let codex_home = tempdir()?; + let storage = FileAuthStorage::new(codex_home.path().to_path_buf()); + login_with_bedrock_api_key( + codex_home.path(), + "bedrock-api-key-test", + "us-east-1", + AuthCredentialsStoreMode::File, + )?; + + crate::auth::login_with_api_key( + codex_home.path(), + "sk-test-key", + AuthCredentialsStoreMode::File, + )?; + + assert_eq!(storage.load()?, Some(api_key_auth())); + Ok(()) +} diff --git a/codex-rs/login/src/auth/manager.rs b/codex-rs/login/src/auth/manager.rs index d7c99b6cbf..9708a5a0a1 100644 --- a/codex-rs/login/src/auth/manager.rs +++ b/codex-rs/login/src/auth/manager.rs @@ -30,6 +30,7 @@ use super::access_token::classify_codex_access_token; use super::external_bearer::BearerTokenRefresher; use super::revoke::revoke_auth_tokens_with_auth_route_config; pub use crate::auth::agent_identity::AgentIdentityAuth; +pub use crate::auth::bedrock_api_key::BedrockApiKeyAuth; pub use crate::auth::personal_access_token::PersonalAccessTokenAuth; pub use crate::auth::storage::AgentIdentityAuthRecord; pub use crate::auth::storage::AuthDotJson; @@ -58,12 +59,14 @@ pub enum CodexAuth { ChatgptAuthTokens(ChatgptAuthTokens), AgentIdentity(AgentIdentityAuth), PersonalAccessToken(PersonalAccessTokenAuth), + BedrockApiKey(BedrockApiKeyAuth), } impl PartialEq for CodexAuth { fn eq(&self, other: &Self) -> bool { match (self, other) { (Self::PersonalAccessToken(a), Self::PersonalAccessToken(b)) => a == b, + (Self::BedrockApiKey(a), Self::BedrockApiKey(b)) => a == b, _ => self.api_auth_mode() == other.api_auth_mode(), } } @@ -240,6 +243,14 @@ impl CodexAuth { }; return Self::from_personal_access_token(personal_access_token).await; } + if auth_mode == ApiAuthMode::BedrockApiKey { + let Some(auth) = auth_dot_json.bedrock_api_key else { + return Err(std::io::Error::other( + "Bedrock API key auth is missing a Bedrock API key.", + )); + }; + return Ok(Self::BedrockApiKey(auth)); + } let storage_mode = auth_dot_json.storage_mode(auth_credentials_store_mode); let state = ChatgptAuthState { @@ -259,6 +270,7 @@ impl CodexAuth { ApiAuthMode::PersonalAccessToken => { unreachable!("personal access token mode is handled above") } + ApiAuthMode::BedrockApiKey => unreachable!("bedrock api key mode is handled above"), } } @@ -331,6 +343,7 @@ impl CodexAuth { Self::Chatgpt(_) | Self::ChatgptAuthTokens(_) => AuthMode::Chatgpt, Self::AgentIdentity(_) => AuthMode::AgentIdentity, Self::PersonalAccessToken(_) => AuthMode::PersonalAccessToken, + Self::BedrockApiKey(_) => AuthMode::BedrockApiKey, } } @@ -341,6 +354,7 @@ impl CodexAuth { Self::ChatgptAuthTokens(_) => ApiAuthMode::ChatgptAuthTokens, Self::AgentIdentity(_) => ApiAuthMode::AgentIdentity, Self::PersonalAccessToken(_) => ApiAuthMode::PersonalAccessToken, + Self::BedrockApiKey(_) => ApiAuthMode::BedrockApiKey, } } @@ -381,7 +395,8 @@ impl CodexAuth { Self::Chatgpt(_) | Self::ChatgptAuthTokens(_) | Self::AgentIdentity(_) - | Self::PersonalAccessToken(_) => None, + | Self::PersonalAccessToken(_) + | Self::BedrockApiKey(_) => None, } } @@ -410,6 +425,9 @@ impl CodexAuth { "agent identity auth does not expose a bearer token", )), Self::PersonalAccessToken(auth) => Ok(auth.access_token().to_string()), + Self::BedrockApiKey(_) => Err(std::io::Error::other( + "Bedrock API key auth does not expose a Codex bearer token", + )), } } @@ -482,7 +500,10 @@ impl CodexAuth { let state = match self { Self::Chatgpt(auth) => &auth.state, Self::ChatgptAuthTokens(auth) => &auth.state, - Self::ApiKey(_) | Self::AgentIdentity(_) | Self::PersonalAccessToken(_) => return None, + Self::ApiKey(_) + | Self::AgentIdentity(_) + | Self::PersonalAccessToken(_) + | Self::BedrockApiKey(_) => return None, }; #[expect(clippy::unwrap_used)] state.auth_dot_json.lock().unwrap().clone() @@ -507,6 +528,7 @@ impl CodexAuth { last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; let state = ChatgptAuthState { @@ -621,6 +643,7 @@ pub fn login_with_api_key( last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth(codex_home, &auth_dot_json, auth_credentials_store_mode) } @@ -661,6 +684,7 @@ pub async fn login_with_access_token_with_auth_route_config( last_refresh: None, agent_identity: None, personal_access_token: Some(access_token.to_string()), + bedrock_api_key: None, } } CodexAccessToken::AgentIdentityJwt(jwt) => { @@ -676,6 +700,7 @@ pub async fn login_with_access_token_with_auth_route_config( last_refresh: None, agent_identity: Some(jwt.to_string()), personal_access_token: None, + bedrock_api_key: None, } } }; @@ -756,7 +781,8 @@ pub async fn enforce_login_restrictions_with_auth_route_config( if let Some(required_method) = config.forced_login_method { let method_violation = match (required_method, auth.auth_mode()) { - (ForcedLoginMethod::Api, AuthMode::ApiKey) => None, + (ForcedLoginMethod::Api, AuthMode::ApiKey) + | (ForcedLoginMethod::Api, AuthMode::BedrockApiKey) => None, (ForcedLoginMethod::Chatgpt, AuthMode::Chatgpt) | (ForcedLoginMethod::Chatgpt, AuthMode::ChatgptAuthTokens) | (ForcedLoginMethod::Chatgpt, AuthMode::AgentIdentity) @@ -768,7 +794,8 @@ pub async fn enforce_login_restrictions_with_auth_route_config( "API key login is required, but ChatGPT is currently being used. Logging out." .to_string(), ), - (ForcedLoginMethod::Chatgpt, AuthMode::ApiKey) => Some( + (ForcedLoginMethod::Chatgpt, AuthMode::ApiKey) + | (ForcedLoginMethod::Chatgpt, AuthMode::BedrockApiKey) => Some( "ChatGPT login is required, but an API key is currently being used. Logging out." .to_string(), ), @@ -785,7 +812,9 @@ pub async fn enforce_login_restrictions_with_auth_route_config( if let Some(expected_account_ids) = config.forced_chatgpt_workspace_id.as_deref() { let chatgpt_account_id = match &auth { - CodexAuth::ApiKey(_) | CodexAuth::PersonalAccessToken(_) => return Ok(()), + CodexAuth::ApiKey(_) + | CodexAuth::PersonalAccessToken(_) + | CodexAuth::BedrockApiKey(_) => return Ok(()), CodexAuth::AgentIdentity(_) => auth.get_account_id(), CodexAuth::Chatgpt(_) | CodexAuth::ChatgptAuthTokens(_) => { let token_data = match auth.get_token_data() { @@ -1110,6 +1139,7 @@ impl AuthDotJson { last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }) } @@ -1126,13 +1156,16 @@ impl AuthDotJson { Self::from_external_tokens(&external) } - fn resolved_mode(&self) -> ApiAuthMode { + pub(super) fn resolved_mode(&self) -> ApiAuthMode { if let Some(mode) = self.auth_mode { return mode; } if self.personal_access_token.is_some() { return ApiAuthMode::PersonalAccessToken; } + if self.bedrock_api_key.is_some() { + return ApiAuthMode::BedrockApiKey; + } if self.openai_api_key.is_some() { return ApiAuthMode::ApiKey; } @@ -1686,6 +1719,7 @@ impl AuthManager { _ => false, }, (ApiAuthMode::PersonalAccessToken, ApiAuthMode::PersonalAccessToken) => a == b, + (ApiAuthMode::BedrockApiKey, ApiAuthMode::BedrockApiKey) => a == b, _ => false, }, _ => false, @@ -1951,7 +1985,8 @@ impl AuthManager { } CodexAuth::ApiKey(_) | CodexAuth::AgentIdentity(_) - | CodexAuth::PersonalAccessToken(_) => Ok(()), + | CodexAuth::PersonalAccessToken(_) + | CodexAuth::BedrockApiKey(_) => Ok(()), }; if let Err(RefreshTokenError::Permanent(error)) = &result { self.record_permanent_refresh_failure_if_unchanged(&attempted_auth, error); diff --git a/codex-rs/login/src/auth/mod.rs b/codex-rs/login/src/auth/mod.rs index 949377baf7..9292b3376d 100644 --- a/codex-rs/login/src/auth/mod.rs +++ b/codex-rs/login/src/auth/mod.rs @@ -1,5 +1,6 @@ mod access_token; mod agent_identity; +mod bedrock_api_key; pub mod default_client; pub mod error; mod personal_access_token; @@ -10,6 +11,8 @@ mod external_bearer; mod manager; mod revoke; +pub use bedrock_api_key::BedrockApiKeyAuth; +pub use bedrock_api_key::login_with_bedrock_api_key; pub use error::RefreshTokenFailedError; pub use error::RefreshTokenFailedReason; pub use manager::*; diff --git a/codex-rs/login/src/auth/storage.rs b/codex-rs/login/src/auth/storage.rs index d0e606a8cc..2edb4ca301 100644 --- a/codex-rs/login/src/auth/storage.rs +++ b/codex-rs/login/src/auth/storage.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use std::sync::Mutex; use tracing::warn; +use super::BedrockApiKeyAuth; use crate::token_data::TokenData; use codex_agent_identity::AgentIdentityJwtClaims; use codex_agent_identity::decode_agent_identity_jwt; @@ -48,6 +49,9 @@ pub struct AuthDotJson { #[serde(default, skip_serializing_if = "Option::is_none")] pub personal_access_token: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bedrock_api_key: Option, } #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] diff --git a/codex-rs/login/src/auth/storage_tests.rs b/codex-rs/login/src/auth/storage_tests.rs index debe0e18d3..c3491a8c4a 100644 --- a/codex-rs/login/src/auth/storage_tests.rs +++ b/codex-rs/login/src/auth/storage_tests.rs @@ -20,6 +20,7 @@ async fn file_storage_load_returns_auth_dot_json() -> anyhow::Result<()> { last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; storage @@ -42,6 +43,7 @@ async fn file_storage_save_persists_auth_dot_json() -> anyhow::Result<()> { last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; let file = get_auth_file(codex_home.path()); @@ -76,6 +78,7 @@ async fn file_storage_round_trips_agent_identity_auth() -> anyhow::Result<()> { last_refresh: None, agent_identity: Some(agent_identity), personal_access_token: None, + bedrock_api_key: None, }; storage.save(&auth_dot_json)?; @@ -96,6 +99,7 @@ async fn file_storage_round_trips_personal_access_token_auth() -> anyhow::Result last_refresh: None, agent_identity: None, personal_access_token: Some("at-example".to_string()), + bedrock_api_key: None, }; storage.save(&auth_dot_json)?; @@ -146,6 +150,7 @@ fn file_storage_delete_removes_auth_file() -> anyhow::Result<()> { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; let storage = create_auth_storage(dir.path().to_path_buf(), AuthCredentialsStoreMode::File); storage.save(&auth_dot_json)?; @@ -171,6 +176,7 @@ fn ephemeral_storage_save_load_delete_is_in_memory_only() -> anyhow::Result<()> last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; storage.save(&auth_dot_json)?; @@ -271,6 +277,7 @@ fn auth_with_prefix(prefix: &str) -> AuthDotJson { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, } } @@ -297,6 +304,7 @@ fn keyring_auth_storage_load_returns_deserialized_auth() -> anyhow::Result<()> { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; seed_keyring_with_auth( &mock_keyring, @@ -341,6 +349,7 @@ fn keyring_auth_storage_save_persists_and_removes_fallback_file() -> anyhow::Res last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; storage.save(&auth)?; diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index 28c19ee211..ac0af3d271 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -43,6 +43,7 @@ pub use auth::load_auth_dot_json; pub use auth::login_with_access_token; pub use auth::login_with_access_token_with_auth_route_config; pub use auth::login_with_api_key; +pub use auth::login_with_bedrock_api_key; pub use auth::logout; pub use auth::logout_with_revoke; pub use auth::read_codex_access_token_from_env; diff --git a/codex-rs/login/src/server.rs b/codex-rs/login/src/server.rs index be963395f7..1aa22a13f4 100644 --- a/codex-rs/login/src/server.rs +++ b/codex-rs/login/src/server.rs @@ -847,6 +847,7 @@ pub(crate) async fn persist_tokens_async( last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth(&codex_home, &auth, auth_credentials_store_mode)?; Ok::<_, io::Error>((previous_auth, auth)) @@ -1360,6 +1361,7 @@ mod tests { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, } } diff --git a/codex-rs/login/tests/suite/auth_refresh.rs b/codex-rs/login/tests/suite/auth_refresh.rs index b30d738cd4..c86338e701 100644 --- a/codex-rs/login/tests/suite/auth_refresh.rs +++ b/codex-rs/login/tests/suite/auth_refresh.rs @@ -56,6 +56,7 @@ async fn refresh_token_succeeds_updates_storage() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -121,6 +122,7 @@ async fn refresh_token_refreshes_when_auth_is_unchanged() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -187,6 +189,7 @@ async fn auth_refreshes_when_access_token_is_near_expiry() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -238,6 +241,7 @@ async fn auth_skips_access_token_outside_refresh_window() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -275,6 +279,7 @@ async fn refresh_token_skips_refresh_when_auth_changed() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -286,6 +291,7 @@ async fn refresh_token_skips_refresh_when_auth_changed() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth( ctx.codex_home.path(), @@ -342,6 +348,7 @@ async fn refresh_token_errors_on_account_mismatch() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -354,6 +361,7 @@ async fn refresh_token_errors_on_account_mismatch() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth( ctx.codex_home.path(), @@ -414,6 +422,7 @@ async fn returns_fresh_tokens_as_is() -> Result<()> { last_refresh: Some(stale_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -463,6 +472,7 @@ async fn refreshes_token_when_access_token_is_expired() -> Result<()> { last_refresh: Some(fresh_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -514,6 +524,7 @@ async fn auth_reloads_disk_auth_when_cached_auth_is_stale() -> Result<()> { last_refresh: Some(stale_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -526,6 +537,7 @@ async fn auth_reloads_disk_auth_when_cached_auth_is_stale() -> Result<()> { last_refresh: Some(fresh_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth( ctx.codex_home.path(), @@ -579,6 +591,7 @@ async fn auth_reloads_disk_auth_without_calling_expired_refresh_token() -> Resul last_refresh: Some(stale_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -591,6 +604,7 @@ async fn auth_reloads_disk_auth_without_calling_expired_refresh_token() -> Resul last_refresh: Some(fresh_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth( ctx.codex_home.path(), @@ -642,6 +656,7 @@ async fn refresh_token_returns_permanent_error_for_expired_refresh_token() -> Re last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -696,6 +711,7 @@ async fn refresh_token_does_not_retry_after_permanent_failure() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -764,6 +780,7 @@ async fn refresh_token_does_not_retry_after_bad_request_reused_failure() -> Resu last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -832,6 +849,7 @@ async fn refresh_token_reloads_changed_auth_after_permanent_failure() -> Result< last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -855,6 +873,7 @@ async fn refresh_token_reloads_changed_auth_after_permanent_failure() -> Result< last_refresh: Some(fresh_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth( ctx.codex_home.path(), @@ -915,6 +934,7 @@ async fn refresh_token_returns_transient_error_on_server_failure() -> Result<()> last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -969,6 +989,7 @@ async fn unauthorized_recovery_reloads_then_refreshes_tokens() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -980,6 +1001,7 @@ async fn unauthorized_recovery_reloads_then_refreshes_tokens() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth( ctx.codex_home.path(), @@ -1065,6 +1087,7 @@ async fn unauthorized_recovery_errors_on_account_mismatch() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&initial_auth).await?; @@ -1077,6 +1100,7 @@ async fn unauthorized_recovery_errors_on_account_mismatch() -> Result<()> { last_refresh: Some(initial_last_refresh), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth( ctx.codex_home.path(), @@ -1136,6 +1160,7 @@ async fn unauthorized_recovery_requires_chatgpt_auth() -> Result<()> { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; ctx.write_auth(&auth).await?; diff --git a/codex-rs/login/tests/suite/logout.rs b/codex-rs/login/tests/suite/logout.rs index 2dd1bef83f..5d75803977 100644 --- a/codex-rs/login/tests/suite/logout.rs +++ b/codex-rs/login/tests/suite/logout.rs @@ -196,6 +196,7 @@ fn chatgpt_auth_with_refresh_token(refresh_token: &str) -> AuthDotJson { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, } } diff --git a/codex-rs/mcp-server/Cargo.toml b/codex-rs/mcp-server/Cargo.toml index 29b2d6c7d7..9d568722e8 100644 --- a/codex-rs/mcp-server/Cargo.toml +++ b/codex-rs/mcp-server/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true } codex-arg0 = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } +codex-home = { workspace = true } codex-exec-server = { workspace = true } codex-extension-api = { workspace = true } codex-login = { workspace = true } diff --git a/codex-rs/mcp-server/src/message_processor.rs b/codex-rs/mcp-server/src/message_processor.rs index 2f85b35143..ce19575a9c 100644 --- a/codex-rs/mcp-server/src/message_processor.rs +++ b/codex-rs/mcp-server/src/message_processor.rs @@ -7,6 +7,7 @@ use codex_core::ThreadManager; use codex_core::config::Config; use codex_exec_server::EnvironmentManager; use codex_extension_api::empty_extension_registry; +use codex_home::CodexHomeUserInstructionsProvider; use codex_login::AuthManager; use codex_login::default_client::USER_AGENT_SUFFIX; use codex_login::default_client::get_codex_user_agent; @@ -62,12 +63,16 @@ impl MessageProcessor { /*enable_codex_api_key_env*/ false, ) .await; + let user_instructions_provider = Arc::new(CodexHomeUserInstructionsProvider::new( + config.codex_home.clone(), + )); let thread_manager = Arc::new(ThreadManager::new( config.as_ref(), auth_manager, SessionSource::Mcp, environment_manager, empty_extension_registry(), + user_instructions_provider, /*analytics_events_client*/ None, codex_core::thread_store_from_config(config.as_ref(), state_db.clone()), state_db.clone(), diff --git a/codex-rs/memories/write/src/runtime.rs b/codex-rs/memories/write/src/runtime.rs index 0b6718f01e..8dae6ca617 100644 --- a/codex-rs/memories/write/src/runtime.rs +++ b/codex-rs/memories/write/src/runtime.rs @@ -7,6 +7,7 @@ use codex_core::StartThreadOptions; use codex_core::ThreadManager; use codex_core::config::Config; use codex_core::content_items_to_text; +use codex_core::detached_memory_responses_metadata; use codex_core::resolve_installation_id; use codex_features::Feature; use codex_login::AuthManager; @@ -49,7 +50,6 @@ pub(crate) struct StageOneRequestContext { pub(crate) reasoning_effort: Option, pub(crate) reasoning_summary: ReasoningSummary, pub(crate) service_tier: Option, - pub(crate) turn_metadata_header: Option, } impl StageOneRequestContext { @@ -199,8 +199,6 @@ impl MemoryStartupContext { .get_models_manager() .get_model_info(model_name, &config.to_models_manager_config()) .await; - let turn_metadata_header = - codex_core::build_turn_metadata_header(&config.cwd, /*sandbox*/ None).await; let reasoning_summary = config .model_reasoning_summary .unwrap_or(model_info.default_reasoning_summary); @@ -214,7 +212,6 @@ impl MemoryStartupContext { reasoning_effort: Some(reasoning_effort), reasoning_summary, service_tier: config_snapshot.service_tier, - turn_metadata_header, } } @@ -227,14 +224,13 @@ impl MemoryStartupContext { let installation_id = resolve_installation_id(&config.codex_home).await?; let config_snapshot = self.thread.config_snapshot().await; let session_source = config_snapshot.session_source; + let session_id = SessionId::from(self.thread_id); + let session_id_string = session_id.to_string(); let model_client = ModelClient::new( Some(Arc::clone(&self.auth_manager)), - SessionId::from(self.thread_id), // We use thread_id to detach this query from the foreground user session. self.thread_id, - installation_id, config.model_provider.clone(), - session_source, - config_snapshot.parent_thread_id, + session_source.clone(), config.model_verbosity, config.features.enabled(Feature::EnableRequestCompression), config.features.enabled(Feature::RuntimeMetrics), @@ -244,16 +240,25 @@ impl MemoryStartupContext { let mut client_session = model_client.new_session(); let window_id = format!("{}:0", self.thread_id); + let responses_metadata = detached_memory_responses_metadata( + installation_id, + session_id_string, + self.thread_id.to_string(), + window_id, + &session_source, + &config.cwd, + /*sandbox*/ None, + ) + .await; let mut stream = client_session .stream( - &window_id, prompt, &context.model_info, &context.session_telemetry, context.reasoning_effort.clone(), context.reasoning_summary, context.service_tier.clone(), - context.turn_metadata_header.as_deref(), + &responses_metadata, &InferenceTraceContext::disabled(), ) .await?; diff --git a/codex-rs/model-provider/src/auth.rs b/codex-rs/model-provider/src/auth.rs index 4f87d46bae..8f5e02b175 100644 --- a/codex-rs/model-provider/src/auth.rs +++ b/codex-rs/model-provider/src/auth.rs @@ -8,11 +8,15 @@ use codex_api::SharedAuthProvider; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_model_provider_info::ModelProviderInfo; +use codex_protocol::error::CodexErr; use http::HeaderMap; use http::HeaderValue; use crate::bearer_auth_provider::BearerAuthProvider; +const BEDROCK_API_KEY_UNSUPPORTED_MESSAGE: &str = + "Bedrock API key auth is only supported by the Amazon Bedrock model provider"; + #[derive(Clone, Debug)] struct AgentIdentityAuthProvider { auth: codex_login::auth::AgentIdentityAuth, @@ -79,6 +83,12 @@ pub(crate) fn resolve_provider_auth( auth: Option<&CodexAuth>, provider: &ModelProviderInfo, ) -> codex_protocol::error::Result { + if matches!(auth, Some(CodexAuth::BedrockApiKey(_))) { + return Err(CodexErr::UnsupportedOperation( + BEDROCK_API_KEY_UNSUPPORTED_MESSAGE.to_string(), + )); + } + if let Some(auth) = bearer_auth_for_provider(provider)? { return Ok(Arc::new(auth)); } @@ -109,6 +119,7 @@ pub fn auth_provider_from_auth(auth: &CodexAuth) -> SharedAuthProvider { CodexAuth::AgentIdentity(auth) => { Arc::new(AgentIdentityAuthProvider { auth: auth.clone() }) } + CodexAuth::BedrockApiKey(_) => unreachable!("{BEDROCK_API_KEY_UNSUPPORTED_MESSAGE}"), CodexAuth::ApiKey(_) | CodexAuth::Chatgpt(_) | CodexAuth::ChatgptAuthTokens(_) @@ -122,8 +133,10 @@ pub fn auth_provider_from_auth(auth: &CodexAuth) -> SharedAuthProvider { #[cfg(test)] mod tests { + use codex_login::auth::BedrockApiKeyAuth; use codex_model_provider_info::WireApi; use codex_model_provider_info::create_oss_provider_with_base_url; + use pretty_assertions::assert_eq; use super::*; @@ -135,4 +148,21 @@ mod tests { assert!(auth.to_auth_headers().is_empty()); } + + #[test] + fn openai_provider_rejects_bedrock_api_key_auth() { + let provider = ModelProviderInfo::create_openai_provider(/*base_url*/ None); + let auth = CodexAuth::BedrockApiKey(BedrockApiKeyAuth { + api_key: "bedrock-api-key-test".to_string(), + region: "us-east-1".to_string(), + }); + + match resolve_provider_auth(Some(&auth), &provider) { + Err(CodexErr::UnsupportedOperation(message)) => { + assert_eq!(message, BEDROCK_API_KEY_UNSUPPORTED_MESSAGE); + } + Err(err) => panic!("unexpected auth error: {err:?}"), + Ok(_) => panic!("Bedrock API key auth should be rejected"), + } + } } diff --git a/codex-rs/model-provider/src/provider.rs b/codex-rs/model-provider/src/provider.rs index e786192433..95427465eb 100644 --- a/codex-rs/model-provider/src/provider.rs +++ b/codex-rs/model-provider/src/provider.rs @@ -51,6 +51,7 @@ pub struct ProviderAccountState { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProviderAccountError { MissingChatgptAccountDetails, + UnsupportedBedrockApiKeyAuth, } impl fmt::Display for ProviderAccountError { @@ -62,6 +63,12 @@ impl fmt::Display for ProviderAccountError { "email and plan type are required for chatgpt authentication" ) } + Self::UnsupportedBedrockApiKeyAuth => { + write!( + f, + "Bedrock API key auth is only supported by the Amazon Bedrock model provider" + ) + } } } } @@ -232,6 +239,9 @@ impl ModelProvider for ConfiguredModelProvider { }) .map(|auth| match &auth { CodexAuth::ApiKey(_) => Ok(ProviderAccount::ApiKey), + CodexAuth::BedrockApiKey(_) => { + Err(ProviderAccountError::UnsupportedBedrockApiKeyAuth) + } CodexAuth::Chatgpt(_) | CodexAuth::ChatgptAuthTokens(_) | CodexAuth::AgentIdentity(_) @@ -287,6 +297,7 @@ impl ModelProvider for ConfiguredModelProvider { mod tests { use std::num::NonZeroU64; + use codex_login::auth::BedrockApiKeyAuth; use codex_model_provider_info::ModelProviderAwsAuthInfo; use codex_model_provider_info::WireApi; use codex_models_manager::manager::RefreshStrategy; @@ -374,6 +385,13 @@ mod tests { .expect("valid model") } + fn bedrock_api_key_auth() -> CodexAuth { + CodexAuth::BedrockApiKey(BedrockApiKeyAuth { + api_key: "bedrock-api-key-test".to_string(), + region: "us-east-1".to_string(), + }) + } + #[test] fn configured_provider_uses_default_capabilities() { let provider = create_model_provider( @@ -491,6 +509,19 @@ mod tests { ); } + #[test] + fn openai_provider_rejects_bedrock_api_key_account_state() { + let provider = create_model_provider( + ModelProviderInfo::create_openai_provider(/*base_url*/ None), + Some(AuthManager::from_auth_for_testing(bedrock_api_key_auth())), + ); + + assert_eq!( + provider.account_state(), + Err(ProviderAccountError::UnsupportedBedrockApiKeyAuth) + ); + } + #[test] fn custom_non_openai_provider_returns_no_account_state() { let provider = create_model_provider( diff --git a/codex-rs/models-manager/src/manager_tests.rs b/codex-rs/models-manager/src/manager_tests.rs index 2e39ef3160..6f3a6a27f4 100644 --- a/codex-rs/models-manager/src/manager_tests.rs +++ b/codex-rs/models-manager/src/manager_tests.rs @@ -212,6 +212,7 @@ c2ln", last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; std::fs::create_dir_all(codex_home).expect("codex home should be created"); std::fs::write( diff --git a/codex-rs/models-manager/src/model_info.rs b/codex-rs/models-manager/src/model_info.rs index 81a5c6e5ed..566953dfc2 100644 --- a/codex-rs/models-manager/src/model_info.rs +++ b/codex-rs/models-manager/src/model_info.rs @@ -94,6 +94,7 @@ pub fn model_info_from_slug(slug: &str) -> ModelInfo { context_window: Some(272_000), max_context_window: Some(272_000), auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), input_modalities: default_input_modalities(), diff --git a/codex-rs/otel/src/lib.rs b/codex-rs/otel/src/lib.rs index 586dfa15a7..fe7a171ce9 100644 --- a/codex-rs/otel/src/lib.rs +++ b/codex-rs/otel/src/lib.rs @@ -54,7 +54,8 @@ pub enum TelemetryAuthMode { impl From for TelemetryAuthMode { fn from(mode: codex_app_server_protocol::AuthMode) -> Self { match mode { - codex_app_server_protocol::AuthMode::ApiKey => Self::ApiKey, + codex_app_server_protocol::AuthMode::ApiKey + | codex_app_server_protocol::AuthMode::BedrockApiKey => Self::ApiKey, codex_app_server_protocol::AuthMode::Chatgpt | codex_app_server_protocol::AuthMode::ChatgptAuthTokens | codex_app_server_protocol::AuthMode::AgentIdentity diff --git a/codex-rs/otel/src/metrics/client.rs b/codex-rs/otel/src/metrics/client.rs index 9d5ad5d166..a36157aae3 100644 --- a/codex-rs/otel/src/metrics/client.rs +++ b/codex-rs/otel/src/metrics/client.rs @@ -12,6 +12,7 @@ use crate::metrics::validation::validate_tags; use codex_utils_string::sanitize_metric_tag_value; use opentelemetry::KeyValue; use opentelemetry::metrics::Counter; +use opentelemetry::metrics::Gauge; use opentelemetry::metrics::Histogram; use opentelemetry::metrics::Meter; use opentelemetry::metrics::MeterProvider as _; @@ -89,6 +90,7 @@ struct MetricsClientInner { meter_provider: SdkMeterProvider, meter: Meter, counters: Mutex>>, + gauges: Mutex>>, histograms: Mutex>>, duration_histograms: Mutex>>, runtime_reader: Option>, @@ -146,6 +148,35 @@ impl MetricsClientInner { Ok(()) } + fn gauge( + &self, + name: &str, + description: Option<&str>, + value: i64, + tags: &[(&str, &str)], + ) -> Result<()> { + validate_metric_name(name)?; + let attributes = self.attributes(tags)?; + + let mut gauges = self + .gauges + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let key = InstrumentKey { + name: name.to_string(), + description: description.map(str::to_string), + }; + let gauge = gauges.entry(key).or_insert_with(|| { + let builder = self.meter.i64_gauge(name.to_string()); + match description { + Some(description) => builder.with_description(description.to_string()).build(), + None => builder.build(), + } + }); + gauge.record(value, &attributes); + Ok(()) + } + fn duration_histogram(&self, name: &str, value: i64, tags: &[(&str, &str)]) -> Result<()> { validate_metric_name(name)?; let attributes = self.attributes(tags)?; @@ -253,6 +284,7 @@ impl MetricsClient { meter_provider, meter, counters: Mutex::new(HashMap::new()), + gauges: Mutex::new(HashMap::new()), histograms: Mutex::new(HashMap::new()), duration_histograms: Mutex::new(HashMap::new()), runtime_reader, @@ -281,6 +313,22 @@ impl MetricsClient { self.0.histogram(name, value, tags) } + /// Send a single gauge measurement. + pub fn gauge(&self, name: &str, value: i64, tags: &[(&str, &str)]) -> Result<()> { + self.0.gauge(name, /*description*/ None, value, tags) + } + + /// Send a single gauge measurement with an instrument description. + pub fn gauge_with_description( + &self, + name: &str, + description: &str, + value: i64, + tags: &[(&str, &str)], + ) -> Result<()> { + self.0.gauge(name, Some(description), value, tags) + } + /// Record a duration in milliseconds using a histogram. pub fn record_duration( &self, diff --git a/codex-rs/otel/tests/suite/otlp_http_loopback.rs b/codex-rs/otel/tests/suite/otlp_http_loopback.rs index 4c2dd36f76..1f41336916 100644 --- a/codex-rs/otel/tests/suite/otlp_http_loopback.rs +++ b/codex-rs/otel/tests/suite/otlp_http_loopback.rs @@ -186,6 +186,12 @@ fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> { ))?; metrics.counter("codex.turns", /*inc*/ 1, &[("source", "test")])?; + metrics.gauge_with_description( + "codex.active", + "Number of active Codex operations.", + /*value*/ 1, + &[("component", "test")], + )?; metrics.shutdown()?; server.join().expect("server join"); @@ -220,6 +226,16 @@ fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> { "expected metric name not found; body prefix: {}", &body.chars().take(2000).collect::() ); + assert!( + body.contains("codex.active"), + "expected gauge not found; body prefix: {}", + &body.chars().take(2000).collect::() + ); + assert!( + body.contains("component") && body.contains("test"), + "expected gauge tag not found; body prefix: {}", + &body.chars().take(2000).collect::() + ); Ok(()) } diff --git a/codex-rs/otel/tests/suite/send.rs b/codex-rs/otel/tests/suite/send.rs index 3a268d3c7d..e2b8f2103e 100644 --- a/codex-rs/otel/tests/suite/send.rs +++ b/codex-rs/otel/tests/suite/send.rs @@ -24,6 +24,12 @@ fn send_builds_payload_with_tags_and_histograms() -> Result<()> { /*value*/ 25, &[("tool", "shell")], )?; + metrics.gauge_with_description( + "codex.active", + "Number of active Codex operations.", + /*value*/ 2, + &[("component", "test")], + )?; metrics.shutdown()?; let resource_metrics = latest_metrics(&exporter); @@ -80,6 +86,27 @@ fn send_builds_payload_with_tags_and_histograms() -> Result<()> { ]); assert_eq!(histogram_attrs, expected_histogram_attributes); + let gauge = find_metric(&resource_metrics, "codex.active").expect("gauge metric missing"); + assert_eq!(gauge.description(), "Number of active Codex operations."); + let gauge_point = match gauge.data() { + opentelemetry_sdk::metrics::data::AggregatedMetrics::I64(data) => match data { + opentelemetry_sdk::metrics::data::MetricData::Gauge(gauge) => { + gauge.data_points().next().expect("gauge point") + } + _ => panic!("unexpected gauge aggregation"), + }, + _ => panic!("unexpected gauge metric data type"), + }; + assert_eq!(gauge_point.value(), 2); + assert_eq!( + attributes_to_map(gauge_point.attributes()), + BTreeMap::from([ + ("component".to_string(), "test".to_string()), + ("env".to_string(), "prod".to_string()), + ("service".to_string(), "codex-cli".to_string()), + ]) + ); + Ok(()) } diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 517c33cac9..5df6d1e77f 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -4,6 +4,7 @@ use std::num::NonZeroUsize; use std::path::Path; use codex_utils_image::PromptImageMode; +use codex_utils_image::data_url_from_bytes; use codex_utils_image::load_for_prompt_bytes; use serde::Deserialize; use serde::Deserializer; @@ -1088,26 +1089,12 @@ pub fn local_image_content_items_with_label_number( }; match load_for_prompt_bytes(path, file_bytes, mode) { - Ok(image) => { - let mut items = Vec::with_capacity(3); - if let Some(label_number) = label_number { - items.push(ContentItem::InputText { - text: local_image_open_tag_text_with_path(label_number, path), - }); - } - items.push(ContentItem::InputImage { - image_url: image.into_data_url(), - detail: Some(detail), - }); - if label_number.is_some() { - items.push(ContentItem::InputText { - text: LOCAL_IMAGE_CLOSE_TAG.to_string(), - }); - } - items - } + Ok(image) => local_image_content_items(path, image.into_data_url(), label_number, detail), Err(err) => match &err { - ImageProcessingError::Read { .. } | ImageProcessingError::Encode { .. } => { + ImageProcessingError::Read { .. } + | ImageProcessingError::Encode { .. } + | ImageProcessingError::InvalidDataUrl { .. } + | ImageProcessingError::ImageTooLarge { .. } => { vec![local_image_error_placeholder(path, &err)] } ImageProcessingError::Decode { .. } if err.is_invalid_image() => { @@ -1123,6 +1110,36 @@ pub fn local_image_content_items_with_label_number( } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LocalImagePreparation { + Process, + Defer, +} + +fn local_image_content_items( + path: &std::path::Path, + image_url: String, + label_number: Option, + detail: ImageDetail, +) -> Vec { + let mut items = Vec::with_capacity(3); + if let Some(label_number) = label_number { + items.push(ContentItem::InputText { + text: local_image_open_tag_text_with_path(label_number, path), + }); + } + items.push(ContentItem::InputImage { + image_url, + detail: Some(detail), + }); + if label_number.is_some() { + items.push(ContentItem::InputText { + text: LOCAL_IMAGE_CLOSE_TAG.to_string(), + }); + } + items +} + impl From for ResponseItem { fn from(item: ResponseInputItem) -> Self { match item { @@ -1235,6 +1252,15 @@ pub enum ReasoningItemContent { impl From> for ResponseInputItem { fn from(items: Vec) -> Self { + Self::from_user_input(items, LocalImagePreparation::Process) + } +} + +impl ResponseInputItem { + pub fn from_user_input( + items: Vec, + local_image_preparation: LocalImagePreparation, + ) -> Self { let mut image_index = 0; Self::Message { role: "user".to_string(), @@ -1256,12 +1282,22 @@ impl From> for ResponseInputItem { image_index += 1; let detail = detail.unwrap_or(DEFAULT_IMAGE_DETAIL); match std::fs::read(&path) { - Ok(file_bytes) => local_image_content_items_with_label_number( - &path, - file_bytes, - Some(image_index), - detail, - ), + Ok(file_bytes) => match local_image_preparation { + LocalImagePreparation::Process => { + local_image_content_items_with_label_number( + &path, + file_bytes, + Some(image_index), + detail, + ) + } + LocalImagePreparation::Defer => local_image_content_items( + &path, + data_url_from_bytes("application/octet-stream", &file_bytes), + Some(image_index), + detail, + ), + }, Err(err) => vec![local_image_error_placeholder(&path, err)], } } diff --git a/codex-rs/protocol/src/openai_models.rs b/codex-rs/protocol/src/openai_models.rs index 1553ef3435..5b95fee29d 100644 --- a/codex-rs/protocol/src/openai_models.rs +++ b/codex-rs/protocol/src/openai_models.rs @@ -389,6 +389,9 @@ pub struct ModelInfo { /// context window when available. #[serde(default, skip_serializing_if = "Option::is_none")] pub auto_compact_token_limit: Option, + /// Opaque identifier for compaction-compatible model configurations. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub comp_hash: Option, /// Percentage of the context window considered usable for inputs, after /// reserving headroom for system prompts, tool overhead, and model output. #[serde(default = "default_effective_context_window_percent")] @@ -671,6 +674,7 @@ mod tests { context_window: None, max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: vec![], input_modalities: default_input_modalities(), @@ -940,6 +944,7 @@ mod tests { assert_eq!(model.web_search_tool_type, WebSearchToolType::Text); assert!(!model.supports_search_tool); assert!(!model.use_responses_lite); + assert_eq!(model.comp_hash, None); assert_eq!(model.auto_review_model_override, None); assert_eq!(model.tool_mode, None); } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 01e664c734..059c5692ce 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2941,6 +2941,8 @@ pub struct TurnContextItem { #[serde(default, skip_serializing_if = "Option::is_none")] pub file_system_sandbox_policy: Option, pub model: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub comp_hash: Option, #[serde(skip_serializing_if = "Option::is_none")] pub personality: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -5162,6 +5164,7 @@ mod tests { assert_eq!(item.network, None); assert_eq!(item.file_system_sandbox_policy, None); + assert_eq!(item.comp_hash, None); Ok(()) } @@ -5222,6 +5225,7 @@ mod tests { }, ])), model: "gpt-5".to_string(), + comp_hash: None, personality: None, collaboration_mode: None, multi_agent_version: None, diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs index 9e6db0fb8d..23be07996c 100644 --- a/codex-rs/rollout/src/lib.rs +++ b/codex-rs/rollout/src/lib.rs @@ -73,6 +73,7 @@ pub use session_index::append_thread_name; pub use session_index::find_thread_meta_by_name_str; pub use session_index::find_thread_name_by_id; pub use session_index::find_thread_names_by_ids; +pub use session_index::remove_thread_name_entries; pub use state_db::StateDbHandle; pub use state_db::sqlite_telemetry_recorder; diff --git a/codex-rs/rollout/src/recorder_tests.rs b/codex-rs/rollout/src/recorder_tests.rs index 07029abe37..68896ad202 100644 --- a/codex-rs/rollout/src/recorder_tests.rs +++ b/codex-rs/rollout/src/recorder_tests.rs @@ -1144,6 +1144,7 @@ async fn resume_candidate_matches_cwd_reads_latest_turn_context() -> std::io::Re network: None, file_system_sandbox_policy: None, model: "test-model".to_string(), + comp_hash: None, personality: None, collaboration_mode: None, multi_agent_version: None, diff --git a/codex-rs/rollout/src/session_index.rs b/codex-rs/rollout/src/session_index.rs index e227515357..96ff9d825e 100644 --- a/codex-rs/rollout/src/session_index.rs +++ b/codex-rs/rollout/src/session_index.rs @@ -1,21 +1,25 @@ use std::collections::HashMap; use std::collections::HashSet; use std::fs::File; +use std::io::ErrorKind; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; +use std::io::Write; use std::path::Path; use std::path::PathBuf; +use std::sync::LazyLock; +use std::sync::Mutex; use codex_protocol::ThreadId; use codex_protocol::protocol::SessionMetaLine; use serde::Deserialize; use serde::Serialize; use tokio::io::AsyncBufReadExt; -use tokio::io::AsyncWriteExt; const SESSION_INDEX_FILE: &str = "session_index.jsonl"; const READ_CHUNK_SIZE: usize = 8192; +static SESSION_INDEX_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SessionIndexEntry { @@ -25,7 +29,7 @@ pub struct SessionIndexEntry { } /// Append a thread name update to the session index. -/// The index is append-only; the most recent entry wins when resolving names or ids. +/// Name updates are append-only; the most recent entry wins when resolving names or ids. pub async fn append_thread_name( codex_home: &Path, thread_id: ThreadId, @@ -46,24 +50,60 @@ pub async fn append_thread_name( } /// Append a raw session index entry to `session_index.jsonl`. -/// The file is append-only; consumers scan from the end to find the newest match. +/// Consumers scan from the end to find the newest match. pub async fn append_session_index_entry( codex_home: &Path, entry: &SessionIndexEntry, ) -> std::io::Result<()> { + let _guard = SESSION_INDEX_LOCK + .lock() + .map_err(|err| std::io::Error::other(err.to_string()))?; let path = session_index_path(codex_home); - let mut file = tokio::fs::OpenOptions::new() + let mut file = std::fs::OpenOptions::new() .create(true) .append(true) - .open(&path) - .await?; + .open(&path)?; let mut line = serde_json::to_string(entry).map_err(std::io::Error::other)?; line.push('\n'); - file.write_all(line.as_bytes()).await?; - file.flush().await?; + file.write_all(line.as_bytes())?; + file.flush()?; Ok(()) } +/// Remove all recorded names for a thread from the session index. +pub async fn remove_thread_name_entries( + codex_home: &Path, + thread_id: ThreadId, +) -> std::io::Result<()> { + let _guard = SESSION_INDEX_LOCK + .lock() + .map_err(|err| std::io::Error::other(err.to_string()))?; + let path = session_index_path(codex_home); + let contents = match std::fs::read_to_string(&path) { + Ok(contents) => contents, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(err), + }; + let mut removed = false; + let mut remaining = String::with_capacity(contents.len()); + for line in contents.lines() { + let should_remove = serde_json::from_str::(line.trim()) + .is_ok_and(|entry| entry.id == thread_id); + if should_remove { + removed = true; + } else { + remaining.push_str(line); + remaining.push('\n'); + } + } + if !removed { + return Ok(()); + } + let temp_path = path.with_extension("jsonl.tmp"); + std::fs::write(&temp_path, remaining)?; + std::fs::rename(temp_path, path) +} + /// Find the latest thread name for a thread id, if any. pub async fn find_thread_name_by_id( codex_home: &Path, diff --git a/codex-rs/rollout/src/state_db.rs b/codex-rs/rollout/src/state_db.rs index fa2905dd4f..c21d7002be 100644 --- a/codex-rs/rollout/src/state_db.rs +++ b/codex-rs/rollout/src/state_db.rs @@ -5,6 +5,7 @@ use crate::list::SortDirection; use crate::list::ThreadSortKey; use crate::metadata; use crate::sqlite_metrics; +use anyhow::Context; use chrono::DateTime; use chrono::Utc; use codex_protocol::ThreadId; @@ -50,7 +51,7 @@ pub async fn init(config: &impl RolloutConfigView) -> Option { { Ok(runtime) => Some(runtime), Err(err) => { - emit_startup_warning(&format!("failed to initialize state runtime: {err}")); + emit_startup_warning(&format!("failed to initialize state runtime: {err:#}")); None } } @@ -109,9 +110,9 @@ async fn try_init_with_roots_inner( let runtime = codex_state::StateRuntime::init(sqlite_home.clone(), default_model_provider_id.clone()) .await - .map_err(|err| { - anyhow::anyhow!( - "failed to initialize state runtime at {}: {err}", + .with_context(|| { + format!( + "failed to initialize state runtime at {}", sqlite_home.display() ) })?; @@ -128,7 +129,10 @@ async fn try_init_with_roots_inner( backfill_gate_started.elapsed(), &backfill_gate_result, ); - backfill_gate_result?; + if let Err(err) = backfill_gate_result { + runtime.close().await; + return Err(err); + } Ok(runtime) } diff --git a/codex-rs/skills/src/assets/samples/plugin-creator/scripts/validate_plugin.py b/codex-rs/skills/src/assets/samples/plugin-creator/scripts/validate_plugin.py index 64518064e5..6f49cb0fed 100644 --- a/codex-rs/skills/src/assets/samples/plugin-creator/scripts/validate_plugin.py +++ b/codex-rs/skills/src/assets/samples/plugin-creator/scripts/validate_plugin.py @@ -307,10 +307,17 @@ def validate_app_manifest(path: Path, errors: list[str]) -> None: if not isinstance(value, dict): errors.append(f"`.app.json` app `{key}` must be an object") continue - reject_companion_unknown_fields(value, {"id"}, f"`.app.json` app `{key}`", errors) + reject_companion_unknown_fields( + value, {"id", "category"}, f"`.app.json` app `{key}`", errors + ) app_id = value.get("id") if not isinstance(app_id, str) or not app_id.strip(): errors.append(f"`.app.json` app `{key}` field `id` must be a non-empty string") + category = value.get("category") + if category is not None and (not isinstance(category, str) or not category.strip()): + errors.append( + f"`.app.json` app `{key}` field `category` must be a non-empty string" + ) def validate_mcp_manifest(path: Path, errors: list[str]) -> None: diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index 47285c5cf5..c1c2e56c93 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -356,6 +356,7 @@ mod tests { network: None, file_system_sandbox_policy: None, model: "gpt-5".to_string(), + comp_hash: None, personality: None, collaboration_mode: None, multi_agent_version: None, @@ -394,6 +395,7 @@ mod tests { network: None, file_system_sandbox_policy: None, model: "gpt-5".to_string(), + comp_hash: None, personality: None, collaboration_mode: None, multi_agent_version: None, @@ -429,6 +431,7 @@ mod tests { network: None, file_system_sandbox_policy: None, model: "gpt-5".to_string(), + comp_hash: None, personality: None, collaboration_mode: None, multi_agent_version: None, @@ -460,6 +463,7 @@ mod tests { network: None, file_system_sandbox_policy: None, model: "gpt-5".to_string(), + comp_hash: None, personality: None, collaboration_mode: None, multi_agent_version: None, diff --git a/codex-rs/state/src/lib.rs b/codex-rs/state/src/lib.rs index cb8c711e65..264b737e9f 100644 --- a/codex-rs/state/src/lib.rs +++ b/codex-rs/state/src/lib.rs @@ -57,15 +57,21 @@ pub use runtime::GoalStore; pub use runtime::GoalUpdate; pub use runtime::MemoryStore; pub use runtime::RemoteControlEnrollmentRecord; +pub use runtime::RuntimeDbBackup; pub use runtime::RuntimeDbPath; pub use runtime::ThreadFilterOptions; +pub use runtime::backup_runtime_db_for_fresh_start; pub use runtime::goals_db_filename; pub use runtime::goals_db_path; +pub use runtime::is_sqlite_corruption_error; pub use runtime::logs_db_filename; pub use runtime::logs_db_path; pub use runtime::memories_db_filename; pub use runtime::memories_db_path; +pub use runtime::runtime_db_path_for_corruption_error; pub use runtime::runtime_db_paths; +pub use runtime::sqlite_error_detail_is_corruption; +pub use runtime::sqlite_error_detail_is_lock; pub use runtime::sqlite_integrity_check; pub use runtime::state_db_filename; pub use runtime::state_db_path; diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 5ae41bcd8b..2937bf0c2e 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -62,6 +62,7 @@ mod backfill; mod goals; mod logs; mod memories; +mod recovery; mod remote_control; #[cfg(test)] mod test_support; @@ -72,6 +73,12 @@ pub use goals::GoalAccountingOutcome; pub use goals::GoalStore; pub use goals::GoalUpdate; pub use memories::MemoryStore; +pub use recovery::RuntimeDbBackup; +pub use recovery::backup_runtime_db_for_fresh_start; +pub use recovery::is_sqlite_corruption_error; +pub use recovery::runtime_db_path_for_corruption_error; +pub use recovery::sqlite_error_detail_is_corruption; +pub use recovery::sqlite_error_detail_is_lock; pub use remote_control::RemoteControlEnrollmentRecord; pub use threads::ThreadFilterOptions; @@ -200,6 +207,7 @@ impl StateRuntime { Ok(db) => Arc::new(db), Err(err) => { warn!("failed to open logs db at {}: {err}", logs_path.display()); + close_sqlite_pools(&[pool.as_ref()]).await; return Err(err); } }; @@ -208,6 +216,7 @@ impl StateRuntime { Ok(db) => Arc::new(db), Err(err) => { warn!("failed to open goals db at {}: {err}", goals_path.display()); + close_sqlite_pools(&[pool.as_ref(), logs_pool.as_ref()]).await; return Err(err); } }; @@ -224,6 +233,7 @@ impl StateRuntime { "failed to open memories db at {}: {err}", memories_path.display() ); + close_sqlite_pools(&[pool.as_ref(), logs_pool.as_ref(), goals_pool.as_ref()]).await; return Err(err); } }; @@ -236,7 +246,16 @@ impl StateRuntime { started.elapsed(), &backfill_state_result, ); - backfill_state_result?; + if let Err(err) = backfill_state_result { + close_sqlite_pools(&[ + pool.as_ref(), + logs_pool.as_ref(), + goals_pool.as_ref(), + memories_pool.as_ref(), + ]) + .await; + return Err(err); + } let started = Instant::now(); let thread_updated_at_millis_result: anyhow::Result> = sqlx::query_scalar("SELECT MAX(threads.updated_at_ms) FROM threads") @@ -250,7 +269,19 @@ impl StateRuntime { started.elapsed(), &thread_updated_at_millis_result, ); - let thread_updated_at_millis = thread_updated_at_millis_result?; + let thread_updated_at_millis = match thread_updated_at_millis_result { + Ok(value) => value, + Err(err) => { + close_sqlite_pools(&[ + pool.as_ref(), + logs_pool.as_ref(), + goals_pool.as_ref(), + memories_pool.as_ref(), + ]) + .await; + return Err(err); + } + }; let thread_updated_at_millis = thread_updated_at_millis.unwrap_or(0); let runtime = Arc::new(Self { thread_goals: GoalStore::new(Arc::clone(&goals_pool)), @@ -283,6 +314,14 @@ impl StateRuntime { &self.memories } + /// Close all SQLite pools and wait for outstanding pool workers to exit. + pub async fn close(&self) { + self.memories.close().await; + self.thread_goals.close().await; + self.logs_pool.close().await; + self.pool.close().await; + } + pub async fn clear_memory_data_in_sqlite_home(sqlite_home: &Path) -> anyhow::Result { let memories_path = MEMORIES_DB.path(sqlite_home); if !tokio::fs::try_exists(&memories_path).await? { @@ -302,6 +341,12 @@ impl StateRuntime { } } +async fn close_sqlite_pools(pools: &[&SqlitePool]) { + for pool in pools { + pool.close().await; + } +} + fn base_sqlite_options(path: &Path) -> SqliteConnectOptions { SqliteConnectOptions::new() .filename(path) @@ -367,7 +412,8 @@ async fn open_sqlite( started.elapsed(), &pool_result, ); - let pool = pool_result?; + let pool = pool_result + .map_err(|source| recovery::RuntimeDbInitError::new(spec.label, "open", path, source))?; let started = Instant::now(); let migrate_result = migrator.run(&pool).await.map_err(anyhow::Error::from); crate::telemetry::record_init_result( @@ -377,7 +423,10 @@ async fn open_sqlite( started.elapsed(), &migrate_result, ); - migrate_result?; + if let Err(source) = migrate_result { + pool.close().await; + return Err(recovery::RuntimeDbInitError::new(spec.label, "migrate", path, source).into()); + } Ok(pool) } diff --git a/codex-rs/state/src/runtime/goals.rs b/codex-rs/state/src/runtime/goals.rs index d0fab47d05..e5769ccd1f 100644 --- a/codex-rs/state/src/runtime/goals.rs +++ b/codex-rs/state/src/runtime/goals.rs @@ -11,6 +11,10 @@ impl GoalStore { pub(crate) fn new(pool: Arc) -> Self { Self { pool } } + + pub(crate) async fn close(&self) { + self.pool.close().await; + } } pub struct GoalUpdate { diff --git a/codex-rs/state/src/runtime/memories.rs b/codex-rs/state/src/runtime/memories.rs index 3101500796..895c254350 100644 --- a/codex-rs/state/src/runtime/memories.rs +++ b/codex-rs/state/src/runtime/memories.rs @@ -35,6 +35,10 @@ impl MemoryStore { Self { pool, state_pool } } + pub(crate) async fn close(&self) { + self.pool.close().await; + } + /// Deletes all persisted memory state in one transaction. /// /// This removes every `stage1_outputs` row and all `jobs` rows for the diff --git a/codex-rs/state/src/runtime/recovery.rs b/codex-rs/state/src/runtime/recovery.rs new file mode 100644 index 0000000000..d9cf0bcdc9 --- /dev/null +++ b/codex-rs/state/src/runtime/recovery.rs @@ -0,0 +1,243 @@ +//! Backup-and-rebuild support for Codex runtime SQLite databases. +//! +//! Codex keeps several independent runtime SQLite databases under one SQLite +//! home. When SQLite reports that one of them is corrupt, automatic recovery +//! moves only that database file and its sidecars into a backup folder so the +//! other databases keep their data. + +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; + +const BACKUP_DIR_NAME: &str = "db-backups"; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RuntimeDbBackup { + /// Path where the runtime database or sidecar lived before it was moved. + pub original_path: PathBuf, + /// Path where the runtime database or sidecar was backed up. + pub backup_path: PathBuf, +} + +#[derive(Debug)] +pub(crate) struct RuntimeDbInitError { + label: &'static str, + operation: &'static str, + path: PathBuf, + source: anyhow::Error, +} + +impl RuntimeDbInitError { + pub(crate) fn new( + label: &'static str, + operation: &'static str, + path: &Path, + source: anyhow::Error, + ) -> Self { + Self { + label, + operation, + path: path.to_path_buf(), + source, + } + } + + fn path(&self) -> &Path { + self.path.as_path() + } +} + +impl std::fmt::Display for RuntimeDbInitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "failed to {} {} at {}: {}", + self.operation, + self.label, + self.path.display(), + self.source + ) + } +} + +impl std::error::Error for RuntimeDbInitError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + +/// Move one Codex runtime SQLite database out of the way so that database can +/// be recreated without discarding unrelated runtime databases. +pub async fn backup_runtime_db_for_fresh_start( + db_path: &Path, +) -> std::io::Result> { + let sqlite_home = db_path.parent().ok_or_else(|| { + std::io::Error::other(format!( + "database path does not have a parent directory: {}", + db_path.display() + )) + })?; + match tokio::fs::metadata(sqlite_home).await { + Ok(metadata) if metadata.is_dir() => backup_runtime_db_files(db_path).await, + Ok(_) => backup_blocking_sqlite_home(sqlite_home).await, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + tokio::fs::create_dir_all(sqlite_home).await?; + Err(std::io::Error::other(format!( + "no Codex runtime database files were found to back up for {}", + db_path.display() + ))) + } + Err(err) => Err(err), + } +} + +pub fn runtime_db_path_for_corruption_error(err: &anyhow::Error) -> Option { + if !is_sqlite_corruption_error(err) { + return None; + } + err.chain() + .find_map(|source| source.downcast_ref::()) + .map(|err| err.path().to_path_buf()) +} + +pub fn is_sqlite_corruption_error(err: &anyhow::Error) -> bool { + err.chain().any(sqlite_error_source_is_corruption) +} + +fn sqlite_error_source_is_corruption(source: &(dyn std::error::Error + 'static)) -> bool { + let Some(err) = source.downcast_ref::() else { + return false; + }; + let sqlx::Error::Database(database_error) = err else { + return false; + }; + sqlite_error_detail_is_corruption(database_error.message()) + || database_error + .code() + .is_some_and(sqlite_database_code_is_corruption) +} + +fn sqlite_database_code_is_corruption(code: Cow<'_, str>) -> bool { + matches!( + code.as_ref().to_ascii_lowercase().as_str(), + "11" | "26" | "sqlite_corrupt" | "sqlite_notadb" + ) +} + +pub fn sqlite_error_detail_is_corruption(detail: &str) -> bool { + let detail = detail.to_ascii_lowercase(); + detail.contains("database disk image is malformed") + || detail.contains("database schema is malformed") + || detail.contains("database is corrupt") + || detail.contains("file is not a database") + || detail.contains("sqlite_corrupt") + || detail.contains("sqlite_notadb") + || detail.contains("(code: 11)") + || detail.contains("(code: 26)") +} + +pub fn sqlite_error_detail_is_lock(detail: &str) -> bool { + let detail = detail.to_ascii_lowercase(); + detail.contains("database is locked") || detail.contains("database is busy") +} + +async fn backup_runtime_db_files(db_path: &Path) -> std::io::Result> { + let sqlite_home = db_path.parent().ok_or_else(|| { + std::io::Error::other(format!( + "database path does not have a parent directory: {}", + db_path.display() + )) + })?; + backup_sqlite_paths(sqlite_home, sqlite_paths(db_path)).await +} + +async fn backup_sqlite_paths( + sqlite_home: &Path, + paths: impl IntoIterator, +) -> std::io::Result> { + let backup_dir = create_unique_backup_dir(sqlite_home.join(BACKUP_DIR_NAME).as_path()).await?; + let mut backups = Vec::new(); + + for path in paths { + if tokio::fs::try_exists(path.as_path()).await? { + let backup_path = backup_dir.join(file_name(path.as_path())?); + tokio::fs::rename(path.as_path(), backup_path.as_path()).await?; + backups.push(RuntimeDbBackup { + original_path: path, + backup_path, + }); + } + } + + if backups.is_empty() { + let _ = tokio::fs::remove_dir(backup_dir).await; + return Err(std::io::Error::other( + "no Codex runtime database files were found to back up", + )); + } + + Ok(backups) +} + +async fn backup_blocking_sqlite_home(sqlite_home: &Path) -> std::io::Result> { + let parent = sqlite_home.parent().ok_or_else(|| { + std::io::Error::other(format!( + "cannot create a backup folder for {}", + sqlite_home.display() + )) + })?; + let mut backup_dir_name = file_name(sqlite_home)?.to_os_string(); + backup_dir_name.push(format!(".{BACKUP_DIR_NAME}")); + let backup_parent = parent.join(backup_dir_name); + let backup_dir = create_unique_backup_dir(backup_parent.as_path()).await?; + let backup_path = backup_dir.join(file_name(sqlite_home)?); + tokio::fs::rename(sqlite_home, backup_path.as_path()).await?; + tokio::fs::create_dir_all(sqlite_home).await?; + Ok(vec![RuntimeDbBackup { + original_path: sqlite_home.to_path_buf(), + backup_path, + }]) +} + +fn sqlite_paths(db_path: &Path) -> Vec { + let mut wal_path = db_path.as_os_str().to_os_string(); + wal_path.push("-wal"); + let mut shm_path = db_path.as_os_str().to_os_string(); + shm_path.push("-shm"); + vec![ + db_path.to_path_buf(), + PathBuf::from(wal_path), + PathBuf::from(shm_path), + ] +} + +async fn create_unique_backup_dir(backup_parent: &Path) -> std::io::Result { + tokio::fs::create_dir_all(backup_parent).await?; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |duration| duration.as_secs()); + let mut sequence = 0_u32; + loop { + let backup_dir = backup_parent.join(format!("sqlite-{timestamp}-{sequence}")); + match tokio::fs::create_dir(backup_dir.as_path()).await { + Ok(()) => return Ok(backup_dir), + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + sequence += 1; + } + Err(err) => return Err(err), + } + } +} + +fn file_name(path: &Path) -> std::io::Result<&std::ffi::OsStr> { + path.file_name().ok_or_else(|| { + std::io::Error::other(format!( + "cannot create a backup name for {}", + path.display() + )) + }) +} + +#[cfg(test)] +#[path = "recovery_tests.rs"] +mod tests; diff --git a/codex-rs/state/src/runtime/recovery_tests.rs b/codex-rs/state/src/runtime/recovery_tests.rs new file mode 100644 index 0000000000..7424dbcfe1 --- /dev/null +++ b/codex-rs/state/src/runtime/recovery_tests.rs @@ -0,0 +1,105 @@ +use super::*; +use crate::runtime::test_support::unique_temp_dir; +use pretty_assertions::assert_eq; + +#[tokio::test] +async fn backup_moves_only_requested_runtime_db_files_to_backup_folder() -> std::io::Result<()> { + let sqlite_home = unique_temp_dir(); + tokio::fs::create_dir_all(sqlite_home.as_path()).await?; + let runtime_paths = super::super::runtime_db_paths(sqlite_home.as_path()); + let mut expected_paths = Vec::new(); + for db_path in runtime_paths.iter().map(|db| db.path.as_path()) { + for path in sqlite_paths(db_path) { + tokio::fs::write(path.as_path(), path.display().to_string()).await?; + expected_paths.push(path); + } + } + let failed_db_path = super::super::logs_db_path(sqlite_home.as_path()); + let failed_paths = sqlite_paths(failed_db_path.as_path()); + + let backups = backup_runtime_db_for_fresh_start(failed_db_path.as_path()).await?; + + assert_eq!(backups.len(), failed_paths.len()); + for path in &failed_paths { + assert!(!tokio::fs::try_exists(path.as_path()).await?); + } + for path in expected_paths + .iter() + .filter(|path| !failed_paths.contains(path)) + { + assert!(tokio::fs::try_exists(path.as_path()).await?); + } + for backup in backups { + assert!( + backup + .backup_path + .starts_with(sqlite_home.join(BACKUP_DIR_NAME)) + ); + assert!(tokio::fs::try_exists(backup.backup_path.as_path()).await?); + } + Ok(()) +} + +#[tokio::test] +async fn backup_replaces_blocking_sqlite_home_file() -> std::io::Result<()> { + let temp_dir = unique_temp_dir(); + tokio::fs::create_dir_all(temp_dir.as_path()).await?; + let sqlite_home = temp_dir.join("sqlite-home"); + tokio::fs::write(sqlite_home.as_path(), b"not-a-directory").await?; + + let backups = backup_runtime_db_for_fresh_start( + super::super::state_db_path(sqlite_home.as_path()).as_path(), + ) + .await?; + + assert_eq!(backups.len(), 1); + assert!(tokio::fs::metadata(sqlite_home.as_path()).await?.is_dir()); + assert!( + backups[0] + .backup_path + .starts_with(temp_dir.join(format!("sqlite-home.{BACKUP_DIR_NAME}"))) + ); + assert!(tokio::fs::try_exists(backups[0].backup_path.as_path()).await?); + Ok(()) +} + +#[test] +fn sqlite_error_detail_classifies_corruption_and_lock_errors() { + assert!(sqlite_error_detail_is_corruption("file is not a database")); + assert!(sqlite_error_detail_is_corruption( + "error returned from database: (code: 11) database disk image is malformed" + )); + assert!(!sqlite_error_detail_is_corruption("database is locked")); + assert!(sqlite_error_detail_is_lock("database is locked")); + assert!(sqlite_error_detail_is_lock("database is busy")); +} + +#[tokio::test] +async fn runtime_db_path_for_corruption_error_returns_failed_database_path() -> std::io::Result<()> +{ + let sqlite_home = unique_temp_dir(); + tokio::fs::create_dir_all(sqlite_home.as_path()).await?; + let path = super::super::state_db_path(sqlite_home.as_path()); + tokio::fs::write(path.as_path(), b"not sqlite").await?; + + let err = match super::super::StateRuntime::init(sqlite_home, "openai".to_string()).await { + Ok(_) => panic!("malformed sqlite should fail to initialize"), + Err(err) => err, + }; + + assert_eq!(runtime_db_path_for_corruption_error(&err), Some(path)); + Ok(()) +} + +#[test] +fn runtime_db_path_for_corruption_error_ignores_corrupt_word_in_path() { + let path = PathBuf::from("/tmp/sqlite_corrupt/state_5.sqlite"); + let err = anyhow::Error::new(RuntimeDbInitError::new( + "state DB", + "open", + path.as_path(), + anyhow::anyhow!("permission denied"), + )); + + assert_eq!(runtime_db_path_for_corruption_error(&err), None); +} diff --git a/codex-rs/state/src/runtime/threads.rs b/codex-rs/state/src/runtime/threads.rs index 1ccfab5e12..98eff16c50 100644 --- a/codex-rs/state/src/runtime/threads.rs +++ b/codex-rs/state/src/runtime/threads.rs @@ -884,17 +884,117 @@ ON CONFLICT(id) DO UPDATE SET self.upsert_thread(&metadata).await } - /// Delete a thread metadata row by id. + /// Delete a thread and all associated state by id. pub async fn delete_thread(&self, thread_id: ThreadId) -> anyhow::Result { - let result = sqlx::query("DELETE FROM threads WHERE id = ?") - .bind(thread_id.to_string()) - .execute(self.pool.as_ref()) - .await?; - let rows_affected = result.rows_affected(); - self.memories.delete_thread_memory(thread_id).await?; - if rows_affected > 0 { - let _ = self.thread_goals.delete_thread_goal(thread_id).await?; + self.delete_threads_strict(&[thread_id]).await + } + + /// Delete a set of threads and all associated state. + /// + /// Spawn edges and thread rows are deleted last so a failed delete can be retried with enough + /// state left to rediscover the same spawned subtree. + pub async fn delete_threads_strict(&self, thread_ids: &[ThreadId]) -> anyhow::Result { + if thread_ids.is_empty() { + return Ok(0); } + + let thread_id_strings = thread_ids + .iter() + .map(ThreadId::to_string) + .collect::>(); + for (thread_id, thread_id_string) in thread_ids.iter().zip(&thread_id_strings) { + sqlx::query("DELETE FROM logs WHERE thread_id = ?") + .bind(thread_id_string) + .execute(self.logs_pool.as_ref()) + .await?; + self.memories.delete_thread_memory(*thread_id).await?; + self.thread_goals.delete_thread_goal(*thread_id).await?; + } + + let now = Utc::now().timestamp(); + let mut tx = self.pool.begin().await?; + for thread_id_string in &thread_id_strings { + for parent_thread_id_string in &thread_id_strings { + // If both the job runner and worker are being deleted, requeueing + // the worker item would leave a running job with no loop to consume it. + sqlx::query( + r#" +UPDATE agent_jobs +SET status = ?, updated_at = ?, completed_at = ?, last_error = ? +WHERE status IN (?, ?) + AND id IN ( + SELECT item.job_id + FROM agent_job_items AS item + JOIN thread_spawn_edges AS edge ON edge.child_thread_id = item.assigned_thread_id + WHERE item.status = ? AND item.assigned_thread_id = ? AND edge.parent_thread_id = ? + ) + "#, + ) + .bind(AgentJobStatus::Cancelled.as_str()) + .bind(now) + .bind(now) + .bind("agent job runner thread was deleted") + .bind(AgentJobStatus::Pending.as_str()) + .bind(AgentJobStatus::Running.as_str()) + .bind(AgentJobItemStatus::Running.as_str()) + .bind(thread_id_string) + .bind(parent_thread_id_string) + .execute(&mut *tx) + .await?; + } + sqlx::query("DELETE FROM thread_dynamic_tools WHERE thread_id = ?") + .bind(thread_id_string) + .execute(&mut *tx) + .await?; + sqlx::query( + r#" +UPDATE agent_job_items +SET + status = ?, + assigned_thread_id = NULL, + updated_at = ?, + last_error = ? +WHERE assigned_thread_id = ? AND status = ? + "#, + ) + .bind(AgentJobItemStatus::Pending.as_str()) + .bind(now) + .bind("assigned thread was deleted") + .bind(thread_id_string) + .bind(AgentJobItemStatus::Running.as_str()) + .execute(&mut *tx) + .await?; + sqlx::query( + r#" +UPDATE agent_job_items +SET assigned_thread_id = NULL, updated_at = ? +WHERE assigned_thread_id = ? + "#, + ) + .bind(now) + .bind(thread_id_string) + .execute(&mut *tx) + .await?; + } + for thread_id_string in &thread_id_strings { + sqlx::query( + "DELETE FROM thread_spawn_edges WHERE parent_thread_id = ? OR child_thread_id = ?", + ) + .bind(thread_id_string) + .bind(thread_id_string) + .execute(&mut *tx) + .await?; + } + let mut rows_affected = 0; + for thread_id_string in &thread_id_strings { + rows_affected += sqlx::query("DELETE FROM threads WHERE id = ?") + .bind(thread_id_string) + .execute(&mut *tx) + .await? + .rows_affected(); + } + tx.commit().await?; + Ok(rows_affected) } } @@ -1136,12 +1236,14 @@ mod tests { use crate::DirectionalThreadSpawnEdgeStatus; use crate::runtime::test_support::test_thread_metadata; use crate::runtime::test_support::unique_temp_dir; + use anyhow::Result; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::GitInfo; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use pretty_assertions::assert_eq; + use serde_json::json; use std::path::PathBuf; #[tokio::test] @@ -1182,6 +1284,180 @@ mod tests { assert_eq!(memory_mode, "disabled"); } + #[tokio::test] + async fn delete_thread_cleans_associated_state() -> Result<()> { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()).await?; + let thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000401")?; + let child_thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000402")?; + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.clone(), + )) + .await?; + seed_thread_cleanup_state(&runtime, thread_id, child_thread_id).await?; + sqlx::query("INSERT INTO thread_dynamic_tools (thread_id, position, name, description, input_schema) VALUES (?, ?, ?, ?, ?)") + .bind(thread_id.to_string()) + .bind(0_i64) + .bind("test_tool") + .bind("test dynamic tool") + .bind("{}") + .execute(runtime.pool.as_ref()) + .await?; + runtime + .create_agent_job( + &AgentJobCreateParams { + id: "job-1".to_string(), + name: "test-job".to_string(), + instruction: "Return a result".to_string(), + auto_export: true, + max_runtime_seconds: None, + output_schema_json: None, + input_headers: vec!["path".to_string()], + input_csv_path: "/tmp/in.csv".to_string(), + output_csv_path: "/tmp/out.csv".to_string(), + }, + &[AgentJobItemCreateParams { + item_id: "item-1".to_string(), + row_index: 0, + source_id: None, + row_json: json!({"path": "file-1"}), + }], + ) + .await?; + runtime.mark_agent_job_running("job-1").await?; + runtime + .mark_agent_job_item_running_with_thread( + "job-1", + "item-1", + &child_thread_id.to_string(), + ) + .await?; + + let rows = runtime + .delete_threads_strict(&[thread_id, child_thread_id]) + .await?; + + assert_eq!(rows, 1); + assert!(runtime.get_thread(thread_id).await?.is_none()); + let dynamic_tool_count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM thread_dynamic_tools WHERE thread_id = ?") + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await?; + assert_eq!(dynamic_tool_count, 0); + assert_thread_cleanup_state(&runtime, thread_id).await?; + let job_item = runtime + .get_agent_job_item("job-1", "item-1") + .await? + .expect("job item should exist"); + assert_eq!(job_item.status, AgentJobItemStatus::Pending); + assert_eq!(job_item.assigned_thread_id, None); + assert_eq!( + job_item.last_error, + Some("assigned thread was deleted".to_string()) + ); + let job = runtime + .get_agent_job("job-1") + .await? + .expect("job should exist"); + assert_eq!(job.status, AgentJobStatus::Cancelled); + + let missing_thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000403")?; + let missing_child_thread_id = + ThreadId::from_string("00000000-0000-0000-0000-000000000404")?; + seed_thread_cleanup_state(&runtime, missing_thread_id, missing_child_thread_id).await?; + + assert_eq!(runtime.delete_thread(missing_thread_id).await?, 0); + assert_thread_cleanup_state(&runtime, missing_thread_id).await?; + Ok(()) + } + + #[tokio::test] + async fn delete_thread_keeps_retry_graph_on_cleanup_failure() -> Result<()> { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()).await?; + let thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000405")?; + let child_thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000406")?; + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.clone(), + )) + .await?; + seed_thread_cleanup_state(&runtime, thread_id, child_thread_id).await?; + + runtime.logs_pool.close().await; + runtime + .delete_thread(thread_id) + .await + .expect_err("closed log db should fail deletion"); + + assert!(runtime.get_thread(thread_id).await?.is_some()); + assert_eq!( + runtime.list_thread_spawn_descendants(thread_id).await?, + vec![child_thread_id] + ); + Ok(()) + } + + async fn seed_thread_cleanup_state( + runtime: &StateRuntime, + thread_id: ThreadId, + child_thread_id: ThreadId, + ) -> Result<()> { + runtime + .upsert_thread_spawn_edge( + thread_id, + child_thread_id, + DirectionalThreadSpawnEdgeStatus::Closed, + ) + .await?; + runtime + .thread_goals() + .replace_thread_goal( + thread_id, + "test goal", + crate::ThreadGoalStatus::Active, + /*token_budget*/ None, + ) + .await?; + sqlx::query("INSERT INTO logs (ts, ts_nanos, level, target, feedback_log_body, thread_id) VALUES (1, 0, 'INFO', 'test', 'feedback log', ?)") + .bind(thread_id.to_string()) + .execute(runtime.logs_pool.as_ref()) + .await?; + Ok(()) + } + + async fn assert_thread_cleanup_state( + runtime: &StateRuntime, + thread_id: ThreadId, + ) -> Result<()> { + let spawn_edge_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM thread_spawn_edges WHERE parent_thread_id = ? OR child_thread_id = ?", + ) + .bind(thread_id.to_string()) + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await?; + assert_eq!(spawn_edge_count, 0); + assert_eq!( + runtime.thread_goals().get_thread_goal(thread_id).await?, + None + ); + let logs = runtime + .query_logs(&LogQuery { + thread_ids: vec![thread_id.to_string()], + ..Default::default() + }) + .await?; + assert!(logs.is_empty()); + Ok(()) + } + #[tokio::test] async fn list_threads_updated_after_returns_oldest_changes_first() { let codex_home = unique_temp_dir(); diff --git a/codex-rs/thread-manager-sample/src/main.rs b/codex-rs/thread-manager-sample/src/main.rs index 6f0be601ea..f96e3021a0 100644 --- a/codex-rs/thread-manager-sample/src/main.rs +++ b/codex-rs/thread-manager-sample/src/main.rs @@ -16,6 +16,7 @@ use codex_core_api::AskForApproval; use codex_core_api::AuthCredentialsStoreMode; use codex_core_api::AuthManager; use codex_core_api::AutoCompactTokenLimitScope; +use codex_core_api::CodexHomeUserInstructionsProvider; use codex_core_api::CodexThread; use codex_core_api::Config; use codex_core_api::ConfigLayerStack; @@ -120,12 +121,16 @@ async fn run_main(arg0_paths: Arg0DispatchPaths) -> anyhow::Result<()> { .await?, ); let installation_id = resolve_installation_id(&config.codex_home).await?; + let user_instructions_provider = Arc::new(CodexHomeUserInstructionsProvider::new( + config.codex_home.clone(), + )); let thread_manager = ThreadManager::new( &config, auth_manager, SessionSource::Exec, environment_manager, empty_extension_registry(), + user_instructions_provider, /*analytics_events_client*/ None, Arc::clone(&thread_store), state_db, @@ -184,7 +189,6 @@ fn new_config(model: Option, arg0_paths: Arg0DispatchPaths) -> anyhow::R enforce_residency: Constrained::allow_any(/*initial_value*/ None), hide_agent_reasoning: false, show_raw_agent_reasoning: false, - user_instructions: None, base_instructions: None, developer_instructions: None, guardian_policy_config: None, diff --git a/codex-rs/thread-store/src/in_memory.rs b/codex-rs/thread-store/src/in_memory.rs index 2ab5103663..07156fe355 100644 --- a/codex-rs/thread-store/src/in_memory.rs +++ b/codex-rs/thread-store/src/in_memory.rs @@ -18,6 +18,7 @@ use codex_protocol::protocol::ThreadMemoryMode; use crate::AppendThreadItemsParams; use crate::ArchiveThreadParams; use crate::CreateThreadParams; +use crate::DeleteThreadParams; use crate::ListThreadsParams; use crate::LoadThreadHistoryParams; use crate::ReadThreadByRolloutPathParams; @@ -115,6 +116,7 @@ pub struct InMemoryThreadStoreCalls { pub update_thread_metadata: usize, pub archive_thread: usize, pub unarchive_thread: usize, + pub delete_thread: usize, } /// In-memory [`ThreadStore`] implementation for tests and debug configs. @@ -333,6 +335,25 @@ impl ThreadStore for InMemoryThreadStore { state.calls.unarchive_thread += 1; stored_thread_from_state(&state, params.thread_id, /*include_history*/ false) } + + async fn delete_thread(&self, params: DeleteThreadParams) -> ThreadStoreResult<()> { + let mut state = self.state.lock().await; + state.calls.delete_thread += 1; + let existed = state.histories.remove(¶ms.thread_id).is_some(); + state.created_threads.remove(¶ms.thread_id); + state.names.remove(¶ms.thread_id); + state.metadata_updates.remove(¶ms.thread_id); + state + .rollout_paths + .retain(|_, thread_id| *thread_id != params.thread_id); + if existed { + Ok(()) + } else { + Err(ThreadStoreError::ThreadNotFound { + thread_id: params.thread_id, + }) + } + } } fn stored_thread_from_state( diff --git a/codex-rs/thread-store/src/lib.rs b/codex-rs/thread-store/src/lib.rs index 84b72f23f0..284614bf19 100644 --- a/codex-rs/thread-store/src/lib.rs +++ b/codex-rs/thread-store/src/lib.rs @@ -25,6 +25,7 @@ pub use types::AppendThreadItemsParams; pub use types::ArchiveThreadParams; pub use types::ClearableField; pub use types::CreateThreadParams; +pub use types::DeleteThreadParams; pub use types::ExtraConfig; pub use types::GitInfoPatch; pub use types::ItemPage; diff --git a/codex-rs/thread-store/src/local/delete_thread.rs b/codex-rs/thread-store/src/local/delete_thread.rs new file mode 100644 index 0000000000..ecd962f647 --- /dev/null +++ b/codex-rs/thread-store/src/local/delete_thread.rs @@ -0,0 +1,210 @@ +//! Local hard-delete support for persisted threads. +//! +//! Existing rollout files are deleted before this operation reports success. A rollout file that +//! vanishes after discovery counts as already deleted. SQLite cleanup happens at the app-server +//! layer after every associated rollout has been removed so failed deletes can be retried. + +use std::io::ErrorKind; +use std::path::Path; + +use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +use codex_rollout::SESSIONS_SUBDIR; +use codex_rollout::find_archived_thread_path_by_id_str; +use codex_rollout::find_thread_path_by_id_str; +use codex_rollout::remove_thread_name_entries; + +use super::LocalThreadStore; +use super::helpers::matching_rollout_file_name; +use super::helpers::scoped_rollout_path; +use crate::DeleteThreadParams; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn delete_thread( + store: &LocalThreadStore, + params: DeleteThreadParams, +) -> ThreadStoreResult<()> { + let thread_id = params.thread_id; + let thread_id_str = thread_id.to_string(); + let state_db_ctx = store.state_db().await; + let mut rollout_paths = Vec::new(); + + match find_thread_path_by_id_str( + store.config.codex_home.as_path(), + thread_id_str.as_str(), + state_db_ctx.as_deref(), + ) + .await + { + Ok(Some(path)) => rollout_paths.push(path), + Ok(None) => {} + Err(err) => { + return Err(ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + }); + } + } + + match find_archived_thread_path_by_id_str( + store.config.codex_home.as_path(), + thread_id_str.as_str(), + state_db_ctx.as_deref(), + ) + .await + { + Ok(Some(path)) => { + if !rollout_paths.contains(&path) { + rollout_paths.push(path); + } + } + Ok(None) => {} + Err(err) => { + return Err(ThreadStoreError::InvalidRequest { + message: format!("failed to locate archived thread id {thread_id}: {err}"), + }); + } + } + + let found_rollout_path = !rollout_paths.is_empty(); + for rollout_path in rollout_paths { + delete_rollout_file(store, rollout_path.as_path(), thread_id)?; + } + remove_thread_name_entries(store.config.codex_home.as_path(), thread_id) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to delete thread name index entries for {thread_id}: {err}"), + })?; + + if !found_rollout_path { + return Err(ThreadStoreError::ThreadNotFound { thread_id }); + } + + store.live_recorders.lock().await.remove(&thread_id); + + Ok(()) +} + +fn delete_rollout_file( + store: &LocalThreadStore, + rollout_path: &Path, + thread_id: codex_protocol::ThreadId, +) -> ThreadStoreResult { + let plain_path = codex_rollout::plain_rollout_path(rollout_path); + let compressed_path = plain_path.with_extension("jsonl.zst"); + let deleted_plain = delete_rollout_path(store, plain_path.as_path(), thread_id)?; + let deleted_compressed = delete_rollout_path(store, compressed_path.as_path(), thread_id)?; + Ok(deleted_plain || deleted_compressed) +} + +fn delete_rollout_path( + store: &LocalThreadStore, + rollout_path: &Path, + thread_id: codex_protocol::ThreadId, +) -> ThreadStoreResult { + let canonical_rollout_path = scoped_rollout_path( + store.config.codex_home.join(SESSIONS_SUBDIR), + rollout_path, + "sessions", + ) + .or_else(|_| { + scoped_rollout_path( + store.config.codex_home.join(ARCHIVED_SESSIONS_SUBDIR), + rollout_path, + "archived sessions", + ) + }) + .or_else(|err| match rollout_path.try_exists() { + Ok(false) => Ok(rollout_path.to_path_buf()), + Ok(true) | Err(_) => Err(err), + })?; + matching_rollout_file_name(&canonical_rollout_path, thread_id, rollout_path)?; + match std::fs::remove_file(&canonical_rollout_path) { + Ok(()) => Ok(true), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(false), + Err(err) => Err(ThreadStoreError::Internal { + message: format!( + "failed to delete rollout file `{}`: {err}", + canonical_rollout_path.display() + ), + }), + } +} + +#[cfg(test)] +mod tests { + use codex_protocol::ThreadId; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_archived_session_file; + use crate::local::test_support::write_session_file; + + #[tokio::test] + async fn delete_thread_removes_active_and_archived_rollouts() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", Uuid::from_u128(301)) + .expect("session file"); + let compressed_path = active_path.with_extension("jsonl.zst"); + std::fs::write(&compressed_path, b"compressed sibling").expect("compressed sibling"); + let cases = [ + (Uuid::from_u128(301), active_path), + ( + Uuid::from_u128(302), + write_archived_session_file( + home.path(), + "2025-01-03T12-00-00", + Uuid::from_u128(302), + ) + .expect("archived session file"), + ), + ]; + + for (uuid, path) in cases { + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + store + .delete_thread(DeleteThreadParams { thread_id }) + .await + .expect("delete thread"); + + assert!(!path.exists()); + } + assert!(!compressed_path.exists()); + } + + #[tokio::test] + async fn delete_rollout_file_treats_vanished_path_as_already_deleted() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let uuid = Uuid::from_u128(305); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + std::fs::remove_file(&path).expect("remove session file"); + + assert!(!delete_rollout_file(&store, path.as_path(), thread_id).expect("delete rollout")); + } + + #[tokio::test] + async fn delete_thread_reports_missing_thread() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let thread_id = + ThreadId::from_string("00000000-0000-0000-0000-000000000304").expect("valid thread id"); + + let err = store + .delete_thread(DeleteThreadParams { thread_id }) + .await + .expect_err("missing thread should fail"); + assert_eq!( + err.to_string(), + "thread 00000000-0000-0000-0000-000000000304 not found" + ); + } +} diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs index cef4496a02..3000fee9d9 100644 --- a/codex-rs/thread-store/src/local/mod.rs +++ b/codex-rs/thread-store/src/local/mod.rs @@ -1,5 +1,6 @@ mod archive_thread; mod create_thread; +mod delete_thread; mod helpers; mod list_threads; mod live_writer; @@ -24,6 +25,7 @@ use tokio::sync::Mutex; use crate::AppendThreadItemsParams; use crate::ArchiveThreadParams; use crate::CreateThreadParams; +use crate::DeleteThreadParams; use crate::ListThreadsParams; use crate::LoadThreadHistoryParams; use crate::ReadThreadByRolloutPathParams; @@ -288,6 +290,10 @@ impl ThreadStore for LocalThreadStore { ) -> ThreadStoreResult { unarchive_thread::unarchive_thread(self, params).await } + + async fn delete_thread(&self, params: DeleteThreadParams) -> ThreadStoreResult<()> { + delete_thread::delete_thread(self, params).await + } } #[cfg(test)] diff --git a/codex-rs/thread-store/src/store.rs b/codex-rs/thread-store/src/store.rs index 7276a980da..6373dd9b18 100644 --- a/codex-rs/thread-store/src/store.rs +++ b/codex-rs/thread-store/src/store.rs @@ -5,6 +5,7 @@ use std::any::Any; use crate::AppendThreadItemsParams; use crate::ArchiveThreadParams; use crate::CreateThreadParams; +use crate::DeleteThreadParams; use crate::ItemPage; use crate::ListItemsParams; use crate::ListThreadsParams; @@ -119,4 +120,7 @@ pub trait ThreadStore: Any + Send + Sync { &self, params: ArchiveThreadParams, ) -> ThreadStoreResult; + + /// Deletes a thread's persisted rollout data and associated metadata. + async fn delete_thread(&self, params: DeleteThreadParams) -> ThreadStoreResult<()>; } diff --git a/codex-rs/thread-store/src/types.rs b/codex-rs/thread-store/src/types.rs index 1f18adf081..853d866bc8 100644 --- a/codex-rs/thread-store/src/types.rs +++ b/codex-rs/thread-store/src/types.rs @@ -653,6 +653,13 @@ pub struct ArchiveThreadParams { pub thread_id: ThreadId, } +/// Parameters for deleting a thread. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeleteThreadParams { + /// Thread id to delete. + pub thread_id: ThreadId, +} + #[cfg(test)] mod tests { use pretty_assertions::assert_eq; diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs index 848bab3e1c..51c313eb0e 100644 --- a/codex-rs/tools/src/lib.rs +++ b/codex-rs/tools/src/lib.rs @@ -100,7 +100,6 @@ pub use tool_output::ToolOutput; pub use tool_payload::ToolPayload; pub use tool_search::ToolSearchEntry; pub use tool_search::ToolSearchInfo; -pub use tool_search::default_tool_search_text; pub use tool_spec::ResponsesApiWebSearchFilters; pub use tool_spec::ResponsesApiWebSearchUserLocation; pub use tool_spec::ToolSpec; diff --git a/codex-rs/tools/src/request_plugin_install.rs b/codex-rs/tools/src/request_plugin_install.rs index 70e9cb093d..5d6352b7f8 100644 --- a/codex-rs/tools/src/request_plugin_install.rs +++ b/codex-rs/tools/src/request_plugin_install.rs @@ -47,6 +47,10 @@ pub struct RequestPluginInstallMeta<'a> { pub tool_name: &'a str, #[serde(skip_serializing_if = "Option::is_none")] pub install_url: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub remote_plugin_id: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub app_connector_ids: Option<&'a [String]>, } pub fn build_request_plugin_install_elicitation_request( @@ -57,8 +61,6 @@ pub fn build_request_plugin_install_elicitation_request( suggest_reason: &str, tool: &DiscoverableTool, ) -> McpServerElicitationRequestParams { - let tool_name = tool.name().to_string(); - let install_url = tool.install_url().map(ToString::to_string); let message = suggest_reason.to_string(); McpServerElicitationRequestParams { @@ -70,9 +72,7 @@ pub fn build_request_plugin_install_elicitation_request( args.tool_type, args.action_type, suggest_reason, - tool.id(), - tool_name.as_str(), - install_url.as_deref(), + tool, ))), message, requested_schema: McpElicitationSchema { @@ -108,19 +108,26 @@ fn build_request_plugin_install_meta<'a>( tool_type: DiscoverableToolType, action_type: DiscoverableToolAction, suggest_reason: &'a str, - tool_id: &'a str, - tool_name: &'a str, - install_url: Option<&'a str>, + tool: &'a DiscoverableTool, ) -> RequestPluginInstallMeta<'a> { + let (remote_plugin_id, app_connector_ids) = match tool { + DiscoverableTool::Connector(_) => (None, None), + DiscoverableTool::Plugin(plugin) => ( + plugin.remote_plugin_id.as_deref(), + Some(plugin.app_connector_ids.as_slice()), + ), + }; RequestPluginInstallMeta { codex_approval_kind: REQUEST_PLUGIN_INSTALL_APPROVAL_KIND_VALUE, persist: REQUEST_PLUGIN_INSTALL_PERSIST_ALWAYS_VALUE, tool_type, suggest_type: action_type, suggest_reason, - tool_id, - tool_name, - install_url, + tool_id: tool.id(), + tool_name: tool.name(), + install_url: tool.install_url(), + remote_plugin_id, + app_connector_ids, } } diff --git a/codex-rs/tools/src/request_plugin_install_tests.rs b/codex-rs/tools/src/request_plugin_install_tests.rs index ff2370ade7..ab8af9be67 100644 --- a/codex-rs/tools/src/request_plugin_install_tests.rs +++ b/codex-rs/tools/src/request_plugin_install_tests.rs @@ -57,6 +57,8 @@ fn build_request_plugin_install_elicitation_request_uses_expected_shape() { install_url: Some( "https://chatgpt.com/apps/google-calendar/connector_2128aebfecb84f64a069897515042a44" ), + remote_plugin_id: None, + app_connector_ids: None, })), message: "Plan and reference events from your calendar".to_string(), requested_schema: McpElicitationSchema { @@ -71,15 +73,16 @@ fn build_request_plugin_install_elicitation_request_uses_expected_shape() { } #[test] -fn build_request_plugin_install_elicitation_request_for_plugin_omits_install_url() { +fn build_request_plugin_install_elicitation_request_injects_plugin_metadata() { let args = RequestPluginInstallArgs { tool_type: DiscoverableToolType::Plugin, action_type: DiscoverableToolAction::Install, - tool_id: "sample@openai-curated".to_string(), + tool_id: "sample@openai-curated-remote".to_string(), suggest_reason: "Use the sample plugin's skills and MCP server".to_string(), }; let plugin = DiscoverableTool::Plugin(Box::new(DiscoverablePluginInfo { - id: "sample@openai-curated".to_string(), + id: "sample@openai-curated-remote".to_string(), + remote_plugin_id: Some("plugins~Plugin_sample".to_string()), name: "Sample Plugin".to_string(), description: Some("Includes skills, MCP servers, and apps.".to_string()), has_skills: true, @@ -109,9 +112,11 @@ fn build_request_plugin_install_elicitation_request_for_plugin_omits_install_url tool_type: DiscoverableToolType::Plugin, suggest_type: DiscoverableToolAction::Install, suggest_reason: "Use the sample plugin's skills and MCP server", - tool_id: "sample@openai-curated", + tool_id: "sample@openai-curated-remote", tool_name: "Sample Plugin", install_url: None, + remote_plugin_id: Some("plugins~Plugin_sample"), + app_connector_ids: Some(&["connector_calendar".to_string()]), })), message: "Use the sample plugin's skills and MCP server".to_string(), requested_schema: McpElicitationSchema { @@ -127,13 +132,28 @@ fn build_request_plugin_install_elicitation_request_for_plugin_omits_install_url #[test] fn build_request_plugin_install_meta_uses_expected_shape() { + let connector = DiscoverableTool::Connector(Box::new(AppInfo { + id: "connector_68df038e0ba48191908c8434991bbac2".to_string(), + name: "Gmail".to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some( + "https://chatgpt.com/apps/gmail/connector_68df038e0ba48191908c8434991bbac2".to_string(), + ), + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + })); let meta = build_request_plugin_install_meta( DiscoverableToolType::Connector, DiscoverableToolAction::Install, "Find and reference emails from your inbox", - "connector_68df038e0ba48191908c8434991bbac2", - "Gmail", - Some("https://chatgpt.com/apps/gmail/connector_68df038e0ba48191908c8434991bbac2"), + &connector, ); assert_eq!( @@ -149,6 +169,8 @@ fn build_request_plugin_install_meta_uses_expected_shape() { install_url: Some( "https://chatgpt.com/apps/gmail/connector_68df038e0ba48191908c8434991bbac2" ), + remote_plugin_id: None, + app_connector_ids: None, }, ); } diff --git a/codex-rs/tools/src/tool_config_tests.rs b/codex-rs/tools/src/tool_config_tests.rs index 0e3917d697..7a4738772c 100644 --- a/codex-rs/tools/src/tool_config_tests.rs +++ b/codex-rs/tools/src/tool_config_tests.rs @@ -39,6 +39,7 @@ fn model_with_shell_type(shell_type: ConfigShellToolType) -> ModelInfo { context_window: None, max_context_window: None, auto_compact_token_limit: None, + comp_hash: None, effective_context_window_percent: 95, experimental_supported_tools: Vec::new(), input_modalities: codex_protocol::openai_models::default_input_modalities(), diff --git a/codex-rs/tools/src/tool_discovery.rs b/codex-rs/tools/src/tool_discovery.rs index ac5aac904b..2584d7f17a 100644 --- a/codex-rs/tools/src/tool_discovery.rs +++ b/codex-rs/tools/src/tool_discovery.rs @@ -93,6 +93,7 @@ pub fn filter_request_plugin_install_discoverable_tools_for_client( #[derive(Clone, Debug, PartialEq, Eq)] pub struct DiscoverablePluginInfo { pub id: String, + pub remote_plugin_id: Option, pub name: String, pub description: Option, pub has_skills: bool, diff --git a/codex-rs/tools/src/tool_discovery_tests.rs b/codex-rs/tools/src/tool_discovery_tests.rs index 6e45260c0e..5a3310057c 100644 --- a/codex-rs/tools/src/tool_discovery_tests.rs +++ b/codex-rs/tools/src/tool_discovery_tests.rs @@ -37,6 +37,7 @@ fn filter_request_plugin_install_discoverable_tools_for_codex_tui_omits_plugins( })), DiscoverableTool::Plugin(Box::new(DiscoverablePluginInfo { id: "slack@openai-curated".to_string(), + remote_plugin_id: None, name: "Slack".to_string(), description: Some("Search Slack messages".to_string()), has_skills: true, diff --git a/codex-rs/tools/src/tool_executor.rs b/codex-rs/tools/src/tool_executor.rs index 708ed1c3b1..294b361f09 100644 --- a/codex-rs/tools/src/tool_executor.rs +++ b/codex-rs/tools/src/tool_executor.rs @@ -58,7 +58,7 @@ pub trait ToolExecutor: Send + Sync { fn search_info(&self) -> Option { let spec = self.spec(); - ToolSearchInfo::from_tool_spec(&self.tool_name(), spec, /*source_info*/ None) + ToolSearchInfo::from_tool_spec(spec, /*source_info*/ None) } fn supports_parallel_tool_calls(&self) -> bool { diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs index fe57f23ce9..f2eed18fac 100644 --- a/codex-rs/tools/src/tool_output.rs +++ b/codex-rs/tools/src/tool_output.rs @@ -93,6 +93,7 @@ where pub struct JsonToolOutput { value: JsonValue, success: Option, + contains_external_context: bool, } impl JsonToolOutput { @@ -100,11 +101,21 @@ impl JsonToolOutput { Self { value, success: Some(true), + contains_external_context: false, } } pub fn with_success(value: JsonValue, success: Option) -> Self { - Self { value, success } + Self { + value, + success, + contains_external_context: false, + } + } + + pub fn with_external_context(mut self) -> Self { + self.contains_external_context = true; + self } } @@ -117,6 +128,10 @@ impl ToolOutput for JsonToolOutput { self.success.unwrap_or(true) } + fn contains_external_context(&self) -> bool { + self.contains_external_context + } + fn to_response_item(&self, call_id: &str, payload: &ToolPayload) -> ResponseInputItem { let output = FunctionCallOutputPayload { body: FunctionCallOutputBody::Text(self.value.to_string()), diff --git a/codex-rs/tools/src/tool_search.rs b/codex-rs/tools/src/tool_search.rs index ecb5504d4f..1083d9ea92 100644 --- a/codex-rs/tools/src/tool_search.rs +++ b/codex-rs/tools/src/tool_search.rs @@ -2,7 +2,6 @@ use crate::JsonSchema; use crate::LoadableToolSpec; use crate::ResponsesApiNamespaceTool; use crate::ResponsesApiTool; -use crate::ToolName; use crate::ToolSearchSourceInfo; use crate::ToolSpec; use crate::default_namespace_description; @@ -21,11 +20,10 @@ pub struct ToolSearchInfo { impl ToolSearchInfo { pub fn from_tool_spec( - tool_name: &ToolName, spec: ToolSpec, source_info: Option, ) -> Option { - let search_text = default_tool_search_text(tool_name, &spec); + let search_text = default_tool_search_text(&spec); Self::from_spec(search_text, spec, source_info) } @@ -67,13 +65,8 @@ impl ToolSearchInfo { } } -pub fn default_tool_search_text(tool_name: &ToolName, spec: &ToolSpec) -> String { +fn default_tool_search_text(spec: &ToolSpec) -> String { let mut parts = Vec::new(); - push_search_part(&mut parts, tool_name.to_string()); - push_search_part(&mut parts, tool_name.name.replace('_', " ")); - if let Some(namespace) = &tool_name.namespace { - push_search_part(&mut parts, namespace.clone()); - } match spec { ToolSpec::Function(tool) => append_function_search_text(tool, &mut parts), @@ -137,3 +130,7 @@ fn push_search_part(parts: &mut Vec, part: String) { parts.push(part.to_string()); } } + +#[cfg(test)] +#[path = "tool_search_tests.rs"] +mod tests; diff --git a/codex-rs/tools/src/tool_search_tests.rs b/codex-rs/tools/src/tool_search_tests.rs new file mode 100644 index 0000000000..bf910c57f1 --- /dev/null +++ b/codex-rs/tools/src/tool_search_tests.rs @@ -0,0 +1,48 @@ +use super::*; +use pretty_assertions::assert_eq; +use std::collections::BTreeMap; + +#[test] +fn default_search_text_uses_model_visible_namespace_metadata_once() { + let mut schedule_schema = JsonSchema::object( + BTreeMap::from([( + "timezone".to_string(), + JsonSchema::string(Some("IANA timezone.".to_string())), + )]), + /*required*/ None, + /*additional_properties*/ None, + ); + schedule_schema.description = Some("Schedule settings.".to_string()); + let mut parameters = JsonSchema::object( + BTreeMap::from([ + ( + "mode".to_string(), + JsonSchema::string(Some("Update mode.".to_string())), + ), + ("schedule".to_string(), schedule_schema), + ]), + /*required*/ None, + /*additional_properties*/ None, + ); + parameters.description = Some("Automation options.".to_string()); + let spec = ToolSpec::Namespace(crate::ResponsesApiNamespace { + name: "codex_app".to_string(), + description: "Manage Codex automations.".to_string(), + tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { + name: "automation_update".to_string(), + description: "Create or update automations.".to_string(), + strict: false, + defer_loading: None, + parameters, + output_schema: None, + })], + }); + + let search_info = ToolSearchInfo::from_tool_spec(spec, /*source_info*/ None) + .expect("namespace should be searchable"); + + assert_eq!( + search_info.entry.search_text, + "codex_app Manage Codex automations. automation_update automation update Create or update automations. Automation options. mode Update mode. schedule Schedule settings. timezone IANA timezone." + ); +} diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index b239ff4895..0df5b81d5e 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -37,8 +37,6 @@ use crate::cwd_prompt::CwdPromptAction; use crate::diff_render::DiffSummary; use crate::exec_command::split_command_string; use crate::exec_command::strip_bash_lc_and_escape; -use crate::external_agent_config_migration_startup::ExternalAgentConfigMigrationStartupOutcome; -use crate::external_agent_config_migration_startup::handle_external_agent_config_migration_prompt_if_needed; use crate::external_editor; use crate::file_search::FileSearchManager; use crate::history_cell; @@ -53,8 +51,6 @@ use crate::legacy_core::config::ConfigBuilder; use crate::legacy_core::config::ConfigOverrides; use crate::legacy_core::config::PermissionProfileSnapshot; use crate::legacy_core::config::edit::ConfigEditsBuilder; -#[cfg(target_os = "windows")] -use crate::legacy_core::windows_sandbox::WindowsSandboxLevelExt; use crate::model_catalog::ModelCatalog; use crate::model_migration::ModelMigrationOutcome; use crate::model_migration::migration_copy_for_models; @@ -401,7 +397,7 @@ const COMMIT_ANIMATION_TICK: Duration = tui::TARGET_FRAME_INTERVAL; pub struct AppExitInfo { pub token_usage: TokenUsage, pub thread_id: Option, - pub thread_name: Option, + pub resume_hint: Option, pub update_action: Option, pub exit_reason: ExitReason, } @@ -411,7 +407,7 @@ impl AppExitInfo { Self { token_usage: TokenUsage::default(), thread_id: None, - thread_name: None, + resume_hint: None, update_action: None, exit_reason: ExitReason::Fatal(message.into()), } @@ -437,10 +433,7 @@ fn session_summary( rollout_path: Option<&Path>, ) -> Option { let usage_line = (!token_usage.is_zero()).then(|| token_usage.to_string()); - let resumable_thread = resumable_thread(thread_id, thread_name, rollout_path); - let resume_hint = resumable_thread.as_ref().and_then(|thread| { - codex_utils_cli::resume_hint(thread.thread_name.as_deref(), Some(thread.thread_id)) - }); + let resume_hint = resume_hint_for_resumable_thread(thread_id, thread_name, rollout_path); if usage_line.is_none() && resume_hint.is_none() { return None; @@ -471,6 +464,15 @@ fn resumable_thread( }) } +fn resume_hint_for_resumable_thread( + thread_id: Option, + thread_name: Option, + rollout_path: Option<&Path>, +) -> Option { + let thread = resumable_thread(thread_id, thread_name, rollout_path)?; + codex_utils_cli::resume_hint(thread.thread_name.as_deref(), Some(thread.thread_id)) +} + fn rollout_path_is_resumable(rollout_path: &Path) -> bool { std::fs::metadata(rollout_path).is_ok_and(|metadata| metadata.is_file() && metadata.len() > 0) } @@ -764,7 +766,6 @@ impl App { session_selection: SessionSelection, feedback: codex_feedback::CodexFeedback, is_first_run: bool, - entered_trust_nux: bool, should_prompt_windows_sandbox_nux_at_startup: bool, app_server_target: AppServerTarget, state_db: Option, @@ -786,39 +787,6 @@ impl App { let harness_overrides = normalize_harness_overrides_for_cwd(harness_overrides, &config.cwd)?; - let external_agent_config_migration_outcome = - handle_external_agent_config_migration_prompt_if_needed( - tui, - &mut app_server, - &mut config, - &cli_kv_overrides, - &harness_overrides, - &cloud_config_bundle, - entered_trust_nux, - ) - .await?; - let external_agent_config_migration_message = match external_agent_config_migration_outcome - { - ExternalAgentConfigMigrationStartupOutcome::Continue { success_message } => { - success_message - } - ExternalAgentConfigMigrationStartupOutcome::ExitRequested => { - app_server - .shutdown() - .await - .inspect_err(|err| { - tracing::warn!("app-server shutdown failed: {err}"); - }) - .ok(); - return Ok(AppExitInfo { - token_usage: TokenUsage::default(), - thread_id: None, - thread_name: None, - update_action: None, - exit_reason: ExitReason::UserRequested, - }); - } - }; let bootstrap = match startup_bootstrap { Some(bootstrap) => bootstrap, None => app_server.bootstrap(&config).await?, @@ -1014,10 +982,6 @@ impl App { }; chat_widget.remote_connection = remote_connection; let thread_and_widget_ms = thread_and_widget_started_at.elapsed().as_millis(); - if let Some(message) = external_agent_config_migration_message { - chat_widget.add_info_message(message, /*hint*/ None); - } - chat_widget .maybe_prompt_windows_sandbox_enable(should_prompt_windows_sandbox_nux_at_startup); @@ -1103,7 +1067,7 @@ See the Codex keymap documentation for supported actions and examples." #[cfg(target_os = "windows")] { let startup_permission_profile = app.config.permissions.effective_permission_profile(); - let should_check = WindowsSandboxLevel::from_config(&app.config) + let should_check = crate::windows_sandbox::level_from_config(&app.config) != WindowsSandboxLevel::Disabled && managed_filesystem_sandbox_is_restricted(&startup_permission_profile) && !app @@ -1258,15 +1222,16 @@ See the Codex keymap documentation for supported actions and examples." return Err(err); } }; - let resumable_thread = resumable_thread( - app.chat_widget.thread_id(), + let thread_id = app.chat_widget.thread_id().or(app.primary_thread_id); + let resume_hint = resume_hint_for_resumable_thread( + thread_id, app.chat_widget.thread_name(), app.chat_widget.rollout_path().as_deref(), ); Ok(AppExitInfo { token_usage: app.token_usage(), - thread_id: resumable_thread.as_ref().map(|thread| thread.thread_id), - thread_name: resumable_thread.and_then(|thread| thread.thread_name), + thread_id, + resume_hint, update_action: app.pending_update_action, exit_reason, }) diff --git a/codex-rs/tui/src/app/app_server_event_targets.rs b/codex-rs/tui/src/app/app_server_event_targets.rs index 800560d520..fa2bab0111 100644 --- a/codex-rs/tui/src/app/app_server_event_targets.rs +++ b/codex-rs/tui/src/app/app_server_event_targets.rs @@ -49,6 +49,7 @@ pub(super) fn server_notification_thread_target( Some(notification.thread_id.as_str()) } ServerNotification::ThreadArchived(notification) => Some(notification.thread_id.as_str()), + ServerNotification::ThreadDeleted(notification) => Some(notification.thread_id.as_str()), ServerNotification::ThreadUnarchived(notification) => Some(notification.thread_id.as_str()), ServerNotification::ThreadClosed(notification) => Some(notification.thread_id.as_str()), ServerNotification::ThreadNameUpdated(notification) => { diff --git a/codex-rs/tui/src/app/app_server_events.rs b/codex-rs/tui/src/app/app_server_events.rs index 268dbd305b..6f6940c067 100644 --- a/codex-rs/tui/src/app/app_server_events.rs +++ b/codex-rs/tui/src/app/app_server_events.rs @@ -93,16 +93,25 @@ impl App { return; } ServerNotification::ExternalAgentConfigImportCompleted(_) => { - let cwd = self.chat_widget.config_ref().cwd.to_path_buf(); + let should_report_completion = + app_server_client.consume_external_agent_config_import_completion(); if let Err(err) = self.refresh_in_memory_config_from_disk().await { tracing::warn!( error = %err, "failed to refresh config after external agent config import" ); } + let cwd = self.chat_widget.config_ref().cwd.to_path_buf(); self.chat_widget.refresh_plugin_mentions(); self.chat_widget.submit_op(AppCommand::reload_user_config()); self.fetch_plugins_list(app_server_client, cwd); + if should_report_completion { + self.chat_widget.add_info_message( + crate::external_agent_config_migration_flow::EXTERNAL_AGENT_CONFIG_MIGRATION_FINISHED_MESSAGE + .to_string(), + /*hint*/ None, + ); + } return; } ServerNotification::AppListUpdated(notification) => { diff --git a/codex-rs/tui/src/app/config_persistence.rs b/codex-rs/tui/src/app/config_persistence.rs index 7d1f6047e0..80ed84d01e 100644 --- a/codex-rs/tui/src/app/config_persistence.rs +++ b/codex-rs/tui/src/app/config_persistence.rs @@ -964,7 +964,7 @@ impl App { fn propagate_windows_sandbox_turn_context(&self) { #[cfg(target_os = "windows")] { - let windows_sandbox_level = WindowsSandboxLevel::from_config(&self.config); + let windows_sandbox_level = crate::windows_sandbox::level_from_config(&self.config); self.app_event_tx .send(AppEvent::CodexOp(AppCommand::override_turn_context( /*cwd*/ None, diff --git a/codex-rs/tui/src/app/event_dispatch.rs b/codex-rs/tui/src/app/event_dispatch.rs index 2d97a513c0..48ad956789 100644 --- a/codex-rs/tui/src/app/event_dispatch.rs +++ b/codex-rs/tui/src/app/event_dispatch.rs @@ -6,6 +6,7 @@ use super::resize_reflow::trailing_run_start; use super::*; use crate::config_update::format_config_error; +use crate::external_agent_config_migration_flow::ExternalAgentConfigMigrationFlowOutcome; #[cfg(target_os = "windows")] use codex_config::types::WindowsSandboxModeToml; @@ -110,6 +111,31 @@ impl App { // Leaving alt-screen may blank the inline viewport; force a redraw either way. tui.frame_requester().schedule_frame(); } + AppEvent::OpenExternalAgentConfigMigration => { + match crate::external_agent_config_migration_flow::handle_external_agent_config_migration_prompt( + tui, + app_server, + &self.config, + ) + .await + { + Ok(ExternalAgentConfigMigrationFlowOutcome::Started(message)) => { + self.chat_widget.add_info_message(message, /*hint*/ None); + } + Ok(ExternalAgentConfigMigrationFlowOutcome::NoItems) => { + self.chat_widget.add_info_message( + crate::external_agent_config_migration_flow::EXTERNAL_AGENT_CONFIG_MIGRATION_NO_ITEMS_MESSAGE + .to_string(), + /*hint*/ None, + ); + } + Ok(ExternalAgentConfigMigrationFlowOutcome::Cancelled) => {} + Err(error_message) => { + self.chat_widget.add_error_message(error_message); + } + } + tui.frame_requester().schedule_frame(); + } AppEvent::ResumeSessionByIdOrName(id_or_name) => { match crate::lookup_session_target_with_app_server(app_server, &id_or_name).await? { Some(target_session) => { @@ -127,6 +153,9 @@ impl App { AppEvent::ArchiveCurrentThread => { return Ok(self.archive_current_thread(app_server).await); } + AppEvent::DeleteCurrentThread => { + return Ok(self.delete_current_thread(app_server).await); + } AppEvent::ForkCurrentSession => { self.session_telemetry.counter( "codex.thread.fork", @@ -920,9 +949,7 @@ impl App { // If the elevated setup already ran on this machine, don't prompt for // elevation again - just flip the config to use the elevated path. - if crate::legacy_core::windows_sandbox::sandbox_setup_is_complete( - codex_home.as_path(), - ) { + if crate::windows_sandbox::sandbox_setup_is_complete(codex_home.as_path()) { tx.send(AppEvent::EnableWindowsSandboxForAgentMode { preset, mode: WindowsSandboxEnableMode::Elevated, @@ -935,7 +962,7 @@ impl App { self.windows_sandbox.setup_started_at = Some(Instant::now()); let session_telemetry = self.session_telemetry.clone(); tokio::task::spawn_blocking(move || { - let result = crate::legacy_core::windows_sandbox::run_elevated_setup( + let result = crate::windows_sandbox::run_elevated_setup( &permission_profile, workspace_roots.as_slice(), command_cwd.as_path(), @@ -959,9 +986,7 @@ impl App { let mut code_tag: Option = None; let mut message_tag: Option = None; if let Some((code, message)) = - crate::legacy_core::windows_sandbox::elevated_setup_failure_details( - &err, - ) + crate::windows_sandbox::elevated_setup_failure_details(&err) { code_tag = Some(code); message_tag = Some(message); @@ -974,7 +999,7 @@ impl App { tags.push(("message", message)); } session_telemetry.counter( - crate::legacy_core::windows_sandbox::elevated_setup_failure_metric_name( + crate::windows_sandbox::elevated_setup_failure_metric_name( &err, ), /*inc*/ 1, @@ -1045,12 +1070,12 @@ impl App { self.chat_widget.show_windows_sandbox_setup_status(); tokio::task::spawn_blocking(move || { if let Err(err) = - crate::legacy_core::windows_sandbox::run_legacy_setup_preflight( + codex_windows_sandbox::run_windows_sandbox_legacy_preflight( &permission_profile, workspace_roots.as_slice(), + codex_home.as_path(), command_cwd.as_path(), &env_map, - codex_home.as_path(), ) { session_telemetry.counter( @@ -1094,7 +1119,7 @@ impl App { tokio::task::spawn_blocking(move || { let requested_path = PathBuf::from(path); - let event = match crate::legacy_core::grant_read_root_non_elevated( + let event = match crate::windows_sandbox::grant_read_root_non_elevated( &permission_profile, workspace_roots.as_slice(), command_cwd.as_path(), @@ -1190,7 +1215,7 @@ impl App { self.config.permissions.windows_sandbox_mode, ); let windows_sandbox_level = - WindowsSandboxLevel::from_config(&self.config); + crate::windows_sandbox::level_from_config(&self.config); if let Some((sample_paths, extra_count, failed_scan)) = self.chat_widget.world_writable_warning_details() { @@ -1533,7 +1558,7 @@ impl App { return Ok(AppRunControl::Continue); } - let should_check = WindowsSandboxLevel::from_config(&self.config) + let should_check = crate::windows_sandbox::level_from_config(&self.config) != WindowsSandboxLevel::Disabled && permission_profile_is_managed_restricted && !self.chat_widget.world_writable_warning_hidden(); @@ -2247,4 +2272,31 @@ impl App { } } } + + pub(super) async fn delete_current_thread( + &mut self, + app_server: &mut AppServerSession, + ) -> AppRunControl { + let Some(thread_id) = self.active_thread_id.or(self.chat_widget.thread_id()) else { + self.chat_widget + .add_error_message("A thread must start before it can be deleted.".to_string()); + return AppRunControl::Continue; + }; + if self.side_threads.contains_key(&thread_id) { + self.chat_widget.add_error_message( + "'/delete' is unavailable in side conversations. Press Ctrl+C to return to the main thread first." + .to_string(), + ); + return AppRunControl::Continue; + } + + match app_server.thread_delete(thread_id).await { + Ok(()) => AppRunControl::Exit(ExitReason::UserRequested), + Err(err) => { + self.chat_widget + .add_error_message(format!("Failed to delete current thread: {err}")); + AppRunControl::Continue + } + } + } } diff --git a/codex-rs/tui/src/app/startup_prompts.rs b/codex-rs/tui/src/app/startup_prompts.rs index 4834656617..57f70a559e 100644 --- a/codex-rs/tui/src/app/startup_prompts.rs +++ b/codex-rs/tui/src/app/startup_prompts.rs @@ -343,7 +343,7 @@ pub(super) async fn handle_model_migration_prompt_if_needed( return Some(AppExitInfo { token_usage: TokenUsage::default(), thread_id: None, - thread_name: None, + resume_hint: None, update_action: None, exit_reason: ExitReason::UserRequested, }); diff --git a/codex-rs/tui/src/app/test_support.rs b/codex-rs/tui/src/app/test_support.rs index 4e5aa8b10c..e0eafaa23b 100644 --- a/codex-rs/tui/src/app/test_support.rs +++ b/codex-rs/tui/src/app/test_support.rs @@ -6,12 +6,14 @@ use super::*; use crate::chatwidget::tests::make_chatwidget_manual_with_sender; +use codex_models_manager::test_support::construct_model_info_offline_for_tests; +use codex_models_manager::test_support::get_model_offline_for_tests; pub(super) async fn make_test_app() -> App { let (chat_widget, app_event_tx, _rx, _op_rx) = make_chatwidget_manual_with_sender().await; let config = chat_widget.config_ref().clone(); let file_search = FileSearchManager::new(config.cwd.to_path_buf(), app_event_tx.clone()); - let model = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model = get_model_offline_for_tests(config.model.as_deref()); let session_telemetry = test_session_telemetry(&config, model.as_str()); App { @@ -68,7 +70,8 @@ pub(super) async fn make_test_app() -> App { } fn test_session_telemetry(config: &Config, model: &str) -> SessionTelemetry { - let model_info = crate::legacy_core::test_support::construct_model_info_offline(model, config); + let model_info = + construct_model_info_offline_for_tests(model, &config.to_models_manager_config()); SessionTelemetry::new( ThreadId::new(), model, diff --git a/codex-rs/tui/src/app/tests.rs b/codex-rs/tui/src/app/tests.rs index ed89c72878..e8ed3e33da 100644 --- a/codex-rs/tui/src/app/tests.rs +++ b/codex-rs/tui/src/app/tests.rs @@ -75,6 +75,8 @@ use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput; use codex_app_server_protocol::UserInput as AppServerUserInput; use codex_app_server_protocol::WarningNotification; +use codex_models_manager::test_support::construct_model_info_offline_for_tests; +use codex_models_manager::test_support::get_model_offline_for_tests; use codex_otel::SessionTelemetry; use codex_protocol::ThreadId; use codex_protocol::config_types::CollaborationMode; @@ -307,7 +309,7 @@ async fn enqueue_primary_thread_session_replays_turns_before_initial_prompt_subm let thread_id = ThreadId::new(); let initial_prompt = "follow-up after replay".to_string(); let config = app.config.clone(); - let model = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model = get_model_offline_for_tests(config.model.as_deref()); app.chat_widget = ChatWidget::new_with_app_event(ChatWidgetInit { config, frame_requester: crate::tui::FrameRequester::test_dummy(), @@ -4024,7 +4026,7 @@ async fn make_test_app() -> App { let (chat_widget, app_event_tx, _rx, _op_rx) = make_chatwidget_manual_with_sender().await; let config = chat_widget.config_ref().clone(); let file_search = FileSearchManager::new(config.cwd.to_path_buf(), app_event_tx.clone()); - let model = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model = get_model_offline_for_tests(config.model.as_deref()); let session_telemetry = test_session_telemetry(&config, model.as_str()); App { @@ -4088,7 +4090,7 @@ async fn make_test_app_with_channels() -> ( let (chat_widget, app_event_tx, rx, op_rx) = make_chatwidget_manual_with_sender().await; let config = chat_widget.config_ref().clone(); let file_search = FileSearchManager::new(config.cwd.to_path_buf(), app_event_tx.clone()); - let model = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model = get_model_offline_for_tests(config.model.as_deref()); let session_telemetry = test_session_telemetry(&config, model.as_str()); ( @@ -4607,7 +4609,8 @@ fn lines_to_single_string(lines: &[Line<'_>]) -> String { } fn test_session_telemetry(config: &Config, model: &str) -> SessionTelemetry { - let model_info = crate::legacy_core::test_support::construct_model_info_offline(model, config); + let model_info = + construct_model_info_offline_for_tests(model, &config.to_models_manager_config()); SessionTelemetry::new( ThreadId::new(), model, diff --git a/codex-rs/tui/src/app/tests/model_catalog.rs b/codex-rs/tui/src/app/tests/model_catalog.rs index 961c65159a..ac4a50af8f 100644 --- a/codex-rs/tui/src/app/tests/model_catalog.rs +++ b/codex-rs/tui/src/app/tests/model_catalog.rs @@ -6,7 +6,7 @@ use pretty_assertions::assert_eq; use tokio::sync::mpsc::unbounded_channel; fn all_model_presets() -> Vec { - crate::legacy_core::test_support::all_model_presets().clone() + crate::test_support::TEST_MODEL_PRESETS.clone() } fn model_availability_nux_config(shown_count: &[(&str, u32)]) -> ModelAvailabilityNuxConfig { diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index 41fa6bdce4..ee6364c0de 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -221,12 +221,18 @@ pub(crate) enum AppEvent { /// Open the resume picker inside the running TUI session. OpenResumePicker, + /// Open the external agent migration picker inside the running TUI session. + OpenExternalAgentConfigMigration, + /// Resume a thread by UUID or thread name inside the running TUI session. ResumeSessionByIdOrName(String), /// Archive the current active main thread and exit after it succeeds. ArchiveCurrentThread, + /// Permanently delete the current active main thread and exit after it succeeds. + DeleteCurrentThread, + /// Fork the current session into a new thread. ForkCurrentSession, diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index e29b40111c..47184cfee2 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -53,6 +53,8 @@ use codex_app_server_protocol::ThreadBackgroundTerminalsCleanParams; use codex_app_server_protocol::ThreadBackgroundTerminalsCleanResponse; use codex_app_server_protocol::ThreadCompactStartParams; use codex_app_server_protocol::ThreadCompactStartResponse; +use codex_app_server_protocol::ThreadDeleteParams; +use codex_app_server_protocol::ThreadDeleteResponse; use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadForkResponse; use codex_app_server_protocol::ThreadGoalClearParams; @@ -128,12 +130,16 @@ use color_eyre::eyre::Result; use color_eyre::eyre::WrapErr; use std::collections::HashMap; use std::path::PathBuf; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; use uuid::Uuid; const JSONRPC_INVALID_REQUEST: i64 = -32600; const JSONRPC_METHOD_NOT_FOUND: i64 = -32601; +pub(crate) const EXTERNAL_AGENT_CONFIG_IMPORT_IN_PROGRESS_MESSAGE: &str = + "A previous agent import is still running. Wait for it to finish before importing again."; const THREAD_SETTINGS_UPDATE_METHOD: &str = "thread/settings/update"; fn bootstrap_request_error(context: &'static str, err: TypedRequestError) -> color_eyre::Report { @@ -176,6 +182,7 @@ pub(crate) struct AppServerSession { thread_settings_update_supported: bool, default_model: Option, available_models: Vec, + external_agent_config_import_completion_pending: AtomicBool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -219,6 +226,7 @@ impl AppServerSession { thread_settings_update_supported: true, default_model: None, available_models: Vec::new(), + external_agent_config_import_completion_pending: AtomicBool::new(false), } } @@ -235,6 +243,10 @@ impl AppServerSession { matches!(self.thread_params_mode, ThreadParamsMode::Remote) } + pub(crate) fn uses_embedded_app_server(&self) -> bool { + matches!(&self.client, AppServerClient::InProcess(_)) + } + pub(crate) fn server_version(&self) -> Option<&str> { let AppServerClient::Remote(client) = &self.client else { return None; @@ -357,21 +369,48 @@ impl AppServerSession { self.client .request_typed(ClientRequest::ExternalAgentConfigDetect { request_id, params }) .await - .wrap_err("externalAgentConfig/detect failed during TUI startup") + .wrap_err("externalAgentConfig/detect failed during agent import") } pub(crate) async fn external_agent_config_import( &mut self, migration_items: Vec, - ) -> Result { + ) -> Result<()> { + // Mark the import active before sending the request so a fast completion notification + // cannot arrive before the TUI records it. + if self + .external_agent_config_import_completion_pending + .swap(true, Ordering::Relaxed) + { + color_eyre::eyre::bail!(EXTERNAL_AGENT_CONFIG_IMPORT_IN_PROGRESS_MESSAGE); + } let request_id = self.next_request_id(); - self.client + let response: Result = self + .client .request_typed(ClientRequest::ExternalAgentConfigImport { request_id, params: ExternalAgentConfigImportParams { migration_items }, }) .await - .wrap_err("externalAgentConfig/import failed during TUI startup") + .wrap_err("externalAgentConfig/import failed during agent import"); + match response { + Ok(_) => Ok(()), + Err(err) => { + self.external_agent_config_import_completion_pending + .store(false, Ordering::Relaxed); + Err(err) + } + } + } + + pub(crate) fn external_agent_config_import_in_progress(&self) -> bool { + self.external_agent_config_import_completion_pending + .load(Ordering::Relaxed) + } + + pub(crate) fn consume_external_agent_config_import_completion(&self) -> bool { + self.external_agent_config_import_completion_pending + .swap(false, Ordering::Relaxed) } pub(crate) async fn next_event(&mut self) -> Option { @@ -589,6 +628,21 @@ impl AppServerSession { Ok(()) } + pub(crate) async fn thread_delete(&mut self, thread_id: ThreadId) -> Result<()> { + let request_id = self.next_request_id(); + let _: ThreadDeleteResponse = self + .client + .request_typed(ClientRequest::ThreadDelete { + request_id, + params: ThreadDeleteParams { + thread_id: thread_id.to_string(), + }, + }) + .await + .wrap_err("failed to delete session")?; + Ok(()) + } + pub(crate) async fn thread_unarchive(&mut self, thread_id: ThreadId) -> Result { let request_id = self.next_request_id(); let response: ThreadUnarchiveResponse = self @@ -1199,6 +1253,7 @@ pub(crate) fn status_account_display_from_auth_mode( email: None, plan: plan_type.map(plan_type_display_name), }), + Some(AuthMode::BedrockApiKey) => None, None => None, } } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 6fd95e8dd5..d8326f3101 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -59,11 +59,7 @@ use crate::bottom_pane::TerminalTitleSetupView; use crate::diff_model::FileChange; use crate::git_action_directives::parse_assistant_markdown; use crate::legacy_core::config::Config; -use crate::legacy_core::config::Constrained; -use crate::legacy_core::config::ConstraintResult; use crate::legacy_core::config::PermissionProfileSnapshot; -#[cfg(any(target_os = "windows", test))] -use crate::legacy_core::windows_sandbox::WindowsSandboxLevelExt; use crate::mention_codec::LinkedMention; use crate::mention_codec::encode_history_mentions; use crate::model_catalog::ModelCatalog; @@ -126,6 +122,8 @@ use codex_app_server_protocol::TurnPlanStepStatus; use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput; use codex_config::ConfigLayerStackOrdering; +use codex_config::Constrained; +use codex_config::ConstraintResult; use codex_config::types::ApprovalsReviewer; use codex_config::types::Notifications; use codex_config::types::WindowsSandboxModeToml; @@ -265,7 +263,7 @@ use crate::app_event::AppEvent; use crate::app_event::ExitMode; use crate::app_event::PermissionProfileSelection; use crate::app_event::RateLimitRefreshOrigin; -#[cfg(any(target_os = "windows", test))] +#[cfg(target_os = "windows")] use crate::app_event::WindowsSandboxEnableMode; use crate::app_event_sender::AppEventSender; use crate::auto_review_denials; diff --git a/codex-rs/tui/src/chatwidget/constructor.rs b/codex-rs/tui/src/chatwidget/constructor.rs index efd252c15d..10a785ed2b 100644 --- a/codex-rs/tui/src/chatwidget/constructor.rs +++ b/codex-rs/tui/src/chatwidget/constructor.rs @@ -251,13 +251,12 @@ impl ChatWidget { .bottom_pane .set_queued_message_edit_binding(widget.queued_message_edit_hint_binding); #[cfg(target_os = "windows")] - widget.bottom_pane.set_windows_degraded_sandbox_active( - crate::legacy_core::windows_sandbox::ELEVATED_SANDBOX_NUX_ENABLED - && matches!( - WindowsSandboxLevel::from_config(&widget.config), - WindowsSandboxLevel::RestrictedToken - ), - ); + widget + .bottom_pane + .set_windows_degraded_sandbox_active(matches!( + crate::windows_sandbox::level_from_config(&widget.config), + WindowsSandboxLevel::RestrictedToken + )); widget.update_collaboration_mode_indicator(); widget diff --git a/codex-rs/tui/src/chatwidget/permission_popups.rs b/codex-rs/tui/src/chatwidget/permission_popups.rs index c0d492d7aa..1283a9e490 100644 --- a/codex-rs/tui/src/chatwidget/permission_popups.rs +++ b/codex-rs/tui/src/chatwidget/permission_popups.rs @@ -29,7 +29,7 @@ impl ChatWidget { let presets: Vec = builtin_approval_presets(); #[cfg(target_os = "windows")] - let windows_sandbox_level = WindowsSandboxLevel::from_config(&self.config); + let windows_sandbox_level = crate::windows_sandbox::level_from_config(&self.config); #[cfg(target_os = "windows")] let windows_degraded_sandbox_enabled = matches!(windows_sandbox_level, WindowsSandboxLevel::RestrictedToken); @@ -37,9 +37,7 @@ impl ChatWidget { let windows_degraded_sandbox_enabled = false; let show_elevate_sandbox_hint = - crate::legacy_core::windows_sandbox::ELEVATED_SANDBOX_NUX_ENABLED - && windows_degraded_sandbox_enabled - && presets.iter().any(|preset| preset.id == "auto"); + windows_degraded_sandbox_enabled && presets.iter().any(|preset| preset.id == "auto"); let guardian_disabled_reason = |enabled: bool| { let mut next_features = self.config.features.get().clone(); @@ -326,13 +324,13 @@ impl ChatWidget { if approvals_reviewer == ApprovalsReviewer::User && preset.id == "auto" { #[cfg(target_os = "windows")] { - if WindowsSandboxLevel::from_config(&self.config) == WindowsSandboxLevel::Disabled { + if crate::windows_sandbox::level_from_config(&self.config) + == WindowsSandboxLevel::Disabled + { let preset = preset.clone(); - if crate::legacy_core::windows_sandbox::ELEVATED_SANDBOX_NUX_ENABLED - && crate::legacy_core::windows_sandbox::sandbox_setup_is_complete( - self.config.codex_home.as_path(), - ) - { + if crate::windows_sandbox::sandbox_setup_is_complete( + self.config.codex_home.as_path(), + ) { return vec![Box::new(move |tx| { tx.send(AppEvent::EnableWindowsSandboxForAgentMode { preset: preset.clone(), diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index 3682f245f5..51e4cedf85 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -224,6 +224,7 @@ impl ChatWidget { | ServerNotification::ThreadStarted(_) | ServerNotification::ThreadStatusChanged(_) | ServerNotification::ThreadArchived(_) + | ServerNotification::ThreadDeleted(_) | ServerNotification::ThreadUnarchived(_) | ServerNotification::RawResponseItemCompleted(_) | ServerNotification::CommandExecOutputDelta(_) diff --git a/codex-rs/tui/src/chatwidget/settings.rs b/codex-rs/tui/src/chatwidget/settings.rs index c05788acaf..bed42e2bb4 100644 --- a/codex-rs/tui/src/chatwidget/settings.rs +++ b/codex-rs/tui/src/chatwidget/settings.rs @@ -58,13 +58,11 @@ impl ChatWidget { pub(crate) fn set_windows_sandbox_mode(&mut self, mode: Option) { self.config.permissions.windows_sandbox_mode = mode; #[cfg(target_os = "windows")] - self.bottom_pane.set_windows_degraded_sandbox_active( - crate::legacy_core::windows_sandbox::ELEVATED_SANDBOX_NUX_ENABLED - && matches!( - WindowsSandboxLevel::from_config(&self.config), - WindowsSandboxLevel::RestrictedToken - ), - ); + self.bottom_pane + .set_windows_degraded_sandbox_active(matches!( + crate::windows_sandbox::level_from_config(&self.config), + WindowsSandboxLevel::RestrictedToken + )); } #[cfg_attr(not(target_os = "windows"), allow(dead_code))] @@ -121,13 +119,11 @@ impl ChatWidget { feature, Feature::WindowsSandbox | Feature::WindowsSandboxElevated ) { - self.bottom_pane.set_windows_degraded_sandbox_active( - crate::legacy_core::windows_sandbox::ELEVATED_SANDBOX_NUX_ENABLED - && matches!( - WindowsSandboxLevel::from_config(&self.config), - WindowsSandboxLevel::RestrictedToken - ), - ); + self.bottom_pane + .set_windows_degraded_sandbox_active(matches!( + crate::windows_sandbox::level_from_config(&self.config), + WindowsSandboxLevel::RestrictedToken + )); } enabled } diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs index e7776a3007..5d0efea2fc 100644 --- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs +++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs @@ -193,6 +193,34 @@ impl ChatWidget { }); self.request_redraw(); } + SlashCommand::Delete => { + self.bottom_pane.show_selection_view(SelectionViewParams { + title: Some("Delete this session?".to_string()), + subtitle: Some( + "Cannot be undone. Subagent threads will also be deleted.".to_string(), + ), + footer_hint: Some(standard_popup_hint_line()), + items: vec![ + SelectionItem { + name: "No, keep this session".to_string(), + description: Some("Return to the current session".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: "Yes, delete and exit".to_string(), + description: Some("Permanently delete this session now".to_string()), + actions: vec![Box::new(|tx| { + tx.send(AppEvent::DeleteCurrentThread); + })], + dismiss_on_select: true, + ..Default::default() + }, + ], + ..Default::default() + }); + self.request_redraw(); + } SlashCommand::Clear => { self.app_event_tx.send(AppEvent::ClearUi); } @@ -289,12 +317,11 @@ impl ChatWidget { SlashCommand::ElevateSandbox => { #[cfg(target_os = "windows")] { - let windows_sandbox_level = WindowsSandboxLevel::from_config(&self.config); + let windows_sandbox_level = + crate::windows_sandbox::level_from_config(&self.config); let windows_degraded_sandbox_enabled = matches!(windows_sandbox_level, WindowsSandboxLevel::RestrictedToken); - if !windows_degraded_sandbox_enabled - || !crate::legacy_core::windows_sandbox::ELEVATED_SANDBOX_NUX_ENABLED - { + if !windows_degraded_sandbox_enabled { // This command should not be visible/recognized outside degraded mode, // but guard anyway in case something dispatches it directly. return; @@ -398,6 +425,10 @@ impl ChatWidget { SlashCommand::Skills => { self.open_skills_menu(); } + SlashCommand::Import => { + self.app_event_tx + .send(AppEvent::OpenExternalAgentConfigMigration); + } SlashCommand::Hooks => { self.add_hooks_output(); } @@ -936,7 +967,7 @@ impl ChatWidget { fn builtin_command_flags(&self) -> BuiltinCommandFlags { #[cfg(target_os = "windows")] let allow_elevate_sandbox = { - let windows_sandbox_level = WindowsSandboxLevel::from_config(&self.config); + let windows_sandbox_level = crate::windows_sandbox::level_from_config(&self.config); matches!(windows_sandbox_level, WindowsSandboxLevel::RestrictedToken) }; #[cfg(not(target_os = "windows"))] @@ -982,6 +1013,7 @@ impl ChatWidget { SlashCommand::Feedback | SlashCommand::New | SlashCommand::Archive + | SlashCommand::Delete | SlashCommand::Clear | SlashCommand::Resume | SlashCommand::Fork @@ -1010,6 +1042,7 @@ impl ChatWidget { | SlashCommand::Logout | SlashCommand::Mention | SlashCommand::Skills + | SlashCommand::Import | SlashCommand::Hooks | SlashCommand::Title | SlashCommand::Statusline diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_delete_confirmation_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_delete_confirmation_popup.snap new file mode 100644 index 0000000000..de5c0610a0 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_delete_confirmation_popup.snap @@ -0,0 +1,11 @@ +--- +source: tui/src/chatwidget/tests/slash_commands.rs +expression: popup +--- + Delete this session? + Cannot be undone. Subagent threads will also be deleted. + +› 1. No, keep this session Return to the current session + 2. Yes, delete and exit Permanently delete this session now + + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index ba89379e73..c20768e10e 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -21,8 +21,6 @@ pub(super) use crate::diff_model::FileChange; pub(super) use crate::history_cell::UserHistoryCell; pub(super) use crate::legacy_core::config::Config; pub(super) use crate::legacy_core::config::ConfigBuilder; -pub(super) use crate::legacy_core::config::Constrained; -pub(super) use crate::legacy_core::config::ConstraintError; pub(super) use crate::model_catalog::ModelCatalog; pub(super) use crate::test_backend::VT100Backend; pub(super) use crate::test_support::PathBufExt; @@ -120,6 +118,8 @@ pub(super) use codex_app_server_protocol::UserInput; pub(super) use codex_app_server_protocol::UserInput as AppServerUserInput; pub(super) use codex_app_server_protocol::WarningNotification; pub(super) use codex_config::ConfigLayerStack; +pub(super) use codex_config::Constrained; +pub(super) use codex_config::ConstraintError; pub(super) use codex_config::RequirementSource; pub(super) use codex_config::types::ApprovalsReviewer; pub(super) use codex_config::types::Notifications; @@ -129,6 +129,8 @@ pub(super) use codex_core_skills::model::SkillMetadata; pub(super) use codex_features::FEATURES; pub(super) use codex_features::Feature; pub(super) use codex_git_utils::CommitLogEntry; +pub(super) use codex_models_manager::test_support::construct_model_info_offline_for_tests; +pub(super) use codex_models_manager::test_support::get_model_offline_for_tests; pub(super) use codex_otel::RuntimeMetricsSummary; pub(super) use codex_otel::SessionTelemetry; pub(super) use codex_protocol::ThreadId; diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index 214ca969aa..3dfc373b59 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -19,7 +19,6 @@ pub(super) async fn test_config() -> Config { config.cwd = PathBuf::from(test_path_display("/tmp/project")).abs(); config.config_layer_stack = ConfigLayerStack::default(); config.startup_warnings.clear(); - config.user_instructions = None; config } @@ -119,7 +118,8 @@ pub(super) fn snapshot(percent: f64) -> RateLimitSnapshot { } pub(super) fn test_session_telemetry(config: &Config, model: &str) -> SessionTelemetry { - let model_info = crate::legacy_core::test_support::construct_model_info_offline(model, config); + let model_info = + construct_model_info_offline_for_tests(model, &config.to_models_manager_config()); SessionTelemetry::new( ThreadId::new(), model, @@ -136,7 +136,7 @@ pub(super) fn test_session_telemetry(config: &Config, model: &str) -> SessionTel pub(super) fn test_model_catalog(_config: &Config) -> Arc { Arc::new(ModelCatalog::new( - crate::legacy_core::test_support::all_model_presets().clone(), + crate::test_support::TEST_MODEL_PRESETS.clone(), )) } @@ -152,9 +152,9 @@ pub(super) async fn make_chatwidget_manual( let app_event_tx = AppEventSender::new(tx_raw); let (op_tx, op_rx) = unbounded_channel::(); let mut cfg = test_config().await; - let resolved_model = model_override.map(str::to_owned).unwrap_or_else(|| { - crate::legacy_core::test_support::get_model_offline(cfg.model.as_deref()) - }); + let resolved_model = model_override + .map(str::to_owned) + .unwrap_or_else(|| get_model_offline_for_tests(cfg.model.as_deref())); if let Some(model) = model_override { cfg.model = Some(model.to_string()); } @@ -1414,7 +1414,7 @@ pub(super) fn plugins_test_detail( description: Option<&str>, skills: &[&str], hooks: &[(codex_app_server_protocol::HookEventName, usize)], - apps: &[(&str, bool)], + apps: &[&str], mcp_servers: &[&str], ) -> PluginDetail { PluginDetail { @@ -1449,12 +1449,12 @@ pub(super) fn plugins_test_detail( .collect(), apps: apps .iter() - .map(|(name, needs_auth)| AppSummary { + .map(|name| AppSummary { id: format!("{name}-id"), name: (*name).to_string(), description: Some(format!("{name} app")), install_url: Some(format!("https://example.test/{name}")), - needs_auth: *needs_auth, + category: None, }) .collect(), app_templates: Vec::new(), diff --git a/codex-rs/tui/src/chatwidget/tests/plan_mode.rs b/codex-rs/tui/src/chatwidget/tests/plan_mode.rs index 74cd527038..984c71d1ab 100644 --- a/codex-rs/tui/src/chatwidget/tests/plan_mode.rs +++ b/codex-rs/tui/src/chatwidget/tests/plan_mode.rs @@ -1447,7 +1447,7 @@ async fn collaboration_modes_defaults_to_code_on_startup() { assert_eq!(chat.active_collaboration_mode_kind(), ModeKind::Default); assert_eq!( chat.current_model(), - crate::legacy_core::test_support::get_model_offline(chat.config.model.as_deref()) + get_model_offline_for_tests(chat.config.model.as_deref()) ); } @@ -1482,7 +1482,7 @@ async fn make_startup_chat_with_cli_overrides( .build() .await .expect("config"); - let resolved_model = crate::legacy_core::test_support::get_model_offline(cfg.model.as_deref()); + let resolved_model = get_model_offline_for_tests(cfg.model.as_deref()); let session_telemetry = test_session_telemetry(&cfg, resolved_model.as_str()); let init = ChatWidgetInit { config: cfg.clone(), diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index ae44cf00cd..736be9fa2a 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -69,7 +69,7 @@ async fn experimental_mode_plan_is_ignored_on_startup() { .build() .await .expect("config"); - let resolved_model = crate::legacy_core::test_support::get_model_offline(cfg.model.as_deref()); + let resolved_model = get_model_offline_for_tests(cfg.model.as_deref()); let session_telemetry = test_session_telemetry(&cfg, resolved_model.as_str()); let init = ChatWidgetInit { config: cfg.clone(), @@ -654,7 +654,7 @@ async fn plugin_detail_popup_snapshot_shows_install_actions_and_capability_summa (codex_app_server_protocol::HookEventName::PreToolUse, 1), (codex_app_server_protocol::HookEventName::Stop, 2), ], - &[("Figma", true), ("Slack", false)], + &["Figma", "Slack"], &["figma-mcp", "docs-mcp"], ), }), @@ -698,7 +698,7 @@ async fn plugin_detail_popup_hides_disclosure_for_installed_plugins() { (codex_app_server_protocol::HookEventName::PreToolUse, 1), (codex_app_server_protocol::HookEventName::Stop, 2), ], - &[("Figma", true), ("Slack", false)], + &["Figma", "Slack"], &["figma-mcp", "docs-mcp"], ), }), diff --git a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs index 02faef89f4..e61ffd48fe 100644 --- a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs +++ b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs @@ -1861,6 +1861,18 @@ async fn slash_resume_opens_picker() { assert_matches!(rx.try_recv(), Ok(AppEvent::OpenResumePicker)); } +#[tokio::test] +async fn slash_import_opens_claude_code_import_picker() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + + chat.dispatch_command(SlashCommand::Import); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::OpenExternalAgentConfigMigration) + ); +} + #[tokio::test] async fn slash_archive_confirmation_requests_current_thread_archive() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; @@ -1879,6 +1891,24 @@ async fn slash_archive_confirmation_requests_current_thread_archive() { assert_matches!(rx.try_recv(), Ok(AppEvent::ArchiveCurrentThread)); } +#[tokio::test] +async fn slash_delete_confirmation_requests_current_thread_delete() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + + chat.dispatch_command(SlashCommand::Delete); + + assert!(chat.bottom_pane.has_active_view()); + assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("slash_delete_confirmation_popup", popup); + + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + assert_matches!(rx.try_recv(), Ok(AppEvent::DeleteCurrentThread)); +} + #[tokio::test] async fn slash_resume_with_arg_requests_named_session() { let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs index 3d08daa427..b7cd476834 100644 --- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs +++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs @@ -411,7 +411,7 @@ async fn configured_pet_load_is_deferred_until_after_construction() { let mut cfg = test_config().await; cfg.tui_pet = Some(crate::pets::DEFAULT_PET_ID.to_string()); crate::pets::write_test_pack(&cfg.codex_home); - let resolved_model = crate::legacy_core::test_support::get_model_offline(cfg.model.as_deref()); + let resolved_model = get_model_offline_for_tests(cfg.model.as_deref()); let session_telemetry = test_session_telemetry(&cfg, resolved_model.as_str()); let init = ChatWidgetInit { config: cfg.clone(), diff --git a/codex-rs/tui/src/chatwidget/windows_sandbox_prompts.rs b/codex-rs/tui/src/chatwidget/windows_sandbox_prompts.rs index e92180d490..89ea3a9bc5 100644 --- a/codex-rs/tui/src/chatwidget/windows_sandbox_prompts.rs +++ b/codex-rs/tui/src/chatwidget/windows_sandbox_prompts.rs @@ -15,7 +15,7 @@ impl ChatWidget { #[cfg(any(target_os = "windows", test))] pub(super) fn elevated_windows_sandbox_setup_required(&self) -> bool { - WindowsSandboxLevel::from_config(&self.config) == WindowsSandboxLevel::Elevated + crate::windows_sandbox::level_from_config(&self.config) == WindowsSandboxLevel::Elevated && self .config .config_layer_stack @@ -23,9 +23,7 @@ impl ChatWidget { .windows_sandbox_mode .source .is_some() - && !crate::legacy_core::windows_sandbox::sandbox_setup_is_complete( - self.config.codex_home.as_path(), - ) + && !crate::windows_sandbox::sandbox_setup_is_complete(self.config.codex_home.as_path()) } #[cfg(target_os = "windows")] @@ -226,54 +224,6 @@ impl ChatWidget { ) { use ratatui_macros::line; - if !crate::legacy_core::windows_sandbox::ELEVATED_SANDBOX_NUX_ENABLED { - // Legacy flow (pre-NUX): explain the experimental sandbox and let the user enable it - // directly (no elevation prompts). - let mut header = ColumnRenderable::new(); - header.push(*Box::new( - Paragraph::new(vec![ - line!["Agent mode on Windows uses an experimental sandbox to limit network and filesystem access.".bold()], - line!["Learn more: https://developers.openai.com/codex/windows"], - ]) - .wrap(Wrap { trim: false }), - )); - - let preset_clone = preset; - let items = vec![ - SelectionItem { - name: "Enable experimental sandbox".to_string(), - description: None, - actions: vec![Box::new(move |tx| { - tx.send(AppEvent::EnableWindowsSandboxForAgentMode { - preset: preset_clone.clone(), - mode: WindowsSandboxEnableMode::Legacy, - profile_selection: profile_selection.clone(), - }); - })], - dismiss_on_select: true, - ..Default::default() - }, - SelectionItem { - name: "Go back".to_string(), - description: None, - actions: vec![Box::new(|tx| { - tx.send(AppEvent::OpenApprovalsPopup); - })], - dismiss_on_select: true, - ..Default::default() - }, - ]; - - self.bottom_pane.show_selection_view(SelectionViewParams { - title: None, - footer_hint: Some(standard_popup_hint_line()), - items, - header: Box::new(header), - ..Default::default() - }); - return; - } - self.session_telemetry.counter( "codex.windows_sandbox.elevated_prompt_shown", /*inc*/ 1, @@ -508,7 +458,7 @@ impl ChatWidget { #[cfg(target_os = "windows")] pub(crate) fn maybe_prompt_windows_sandbox_enable(&mut self, show_now: bool) { - let windows_sandbox_level = WindowsSandboxLevel::from_config(&self.config); + let windows_sandbox_level = crate::windows_sandbox::level_from_config(&self.config); let setup_is_required = windows_sandbox_level == WindowsSandboxLevel::Disabled || self.elevated_windows_sandbox_setup_required(); if show_now diff --git a/codex-rs/tui/src/debug_config.rs b/codex-rs/tui/src/debug_config.rs index ebd2042a10..d1dde975f3 100644 --- a/codex-rs/tui/src/debug_config.rs +++ b/codex-rs/tui/src/debug_config.rs @@ -544,8 +544,6 @@ mod tests { use super::render_debug_config_lines; use super::sandbox_mode_is_allowed_by_permissions; use super::session_all_proxy_url; - use crate::legacy_core::config::Constrained; - use crate::legacy_core::config::ConstraintError; use crate::legacy_core::config::Permissions; use codex_app_server_protocol::AskForApproval; use codex_app_server_protocol::ConfigLayerSource; @@ -553,7 +551,9 @@ mod tests { use codex_config::ConfigLayerStack; use codex_config::ConfigRequirements; use codex_config::ConfigRequirementsToml; + use codex_config::Constrained; use codex_config::ConstrainedWithSource; + use codex_config::ConstraintError; use codex_config::FeatureRequirementsToml; use codex_config::FilesystemConstraints; use codex_config::HookEventsToml; diff --git a/codex-rs/tui/src/external_agent_config_migration.rs b/codex-rs/tui/src/external_agent_config_migration.rs index 1ce9d44751..d1c4582d68 100644 --- a/codex-rs/tui/src/external_agent_config_migration.rs +++ b/codex-rs/tui/src/external_agent_config_migration.rs @@ -1,10 +1,8 @@ use crate::diff_render::display_path_for; -use crate::key_hint; -use crate::line_truncation::truncate_line_with_ellipsis_if_overflow; -use crate::render::Insets; -use crate::render::RectExt as _; -use crate::selection_list::selection_option_row_with_dim; -use crate::style::accent_style; +use crate::external_agent_config_migration_model::ExternalAgentConfigMigrationGroupModel; +use crate::external_agent_config_migration_model::external_agent_config_migration_groups; +use crate::external_agent_config_migration_model::external_agent_config_migration_item_detail; +use crate::external_agent_config_migration_model::external_agent_config_migration_item_label; use crate::tui::FrameRequester; use crate::tui::Tui; use crate::tui::TuiEvent; @@ -14,25 +12,16 @@ use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use crossterm::event::KeyModifiers; -use ratatui::buffer::Buffer; -use ratatui::layout::Constraint; -use ratatui::layout::Layout; -use ratatui::layout::Rect; use ratatui::prelude::Stylize as _; use ratatui::text::Line; -use ratatui::widgets::Clear; -use ratatui::widgets::Paragraph; -use ratatui::widgets::Widget; -use ratatui::widgets::WidgetRef; -use ratatui::widgets::Wrap; use tokio_stream::StreamExt; +mod render; + #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum ExternalAgentConfigMigrationOutcome { Proceed(Vec), Skip, - SkipForever, - Exit, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -44,34 +33,26 @@ enum FocusArea { #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ActionMenuOption { Proceed, + Customize, Skip, - SkipForever, + Back, } impl ActionMenuOption { fn label(self) -> &'static str { match self { - Self::Proceed => "Proceed with selected", - Self::Skip => "Skip for now", - Self::SkipForever => "Don't ask again", + Self::Proceed => "Import selected", + Self::Customize => "Customize selection", + Self::Skip => "Cancel", + Self::Back => "Review selection", } } +} - fn previous(self) -> Option { - match self { - Self::Proceed => None, - Self::Skip => Some(Self::Proceed), - Self::SkipForever => Some(Self::Skip), - } - } - - fn next(self) -> Option { - match self { - Self::Proceed => Some(Self::Skip), - Self::Skip => Some(Self::SkipForever), - Self::SkipForever => None, - } - } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MigrationView { + Summary, + Customize, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -136,6 +117,8 @@ pub(crate) async fn run_external_agent_config_migration_prompt( struct ExternalAgentConfigMigrationScreen { request_frame: FrameRequester, items: Vec, + groups: Vec, + view: MigrationView, selected_item_idx: Option, scroll_top: usize, focus: FocusArea, @@ -151,44 +134,67 @@ impl ExternalAgentConfigMigrationScreen { } fn first_available_action(&self) -> ActionMenuOption { - if self.proceed_enabled() { - ActionMenuOption::Proceed - } else { - ActionMenuOption::Skip + match self.available_actions().first() { + Some(action) => *action, + None => ActionMenuOption::Back, + } + } + + fn last_available_action(&self) -> ActionMenuOption { + match self.available_actions().last() { + Some(action) => *action, + None => ActionMenuOption::Back, } } fn previous_available_action(&self, action: ActionMenuOption) -> Option { - let mut candidate = action.previous(); - while let Some(option) = candidate { - if option != ActionMenuOption::Proceed || self.proceed_enabled() { - return Some(option); - } - candidate = option.previous(); - } - None + let actions = self.available_actions(); + actions + .iter() + .position(|candidate| *candidate == action) + .and_then(|idx| idx.checked_sub(1)) + .and_then(|idx| actions.get(idx)) + .copied() } fn next_available_action(&self, action: ActionMenuOption) -> Option { - let mut candidate = action.next(); - while let Some(option) = candidate { - if option != ActionMenuOption::Proceed || self.proceed_enabled() { - return Some(option); + let actions = self.available_actions(); + actions + .iter() + .position(|candidate| *candidate == action) + .and_then(|idx| actions.get(idx + 1)) + .copied() + } + + fn available_actions(&self) -> Vec { + match self.view { + MigrationView::Summary => { + let mut actions = Vec::new(); + if self.proceed_enabled() { + actions.push(ActionMenuOption::Proceed); + } + actions.extend([ActionMenuOption::Customize, ActionMenuOption::Skip]); + actions } - candidate = option.next(); + MigrationView::Customize => vec![ActionMenuOption::Back], } - None } fn normalize_highlighted_action(&mut self) { - if self.highlighted_action == ActionMenuOption::Proceed && !self.proceed_enabled() { + if !self.available_actions().contains(&self.highlighted_action) { self.highlighted_action = self.first_available_action(); } } fn display_description(item: &ExternalAgentConfigMigrationItem) -> String { + // App-server descriptions use migration vocabulary. Normalize that prefix so the TUI + // consistently uses the user-facing import vocabulary. + let description = item + .description + .strip_prefix("Migrate ") + .map_or_else(|| item.description.clone(), |rest| format!("Import {rest}")); let Some(cwd) = item.cwd.as_deref() else { - return item.description.clone(); + return description; }; fn reformat_description( @@ -207,33 +213,23 @@ impl ExternalAgentConfigMigrationScreen { )) } - if let Some(reformatted) = - reformat_description(&item.description, "Migrate ", " into ", cwd) - { + if let Some(reformatted) = reformat_description(&description, "Import ", " into ", cwd) { return reformatted; } if let Some(reformatted) = - reformat_description(&item.description, "Migrate skills from ", " to ", cwd) + reformat_description(&description, "Import skills from ", " to ", cwd) { return reformatted; } - if let Some(reformatted) = reformat_description(&item.description, "Migrate ", " to ", cwd) - { + if let Some(reformatted) = reformat_description(&description, "Import ", " to ", cwd) { return reformatted; } - if let Some(reformatted) = reformat_description(&item.description, "Import ", " to ", cwd) { - return reformatted; - } - - if let Some(source) = item - .description - .strip_prefix("Migrate enabled plugins from ") - { + if let Some(source) = description.strip_prefix("Import enabled plugins from ") { let description = format!( - "Migrate enabled plugins from {}", + "Import enabled plugins from {}", display_path_for(std::path::Path::new(source), cwd) ); if let Some(details) = &item.details { @@ -260,7 +256,7 @@ impl ExternalAgentConfigMigrationScreen { return description; } - item.description.clone() + description } fn new( @@ -269,6 +265,7 @@ impl ExternalAgentConfigMigrationScreen { selected_items: &[ExternalAgentConfigMigrationItem], error: Option, ) -> Self { + let groups = external_agent_config_migration_groups(items); let items = items .iter() .cloned() @@ -277,18 +274,22 @@ impl ExternalAgentConfigMigrationScreen { item, }) .collect::>(); - let selected_item_idx = (!items.is_empty()).then_some(0); - Self { + let selected_item_idx = (!groups.is_empty()).then_some(0); + let mut screen = Self { request_frame, items, + groups, + view: MigrationView::Summary, selected_item_idx, scroll_top: 0, - focus: FocusArea::Items, + focus: FocusArea::Actions, highlighted_action: ActionMenuOption::Proceed, done: false, outcome: ExternalAgentConfigMigrationOutcome::Skip, error, - } + }; + screen.normalize_highlighted_action(); + screen } fn plugin_detail_lines(plugin_groups: &[PluginsMigration]) -> Vec> { @@ -341,12 +342,6 @@ impl ExternalAgentConfigMigrationScreen { fn proceed(&mut self) { let selected = self.selected_items(); - if selected.is_empty() { - self.error = Some("Select at least one item or choose a skip option.".to_string()); - self.request_frame.schedule_frame(); - return; - } - self.finish_with(ExternalAgentConfigMigrationOutcome::Proceed(selected)); } @@ -354,14 +349,6 @@ impl ExternalAgentConfigMigrationScreen { self.finish_with(ExternalAgentConfigMigrationOutcome::Skip); } - fn skip_forever(&mut self) { - self.finish_with(ExternalAgentConfigMigrationOutcome::SkipForever); - } - - fn exit(&mut self) { - self.finish_with(ExternalAgentConfigMigrationOutcome::Exit); - } - fn selected_items(&self) -> Vec { self.items .iter() @@ -374,6 +361,22 @@ impl ExternalAgentConfigMigrationScreen { self.items.iter().filter(|item| item.enabled).count() } + fn group_selection_marker( + &self, + group: &ExternalAgentConfigMigrationGroupModel, + ) -> &'static str { + let enabled_count = group + .item_indices + .iter() + .filter(|idx| self.items[**idx].enabled) + .count(); + match enabled_count { + 0 => " ", + count if count == group.item_indices.len() => "x", + _ => "-", + } + } + fn set_all_enabled(&mut self, enabled: bool) { for item in &mut self.items { item.enabled = enabled; @@ -384,7 +387,7 @@ impl ExternalAgentConfigMigrationScreen { } fn toggle_selected_item(&mut self) { - if self.focus != FocusArea::Items { + if self.view != MigrationView::Customize || self.focus != FocusArea::Items { return; } let Some(selected_idx) = self.selected_item_idx else { @@ -393,26 +396,51 @@ impl ExternalAgentConfigMigrationScreen { let Some(item) = self.items.get_mut(selected_idx) else { return; }; - item.enabled = !item.enabled; self.error = None; self.normalize_highlighted_action(); self.request_frame.schedule_frame(); } + fn customize(&mut self) { + self.view = MigrationView::Customize; + self.selected_item_idx = (!self.items.is_empty()).then_some(0); + self.scroll_top = 0; + self.focus = FocusArea::Items; + self.highlighted_action = ActionMenuOption::Back; + self.request_frame.schedule_frame(); + } + + fn back_to_summary(&mut self) { + self.view = MigrationView::Summary; + self.selected_item_idx = (!self.groups.is_empty()).then_some(0); + self.scroll_top = 0; + self.focus = FocusArea::Actions; + self.highlighted_action = self.first_available_action(); + self.request_frame.schedule_frame(); + } + fn move_up(&mut self) { + if self.view == MigrationView::Summary { + self.focus = FocusArea::Actions; + self.highlighted_action = self + .previous_available_action(self.highlighted_action) + .unwrap_or_else(|| self.last_available_action()); + self.request_frame.schedule_frame(); + return; + } match self.focus { FocusArea::Items => match self.selected_item_idx { Some(0) => { self.focus = FocusArea::Actions; - self.highlighted_action = ActionMenuOption::SkipForever; + self.highlighted_action = self.last_available_action(); } Some(idx) => { self.selected_item_idx = Some(idx.saturating_sub(1)); } None => { self.focus = FocusArea::Actions; - self.highlighted_action = ActionMenuOption::SkipForever; + self.highlighted_action = self.last_available_action(); } }, FocusArea::Actions => { @@ -431,6 +459,14 @@ impl ExternalAgentConfigMigrationScreen { } fn move_down(&mut self) { + if self.view == MigrationView::Summary { + self.focus = FocusArea::Actions; + self.highlighted_action = self + .next_available_action(self.highlighted_action) + .unwrap_or_else(|| self.first_available_action()); + self.request_frame.schedule_frame(); + return; + } match self.focus { FocusArea::Items => match self.selected_item_idx { Some(idx) if idx + 1 < self.items.len() => { @@ -461,8 +497,9 @@ impl ExternalAgentConfigMigrationScreen { FocusArea::Items => self.toggle_selected_item(), FocusArea::Actions => match self.highlighted_action { ActionMenuOption::Proceed => self.proceed(), + ActionMenuOption::Customize => self.customize(), ActionMenuOption::Skip => self.skip(), - ActionMenuOption::SkipForever => self.skip_forever(), + ActionMenuOption::Back => self.back_to_summary(), }, } } @@ -471,39 +508,47 @@ impl ExternalAgentConfigMigrationScreen { if key_event.kind == KeyEventKind::Release { return; } - if is_ctrl_exit_combo(key_event) { - self.exit(); + self.skip(); return; } match key_event.code { KeyCode::Up | KeyCode::Char('k') => self.move_up(), KeyCode::Down | KeyCode::Char('j') => self.move_down(), - KeyCode::Char('1') => { - self.focus = FocusArea::Actions; - self.highlighted_action = ActionMenuOption::Proceed; - self.proceed(); + KeyCode::Char(number @ '1'..='9') => self.select_numbered_action(number), + KeyCode::Char('c') if self.view == MigrationView::Summary => self.customize(), + KeyCode::Char('b') if self.view == MigrationView::Customize => self.back_to_summary(), + KeyCode::Char(' ') if self.view == MigrationView::Customize => { + self.toggle_selected_item(); } - KeyCode::Char('2') => { - self.focus = FocusArea::Actions; - self.highlighted_action = ActionMenuOption::Skip; - self.skip(); + KeyCode::Char('a') if self.view == MigrationView::Customize => { + self.set_all_enabled(/*enabled*/ true); } - KeyCode::Char('3') => { - self.focus = FocusArea::Actions; - self.highlighted_action = ActionMenuOption::SkipForever; - self.skip_forever(); + KeyCode::Char('n') if self.view == MigrationView::Customize => { + self.set_all_enabled(/*enabled*/ false); } - KeyCode::Char(' ') => self.toggle_selected_item(), - KeyCode::Char('a') => self.set_all_enabled(/*enabled*/ true), - KeyCode::Char('n') => self.set_all_enabled(/*enabled*/ false), KeyCode::Enter => self.confirm_selection(), - KeyCode::Esc => self.skip(), + KeyCode::Esc => match self.view { + MigrationView::Summary => self.skip(), + MigrationView::Customize => self.back_to_summary(), + }, _ => {} } } + fn select_numbered_action(&mut self, number: char) { + let Some(index) = number.to_digit(10).and_then(|number| number.checked_sub(1)) else { + return; + }; + let Some(action) = self.available_actions().get(index as usize).copied() else { + return; + }; + self.focus = FocusArea::Actions; + self.highlighted_action = action; + self.confirm_selection(); + } + fn ensure_selected_item_visible(&mut self) { let Some(selected_idx) = self.selected_item_idx else { self.scroll_top = 0; @@ -534,12 +579,47 @@ impl ExternalAgentConfigMigrationScreen { fn section_title(cwd: Option<&std::path::Path>) -> Line<'static> { match cwd { - Some(cwd) => Line::from(vec!["Project: ".bold(), cwd.display().to_string().dim()]), + Some(cwd) => Line::from(vec![ + "Current project: ".bold(), + cwd.display().to_string().dim(), + ]), None => Line::from("Home".bold()), } } fn build_render_lines(&self) -> Vec { + match self.view { + MigrationView::Summary => self.build_summary_render_lines(), + MigrationView::Customize => self.build_customize_render_lines(), + } + } + + fn build_summary_render_lines(&self) -> Vec { + self.groups + .iter() + .enumerate() + .flat_map(|(idx, group)| { + [ + RenderLineEntry { + item_idx: Some(idx), + kind: RenderLineKind::Item, + line: Line::from(format!( + " [{}] {}", + self.group_selection_marker(group), + group.label + )), + }, + RenderLineEntry { + item_idx: None, + kind: RenderLineKind::ItemDetail, + line: Line::from(format!(" {}", group.description)), + }, + ] + }) + .collect() + } + + fn build_customize_render_lines(&self) -> Vec { let mut lines = Vec::new(); let mut current_scope: Option> = None; for (idx, item) in self.items.iter().enumerate() { @@ -562,12 +642,28 @@ impl ExternalAgentConfigMigrationScreen { lines.push(RenderLineEntry { item_idx: Some(idx), kind: RenderLineKind::Item, - line: Line::from(format!( - " [{}] {}", - if item.enabled { "x" } else { " " }, - Self::display_description(&item.item) - )), + line: Line::from(vec![ + " ".into(), + format!( + "[{}] {}", + if item.enabled { "x" } else { " " }, + external_agent_config_migration_item_label(&item.item) + ) + .into(), + ]), }); + lines.push(RenderLineEntry { + item_idx: None, + kind: RenderLineKind::ItemDetail, + line: Line::from(format!(" {}", Self::display_description(&item.item))), + }); + if let Some(details) = external_agent_config_migration_item_detail(&item.item) { + lines.push(RenderLineEntry { + item_idx: None, + kind: RenderLineKind::ItemDetail, + line: Line::from(format!(" {details}")), + }); + } if let Some(details) = &item.item.details { for line in Self::plugin_detail_lines(&details.plugins) { lines.push(RenderLineEntry { @@ -580,194 +676,25 @@ impl ExternalAgentConfigMigrationScreen { } lines } - - fn render_items(&self, area: Rect, buf: &mut Buffer) { - if area.height == 0 || area.width == 0 { - return; - } - let rows = self.build_render_lines(); - let visible_rows = area.height as usize; - let mut start_idx = self.scroll_top.min(rows.len().saturating_sub(1)); - if let Some(selected_item_idx) = self.selected_item_idx { - let selected_render_idx = self.selected_render_line_index(selected_item_idx); - if selected_render_idx < start_idx { - start_idx = selected_render_idx; - } else if visible_rows > 0 { - let bottom = start_idx + visible_rows - 1; - if selected_render_idx > bottom { - start_idx = selected_render_idx + 1 - visible_rows; - } - } - } - - let mut y = area.y; - for entry in rows.iter().skip(start_idx).take(visible_rows) { - if y >= area.y + area.height { - break; - } - - let selected = - self.focus == FocusArea::Items && self.selected_item_idx == entry.item_idx; - let mut line = entry.line.clone(); - if selected { - line.spans.iter_mut().for_each(|span| { - span.style = span.style.patch(accent_style()); - }); - } else if entry.kind != RenderLineKind::Item && !line.spans.is_empty() { - line.spans.iter_mut().for_each(|span| { - span.style = span.style.dim(); - }); - } - let line = truncate_line_with_ellipsis_if_overflow(line, area.width as usize); - line.render( - Rect { - x: area.x, - y, - width: area.width, - height: 1, - }, - buf, - ); - y = y.saturating_add(1); - } - } -} - -impl WidgetRef for &ExternalAgentConfigMigrationScreen { - fn render_ref(&self, area: Rect, buf: &mut Buffer) { - Clear.render(area, buf); - - let inner_area = area.inset(Insets::vh(/*v*/ 1, /*h*/ 2)); - let error_height = u16::from(self.error.is_some()); - let fixed_height = 1u16 + 2u16 + error_height + 1u16 + 4u16 + 1u16; - let list_height = - self.render_line_count() - .max(1) - .min(inner_area.height.saturating_sub(fixed_height) as usize) as u16; - let [ - header_area, - intro_area, - error_area, - list_area, - list_gap_area, - actions_area, - footer_area, - _spacer_area, - ] = Layout::vertical([ - Constraint::Length(1), - Constraint::Length(2), - Constraint::Length(error_height), - Constraint::Length(list_height), - Constraint::Length(1), - Constraint::Length(4), - Constraint::Length(1), - Constraint::Fill(1), - ]) - .areas(inner_area); - - let heading = Line::from(vec!["> ".into(), "External agent config detected".bold()]); - heading.render(header_area, buf); - - Paragraph::new(vec![ - Line::from("We found settings from another agent that you can add to this project."), - Line::from("Select what to import"), - ]) - .wrap(Wrap { trim: false }) - .render(intro_area, buf); - - if let Some(error) = &self.error { - Paragraph::new(error.clone().red().to_string()) - .wrap(Wrap { trim: false }) - .render(error_area, buf); - } - - self.render_items(list_area, buf); - Clear.render(list_gap_area, buf); - - let [ - actions_intro_area, - proceed_area, - skip_area, - skip_forever_area, - ] = Layout::vertical([ - Constraint::Length(1), - Constraint::Length(1), - Constraint::Length(1), - Constraint::Length(1), - ]) - .areas(actions_area); - let actions_intro = format!( - "Selected {} of {} item(s).", - self.selected_count(), - self.items.len() - ); - Paragraph::new(actions_intro) - .wrap(Wrap { trim: false }) - .render(actions_intro_area, buf); - selection_option_row_with_dim( - /*index*/ 0, - ActionMenuOption::Proceed.label().to_string(), - self.focus == FocusArea::Actions - && self.highlighted_action == ActionMenuOption::Proceed, - /*dim*/ self.focus != FocusArea::Actions || !self.proceed_enabled(), - ) - .render(proceed_area, buf); - selection_option_row_with_dim( - /*index*/ 1, - ActionMenuOption::Skip.label().to_string(), - self.focus == FocusArea::Actions && self.highlighted_action == ActionMenuOption::Skip, - /*dim*/ self.focus != FocusArea::Actions, - ) - .render(skip_area, buf); - selection_option_row_with_dim( - /*index*/ 2, - ActionMenuOption::SkipForever.label().to_string(), - self.focus == FocusArea::Actions - && self.highlighted_action == ActionMenuOption::SkipForever, - /*dim*/ self.focus != FocusArea::Actions, - ) - .render(skip_forever_area, buf); - - Line::from(vec![ - "Use ".dim(), - key_hint::plain(KeyCode::Up).into(), - "/".dim(), - key_hint::plain(KeyCode::Down).into(), - " to move, ".dim(), - key_hint::plain(KeyCode::Char(' ')).into(), - " to toggle, ".dim(), - "1".cyan(), - "/".dim(), - "2".cyan(), - "/".dim(), - "3".cyan(), - " to choose, ".dim(), - "a".cyan(), - "/".dim(), - "n".cyan(), - " for all/none".dim(), - ]) - .render(footer_area, buf); - } } fn is_ctrl_exit_combo(key_event: KeyEvent) -> bool { - key_event.modifiers.contains(KeyModifiers::CONTROL) - && matches!(key_event.code, KeyCode::Char('c') | KeyCode::Char('d')) + matches!(key_event.code, KeyCode::Char('c' | 'd')) + && key_event.modifiers.contains(KeyModifiers::CONTROL) } #[cfg(test)] mod tests { - use super::ActionMenuOption; use super::ExternalAgentConfigMigrationOutcome; use super::ExternalAgentConfigMigrationScreen; - use super::FocusArea; + use super::MigrationView; use crate::custom_terminal::Terminal; use crate::test_backend::VT100Backend; use crate::tui::FrameRequester; use codex_app_server_protocol::ExternalAgentConfigMigrationItem; use codex_app_server_protocol::ExternalAgentConfigMigrationItemType; use codex_app_server_protocol::PluginsMigration; + use codex_app_server_protocol::SessionMigration; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyModifiers; @@ -829,6 +756,19 @@ mod tests { cwd: None, details: None, }, + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Sessions, + description: "Migrate recent chat sessions".to_string(), + cwd: None, + details: Some(codex_app_server_protocol::MigrationDetails { + sessions: vec![SessionMigration { + path: PathBuf::from("/Users/alex/.claude/projects/project/session.jsonl"), + cwd: project_root.clone(), + title: Some("Investigate migration UX".to_string()), + }], + ..Default::default() + }), + }, ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Plugins, description: format!( @@ -864,7 +804,13 @@ mod tests { frame.render_widget_ref(screen, frame.area()); } terminal.flush().expect("flush"); - terminal.backend().to_string() + terminal + .backend() + .to_string() + .lines() + .map(str::trim_end) + .collect::>() + .join("\n") } #[test] @@ -877,13 +823,56 @@ mod tests { /*error*/ None, ); - let rendered = render_screen(&screen, /*width*/ 80, /*height*/ 21); + let rendered = render_screen(&screen, /*width*/ 80, /*height*/ 24); #[cfg(windows)] assert_snapshot!("external_agent_config_migration_prompt_windows", rendered); #[cfg(not(windows))] assert_snapshot!("external_agent_config_migration_prompt", rendered); } + #[test] + fn customize_snapshot() { + let items = sample_items(); + let mut screen = ExternalAgentConfigMigrationScreen::new( + FrameRequester::test_dummy(), + &items, + &items, + /*error*/ None, + ); + screen.customize(); + + let rendered = render_screen(&screen, /*width*/ 80, /*height*/ 30); + #[cfg(windows)] + assert_snapshot!( + "external_agent_config_migration_customize_windows", + rendered + ); + #[cfg(not(windows))] + assert_snapshot!("external_agent_config_migration_customize", rendered); + } + + #[test] + fn customize_action_snapshot() { + let items = sample_items(); + let mut screen = ExternalAgentConfigMigrationScreen::new( + FrameRequester::test_dummy(), + &items, + &items, + /*error*/ None, + ); + screen.customize(); + screen.move_up(); + + let rendered = render_screen(&screen, /*width*/ 80, /*height*/ 30); + #[cfg(windows)] + assert_snapshot!( + "external_agent_config_migration_customize_action_windows", + rendered + ); + #[cfg(not(windows))] + assert_snapshot!("external_agent_config_migration_customize_action", rendered); + } + #[test] fn proceed_returns_selected_items() { let items = sample_items(); @@ -894,9 +883,6 @@ mod tests { /*error*/ None, ); - screen.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); - screen.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); - screen.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); screen.handle_key(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); assert!(screen.is_done()); @@ -916,16 +902,19 @@ mod tests { /*error*/ None, ); + screen.handle_key(KeyEvent::new(KeyCode::Char('c'), KeyModifiers::NONE)); screen.handle_key(KeyEvent::new(KeyCode::Char(' '), KeyModifiers::NONE)); - screen.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); - screen.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); - screen.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); - screen.handle_key(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + screen.handle_key(KeyEvent::new(KeyCode::Char('b'), KeyModifiers::NONE)); + screen.handle_key(KeyEvent::new(KeyCode::Char('1'), KeyModifiers::NONE)); assert!(screen.is_done()); assert_eq!( screen.outcome(), - ExternalAgentConfigMigrationOutcome::Proceed(vec![items[1].clone(), items[2].clone(),]) + ExternalAgentConfigMigrationOutcome::Proceed(vec![ + items[1].clone(), + items[2].clone(), + items[3].clone(), + ]) ); } @@ -946,30 +935,7 @@ mod tests { } #[test] - fn skip_forever_returns_skip_forever_outcome() { - let items = sample_items(); - let mut screen = ExternalAgentConfigMigrationScreen::new( - FrameRequester::test_dummy(), - &items, - &items, - /*error*/ None, - ); - - screen.move_down(); - screen.move_down(); - screen.move_down(); - screen.move_down(); - screen.move_down(); - screen.confirm_selection(); - - assert_eq!( - screen.outcome(), - ExternalAgentConfigMigrationOutcome::SkipForever - ); - } - - #[test] - fn proceed_requires_at_least_one_selected_item() { + fn numeric_shortcuts_follow_visible_actions_when_proceed_is_disabled() { let items = sample_items(); let mut screen = ExternalAgentConfigMigrationScreen::new( FrameRequester::test_dummy(), @@ -978,35 +944,46 @@ mod tests { /*error*/ None, ); + screen.handle_key(KeyEvent::new(KeyCode::Char('c'), KeyModifiers::NONE)); screen.handle_key(KeyEvent::new(KeyCode::Char('n'), KeyModifiers::NONE)); + screen.handle_key(KeyEvent::new(KeyCode::Char('b'), KeyModifiers::NONE)); screen.handle_key(KeyEvent::new(KeyCode::Char('1'), KeyModifiers::NONE)); - assert!(!screen.is_done()); - assert_eq!(screen.highlighted_action, ActionMenuOption::Proceed); - let rendered = render_screen(&screen, /*width*/ 80, /*height*/ 20); - assert!( - rendered.contains("Select at least one item or choose a skip option."), - "expected inline validation error, got:\n{rendered}" - ); + assert_eq!(screen.view, MigrationView::Customize); } #[test] - fn proceed_action_is_skipped_when_no_items_are_selected() { + fn empty_selection_enter_opens_customize_instead_of_proceeding() { let items = sample_items(); let mut screen = ExternalAgentConfigMigrationScreen::new( FrameRequester::test_dummy(), &items, - &items, + &[], /*error*/ None, ); - screen.handle_key(KeyEvent::new(KeyCode::Char('n'), KeyModifiers::NONE)); - screen.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); - screen.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); - screen.handle_key(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); + screen.handle_key(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); - assert_eq!(screen.focus, FocusArea::Actions); - assert_eq!(screen.highlighted_action, ActionMenuOption::Skip); + assert!(!screen.is_done()); + assert_eq!(screen.view, MigrationView::Customize); + } + + #[test] + fn control_exit_shortcuts_cancel_prompt() { + let items = sample_items(); + for key_code in [KeyCode::Char('c'), KeyCode::Char('d')] { + let mut screen = ExternalAgentConfigMigrationScreen::new( + FrameRequester::test_dummy(), + &items, + &items, + /*error*/ None, + ); + + screen.handle_key(KeyEvent::new(key_code, KeyModifiers::CONTROL)); + + assert!(screen.is_done()); + assert_eq!(screen.outcome(), ExternalAgentConfigMigrationOutcome::Skip); + } } #[test] @@ -1025,28 +1002,42 @@ mod tests { ExternalAgentConfigMigrationOutcome::Proceed(items.clone()) ); + let mut customize_screen = ExternalAgentConfigMigrationScreen::new( + FrameRequester::test_dummy(), + &items, + &items, + /*error*/ None, + ); + customize_screen.handle_key(KeyEvent::new(KeyCode::Char('2'), KeyModifiers::NONE)); + assert_eq!(customize_screen.view, MigrationView::Customize); + customize_screen.handle_key(KeyEvent::new(KeyCode::Char('1'), KeyModifiers::NONE)); + assert_eq!(customize_screen.view, MigrationView::Summary); + let mut skip_screen = ExternalAgentConfigMigrationScreen::new( FrameRequester::test_dummy(), &items, &items, /*error*/ None, ); - skip_screen.handle_key(KeyEvent::new(KeyCode::Char('2'), KeyModifiers::NONE)); + skip_screen.handle_key(KeyEvent::new(KeyCode::Char('3'), KeyModifiers::NONE)); assert_eq!( skip_screen.outcome(), ExternalAgentConfigMigrationOutcome::Skip ); + } - let mut skip_forever_screen = ExternalAgentConfigMigrationScreen::new( + #[test] + fn summary_does_not_toggle_selection() { + let items = sample_items(); + let mut screen = ExternalAgentConfigMigrationScreen::new( FrameRequester::test_dummy(), &items, &items, /*error*/ None, ); - skip_forever_screen.handle_key(KeyEvent::new(KeyCode::Char('3'), KeyModifiers::NONE)); - assert_eq!( - skip_forever_screen.outcome(), - ExternalAgentConfigMigrationOutcome::SkipForever - ); + + screen.handle_key(KeyEvent::new(KeyCode::Char(' '), KeyModifiers::NONE)); + + assert_eq!(screen.selected_items(), items); } } diff --git a/codex-rs/tui/src/external_agent_config_migration/render.rs b/codex-rs/tui/src/external_agent_config_migration/render.rs new file mode 100644 index 0000000000..4d518906c4 --- /dev/null +++ b/codex-rs/tui/src/external_agent_config_migration/render.rs @@ -0,0 +1,206 @@ +use super::*; +use crate::key_hint; +use crate::line_truncation::truncate_line_with_ellipsis_if_overflow; +use crate::render::Insets; +use crate::render::RectExt as _; +use crate::selection_list::selection_option_row_with_dim; +use ratatui::buffer::Buffer; +use ratatui::layout::Constraint; +use ratatui::layout::Layout; +use ratatui::layout::Rect; +use ratatui::widgets::Clear; +use ratatui::widgets::Paragraph; +use ratatui::widgets::Widget; +use ratatui::widgets::WidgetRef; +use ratatui::widgets::Wrap; + +impl ExternalAgentConfigMigrationScreen { + fn render_items(&self, area: Rect, buf: &mut Buffer) { + if area.height == 0 || area.width == 0 { + return; + } + let rows = self.build_render_lines(); + let visible_rows = area.height as usize; + let mut start_idx = self.scroll_top.min(rows.len().saturating_sub(1)); + if let Some(selected_item_idx) = self.selected_item_idx { + let selected_render_idx = self.selected_render_line_index(selected_item_idx); + if selected_render_idx < start_idx { + start_idx = selected_render_idx; + } else if visible_rows > 0 { + let bottom = start_idx + visible_rows - 1; + if selected_render_idx > bottom { + start_idx = selected_render_idx + 1 - visible_rows; + } + } + } + + let mut y = area.y; + for entry in rows.iter().skip(start_idx).take(visible_rows) { + if y >= area.y + area.height { + break; + } + + let selected = + self.focus == FocusArea::Items && self.selected_item_idx == entry.item_idx; + let mut line = entry.line.clone(); + if selected { + if let Some(cursor) = line.spans.first_mut() { + cursor.content = "› ".into(); + } + line.spans.iter_mut().for_each(|span| { + span.style = span.style.cyan().bold(); + }); + } else if entry.kind != RenderLineKind::Item && !line.spans.is_empty() { + line.spans.iter_mut().for_each(|span| { + span.style = span.style.dim(); + }); + } + let line = truncate_line_with_ellipsis_if_overflow(line, area.width as usize); + line.render( + Rect { + x: area.x, + y, + width: area.width, + height: 1, + }, + buf, + ); + y = y.saturating_add(1); + } + } +} + +impl WidgetRef for &ExternalAgentConfigMigrationScreen { + fn render_ref(&self, area: Rect, buf: &mut Buffer) { + Clear.render(area, buf); + + let inner_area = area.inset(Insets::vh(/*v*/ 1, /*h*/ 2)); + let error_height = u16::from(self.error.is_some()); + let intro_lines = match self.view { + MigrationView::Summary => vec![ + Line::from("Bring over your setup, current project, and recent chats."), + Line::from("Codex may add files to your current project folder."), + Line::from("Your existing agent setup will not be changed."), + Line::from("Cloud-hosted chat data cannot be imported."), + ], + MigrationView::Customize => vec![ + Line::from("Choose the items to import."), + Line::from("Codex may add files to your current project folder."), + Line::from("Your existing agent setup will not be changed."), + ], + }; + let intro_height = intro_lines.len() as u16; + let actions = self.available_actions(); + let actions_height = actions.len() as u16 + 1; + let fixed_height = 1u16 + intro_height + error_height + 1u16 + actions_height + 1u16; + let list_height = + self.render_line_count() + .max(1) + .min(inner_area.height.saturating_sub(fixed_height) as usize) as u16; + let [ + header_area, + intro_area, + error_area, + list_area, + list_gap_area, + actions_area, + footer_area, + _spacer_area, + ] = Layout::vertical([ + Constraint::Length(1), + Constraint::Length(intro_height), + Constraint::Length(error_height), + Constraint::Length(list_height), + Constraint::Length(1), + Constraint::Length(actions_height), + Constraint::Length(1), + Constraint::Fill(1), + ]) + .areas(inner_area); + + let title = match self.view { + MigrationView::Summary => "Import from another coding agent", + MigrationView::Customize => "Choose what to import", + }; + let heading = Line::from(vec!["> ".into(), title.bold()]); + heading.render(header_area, buf); + + Paragraph::new(intro_lines) + .wrap(Wrap { trim: false }) + .render(intro_area, buf); + + if let Some(error) = &self.error { + Paragraph::new(error.clone().red().to_string()) + .wrap(Wrap { trim: false }) + .render(error_area, buf); + } + + self.render_items(list_area, buf); + Clear.render(list_gap_area, buf); + + let action_areas = Layout::vertical(std::iter::repeat_n( + Constraint::Length(1), + actions.len() + 1, + )) + .split(actions_area); + let item_label = if self.items.len() == 1 { + "item" + } else { + "items" + }; + let actions_intro = format!( + "Selected {} of {} {item_label}.", + self.selected_count(), + self.items.len() + ); + Paragraph::new(actions_intro) + .wrap(Wrap { trim: false }) + .render(action_areas[0], buf); + for (idx, action) in actions.iter().enumerate() { + selection_option_row_with_dim( + idx, + action.label().to_string(), + self.focus == FocusArea::Actions && self.highlighted_action == *action, + /*dim*/ self.focus != FocusArea::Actions, + ) + .render(action_areas[idx + 1], buf); + } + + let footer = match self.view { + MigrationView::Summary => Line::from(vec![ + "Use ".dim(), + key_hint::plain(KeyCode::Up).into(), + "/".dim(), + key_hint::plain(KeyCode::Down).into(), + " to move, ".dim(), + key_hint::plain(KeyCode::Enter).into(), + " to select, ".dim(), + "c".cyan(), + " to customize".dim(), + ]), + MigrationView::Customize if self.focus == FocusArea::Actions => Line::from(vec![ + "Press ".dim(), + key_hint::plain(KeyCode::Enter).into(), + " to continue, ".dim(), + key_hint::plain(KeyCode::Up).into(), + "/".dim(), + key_hint::plain(KeyCode::Down).into(), + " to move, ".dim(), + "b".cyan(), + " to go back".dim(), + ]), + MigrationView::Customize => Line::from(vec![ + "Use ".dim(), + key_hint::plain(KeyCode::Up).into(), + "/".dim(), + key_hint::plain(KeyCode::Down).into(), + " to move, ".dim(), + key_hint::plain(KeyCode::Char(' ')).into(), + " to toggle, ".dim(), + "b".cyan(), + " to go back".dim(), + ]), + }; + footer.render(footer_area, buf); + } +} diff --git a/codex-rs/tui/src/external_agent_config_migration_flow.rs b/codex-rs/tui/src/external_agent_config_migration_flow.rs new file mode 100644 index 0000000000..f0fabe3a01 --- /dev/null +++ b/codex-rs/tui/src/external_agent_config_migration_flow.rs @@ -0,0 +1,125 @@ +use crate::app_server_session::AppServerSession; +use crate::app_server_session::EXTERNAL_AGENT_CONFIG_IMPORT_IN_PROGRESS_MESSAGE; +use crate::external_agent_config_migration::ExternalAgentConfigMigrationOutcome; +use crate::external_agent_config_migration::run_external_agent_config_migration_prompt; +use crate::legacy_core::config::Config; +use crate::tui; +use codex_app_server_protocol::ExternalAgentConfigDetectParams; + +pub(crate) const EXTERNAL_AGENT_CONFIG_MIGRATION_FINISHED_MESSAGE: &str = + "Agent import finished. Run /import again to check for additional items."; +pub(crate) const EXTERNAL_AGENT_CONFIG_MIGRATION_NO_ITEMS_MESSAGE: &str = + "No supported agent setup was found to import."; +pub(crate) const EXTERNAL_AGENT_CONFIG_MIGRATION_REMOTE_UNAVAILABLE_MESSAGE: &str = + "Agent import is unavailable in remote sessions. Start Codex locally and run /import."; +pub(crate) const EXTERNAL_AGENT_CONFIG_MIGRATION_DAEMON_UNAVAILABLE_MESSAGE: &str = "Agent import is unavailable while Codex is connected to the local app-server daemon. Stop the daemon, restart Codex, and run /import."; + +pub(crate) enum ExternalAgentConfigMigrationFlowOutcome { + Started(String), + NoItems, + Cancelled, +} + +fn external_agent_config_migration_success_message(remaining_item_count: usize) -> String { + let message = "Agent import started. You can keep working while it finishes. Imported setup will apply to new chats."; + match remaining_items_handoff(remaining_item_count) { + Some(remaining_items_handoff) => format!("{message} {remaining_items_handoff}"), + None => message.to_string(), + } +} + +fn remaining_items_handoff(remaining_item_count: usize) -> Option { + match remaining_item_count { + 0 => None, + 1 => Some( + "1 additional item remains. After it finishes, run /import again to review it." + .to_string(), + ), + _ => Some(format!( + "{remaining_item_count} additional items remain. After it finishes, run /import again to review them." + )), + } +} + +pub(crate) async fn handle_external_agent_config_migration_prompt( + tui: &mut tui::Tui, + app_server: &mut AppServerSession, + config: &Config, +) -> Result { + if app_server.uses_remote_workspace() { + return Err(EXTERNAL_AGENT_CONFIG_MIGRATION_REMOTE_UNAVAILABLE_MESSAGE.to_string()); + } + if !app_server.uses_embedded_app_server() { + return Err(EXTERNAL_AGENT_CONFIG_MIGRATION_DAEMON_UNAVAILABLE_MESSAGE.to_string()); + } + if app_server.external_agent_config_import_in_progress() { + return Err(EXTERNAL_AGENT_CONFIG_IMPORT_IN_PROGRESS_MESSAGE.to_string()); + } + + let cwd = config.cwd.to_path_buf(); + let detected_items = match app_server + .external_agent_config_detect(ExternalAgentConfigDetectParams { + include_home: true, + cwds: Some(vec![cwd.clone()]), + }) + .await + { + Ok(response) => response.items, + Err(err) => { + tracing::warn!( + error = %err, + cwd = %cwd.display(), + "failed to detect external agent config migrations" + ); + return Err(format!("Could not check for agent setup: {err}")); + } + }; + + if detected_items.is_empty() { + return Ok(ExternalAgentConfigMigrationFlowOutcome::NoItems); + } + + let mut selected_items = detected_items.clone(); + let mut error: Option = None; + + loop { + match run_external_agent_config_migration_prompt( + tui, + &detected_items, + &selected_items, + error.as_deref(), + ) + .await + { + ExternalAgentConfigMigrationOutcome::Proceed(items) => { + selected_items = items.clone(); + match app_server.external_agent_config_import(items).await { + Ok(()) => { + let remaining_item_count = + detected_items.len().saturating_sub(selected_items.len()); + let success_message = + external_agent_config_migration_success_message(remaining_item_count); + return Ok(ExternalAgentConfigMigrationFlowOutcome::Started( + success_message, + )); + } + Err(err) => { + tracing::warn!( + error = %err, + cwd = %cwd.display(), + "failed to import external agent config migration items" + ); + error = Some(format!("Import failed: {err}")); + } + } + } + ExternalAgentConfigMigrationOutcome::Skip => { + return Ok(ExternalAgentConfigMigrationFlowOutcome::Cancelled); + } + } + } +} + +#[cfg(test)] +#[path = "external_agent_config_migration_flow_tests.rs"] +mod tests; diff --git a/codex-rs/tui/src/external_agent_config_migration_flow_tests.rs b/codex-rs/tui/src/external_agent_config_migration_flow_tests.rs new file mode 100644 index 0000000000..913b55607f --- /dev/null +++ b/codex-rs/tui/src/external_agent_config_migration_flow_tests.rs @@ -0,0 +1,21 @@ +use super::*; + +#[test] +fn external_agent_config_migration_messages_snapshot() { + let cases = [0, 1, 2]; + + let messages = cases + .map(external_agent_config_migration_success_message) + .into_iter() + .chain([ + EXTERNAL_AGENT_CONFIG_MIGRATION_FINISHED_MESSAGE.to_string(), + EXTERNAL_AGENT_CONFIG_MIGRATION_NO_ITEMS_MESSAGE.to_string(), + EXTERNAL_AGENT_CONFIG_MIGRATION_REMOTE_UNAVAILABLE_MESSAGE.to_string(), + EXTERNAL_AGENT_CONFIG_MIGRATION_DAEMON_UNAVAILABLE_MESSAGE.to_string(), + EXTERNAL_AGENT_CONFIG_IMPORT_IN_PROGRESS_MESSAGE.to_string(), + ]) + .collect::>() + .join("\n"); + + insta::assert_snapshot!("external_agent_config_migration_messages", messages); +} diff --git a/codex-rs/tui/src/external_agent_config_migration_model.rs b/codex-rs/tui/src/external_agent_config_migration_model.rs new file mode 100644 index 0000000000..d03e238da3 --- /dev/null +++ b/codex-rs/tui/src/external_agent_config_migration_model.rs @@ -0,0 +1,147 @@ +use codex_app_server_protocol::ExternalAgentConfigMigrationItem; +use codex_app_server_protocol::ExternalAgentConfigMigrationItemType; +use std::collections::BTreeSet; + +#[derive(Clone, Debug)] +pub(crate) struct ExternalAgentConfigMigrationGroupModel { + pub(crate) label: String, + pub(crate) description: &'static str, + pub(crate) item_indices: Vec, +} + +pub(crate) fn external_agent_config_migration_groups( + items: &[ExternalAgentConfigMigrationItem], +) -> Vec { + let tools_and_setup = items + .iter() + .enumerate() + .filter_map(|(idx, item)| { + (item.cwd.is_none() && item.item_type != ExternalAgentConfigMigrationItemType::Sessions) + .then_some(idx) + }) + .collect::>(); + let projects = items + .iter() + .enumerate() + .filter_map(|(idx, item)| { + (item.cwd.is_some() && item.item_type != ExternalAgentConfigMigrationItemType::Sessions) + .then_some(idx) + }) + .collect::>(); + let chat_sessions = items + .iter() + .enumerate() + .filter_map(|(idx, item)| { + (item.item_type == ExternalAgentConfigMigrationItemType::Sessions).then_some(idx) + }) + .collect::>(); + + let mut groups = Vec::new(); + if !tools_and_setup.is_empty() { + groups.push(ExternalAgentConfigMigrationGroupModel { + label: "Tools & setup".to_string(), + description: "Settings, instructions, integrations, agents, commands, and skills", + item_indices: tools_and_setup, + }); + } + if !projects.is_empty() { + let project_count = projects + .iter() + .filter_map(|idx| items[*idx].cwd.as_deref()) + .collect::>() + .len(); + groups.push(ExternalAgentConfigMigrationGroupModel { + label: if project_count == 1 { + "Current project".to_string() + } else { + format!("Projects ({project_count})") + }, + description: "Add Codex files alongside your existing project files", + item_indices: projects, + }); + } + if !chat_sessions.is_empty() { + let session_count = chat_sessions + .iter() + .filter_map(|idx| items[*idx].details.as_ref()) + .map(|details| details.sessions.len()) + .sum::(); + groups.push(ExternalAgentConfigMigrationGroupModel { + label: format!("Chat sessions ({session_count})"), + description: "Last 30 days of chats", + item_indices: chat_sessions, + }); + } + groups +} + +pub(crate) fn external_agent_config_migration_item_label( + item: &ExternalAgentConfigMigrationItem, +) -> &'static str { + match item.item_type { + ExternalAgentConfigMigrationItemType::AgentsMd => "Instructions (CLAUDE.md -> AGENTS.md)", + ExternalAgentConfigMigrationItemType::Config => "Settings (settings.json -> config.toml)", + ExternalAgentConfigMigrationItemType::Skills => "Skills", + ExternalAgentConfigMigrationItemType::Plugins => "Plugins", + ExternalAgentConfigMigrationItemType::McpServerConfig => "MCP servers", + ExternalAgentConfigMigrationItemType::Subagents => "Agents", + ExternalAgentConfigMigrationItemType::Hooks => "Hooks", + ExternalAgentConfigMigrationItemType::Commands => "Slash commands", + ExternalAgentConfigMigrationItemType::Sessions => "Recent chat sessions", + } +} + +pub(crate) fn external_agent_config_migration_item_detail( + item: &ExternalAgentConfigMigrationItem, +) -> Option { + let details = item.details.as_ref()?; + match item.item_type { + ExternalAgentConfigMigrationItemType::Plugins => None, + ExternalAgentConfigMigrationItemType::McpServerConfig => Some(format_counted_details( + "MCP server", + details.mcp_servers.len(), + details + .mcp_servers + .iter() + .map(|server| server.name.as_str()), + )), + ExternalAgentConfigMigrationItemType::Subagents => Some(format_counted_details( + "agent", + details.subagents.len(), + details.subagents.iter().map(|agent| agent.name.as_str()), + )), + ExternalAgentConfigMigrationItemType::Hooks => Some(format_counted_details( + "hook", + details.hooks.len(), + details.hooks.iter().map(|hook| hook.name.as_str()), + )), + ExternalAgentConfigMigrationItemType::Commands => Some(format_counted_details( + "slash command", + details.commands.len(), + details.commands.iter().map(|command| command.name.as_str()), + )), + ExternalAgentConfigMigrationItemType::Sessions => Some(format_counted_details( + "chat session", + details.sessions.len(), + details + .sessions + .iter() + .filter_map(|session| session.title.as_deref()), + )), + ExternalAgentConfigMigrationItemType::AgentsMd + | ExternalAgentConfigMigrationItemType::Config + | ExternalAgentConfigMigrationItemType::Skills => None, + } +} + +fn format_counted_details<'a>( + noun: &str, + count: usize, + names: impl Iterator, +) -> String { + let suffix = if count == 1 { "" } else { "s" }; + match names.take(4).collect::>() { + names if names.is_empty() => format!("{count} {noun}{suffix}"), + names => format!("{count} {noun}{suffix}: {}", names.join(", ")), + } +} diff --git a/codex-rs/tui/src/external_agent_config_migration_startup.rs b/codex-rs/tui/src/external_agent_config_migration_startup.rs deleted file mode 100644 index b2afbe7718..0000000000 --- a/codex-rs/tui/src/external_agent_config_migration_startup.rs +++ /dev/null @@ -1,570 +0,0 @@ -use crate::app_server_session::AppServerSession; -use crate::external_agent_config_migration::ExternalAgentConfigMigrationOutcome; -use crate::external_agent_config_migration::run_external_agent_config_migration_prompt; -use crate::legacy_core::config::Config; -use crate::legacy_core::config::ConfigBuilder; -use crate::legacy_core::config::ConfigOverrides; -use crate::legacy_core::config::edit::ConfigEdit; -use crate::legacy_core::config::edit::ConfigEditsBuilder; -use crate::tui; -use codex_app_server_protocol::ExternalAgentConfigDetectParams; -use codex_app_server_protocol::ExternalAgentConfigMigrationItem; -use codex_config::CloudConfigBundleLoader; -use codex_features::Feature; -use color_eyre::eyre::Result; -use color_eyre::eyre::WrapErr; -use std::collections::BTreeSet; -use std::path::Path; -use std::time::SystemTime; -use std::time::UNIX_EPOCH; -use toml::Value as TomlValue; - -const EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS: i64 = 5 * 24 * 60 * 60; - -pub(crate) enum ExternalAgentConfigMigrationStartupOutcome { - Continue { success_message: Option }, - ExitRequested, -} - -pub(crate) fn should_show_external_agent_config_migration_prompt( - config: &Config, - entered_trust_nux: bool, -) -> bool { - entered_trust_nux && config.features.enabled(Feature::ExternalMigration) -} - -fn external_config_migration_project_key(path: &Path) -> String { - path.display().to_string() -} - -fn is_external_config_migration_scope_hidden(config: &Config, cwd: Option<&Path>) -> bool { - match cwd { - Some(cwd) => config - .notices - .external_config_migration_prompts - .projects - .get(&external_config_migration_project_key(cwd)) - .copied() - .unwrap_or(false), - None => config - .notices - .external_config_migration_prompts - .home - .unwrap_or(false), - } -} - -fn external_config_migration_last_prompted_at(config: &Config, cwd: Option<&Path>) -> Option { - match cwd { - Some(cwd) => config - .notices - .external_config_migration_prompts - .project_last_prompted_at - .get(&external_config_migration_project_key(cwd)) - .copied(), - None => { - config - .notices - .external_config_migration_prompts - .home_last_prompted_at - } - } -} - -fn is_external_config_migration_scope_cooling_down( - config: &Config, - cwd: Option<&Path>, - now_unix_seconds: i64, -) -> bool { - external_config_migration_last_prompted_at(config, cwd).is_some_and(|last_prompted_at| { - last_prompted_at.saturating_add(EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS) - > now_unix_seconds - }) -} - -fn visible_external_agent_config_migration_items( - config: &Config, - items: Vec, - now_unix_seconds: i64, -) -> Vec { - items - .into_iter() - .filter(|item| { - !is_external_config_migration_scope_hidden(config, item.cwd.as_deref()) - && !is_external_config_migration_scope_cooling_down( - config, - item.cwd.as_deref(), - now_unix_seconds, - ) - }) - .collect() -} - -fn external_agent_config_migration_success_message( - items: &[ExternalAgentConfigMigrationItem], -) -> String { - if items.iter().any(|item| { - item.item_type == codex_app_server_protocol::ExternalAgentConfigMigrationItemType::Plugins - }) { - "External config migration completed. Plugin migration is still in progress and may take a few minutes." - .to_string() - } else { - "External config migration completed successfully.".to_string() - } -} - -fn unix_seconds_now() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() as i64 -} - -async fn persist_external_agent_config_migration_prompt_shown( - config: &mut Config, - items: &[ExternalAgentConfigMigrationItem], - now_unix_seconds: i64, -) -> Result<()> { - let mut edits = Vec::new(); - if items.iter().any(|item| item.cwd.is_none()) { - edits.push( - ConfigEdit::SetNoticeExternalConfigMigrationPromptHomeLastPromptedAt(now_unix_seconds), - ); - } - - for project in items - .iter() - .filter_map(|item| item.cwd.as_deref()) - .map(external_config_migration_project_key) - { - edits.push( - ConfigEdit::SetNoticeExternalConfigMigrationPromptProjectLastPromptedAt( - project, - now_unix_seconds, - ), - ); - } - - if edits.is_empty() { - return Ok(()); - } - - ConfigEditsBuilder::for_config(config) - .with_edits(edits) - .apply() - .await - .map_err(|err| color_eyre::eyre::eyre!("{err}")) - .wrap_err("Failed to save external config migration prompt timestamp")?; - - if items.iter().any(|item| item.cwd.is_none()) { - config - .notices - .external_config_migration_prompts - .home_last_prompted_at = Some(now_unix_seconds); - } - for project in items - .iter() - .filter_map(|item| item.cwd.as_deref()) - .map(external_config_migration_project_key) - { - config - .notices - .external_config_migration_prompts - .project_last_prompted_at - .insert(project, now_unix_seconds); - } - - Ok(()) -} - -async fn persist_external_agent_config_migration_prompt_dismissal( - config: &mut Config, - items: &[ExternalAgentConfigMigrationItem], -) -> Result<()> { - let hide_home = items.iter().any(|item| item.cwd.is_none()); - let projects = items - .iter() - .filter_map(|item| item.cwd.as_deref()) - .map(external_config_migration_project_key) - .collect::>(); - - let mut edits = Vec::new(); - if hide_home - && !config - .notices - .external_config_migration_prompts - .home - .unwrap_or(false) - { - edits.push(ConfigEdit::SetNoticeHideExternalConfigMigrationPromptHome( - true, - )); - } - for project in &projects { - if !config - .notices - .external_config_migration_prompts - .projects - .get(project) - .copied() - .unwrap_or(false) - { - edits.push( - ConfigEdit::SetNoticeHideExternalConfigMigrationPromptProject( - project.clone(), - true, - ), - ); - } - } - - if edits.is_empty() { - return Ok(()); - } - - ConfigEditsBuilder::for_config(config) - .with_edits(edits) - .apply() - .await - .map_err(|err| color_eyre::eyre::eyre!("{err}")) - .wrap_err("Failed to save external config migration prompt preference")?; - - if hide_home { - config.notices.external_config_migration_prompts.home = Some(true); - } - for project in projects { - config - .notices - .external_config_migration_prompts - .projects - .insert(project, true); - } - - Ok(()) -} - -pub(crate) async fn handle_external_agent_config_migration_prompt_if_needed( - tui: &mut tui::Tui, - app_server: &mut AppServerSession, - config: &mut Config, - cli_kv_overrides: &[(String, TomlValue)], - harness_overrides: &ConfigOverrides, - cloud_config_bundle: &CloudConfigBundleLoader, - entered_trust_nux: bool, -) -> Result { - if !should_show_external_agent_config_migration_prompt(config, entered_trust_nux) { - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - - let now_unix_seconds = unix_seconds_now(); - let detected_items = match app_server - .external_agent_config_detect(ExternalAgentConfigDetectParams { - include_home: true, - cwds: Some(vec![config.cwd.to_path_buf()]), - }) - .await - { - Ok(response) => { - visible_external_agent_config_migration_items(config, response.items, now_unix_seconds) - } - Err(err) => { - tracing::warn!( - error = %err, - cwd = %config.cwd.display(), - "failed to detect external agent config migrations; continuing startup" - ); - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - }; - - if detected_items.is_empty() { - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - - if let Err(err) = persist_external_agent_config_migration_prompt_shown( - config, - &detected_items, - now_unix_seconds, - ) - .await - { - tracing::warn!( - error = %err, - cwd = %config.cwd.display(), - "failed to persist external config migration prompt timestamp" - ); - } - - let mut selected_items = detected_items.clone(); - let mut error: Option = None; - - loop { - match run_external_agent_config_migration_prompt( - tui, - &detected_items, - &selected_items, - error.as_deref(), - ) - .await - { - ExternalAgentConfigMigrationOutcome::Proceed(items) => { - selected_items = items.clone(); - match app_server.external_agent_config_import(items).await { - Ok(_) => { - let success_message = - external_agent_config_migration_success_message(&selected_items); - *config = ConfigBuilder::default() - .codex_home(config.codex_home.to_path_buf()) - .cli_overrides(cli_kv_overrides.to_vec()) - .harness_overrides(harness_overrides.clone()) - .cloud_config_bundle(cloud_config_bundle.clone()) - .build() - .await - .wrap_err("Failed to reload config after external agent migration")?; - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: Some(success_message), - }); - } - Err(err) => { - tracing::warn!( - error = %err, - cwd = %config.cwd.display(), - "failed to import external agent config migration items" - ); - error = Some(format!("Migration failed: {err}")); - } - } - } - ExternalAgentConfigMigrationOutcome::Skip => { - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - ExternalAgentConfigMigrationOutcome::SkipForever => { - match persist_external_agent_config_migration_prompt_dismissal( - config, - &detected_items, - ) - .await - { - Ok(()) => { - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - Err(err) => { - tracing::warn!( - error = %err, - cwd = %config.cwd.display(), - "failed to persist external config migration prompt dismissal" - ); - error = Some(format!("Failed to save preference: {err}")); - } - } - } - ExternalAgentConfigMigrationOutcome::Exit => { - return Ok(ExternalAgentConfigMigrationStartupOutcome::ExitRequested); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use codex_app_server_protocol::ExternalAgentConfigMigrationItemType; - use pretty_assertions::assert_eq; - use std::path::PathBuf; - use tempfile::tempdir; - - #[tokio::test] - async fn visible_external_agent_config_migration_items_omits_hidden_scopes() { - let codex_home = tempdir().expect("temp codex home"); - let mut config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .build() - .await - .expect("config"); - config.notices.external_config_migration_prompts.home = Some(true); - config - .notices - .external_config_migration_prompts - .projects - .insert("/tmp/project".to_string(), true); - - let visible = visible_external_agent_config_migration_items( - &config, - vec![ - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Config, - description: "home".to_string(), - cwd: None, - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::AgentsMd, - description: "project".to_string(), - cwd: Some(PathBuf::from("/tmp/project")), - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Skills, - description: "other project".to_string(), - cwd: Some(PathBuf::from("/tmp/other")), - details: None, - }, - ], - /*now_unix_seconds*/ 1_760_000_000, - ); - - assert_eq!( - visible, - vec![ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Skills, - description: "other project".to_string(), - cwd: Some(PathBuf::from("/tmp/other")), - details: None, - }] - ); - } - - #[tokio::test] - async fn visible_external_agent_config_migration_items_omits_recently_prompted_scopes() { - let codex_home = tempdir().expect("temp codex home"); - let mut config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .build() - .await - .expect("config"); - config - .notices - .external_config_migration_prompts - .home_last_prompted_at = Some(1_760_000_000); - config - .notices - .external_config_migration_prompts - .project_last_prompted_at - .insert("/tmp/project".to_string(), 1_760_000_000); - - let visible = visible_external_agent_config_migration_items( - &config, - vec![ - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Config, - description: "home".to_string(), - cwd: None, - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::AgentsMd, - description: "project".to_string(), - cwd: Some(PathBuf::from("/tmp/project")), - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Skills, - description: "other project".to_string(), - cwd: Some(PathBuf::from("/tmp/other")), - details: None, - }, - ], - /*now_unix_seconds*/ - 1_760_000_000 + EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS - 1, - ); - - assert_eq!( - visible, - vec![ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Skills, - description: "other project".to_string(), - cwd: Some(PathBuf::from("/tmp/other")), - details: None, - }] - ); - } - - #[tokio::test] - async fn external_config_migration_scope_cooldown_expires_after_five_days() { - let codex_home = tempdir().expect("temp codex home"); - let mut config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .build() - .await - .expect("config"); - config - .notices - .external_config_migration_prompts - .home_last_prompted_at = Some(1_760_000_000); - - assert!(is_external_config_migration_scope_cooling_down( - &config, - /*cwd*/ None, - 1_760_000_000 + EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS - 1, - )); - assert!(!is_external_config_migration_scope_cooling_down( - &config, - /*cwd*/ None, - 1_760_000_000 + EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS, - )); - } - - #[test] - fn external_agent_config_migration_success_message_mentions_plugins_when_present() { - let message = external_agent_config_migration_success_message(&[ - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Config, - description: String::new(), - cwd: None, - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Plugins, - description: String::new(), - cwd: None, - details: None, - }, - ]); - - assert_eq!( - message, - "External config migration completed. Plugin migration is still in progress and may take a few minutes." - ); - } - - #[test] - fn external_agent_config_migration_success_message_omits_plugins_copy_when_absent() { - let message = - external_agent_config_migration_success_message(&[ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::AgentsMd, - description: String::new(), - cwd: None, - details: None, - }]); - - assert_eq!(message, "External config migration completed successfully."); - } - - #[tokio::test] - async fn external_agent_config_migration_prompt_requires_trust_nux_entry() { - let codex_home = tempdir().expect("temp codex home"); - let mut config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .build() - .await - .expect("config"); - let _ = config.features.enable(Feature::ExternalMigration); - - assert!(!should_show_external_agent_config_migration_prompt( - &config, /*entered_trust_nux*/ false, - )); - assert!(should_show_external_agent_config_migration_prompt( - &config, /*entered_trust_nux*/ true, - )); - } -} diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 9f5748fb69..82709837d9 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -12,8 +12,6 @@ use crate::legacy_core::config::resolve_bootstrap_system_proxy_config; use crate::legacy_core::config::resolve_oss_provider; use crate::legacy_core::config::resolve_profile_v2_config_path; use crate::legacy_core::format_exec_policy_error_with_source; -#[cfg(target_os = "windows")] -use crate::legacy_core::windows_sandbox::WindowsSandboxLevelExt; use crate::session_resume::ResolveCwdOutcome; use crate::session_resume::resolve_cwd_for_resume_or_fork; pub use crate::startup_error::LocalStateDbStartupError; @@ -67,6 +65,7 @@ use codex_utils_oss::ensure_oss_provider_ready; use codex_utils_oss::get_default_model_for_oss_provider; use color_eyre::eyre::WrapErr; use cwd_prompt::CwdPromptAction; +pub use session_archive_commands::DeleteConfirmation; pub use session_archive_commands::SessionArchiveAction; pub use session_archive_commands::SessionArchiveCommandOptions; pub use session_archive_commands::run_session_archive_command; @@ -134,7 +133,8 @@ mod diff_render; mod exec_cell; mod exec_command; mod external_agent_config_migration; -mod external_agent_config_migration_startup; +mod external_agent_config_migration_flow; +mod external_agent_config_migration_model; mod external_editor; mod file_search; mod frames; @@ -213,6 +213,8 @@ mod version; #[cfg(not(target_os = "linux"))] mod voice; mod width; +#[cfg(any(target_os = "windows", test))] +mod windows_sandbox; mod workspace_command; #[cfg(target_os = "linux")] #[allow(dead_code)] @@ -355,9 +357,11 @@ async fn init_state_db_for_app_server_target( ) -> std::io::Result> { match app_server_target { AppServerTarget::Embedded => state_db::try_init(config).await.map(Some).map_err(|err| { + let database_path = codex_state::runtime_db_path_for_corruption_error(&err) + .unwrap_or_else(|| codex_state::state_db_path(config.sqlite_home.as_path())); std::io::Error::other(LocalStateDbStartupError::new( - codex_state::state_db_path(config.sqlite_home.as_path()), - err.to_string(), + database_path, + format!("{err:#}"), )) }), AppServerTarget::LocalDaemon { .. } | AppServerTarget::Remote { .. } => { @@ -1132,7 +1136,7 @@ pub async fn run_main( let otel_originator = originator().value; let otel = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - crate::legacy_core::otel_init::build_provider( + codex_app_server_client::build_otel_provider( &config, env!("CARGO_PKG_VERSION"), /*service_name_override*/ None, @@ -1155,26 +1159,25 @@ pub async fn run_main( None } }; - crate::legacy_core::otel_init::record_process_start(otel.as_ref(), otel_originator.as_str()); - crate::legacy_core::otel_init::install_sqlite_telemetry( - otel.as_ref(), - otel_originator.as_str(), - ); + if let Some(metrics) = otel.as_ref().and_then(codex_otel::OtelProvider::metrics) { + let _ = codex_otel::record_process_start_once(metrics, otel_originator.as_str()); + let telemetry = + codex_rollout::sqlite_telemetry_recorder(metrics.clone(), otel_originator.as_str()); + let _ = codex_state::install_process_db_telemetry(telemetry); + } let state_db = init_state_db_for_app_server_target(&config, &app_server_target).await?; let effective_toml = config.config_layer_stack.effective_config(); match effective_toml.try_into() { Ok(config_toml) => { - match crate::legacy_core::personality_migration::maybe_migrate_personality( + match codex_app_server_client::migrate_personality_if_needed( &config.codex_home, &config_toml, state_db.clone(), ) .await { - Ok( - crate::legacy_core::personality_migration::PersonalityMigrationStatus::Applied, - ) => { + Ok(true) => { config = load_config_or_exit( cli_kv_overrides.clone(), overrides.clone(), @@ -1184,11 +1187,7 @@ pub async fn run_main( ) .await; } - Ok( - crate::legacy_core::personality_migration::PersonalityMigrationStatus::SkippedMarker - | crate::legacy_core::personality_migration::PersonalityMigrationStatus::SkippedExplicitPersonality - | crate::legacy_core::personality_migration::PersonalityMigrationStatus::SkippedNoSessions, - ) => {} + Ok(false) => {} Err(err) => { tracing::warn!(error = %err, "failed to run personality migration"); } @@ -1400,7 +1399,7 @@ async fn run_ratatui_app( return Ok(AppExitInfo { token_usage: crate::token_usage::TokenUsage::default(), thread_id: None, - thread_name: None, + resume_hint: None, update_action: Some(action), exit_reason: ExitReason::UserRequested, }); @@ -1493,7 +1492,7 @@ async fn run_ratatui_app( return Ok(AppExitInfo { token_usage: crate::token_usage::TokenUsage::default(), thread_id: None, - thread_name: None, + resume_hint: None, update_action: None, exit_reason: ExitReason::UserRequested, }); @@ -1547,7 +1546,7 @@ async fn run_ratatui_app( Ok(AppExitInfo { token_usage: crate::token_usage::TokenUsage::default(), thread_id: None, - thread_name: None, + resume_hint: None, update_action: None, exit_reason: ExitReason::Fatal(format!( "No saved session found with ID {id_str}. Run `codex {action}` without an ID to choose from existing sessions." @@ -1604,7 +1603,7 @@ async fn run_ratatui_app( return Ok(AppExitInfo { token_usage: crate::token_usage::TokenUsage::default(), thread_id: None, - thread_name: None, + resume_hint: None, update_action: None, exit_reason: ExitReason::UserRequested, }); @@ -1665,7 +1664,7 @@ async fn run_ratatui_app( return Ok(AppExitInfo { token_usage: crate::token_usage::TokenUsage::default(), thread_id: None, - thread_name: None, + resume_hint: None, update_action: None, exit_reason: ExitReason::UserRequested, }); @@ -1710,7 +1709,7 @@ async fn run_ratatui_app( return Ok(AppExitInfo { token_usage: crate::token_usage::TokenUsage::default(), thread_id: None, - thread_name: None, + resume_hint: None, update_action: None, exit_reason: ExitReason::UserRequested, }); @@ -1764,7 +1763,7 @@ async fn run_ratatui_app( set_default_client_residency_requirement(config.enforce_residency.value()); let should_show_trust_screen = should_show_trust_screen(&config); #[cfg(target_os = "windows")] - let windows_sandbox_level = WindowsSandboxLevel::from_config(&config); + let windows_sandbox_level = crate::windows_sandbox::level_from_config(&config); #[cfg(target_os = "windows")] let required_elevated_sandbox_needs_setup = windows_sandbox_level == WindowsSandboxLevel::Elevated @@ -1774,9 +1773,7 @@ async fn run_ratatui_app( .windows_sandbox_mode .source .is_some() - && !crate::legacy_core::windows_sandbox::sandbox_setup_is_complete( - config.codex_home.as_path(), - ); + && !crate::windows_sandbox::sandbox_setup_is_complete(config.codex_home.as_path()); #[cfg(target_os = "windows")] let should_prompt_windows_sandbox_nux_at_startup = (trust_decision_was_made && windows_sandbox_level == WindowsSandboxLevel::Disabled) @@ -1834,23 +1831,11 @@ async fn run_ratatui_app( let hooks_request_handle = app_server.request_handle(); let hooks_cwd = config.cwd.to_path_buf(); let startup_prefetch_started_at = Instant::now(); - let should_defer_bootstrap = - external_agent_config_migration_startup::should_show_external_agent_config_migration_prompt( - &config, - should_show_trust_screen_flag, - ); - let (startup_bootstrap, startup_hooks_entry) = if should_defer_bootstrap { - ( - None, - load_startup_hooks_review_entry(hooks_request_handle, hooks_cwd).await, - ) - } else { - let (bootstrap, entry) = tokio::join!( - app_server.bootstrap(&config), - load_startup_hooks_review_entry(hooks_request_handle, hooks_cwd), - ); - (Some(bootstrap?), entry) - }; + let (startup_bootstrap, startup_hooks_entry) = tokio::join!( + app_server.bootstrap(&config), + load_startup_hooks_review_entry(hooks_request_handle, hooks_cwd), + ); + let startup_bootstrap = Some(startup_bootstrap?); let startup_elapsed_before_app = startup_prefetch_started_at.elapsed(); let startup_hooks_browser = match maybe_run_startup_hooks_review( &mut app_server, @@ -1878,7 +1863,6 @@ async fn run_ratatui_app( session_selection, feedback, should_show_trust_screen, // Proxy to: is it a first run in this directory? - should_show_trust_screen_flag, // Preserve the startup-time trust NUX signal before onboarding should_prompt_windows_sandbox_nux_at_startup, app_server_target, state_db, @@ -3015,6 +2999,37 @@ mod tests { ); Ok(()) } + + #[tokio::test] + async fn embedded_state_db_corruption_preserves_failed_database_for_cli_recovery() + -> color_eyre::Result<()> { + let temp_dir = TempDir::new()?; + let mut config = build_config(&temp_dir).await?; + let sqlite_home = temp_dir.path().join("sqlite-home"); + std::fs::create_dir_all(&sqlite_home)?; + let logs_db_path = codex_state::logs_db_path(&sqlite_home); + std::fs::write(&logs_db_path, "not a sqlite database")?; + config.sqlite_home = sqlite_home; + + let err = + match init_state_db_for_app_server_target(&config, &AppServerTarget::Embedded).await { + Ok(_) => panic!("embedded startup should surface state db init failures"), + Err(err) => err, + }; + let startup_error = err + .get_ref() + .and_then(|err| err.downcast_ref::()) + .expect("state db startup failure should retain its typed context"); + + assert_eq!(startup_error.database_path(), logs_db_path.as_path()); + assert!( + codex_state::sqlite_error_detail_is_corruption(startup_error.detail()), + "startup error should preserve the SQLite corruption cause, got: {}", + startup_error.detail() + ); + Ok(()) + } + #[tokio::test] #[serial] async fn windows_shows_trust_prompt_with_sandbox() -> std::io::Result<()> { diff --git a/codex-rs/tui/src/local_chatgpt_auth.rs b/codex-rs/tui/src/local_chatgpt_auth.rs index d7ca01f618..8a9ba58fc9 100644 --- a/codex-rs/tui/src/local_chatgpt_auth.rs +++ b/codex-rs/tui/src/local_chatgpt_auth.rs @@ -110,6 +110,7 @@ mod tests { last_refresh: Some(Utc::now()), agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }; save_auth(codex_home, &auth, AuthCredentialsStoreMode::File) .expect("chatgpt auth should save"); @@ -158,6 +159,7 @@ mod tests { last_refresh: None, agent_identity: None, personal_access_token: None, + bedrock_api_key: None, }, AuthCredentialsStoreMode::File, ) diff --git a/codex-rs/tui/src/main.rs b/codex-rs/tui/src/main.rs index 9c69c7c81a..486dbe6499 100644 --- a/codex-rs/tui/src/main.rs +++ b/codex-rs/tui/src/main.rs @@ -7,14 +7,15 @@ use codex_tui::Cli; use codex_tui::ExitReason; use codex_tui::run_main; use codex_utils_cli::CliConfigOverrides; -use codex_utils_cli::resume_hint; +use std::io::Write; use supports_color::Stream; fn format_exit_messages(exit_info: AppExitInfo, color_enabled: bool) -> Vec { + let is_fatal = matches!(&exit_info.exit_reason, ExitReason::Fatal(_)); let AppExitInfo { token_usage, thread_id, - thread_name, + resume_hint, .. } = exit_info; @@ -23,13 +24,15 @@ fn format_exit_messages(exit_info: AppExitInfo, color_enabled: bool) -> Vec anyhow::Result<()> { /*explicit_remote_endpoint*/ None, ) .await?; - match exit_info.exit_reason { + let is_fatal = match &exit_info.exit_reason { ExitReason::Fatal(message) => { eprintln!("ERROR: {message}"); - std::process::exit(1); + true } - ExitReason::UserRequested => {} - } + ExitReason::UserRequested => false, + }; let color_enabled = supports_color::on(Stream::Stdout).is_some(); for line in format_exit_messages(exit_info, color_enabled) { println!("{line}"); } + if is_fatal { + std::io::stdout().flush()?; + std::process::exit(1); + } Ok(()) }) } diff --git a/codex-rs/tui/src/onboarding/onboarding_screen.rs b/codex-rs/tui/src/onboarding/onboarding_screen.rs index e4c4b346d5..405b37743c 100644 --- a/codex-rs/tui/src/onboarding/onboarding_screen.rs +++ b/codex-rs/tui/src/onboarding/onboarding_screen.rs @@ -36,8 +36,6 @@ use crate::config_update::format_config_error; use crate::config_update::write_trusted_project; use crate::key_hint::KeyBindingListExt; use crate::legacy_core::config::Config; -#[cfg(target_os = "windows")] -use crate::legacy_core::windows_sandbox::WindowsSandboxLevelExt; use crate::onboarding::auth::AuthModeWidget; use crate::onboarding::auth::SignInOption; use crate::onboarding::auth::SignInState; @@ -143,7 +141,7 @@ impl OnboardingScreen { } #[cfg(target_os = "windows")] let show_windows_create_sandbox_hint = - WindowsSandboxLevel::from_config(&config) == WindowsSandboxLevel::Disabled; + crate::windows_sandbox::level_from_config(&config) == WindowsSandboxLevel::Disabled; #[cfg(not(target_os = "windows"))] let show_windows_create_sandbox_hint = false; let highlighted = TrustDirectorySelection::Trust; diff --git a/codex-rs/tui/src/session_archive_commands.rs b/codex-rs/tui/src/session_archive_commands.rs index 4e5c420f63..7de1e577c7 100644 --- a/codex-rs/tui/src/session_archive_commands.rs +++ b/codex-rs/tui/src/session_archive_commands.rs @@ -1,8 +1,10 @@ -//! Shared implementation for `codex archive` and `codex unarchive`. +//! Shared implementation for `codex archive`, `codex delete`, and `codex unarchive`. //! //! The CLI commands are thin app-server clients: resolve a user-provided UUID or exact session -//! name, then call the existing `thread/archive` or `thread/unarchive` RPC. +//! name, then call the corresponding app-server RPC. +use std::io::IsTerminal; +use std::io::Write; use std::sync::Arc; use crate::Cli; @@ -26,16 +28,22 @@ use codex_protocol::ThreadId; use codex_utils_cli::CliConfigOverrides; use codex_utils_home_dir::find_codex_home; use codex_utils_oss::get_default_model_for_oss_provider; -use color_eyre::eyre::ContextCompat; use color_eyre::eyre::Result; use color_eyre::eyre::WrapErr; use color_eyre::eyre::eyre; use super::RemoteAppServerEndpoint; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeleteConfirmation { + Prompt, + Skip, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SessionArchiveAction { Archive, + Delete(DeleteConfirmation), Unarchive, } @@ -52,6 +60,7 @@ fn success_message( ) -> String { let action = match action { SessionArchiveAction::Archive => "Archived", + SessionArchiveAction::Delete(_) => "Deleted", SessionArchiveAction::Unarchive => "Unarchived", }; match session_name { @@ -80,25 +89,30 @@ async fn run_session_archive_action_with_app_server( target: &str, ) -> Result { let resolved = resolve_session_target(app_server, action, target).await?; - match action { + let session_name = match action { SessionArchiveAction::Archive => { app_server.thread_archive(resolved.session_id).await?; - Ok(success_message( - action, - resolved.session_id, - resolved.session_name.as_deref(), - )) + resolved.session_name + } + SessionArchiveAction::Delete(confirmation) => { + if matches!(confirmation, DeleteConfirmation::Prompt) + && !confirm_session_delete(&resolved)? + { + return Ok("Delete cancelled.".to_string()); + } + app_server.thread_delete(resolved.session_id).await?; + resolved.session_name } SessionArchiveAction::Unarchive => { let thread = app_server.thread_unarchive(resolved.session_id).await?; - let session_name = thread.name.or(resolved.session_name); - Ok(success_message( - action, - resolved.session_id, - session_name.as_deref(), - )) + thread.name.or(resolved.session_name) } - } + }; + Ok(success_message( + action, + resolved.session_id, + session_name.as_deref(), + )) } async fn resolve_session_target( @@ -107,61 +121,83 @@ async fn resolve_session_target( target: &str, ) -> Result { if let Ok(session_id) = ThreadId::from_string(target) { + if matches!( + action, + SessionArchiveAction::Delete(DeleteConfirmation::Prompt) + ) { + let thread = app_server + .thread_read(session_id, /*include_turns*/ false) + .await + .with_context(|| { + format!("No active or archived session found matching '{target}'.") + })?; + return Ok(ResolvedSessionTarget { + session_id, + session_name: thread.name, + }); + } return Ok(ResolvedSessionTarget { session_id, session_name: None, }); } - let search_scope = match action { - SessionArchiveAction::Archive => "active", - SessionArchiveAction::Unarchive => "archived", + let (search_scope, archived_values): (&str, &[bool]) = match action { + SessionArchiveAction::Archive => ("active", &[false]), + SessionArchiveAction::Delete(_) => ("active or archived", &[false, true]), + SessionArchiveAction::Unarchive => ("archived", &[true]), }; - let resolved = lookup_session_by_exact_name(app_server, action, target) - .await? - .map(session_target_from_app_server_thread) - .transpose()?; - - resolved.with_context(|| format!("No {search_scope} session found matching '{target}'.")) + for &archived in archived_values { + if let Some(thread) = lookup_session_by_exact_name(app_server, target, archived).await? { + return session_target_from_app_server_thread(thread); + } + } + Err(eyre!( + "No {search_scope} session found matching '{target}'." + )) } async fn lookup_session_by_exact_name( app_server: &mut AppServerSession, - action: SessionArchiveAction, name: &str, + archived: bool, ) -> Result> { - let mut cursor = None; - loop { - let response = app_server - .thread_list(ThreadListParams { - cursor: cursor.clone(), - limit: Some(100), - sort_key: Some(ThreadSortKey::UpdatedAt), - sort_direction: None, - model_providers: None, - source_kinds: Some(super::resume_source_kinds( - /*include_non_interactive*/ false, - )), - archived: Some(matches!(action, SessionArchiveAction::Unarchive)), - cwd: None, - use_state_db_only: false, - search_term: Some(name.to_string()), - }) - .await - .wrap_err("failed to list sessions while resolving session name")?; + // Search is the fast path, but some stores attach renamed titles after applying the filter. + for search_term in [Some(name), None] { + let mut cursor = None; + loop { + let response = app_server + .thread_list(ThreadListParams { + cursor: cursor.clone(), + limit: Some(100), + sort_key: Some(ThreadSortKey::UpdatedAt), + sort_direction: None, + model_providers: None, + source_kinds: Some(super::resume_source_kinds( + /*include_non_interactive*/ false, + )), + archived: Some(archived), + cwd: None, + use_state_db_only: false, + search_term: search_term.map(str::to_string), + }) + .await + .wrap_err("failed to list sessions while resolving session name")?; - if let Some(thread) = response - .data - .into_iter() - .find(|thread| thread.name.as_deref() == Some(name)) - { - return Ok(Some(thread)); + if let Some(thread) = response + .data + .into_iter() + .find(|thread| thread.name.as_deref() == Some(name)) + { + return Ok(Some(thread)); + } + let Some(next_cursor) = response.next_cursor else { + break; + }; + cursor = Some(next_cursor); } - if response.next_cursor.is_none() { - return Ok(None); - } - cursor = response.next_cursor; } + Ok(None) } fn session_target_from_app_server_thread(thread: AppServerThread) -> Result { @@ -173,6 +209,35 @@ fn session_target_from_app_server_thread(thread: AppServerThread) -> Result Result { + if !(std::io::stdin().is_terminal() && std::io::stderr().is_terminal()) { + return Err(eyre!( + "cannot confirm session deletion without an interactive terminal; rerun with --force and a session UUID" + )); + } + + let mut stderr = std::io::stderr().lock(); + match target.session_name.as_deref() { + Some(name) => writeln!( + stderr, + "Permanently delete session '{name}' ({})?", + target.session_id + ), + None => writeln!(stderr, "Permanently delete session {}?", target.session_id), + }?; + writeln!( + stderr, + "This cannot be undone. Subagent threads will also be deleted." + )?; + write!(stderr, "Continue? [y/N]: ")?; + stderr.flush()?; + + let mut input = String::new(); + std::io::stdin().read_line(&mut input)?; + let answer = input.trim(); + Ok(answer.eq_ignore_ascii_case("y") || answer.eq_ignore_ascii_case("yes")) +} + async fn start_app_server_for_archive_command( options: SessionArchiveCommandOptions, ) -> Result { diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 805f266e0a..9ced21b26e 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -26,11 +26,13 @@ pub enum SlashCommand { AutoReview, Memories, Skills, + Import, Hooks, Review, Rename, New, Archive, + Delete, Resume, Fork, App, @@ -89,6 +91,7 @@ impl SlashCommand { SlashCommand::Rename => "rename the current thread", SlashCommand::Resume => "resume a saved chat", SlashCommand::Archive => "archive this session and exit", + SlashCommand::Delete => "permanently delete this session and exit", SlashCommand::Clear => "clear the terminal and start a new chat", SlashCommand::Fork => "fork the current chat", SlashCommand::App => "continue this session in Codex Desktop", @@ -98,6 +101,9 @@ impl SlashCommand { SlashCommand::Diff => "show git diff (including untracked files)", SlashCommand::Mention => "mention a file", SlashCommand::Skills => "use skills to improve how Codex performs specific tasks", + SlashCommand::Import => { + "import setup, this project, and recent chats from another coding agent" + } SlashCommand::Hooks => "view and manage lifecycle hooks", SlashCommand::Status => "show current session configuration and token usage", SlashCommand::DebugConfig => "show config layers and requirement sources for debugging", @@ -185,6 +191,7 @@ impl SlashCommand { match self { SlashCommand::New | SlashCommand::Archive + | SlashCommand::Delete | SlashCommand::Resume | SlashCommand::Fork | SlashCommand::Init @@ -198,6 +205,7 @@ impl SlashCommand { | SlashCommand::SandboxReadRoot | SlashCommand::Experimental | SlashCommand::Memories + | SlashCommand::Import | SlashCommand::Review | SlashCommand::Plan | SlashCommand::Clear diff --git a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize.snap b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize.snap new file mode 100644 index 0000000000..f23691b6f3 --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize.snap @@ -0,0 +1,29 @@ +--- +source: tui/src/external_agent_config_migration.rs +expression: rendered +--- + + > Choose what to import + Choose the items to import. + Codex may add files to your current project folder. + Your existing agent setup will not be changed. + Home + › [x] Settings (settings.json -> config.toml) + Import /Users/alex/.claude/settings.json into /Users/alex/.codex/conf… + [x] Recent chat sessions + Import recent chat sessions + 1 chat session: Investigate migration UX + + Current project: /workspace/project + [x] Plugins + Import enabled plugins from .claude/settings.json (4 marketplaces, 6 … + • acme-tools: deployer, formatter, +1 more + • team-marketplace: asana + • debug: sample + • +1 more marketplaces + [x] Instructions (CLAUDE.md -> AGENTS.md) + Import CLAUDE.md to AGENTS.md + + Selected 4 of 4 items. + 1. Review selection + Use ↑/↓ to move, space to toggle, b to go back diff --git a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize_action.snap b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize_action.snap new file mode 100644 index 0000000000..70df423cee --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize_action.snap @@ -0,0 +1,29 @@ +--- +source: tui/src/external_agent_config_migration.rs +expression: rendered +--- + + > Choose what to import + Choose the items to import. + Codex may add files to your current project folder. + Your existing agent setup will not be changed. + Home + [x] Settings (settings.json -> config.toml) + Import /Users/alex/.claude/settings.json into /Users/alex/.codex/conf… + [x] Recent chat sessions + Import recent chat sessions + 1 chat session: Investigate migration UX + + Current project: /workspace/project + [x] Plugins + Import enabled plugins from .claude/settings.json (4 marketplaces, 6 … + • acme-tools: deployer, formatter, +1 more + • team-marketplace: asana + • debug: sample + • +1 more marketplaces + [x] Instructions (CLAUDE.md -> AGENTS.md) + Import CLAUDE.md to AGENTS.md + + Selected 4 of 4 items. + › 1. Review selection + Press enter to continue, ↑/↓ to move, b to go back diff --git a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize_action_windows.snap b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize_action_windows.snap new file mode 100644 index 0000000000..e6e35384ae --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize_action_windows.snap @@ -0,0 +1,29 @@ +--- +source: tui/src/external_agent_config_migration.rs +expression: rendered +--- + + > Choose what to import + Choose the items to import. + Codex may add files to your current project folder. + Your existing agent setup will not be changed. + Home + [x] Settings (settings.json -> config.toml) + Import /Users/alex/.claude/settings.json into /Users/alex/.codex/conf… + [x] Recent chat sessions + Import recent chat sessions + 1 chat session: Investigate migration UX + + Current project: C:\workspace\project + [x] Plugins + Import enabled plugins from .claude/settings.json (4 marketplaces, 6 … + • acme-tools: deployer, formatter, +1 more + • team-marketplace: asana + • debug: sample + • +1 more marketplaces + [x] Instructions (CLAUDE.md -> AGENTS.md) + Import CLAUDE.md to AGENTS.md + + Selected 4 of 4 items. + › 1. Review selection + Press enter to continue, ↑/↓ to move, b to go back diff --git a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize_windows.snap b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize_windows.snap new file mode 100644 index 0000000000..12dff7a4e5 --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_customize_windows.snap @@ -0,0 +1,29 @@ +--- +source: tui/src/external_agent_config_migration.rs +expression: rendered +--- + + > Choose what to import + Choose the items to import. + Codex may add files to your current project folder. + Your existing agent setup will not be changed. + Home + › [x] Settings (settings.json -> config.toml) + Import /Users/alex/.claude/settings.json into /Users/alex/.codex/conf… + [x] Recent chat sessions + Import recent chat sessions + 1 chat session: Investigate migration UX + + Current project: C:\workspace\project + [x] Plugins + Import enabled plugins from .claude/settings.json (4 marketplaces, 6 … + • acme-tools: deployer, formatter, +1 more + • team-marketplace: asana + • debug: sample + • +1 more marketplaces + [x] Instructions (CLAUDE.md -> AGENTS.md) + Import CLAUDE.md to AGENTS.md + + Selected 4 of 4 items. + 1. Review selection + Use ↑/↓ to move, space to toggle, b to go back diff --git a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt.snap b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt.snap index 5ce3c5ef09..64a5f0b1b0 100644 --- a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt.snap +++ b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt.snap @@ -2,22 +2,21 @@ source: tui/src/external_agent_config_migration.rs expression: rendered --- - > External agent config detected - We found settings from another agent that you can add to this project. - Select what to import - Home - [x] Migrate /Users/alex/.claude/settings.json into /Users/alex/.codex/con… - Project: /workspace/project - [x] Migrate enabled plugins from .claude/settings.json (4 marketplaces, 6… - • acme-tools: deployer, formatter, +1 more - • team-marketplace: asana - • debug: sample - • +1 more marketplaces - [x] Migrate CLAUDE.md to AGENTS.md + > Import from another coding agent + Bring over your setup, current project, and recent chats. + Codex may add files to your current project folder. + Your existing agent setup will not be changed. + Cloud-hosted chat data cannot be imported. + [x] Tools & setup + Settings, instructions, integrations, agents, commands, and skills + [x] Current project + Add Codex files alongside your existing project files + [x] Chat sessions (1) + Last 30 days of chats - Selected 3 of 3 item(s). - 1. Proceed with selected - 2. Skip for now - 3. Don't ask again - Use ↑/↓ to move, space to toggle, 1/2/3 to choose, a/n for all/none + Selected 4 of 4 items. + › 1. Import selected + 2. Customize selection + 3. Cancel + Use ↑/↓ to move, enter to select, c to customize diff --git a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt_windows.snap b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt_windows.snap index 44f41784e6..64a5f0b1b0 100644 --- a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt_windows.snap +++ b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt_windows.snap @@ -3,22 +3,20 @@ source: tui/src/external_agent_config_migration.rs expression: rendered --- - > External agent config detected - We found settings from another agent that you can add to this project. - Select what to import - Home - [x] Migrate /Users/alex/.claude/settings.json into /Users/alex/.codex/con… + > Import from another coding agent + Bring over your setup, current project, and recent chats. + Codex may add files to your current project folder. + Your existing agent setup will not be changed. + Cloud-hosted chat data cannot be imported. + [x] Tools & setup + Settings, instructions, integrations, agents, commands, and skills + [x] Current project + Add Codex files alongside your existing project files + [x] Chat sessions (1) + Last 30 days of chats - Project: C:\workspace\project - [x] Migrate enabled plugins from .claude/settings.json (4 marketplaces, 6… - • acme-tools: deployer, formatter, +1 more - • team-marketplace: asana - • debug: sample - • +1 more marketplaces - [x] Migrate CLAUDE.md to AGENTS.md - - Selected 3 of 3 item(s). - 1. Proceed with selected - 2. Skip for now - 3. Don't ask again - Use ↑/↓ to move, space to toggle, 1/2/3 to choose, a/n for all/none + Selected 4 of 4 items. + › 1. Import selected + 2. Customize selection + 3. Cancel + Use ↑/↓ to move, enter to select, c to customize diff --git a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration_flow__tests__external_agent_config_migration_messages.snap b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration_flow__tests__external_agent_config_migration_messages.snap new file mode 100644 index 0000000000..5e80f8ac29 --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration_flow__tests__external_agent_config_migration_messages.snap @@ -0,0 +1,12 @@ +--- +source: tui/src/external_agent_config_migration_flow_tests.rs +expression: messages +--- +Agent import started. You can keep working while it finishes. Imported setup will apply to new chats. +Agent import started. You can keep working while it finishes. Imported setup will apply to new chats. 1 additional item remains. After it finishes, run /import again to review it. +Agent import started. You can keep working while it finishes. Imported setup will apply to new chats. 2 additional items remain. After it finishes, run /import again to review them. +Agent import finished. Run /import again to check for additional items. +No supported agent setup was found to import. +Agent import is unavailable in remote sessions. Start Codex locally and run /import. +Agent import is unavailable while Codex is connected to the local app-server daemon. Stop the daemon, restart Codex, and run /import. +A previous agent import is still running. Wait for it to finish before importing again. diff --git a/codex-rs/tui/src/startup_error.rs b/codex-rs/tui/src/startup_error.rs index 55b691d836..585cec4b23 100644 --- a/codex-rs/tui/src/startup_error.rs +++ b/codex-rs/tui/src/startup_error.rs @@ -3,24 +3,28 @@ use std::path::PathBuf; #[derive(Debug, thiserror::Error)] #[error( - "failed to initialize sqlite state db at {}: {detail}", - state_db_path.display() + "failed to initialize sqlite local db at {}: {detail}", + database_path.display() )] pub struct LocalStateDbStartupError { - state_db_path: PathBuf, + database_path: PathBuf, detail: String, } impl LocalStateDbStartupError { - pub fn new(state_db_path: PathBuf, detail: String) -> Self { + pub fn new(database_path: PathBuf, detail: String) -> Self { Self { - state_db_path, + database_path, detail, } } + pub fn database_path(&self) -> &Path { + self.database_path.as_path() + } + pub fn state_db_path(&self) -> &Path { - self.state_db_path.as_path() + self.database_path() } pub fn detail(&self) -> &str { diff --git a/codex-rs/tui/src/status/tests.rs b/codex-rs/tui/src/status/tests.rs index 1fc4c4feba..4ab7a2cfa2 100644 --- a/codex-rs/tui/src/status/tests.rs +++ b/codex-rs/tui/src/status/tests.rs @@ -29,6 +29,8 @@ use codex_app_server_protocol::SpendControlLimitSnapshot; use codex_config::LoaderOverrides; use codex_model_provider_info::ModelProviderAwsAuthInfo; use codex_model_provider_info::ModelProviderInfo; +use codex_models_manager::test_support::construct_model_info_offline_for_tests; +use codex_models_manager::test_support::get_model_offline_for_tests; use codex_protocol::ThreadId; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::ReasoningSummary; @@ -147,7 +149,7 @@ fn test_status_account_display() -> Option { fn token_info_for(model_slug: &str, config: &Config, usage: &TokenUsage) -> TokenUsageInfo { let context_window = - crate::legacy_core::test_support::construct_model_info_offline(model_slug, config) + construct_model_info_offline_for_tests(model_slug, &config.to_models_manager_config()) .context_window; TokenUsageInfo { total_token_usage: usage.clone(), @@ -212,7 +214,7 @@ fn permissions_text_for(config: &Config) -> Option { .with_ymd_and_hms(2024, 1, 2, 3, 4, 5) .single() .expect("timestamp"); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let composite = new_status_output( config, test_status_account_display().as_ref(), @@ -287,7 +289,7 @@ async fn status_snapshot_includes_reasoning_details() { }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let reasoning_effort_override = Some(Some(ReasoningEffort::High)); @@ -603,7 +605,7 @@ async fn status_snapshot_shows_active_user_defined_profile() { .with_ymd_and_hms(2024, 1, 2, 3, 4, 5) .single() .expect("timestamp"); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( @@ -648,7 +650,7 @@ async fn status_model_provider_uses_bedrock_runtime_base_url_and_gates_usage_lin .with_ymd_and_hms(2024, 1, 2, 3, 4, 5) .single() .expect("timestamp"); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let runtime_base_url = "https://bedrock-mantle.eu-west-1.api.aws/openai/v1"; let (composite, _handle) = new_status_output_with_rate_limits_handle( @@ -758,7 +760,7 @@ async fn status_snapshot_shows_auto_review_permissions() { .with_ymd_and_hms(2024, 1, 2, 3, 4, 5) .single() .expect("timestamp"); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( @@ -854,7 +856,7 @@ async fn status_snapshot_includes_forked_from() { .single() .expect("valid time"); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let session_id = ThreadId::from_string("0f0f3c13-6cf9-4aa4-8b80-7d49c2f1be2e").expect("session id"); @@ -923,7 +925,7 @@ async fn status_snapshot_includes_monthly_limit() { }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -987,7 +989,7 @@ async fn status_snapshot_includes_enterprise_monthly_credit_limit() { }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1067,7 +1069,7 @@ async fn status_snapshot_uses_generic_limit_labels_for_unsupported_windows() { }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1119,7 +1121,7 @@ async fn status_snapshot_shows_unlimited_credits() { rate_limit_reached_type: None, }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1170,7 +1172,7 @@ async fn status_snapshot_shows_positive_credits() { rate_limit_reached_type: None, }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1221,7 +1223,7 @@ async fn status_snapshot_hides_zero_credits() { rate_limit_reached_type: None, }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1270,7 +1272,7 @@ async fn status_snapshot_hides_when_has_no_credits_flag() { rate_limit_reached_type: None, }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1315,7 +1317,7 @@ async fn status_card_token_usage_excludes_cached_tokens() { .single() .expect("timestamp"); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1378,7 +1380,7 @@ async fn status_snapshot_truncates_in_narrow_terminal() { }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let reasoning_effort_override = Some(Some(ReasoningEffort::High)); let composite = new_status_output( @@ -1428,7 +1430,7 @@ async fn status_snapshot_shows_missing_limits_message() { .single() .expect("timestamp"); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1480,7 +1482,7 @@ async fn status_snapshot_uses_default_reasoning_when_config_empty() { version: "v0.133.0".to_string(), }; - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let (composite, _) = new_status_output_with_rate_limits_handle( &config, @@ -1549,7 +1551,7 @@ async fn status_snapshot_shows_refreshing_limits_notice() { }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output_with_rate_limits( &config, @@ -1621,7 +1623,7 @@ async fn status_snapshot_includes_credits_and_limits() { }; let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1680,7 +1682,7 @@ async fn status_snapshot_shows_unavailable_limits_message() { .expect("timestamp"); let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1738,7 +1740,7 @@ async fn status_snapshot_treats_refreshing_empty_limits_as_unavailable() { .expect("timestamp"); let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output_with_rate_limits( &config, @@ -1807,7 +1809,7 @@ async fn status_snapshot_shows_stale_limits_message() { let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); let now = captured_at + ChronoDuration::minutes(20); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1879,7 +1881,7 @@ async fn status_snapshot_cached_limits_hide_credits_without_flag() { let rate_display = rate_limit_snapshot_display(&snapshot, captured_at); let now = captured_at + ChronoDuration::minutes(20); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = token_info_for(&model_slug, &config, &usage); let composite = new_status_output( &config, @@ -1933,7 +1935,7 @@ async fn status_context_window_uses_last_usage() { .single() .expect("timestamp"); - let model_slug = crate::legacy_core::test_support::get_model_offline(config.model.as_deref()); + let model_slug = get_model_offline_for_tests(config.model.as_deref()); let token_info = TokenUsageInfo { total_token_usage: total_usage.clone(), last_token_usage: last_usage, diff --git a/codex-rs/tui/src/test_support.rs b/codex-rs/tui/src/test_support.rs index 53fd8adf62..44e9899379 100644 --- a/codex-rs/tui/src/test_support.rs +++ b/codex-rs/tui/src/test_support.rs @@ -1,8 +1,23 @@ +//! Test-only helpers shared across the TUI crate. + +use std::sync::LazyLock; + +use codex_models_manager::bundled_models_response; +use codex_protocol::openai_models::ModelPreset; pub(crate) use codex_utils_absolute_path::test_support::PathBufExt; pub(crate) use codex_utils_absolute_path::test_support::test_path_buf; use serde::Serialize; use serde::de::DeserializeOwned; +pub(crate) static TEST_MODEL_PRESETS: LazyLock> = LazyLock::new(|| { + let mut response = bundled_models_response() + .unwrap_or_else(|err| panic!("bundled models.json should parse: {err}")); + response.models.sort_by_key(|model| model.priority); + let mut presets: Vec = response.models.into_iter().map(Into::into).collect(); + ModelPreset::mark_default_by_picker_visibility(&mut presets); + presets +}); + pub(crate) fn test_path_display(path: &str) -> String { test_path_buf(path).display().to_string() } diff --git a/codex-rs/tui/src/windows_sandbox.rs b/codex-rs/tui/src/windows_sandbox.rs new file mode 100644 index 0000000000..ec586e7b1b --- /dev/null +++ b/codex-rs/tui/src/windows_sandbox.rs @@ -0,0 +1,122 @@ +//! TUI-owned Windows sandbox helpers retained while setup still runs in the local client process. +//! +//! TODO: These helpers inspect and modify the TUI host, so they do not support +//! cross-platform remote app servers. Move readiness and setup to the existing +//! `windowsSandbox/*` RPCs while preserving the pending permission profile, +//! use the server platform reported during initialization, and add a remote +//! equivalent for read-root grants. + +use crate::legacy_core::config::Config; +use codex_config::types::WindowsSandboxModeToml; +use codex_features::Feature; +use codex_protocol::config_types::WindowsSandboxLevel; +#[cfg(target_os = "windows")] +use codex_protocol::models::PermissionProfile; +#[cfg(target_os = "windows")] +use codex_utils_absolute_path::AbsolutePathBuf; +#[cfg(target_os = "windows")] +use std::collections::HashMap; +use std::path::Path; +#[cfg(target_os = "windows")] +use std::path::PathBuf; + +pub(crate) fn level_from_config(config: &Config) -> WindowsSandboxLevel { + match config.permissions.windows_sandbox_mode { + Some(WindowsSandboxModeToml::Elevated) => WindowsSandboxLevel::Elevated, + Some(WindowsSandboxModeToml::Unelevated) => WindowsSandboxLevel::RestrictedToken, + None if config.features.enabled(Feature::WindowsSandboxElevated) => { + WindowsSandboxLevel::Elevated + } + None if config.features.enabled(Feature::WindowsSandbox) => { + WindowsSandboxLevel::RestrictedToken + } + None => WindowsSandboxLevel::Disabled, + } +} + +#[cfg(target_os = "windows")] +pub(crate) use codex_windows_sandbox::sandbox_setup_is_complete; + +#[cfg(not(target_os = "windows"))] +pub(crate) fn sandbox_setup_is_complete(_codex_home: &Path) -> bool { + false +} + +#[cfg(target_os = "windows")] +pub(crate) fn run_elevated_setup( + permission_profile: &PermissionProfile, + workspace_roots: &[AbsolutePathBuf], + command_cwd: &Path, + env_map: &HashMap, + codex_home: &Path, +) -> anyhow::Result<()> { + let permissions = codex_windows_sandbox::ResolvedWindowsSandboxPermissions::try_from_permission_profile_for_workspace_roots( + permission_profile, + workspace_roots, + )?; + codex_windows_sandbox::run_elevated_setup( + codex_windows_sandbox::SandboxSetupRequest { + permissions: &permissions, + command_cwd, + env_map, + codex_home, + proxy_enforced: false, + }, + codex_windows_sandbox::SetupRootOverrides::default(), + ) +} + +#[cfg(target_os = "windows")] +pub(crate) fn elevated_setup_failure_details(err: &anyhow::Error) -> Option<(String, String)> { + let failure = codex_windows_sandbox::extract_setup_failure(err)?; + Some(( + failure.code.as_str().to_string(), + codex_windows_sandbox::sanitize_setup_metric_tag_value(&failure.message), + )) +} + +#[cfg(target_os = "windows")] +pub(crate) fn elevated_setup_failure_metric_name(err: &anyhow::Error) -> &'static str { + if codex_windows_sandbox::extract_setup_failure(err).is_some_and(|failure| { + matches!( + failure.code, + codex_windows_sandbox::SetupErrorCode::OrchestratorHelperLaunchCanceled + ) + }) { + "codex.windows_sandbox.elevated_setup_canceled" + } else { + "codex.windows_sandbox.elevated_setup_failure" + } +} + +#[cfg(target_os = "windows")] +pub(crate) fn grant_read_root_non_elevated( + permission_profile: &PermissionProfile, + workspace_roots: &[AbsolutePathBuf], + command_cwd: &Path, + env_map: &HashMap, + codex_home: &Path, + read_root: &Path, +) -> anyhow::Result { + if !read_root.is_absolute() { + anyhow::bail!("path must be absolute: {}", read_root.display()); + } + if !read_root.exists() { + anyhow::bail!("path does not exist: {}", read_root.display()); + } + if !read_root.is_dir() { + anyhow::bail!("path must be a directory: {}", read_root.display()); + } + + let canonical_root = dunce::canonicalize(read_root)?; + codex_windows_sandbox::run_setup_refresh_with_extra_read_roots( + permission_profile, + workspace_roots, + command_cwd, + env_map, + codex_home, + vec![canonical_root.clone()], + /*proxy_enforced*/ false, + )?; + Ok(canonical_root) +} diff --git a/codex-rs/utils/image/src/error.rs b/codex-rs/utils/image/src/error.rs index 28b73f4a7c..aaa3d4f2f3 100644 --- a/codex-rs/utils/image/src/error.rs +++ b/codex-rs/utils/image/src/error.rs @@ -25,6 +25,14 @@ pub enum ImageProcessingError { }, #[error("unsupported image `{mime}`")] UnsupportedImageFormat { mime: String }, + #[error("invalid image data URL: {reason}")] + InvalidDataUrl { reason: String }, + #[error("image {representation} is too large ({size} bytes; max {max} bytes)")] + ImageTooLarge { + representation: &'static str, + size: usize, + max: usize, + }, } impl ImageProcessingError { diff --git a/codex-rs/utils/image/src/image_tests.rs b/codex-rs/utils/image/src/image_tests.rs new file mode 100644 index 0000000000..e60c8c6be0 --- /dev/null +++ b/codex-rs/utils/image/src/image_tests.rs @@ -0,0 +1,341 @@ +use std::io::Cursor; + +use super::*; +use image::GenericImageView; +use image::ImageBuffer; +use image::ImageDecoder; +use image::Rgba; +use image::metadata::Orientation; + +const TEST_RGB_ICC_PROFILE: &[u8] = b"0123456789abcdefRGB "; +const TEST_CMYK_ICC_PROFILE: &[u8] = b"0123456789abcdefCMYK"; +const ROTATE_90_EXIF: &[u8] = &[ + 0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x12, 0x01, 0x03, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + +fn image_bytes(image: &ImageBuffer, Vec>, format: ImageFormat) -> Vec { + let mut encoded = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(image.clone()) + .write_to(&mut encoded, format) + .expect("encode image to bytes"); + encoded.into_inner() +} + +fn image_bytes_with_metadata( + image: &ImageBuffer, Vec>, + format: ImageFormat, + icc_profile: &[u8], +) -> Vec { + let mut encoded = Vec::new(); + match format { + ImageFormat::Png => { + let mut encoder = PngEncoder::new(&mut encoded); + encoder + .set_icc_profile(icc_profile.to_vec()) + .expect("set PNG ICC profile"); + encoder + .set_exif_metadata(ROTATE_90_EXIF.to_vec()) + .expect("set PNG EXIF metadata"); + encoder + .write_image( + image.as_raw(), + image.width(), + image.height(), + ColorType::Rgba8.into(), + ) + .expect("encode PNG with metadata"); + } + ImageFormat::Jpeg => { + let mut encoder = JpegEncoder::new_with_quality(&mut encoded, 90); + encoder + .set_icc_profile(icc_profile.to_vec()) + .expect("set JPEG ICC profile"); + encoder + .set_exif_metadata(ROTATE_90_EXIF.to_vec()) + .expect("set JPEG EXIF metadata"); + encoder + .encode_image(&DynamicImage::ImageRgba8(image.clone())) + .expect("encode JPEG with metadata"); + } + ImageFormat::WebP => { + let mut encoder = WebPEncoder::new_lossless(&mut encoded); + encoder + .set_icc_profile(icc_profile.to_vec()) + .expect("set WebP ICC profile"); + encoder + .set_exif_metadata(ROTATE_90_EXIF.to_vec()) + .expect("set WebP EXIF metadata"); + encoder + .write_image( + image.as_raw(), + image.width(), + image.height(), + ColorType::Rgba8.into(), + ) + .expect("encode WebP with metadata"); + } + _ => panic!("unsupported test format"), + } + encoded +} + +#[tokio::test(flavor = "multi_thread")] +async fn returns_original_image_when_within_bounds() { + for (format, mime) in [ + (ImageFormat::Png, "image/png"), + (ImageFormat::WebP, "image/webp"), + ] { + let image = ImageBuffer::from_pixel(64, 32, Rgba([10u8, 20, 30, 255])); + let original_bytes = image_bytes(&image, format); + + let encoded = load_for_prompt_bytes( + Path::new("in-memory-image"), + original_bytes.clone(), + PromptImageMode::ResizeToFit, + ) + .expect("process image"); + + assert_eq!(encoded.width, 64); + assert_eq!(encoded.height, 32); + assert_eq!(encoded.mime, mime); + assert_eq!(encoded.bytes, original_bytes); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn downscales_large_image() { + for (format, mime) in [ + (ImageFormat::Png, "image/png"), + (ImageFormat::WebP, "image/webp"), + ] { + let image = ImageBuffer::from_pixel(4096, 2048, Rgba([200u8, 10, 10, 255])); + let original_bytes = image_bytes(&image, format); + + let processed = load_for_prompt_bytes( + Path::new("in-memory-image"), + original_bytes, + PromptImageMode::ResizeToFit, + ) + .expect("process image"); + + assert!(processed.width <= MAX_DIMENSION); + assert!(processed.height <= MAX_DIMENSION); + assert_eq!(processed.mime, mime); + + let detected_format = + image::guess_format(&processed.bytes).expect("detect resized output format"); + assert_eq!(detected_format, format); + + let loaded = + image::load_from_memory(&processed.bytes).expect("read resized bytes back into image"); + assert_eq!(loaded.dimensions(), (processed.width, processed.height)); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn downscales_tall_image_to_fit_square_bounds() { + let image = ImageBuffer::from_pixel(1024, 4096, Rgba([200u8, 10, 10, 255])); + let original_bytes = image_bytes(&image, ImageFormat::Png); + + let processed = load_for_prompt_bytes( + Path::new("in-memory-image"), + original_bytes, + PromptImageMode::ResizeToFit, + ) + .expect("process image"); + + assert_eq!(processed.width, 512); + assert_eq!(processed.height, MAX_DIMENSION); + assert_eq!(processed.mime, "image/png"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn resizing_preserves_supported_metadata() { + for format in [ImageFormat::Png, ImageFormat::Jpeg, ImageFormat::WebP] { + let image = ImageBuffer::from_pixel(2050, 2, Rgba([200u8, 10, 10, 255])); + let original_bytes = image_bytes_with_metadata(&image, format, TEST_RGB_ICC_PROFILE); + + let processed = load_for_prompt_bytes( + Path::new("in-memory-image"), + original_bytes, + PromptImageMode::ResizeToFit, + ) + .expect("process image"); + + assert_eq!((processed.width, processed.height), (2048, 2)); + + let mut decoder = ImageReader::with_format(Cursor::new(&processed.bytes), format) + .into_decoder() + .expect("create decoder"); + assert_eq!( + ( + decoder.dimensions(), + decoder.orientation().expect("read orientation"), + decoder.icc_profile().expect("read ICC profile"), + decoder.exif_metadata().expect("read EXIF metadata"), + ), + ( + (2048, 2), + Orientation::Rotate90, + Some(TEST_RGB_ICC_PROFILE.to_vec()), + Some(ROTATE_90_EXIF.to_vec()), + ) + ); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn resizing_drops_non_rgb_icc_profile() { + let image = ImageBuffer::from_pixel(2050, 2, Rgba([200u8, 10, 10, 255])); + let original_bytes = + image_bytes_with_metadata(&image, ImageFormat::Jpeg, TEST_CMYK_ICC_PROFILE); + + let processed = load_for_prompt_bytes( + Path::new("in-memory-image"), + original_bytes, + PromptImageMode::ResizeToFit, + ) + .expect("process image"); + + let mut decoder = ImageReader::with_format(Cursor::new(&processed.bytes), ImageFormat::Jpeg) + .into_decoder() + .expect("create decoder"); + assert_eq!( + ( + decoder.icc_profile().expect("read ICC profile"), + decoder.exif_metadata().expect("read EXIF metadata"), + ), + (None, Some(ROTATE_90_EXIF.to_vec())) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn preserves_large_image_in_original_mode() { + let image = ImageBuffer::from_pixel(4096, 2048, Rgba([180u8, 30, 30, 255])); + let original_bytes = image_bytes(&image, ImageFormat::Png); + + let processed = load_for_prompt_bytes( + Path::new("in-memory-image"), + original_bytes.clone(), + PromptImageMode::Original, + ) + .expect("process image"); + + assert_eq!(processed.width, 4096); + assert_eq!(processed.height, 2048); + assert_eq!(processed.mime, "image/png"); + assert_eq!(processed.bytes, original_bytes); +} + +#[tokio::test(flavor = "multi_thread")] +async fn data_url_processing_preserves_supported_source_bytes() { + let image = ImageBuffer::from_pixel(64, 32, Rgba([10u8, 20, 30, 255])); + let original_bytes = image_bytes(&image, ImageFormat::Png); + let image_url = data_url_from_bytes("image/png", &original_bytes) + .replacen("data:", "DATA:", 1) + .replacen(";base64,", ";BASE64,", 1); + + let processed = load_data_url_for_prompt(&image_url, PromptImageMode::ResizeToFit) + .expect("process data URL image"); + + assert_eq!(processed.width, 64); + assert_eq!(processed.height, 32); + assert_eq!(processed.mime, "image/png"); + assert_eq!(processed.bytes, original_bytes); +} + +#[tokio::test(flavor = "multi_thread")] +async fn data_url_processing_converts_gif_to_png() { + let image = ImageBuffer::from_pixel(64, 32, Rgba([10u8, 20, 30, 255])); + let gif_bytes = image_bytes(&image, ImageFormat::Gif); + let image_url = data_url_from_bytes("image/gif", &gif_bytes); + + let processed = load_data_url_for_prompt(&image_url, PromptImageMode::ResizeToFit) + .expect("process GIF data URL"); + + assert_eq!(processed.mime, "image/png"); + assert_eq!( + image::guess_format(&processed.bytes).expect("detect processed format"), + ImageFormat::Png + ); +} + +#[test] +fn data_url_processing_rejects_malformed_input() { + for image_url in [ + "image/png;base64,AAAA", + "data:image/png;base64", + "data:image/png,AAAA", + "data:image/png;base64,not base64", + ] { + assert!(matches!( + load_data_url_for_prompt(image_url, PromptImageMode::ResizeToFit), + Err(ImageProcessingError::InvalidDataUrl { .. }) + )); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn resize_with_limits_respects_dimension_and_patch_budgets() { + let image = ImageBuffer::from_pixel(2048, 2048, Rgba([200u8, 10, 10, 255])); + let original_bytes = image_bytes(&image, ImageFormat::Png); + let limits = PromptImageResizeLimits { + max_dimension: 2048, + max_patches: 2_500, + }; + + let processed = load_for_prompt_bytes( + Path::new("in-memory-image"), + original_bytes, + PromptImageMode::ResizeWithLimits(limits), + ) + .expect("process image with explicit limits"); + + assert_eq!((processed.width, processed.height), (1600, 1600)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn fails_cleanly_for_invalid_images() { + let err = load_for_prompt_bytes( + Path::new("in-memory-image"), + b"not an image".to_vec(), + PromptImageMode::ResizeToFit, + ) + .expect_err("invalid image should fail"); + assert!(matches!( + err, + ImageProcessingError::Decode { .. } | ImageProcessingError::UnsupportedImageFormat { .. } + )); +} + +#[tokio::test(flavor = "multi_thread")] +async fn reprocesses_updated_file_contents() { + IMAGE_CACHE.clear(); + + let first_image = ImageBuffer::from_pixel(32, 16, Rgba([20u8, 120, 220, 255])); + let first_bytes = image_bytes(&first_image, ImageFormat::Png); + + let first = load_for_prompt_bytes( + Path::new("in-memory-image"), + first_bytes, + PromptImageMode::ResizeToFit, + ) + .expect("process first image"); + + let second_image = ImageBuffer::from_pixel(96, 48, Rgba([50u8, 60, 70, 255])); + let second_bytes = image_bytes(&second_image, ImageFormat::Png); + + let second = load_for_prompt_bytes( + Path::new("in-memory-image"), + second_bytes, + PromptImageMode::ResizeToFit, + ) + .expect("process updated image"); + + assert_eq!(first.width, 32); + assert_eq!(first.height, 16); + assert_eq!(second.width, 96); + assert_eq!(second.height, 48); + assert_ne!(second.bytes, first.bytes); +} diff --git a/codex-rs/utils/image/src/lib.rs b/codex-rs/utils/image/src/lib.rs index a3f595a183..93ae427ea5 100644 --- a/codex-rs/utils/image/src/lib.rs +++ b/codex-rs/utils/image/src/lib.rs @@ -1,3 +1,4 @@ +use std::io::Cursor; use std::num::NonZeroUsize; use std::path::Path; use std::sync::LazyLock; @@ -9,14 +10,24 @@ use codex_utils_cache::sha1_digest; use image::ColorType; use image::DynamicImage; use image::GenericImageView; +use image::ImageDecoder; use image::ImageEncoder; use image::ImageFormat; +use image::ImageReader; use image::codecs::jpeg::JpegEncoder; use image::codecs::png::PngEncoder; use image::codecs::webp::WebPEncoder; use image::imageops::FilterType; + +const DATA_URL_PREFIX: &str = "data:"; +pub const PROMPT_IMAGE_PATCH_SIZE: u32 = 32; /// Maximum width or height used when resizing images before uploading. pub const MAX_DIMENSION: u32 = 2048; +/// Maximum accepted byte length for prompt image input representations. +/// +/// This is a high sanity guard against pathological inputs, not a protocol +/// requirement or target upload size. +pub const MAX_PROMPT_IMAGE_INPUT_BYTES: usize = 1024 * 1024 * 1024; pub mod error; @@ -32,15 +43,32 @@ pub struct EncodedImage { impl EncodedImage { pub fn into_data_url(self) -> String { - let encoded = BASE64_STANDARD.encode(&self.bytes); - format!("data:{};base64,{encoded}", self.mime) + data_url_from_bytes(&self.mime, &self.bytes) } } +/// Wraps image bytes in a data URL without decoding or validating them. +pub fn data_url_from_bytes(mime: &str, bytes: &[u8]) -> String { + let encoded = BASE64_STANDARD.encode(bytes); + format!("data:{mime};base64,{encoded}") +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum PromptImageMode { ResizeToFit, Original, + ResizeWithLimits(PromptImageResizeLimits), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PromptImageResizeLimits { + pub max_dimension: u32, + pub max_patches: usize, +} + +struct ImageMetadata { + icc_profile: Option>, + exif: Option>, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -65,22 +93,70 @@ pub fn load_for_prompt_bytes( }; IMAGE_CACHE.get_or_try_insert_with(key, move || { - let format = match image::guess_format(&file_bytes) { - Ok(ImageFormat::Png) => Some(ImageFormat::Png), - Ok(ImageFormat::Jpeg) => Some(ImageFormat::Jpeg), - Ok(ImageFormat::Gif) => Some(ImageFormat::Gif), - Ok(ImageFormat::WebP) => Some(ImageFormat::WebP), + let guessed_format = image::guess_format(&file_bytes) + .map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?; + let format = match guessed_format { + ImageFormat::Png => Some(ImageFormat::Png), + ImageFormat::Jpeg => Some(ImageFormat::Jpeg), + ImageFormat::Gif => Some(ImageFormat::Gif), + ImageFormat::WebP => Some(ImageFormat::WebP), _ => None, }; - let dynamic = image::load_from_memory(&file_bytes) + let mut decoder = ImageReader::with_format(Cursor::new(&file_bytes), guessed_format) + .into_decoder() + .map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?; + // Preserve the metadata most important for rendering prompt images faithfully: the color + // profile and EXIF data, including orientation. Other format-specific metadata is + // intentionally not copied. + let metadata = ImageMetadata { + // Only RGB profiles are safe across every re-encoding path. For example, JPEG decoding + // can convert CMYK/YCCK pixels to RGB while retaining the source profile; copying it + // would mislabel the output. Bytes 16..20 are the ICC data color space signature. + icc_profile: decoder + .icc_profile() + .ok() + .flatten() + .filter(|profile| profile.get(16..20) == Some(b"RGB ")), + exif: decoder.exif_metadata().ok().flatten(), + }; + let dynamic = DynamicImage::from_decoder(decoder) .map_err(|source| ImageProcessingError::decode_error(&path_buf, source))?; let (width, height) = dynamic.dimensions(); - let encoded = if mode == PromptImageMode::Original - || (width <= MAX_DIMENSION && height <= MAX_DIMENSION) - { + let target_dimensions = match mode { + PromptImageMode::ResizeToFit if width > MAX_DIMENSION || height > MAX_DIMENSION => { + let resized = dynamic.resize(MAX_DIMENSION, MAX_DIMENSION, FilterType::Triangle); + Some((resized.width(), resized.height(), resized)) + } + PromptImageMode::ResizeWithLimits(limits) => { + let (target_width, target_height) = + prompt_image_output_dimensions_for_limits(width, height, limits); + if (target_width, target_height) == (width, height) { + None + } else { + let resized = + dynamic.resize_exact(target_width, target_height, FilterType::Triangle); + Some((target_width, target_height, resized)) + } + } + PromptImageMode::ResizeToFit | PromptImageMode::Original => None, + }; + + let encoded = if let Some((width, height, resized)) = target_dimensions { + let target_format = format + .filter(|format| can_preserve_source_bytes(*format)) + .unwrap_or(ImageFormat::Png); + let (bytes, output_format) = encode_image(&resized, target_format, metadata)?; + let mime = format_to_mime(output_format); + EncodedImage { + bytes, + mime, + width, + height, + } + } else { if let Some(format) = format.filter(|format| can_preserve_source_bytes(*format)) { let mime = format_to_mime(format); EncodedImage { @@ -90,7 +166,7 @@ pub fn load_for_prompt_bytes( height, } } else { - let (bytes, output_format) = encode_image(&dynamic, ImageFormat::Png)?; + let (bytes, output_format) = encode_image(&dynamic, ImageFormat::Png, metadata)?; let mime = format_to_mime(output_format); EncodedImage { bytes, @@ -99,25 +175,107 @@ pub fn load_for_prompt_bytes( height, } } - } else { - let resized = dynamic.resize(MAX_DIMENSION, MAX_DIMENSION, FilterType::Triangle); - let target_format = format - .filter(|format| can_preserve_source_bytes(*format)) - .unwrap_or(ImageFormat::Png); - let (bytes, output_format) = encode_image(&resized, target_format)?; - let mime = format_to_mime(output_format); - EncodedImage { - bytes, - mime, - width: resized.width(), - height: resized.height(), - } }; Ok(encoded) }) } +pub fn load_data_url_for_prompt( + image_url: &str, + mode: PromptImageMode, +) -> Result { + let rest = image_url + .get(..DATA_URL_PREFIX.len()) + .filter(|prefix| prefix.eq_ignore_ascii_case(DATA_URL_PREFIX)) + .and_then(|_| image_url.get(DATA_URL_PREFIX.len()..)) + .ok_or_else(|| ImageProcessingError::InvalidDataUrl { + reason: "missing data: prefix".to_string(), + })?; + let (metadata, encoded) = + rest.split_once(',') + .ok_or_else(|| ImageProcessingError::InvalidDataUrl { + reason: "missing comma separator".to_string(), + })?; + if !metadata + .split(';') + .any(|part| part.eq_ignore_ascii_case("base64")) + { + return Err(ImageProcessingError::InvalidDataUrl { + reason: "only base64 data URLs are supported".to_string(), + }); + } + + if encoded.len() > MAX_PROMPT_IMAGE_INPUT_BYTES { + return Err(ImageProcessingError::ImageTooLarge { + representation: "base64 payload", + size: encoded.len(), + max: MAX_PROMPT_IMAGE_INPUT_BYTES, + }); + } + let file_bytes = + BASE64_STANDARD + .decode(encoded) + .map_err(|source| ImageProcessingError::InvalidDataUrl { + reason: format!("invalid base64 payload: {source}"), + })?; + if file_bytes.len() > MAX_PROMPT_IMAGE_INPUT_BYTES { + return Err(ImageProcessingError::ImageTooLarge { + representation: "decoded input", + size: file_bytes.len(), + max: MAX_PROMPT_IMAGE_INPUT_BYTES, + }); + } + + load_for_prompt_bytes(Path::new(""), file_bytes, mode) +} + +fn prompt_image_output_dimensions_for_limits( + width: u32, + height: u32, + limits: PromptImageResizeLimits, +) -> (u32, u32) { + let width = width.max(1); + let height = height.max(1); + if prompt_image_dimensions_fit(width, height, limits) { + return (width, height); + } + + let max_dimension_scale = + (f64::from(limits.max_dimension) / f64::from(width.max(height))).min(1.0); + let width = ((f64::from(width) * max_dimension_scale).round() as u32).max(1); + let height = ((f64::from(height) * max_dimension_scale).round() as u32).max(1); + if prompt_image_dimensions_fit(width, height, limits) { + return (width, height); + } + + let width_f64 = f64::from(width); + let height_f64 = f64::from(height); + let patch_size = f64::from(PROMPT_IMAGE_PATCH_SIZE); + let mut scale = + (patch_size * patch_size * limits.max_patches as f64 / width_f64 / height_f64).sqrt(); + // Match Responses patch-budget math: shrink by area, then round the scaled + // patch grid down so integer output dimensions remain within the budget. + let scaled_patches_wide = width_f64 * scale / patch_size; + let scaled_patches_high = height_f64 * scale / patch_size; + scale *= (scaled_patches_wide.floor() / scaled_patches_wide) + .min(scaled_patches_high.floor() / scaled_patches_high); + + ( + ((width_f64 * scale).floor() as u32).max(1), + ((height_f64 * scale).floor() as u32).max(1), + ) +} + +fn prompt_image_dimensions_fit(width: u32, height: u32, limits: PromptImageResizeLimits) -> bool { + let patches_wide = width.div_ceil(PROMPT_IMAGE_PATCH_SIZE); + let patches_high = height.div_ceil(PROMPT_IMAGE_PATCH_SIZE); + let patch_count = u64::from(patches_wide) * u64::from(patches_high); + width <= limits.max_dimension + && height <= limits.max_dimension + && patch_count <= limits.max_patches as u64 +} + fn can_preserve_source_bytes(format: ImageFormat) -> bool { // Public API docs explicitly call out non-animated GIF support only. // Preserve byte-for-byte only for formats we can safely pass through. @@ -130,6 +288,7 @@ fn can_preserve_source_bytes(format: ImageFormat) -> bool { fn encode_image( image: &DynamicImage, preferred_format: ImageFormat, + metadata: ImageMetadata, ) -> Result<(Vec, ImageFormat), ImageProcessingError> { let target_format = match preferred_format { ImageFormat::Jpeg => ImageFormat::Jpeg, @@ -138,11 +297,13 @@ fn encode_image( }; let mut buffer = Vec::new(); + let ImageMetadata { icc_profile, exif } = metadata; match target_format { ImageFormat::Png => { let rgba = image.to_rgba8(); - let encoder = PngEncoder::new(&mut buffer); + let mut encoder = PngEncoder::new(&mut buffer); + apply_image_metadata(&mut encoder, icc_profile, exif, target_format)?; encoder .write_image( rgba.as_raw(), @@ -157,6 +318,7 @@ fn encode_image( } ImageFormat::Jpeg => { let mut encoder = JpegEncoder::new_with_quality(&mut buffer, 85); + apply_image_metadata(&mut encoder, icc_profile, exif, target_format)?; encoder .encode_image(image) .map_err(|source| ImageProcessingError::Encode { @@ -166,7 +328,8 @@ fn encode_image( } ImageFormat::WebP => { let rgba = image.to_rgba8(); - let encoder = WebPEncoder::new_lossless(&mut buffer); + let mut encoder = WebPEncoder::new_lossless(&mut buffer); + apply_image_metadata(&mut encoder, icc_profile, exif, target_format)?; encoder .write_image( rgba.as_raw(), @@ -185,6 +348,31 @@ fn encode_image( Ok((buffer, target_format)) } +fn apply_image_metadata( + encoder: &mut impl ImageEncoder, + icc_profile: Option>, + exif: Option>, + format: ImageFormat, +) -> Result<(), ImageProcessingError> { + if let Some(icc_profile) = icc_profile { + encoder + .set_icc_profile(icc_profile) + .map_err(|source| ImageProcessingError::Encode { + format, + source: image::ImageError::Unsupported(source), + })?; + } + if let Some(exif) = exif { + encoder + .set_exif_metadata(exif) + .map_err(|source| ImageProcessingError::Encode { + format, + source: image::ImageError::Unsupported(source), + })?; + } + Ok(()) +} + fn format_to_mime(format: ImageFormat) -> String { match format { ImageFormat::Jpeg => "image/jpeg".to_string(), @@ -195,155 +383,5 @@ fn format_to_mime(format: ImageFormat) -> String { } #[cfg(test)] -mod tests { - use std::io::Cursor; - - use super::*; - use image::GenericImageView; - use image::ImageBuffer; - use image::Rgba; - - fn image_bytes(image: &ImageBuffer, Vec>, format: ImageFormat) -> Vec { - let mut encoded = Cursor::new(Vec::new()); - DynamicImage::ImageRgba8(image.clone()) - .write_to(&mut encoded, format) - .expect("encode image to bytes"); - encoded.into_inner() - } - - #[tokio::test(flavor = "multi_thread")] - async fn returns_original_image_when_within_bounds() { - for (format, mime) in [ - (ImageFormat::Png, "image/png"), - (ImageFormat::WebP, "image/webp"), - ] { - let image = ImageBuffer::from_pixel(64, 32, Rgba([10u8, 20, 30, 255])); - let original_bytes = image_bytes(&image, format); - - let encoded = load_for_prompt_bytes( - Path::new("in-memory-image"), - original_bytes.clone(), - PromptImageMode::ResizeToFit, - ) - .expect("process image"); - - assert_eq!(encoded.width, 64); - assert_eq!(encoded.height, 32); - assert_eq!(encoded.mime, mime); - assert_eq!(encoded.bytes, original_bytes); - } - } - - #[tokio::test(flavor = "multi_thread")] - async fn downscales_large_image() { - for (format, mime) in [ - (ImageFormat::Png, "image/png"), - (ImageFormat::WebP, "image/webp"), - ] { - let image = ImageBuffer::from_pixel(4096, 2048, Rgba([200u8, 10, 10, 255])); - let original_bytes = image_bytes(&image, format); - - let processed = load_for_prompt_bytes( - Path::new("in-memory-image"), - original_bytes, - PromptImageMode::ResizeToFit, - ) - .expect("process image"); - - assert!(processed.width <= MAX_DIMENSION); - assert!(processed.height <= MAX_DIMENSION); - assert_eq!(processed.mime, mime); - - let detected_format = - image::guess_format(&processed.bytes).expect("detect resized output format"); - assert_eq!(detected_format, format); - - let loaded = image::load_from_memory(&processed.bytes) - .expect("read resized bytes back into image"); - assert_eq!(loaded.dimensions(), (processed.width, processed.height)); - } - } - - #[tokio::test(flavor = "multi_thread")] - async fn downscales_tall_image_to_fit_square_bounds() { - let image = ImageBuffer::from_pixel(1024, 4096, Rgba([200u8, 10, 10, 255])); - let original_bytes = image_bytes(&image, ImageFormat::Png); - - let processed = load_for_prompt_bytes( - Path::new("in-memory-image"), - original_bytes, - PromptImageMode::ResizeToFit, - ) - .expect("process image"); - - assert_eq!(processed.width, 512); - assert_eq!(processed.height, MAX_DIMENSION); - assert_eq!(processed.mime, "image/png"); - } - - #[tokio::test(flavor = "multi_thread")] - async fn preserves_large_image_in_original_mode() { - let image = ImageBuffer::from_pixel(4096, 2048, Rgba([180u8, 30, 30, 255])); - let original_bytes = image_bytes(&image, ImageFormat::Png); - - let processed = load_for_prompt_bytes( - Path::new("in-memory-image"), - original_bytes.clone(), - PromptImageMode::Original, - ) - .expect("process image"); - - assert_eq!(processed.width, 4096); - assert_eq!(processed.height, 2048); - assert_eq!(processed.mime, "image/png"); - assert_eq!(processed.bytes, original_bytes); - } - - #[tokio::test(flavor = "multi_thread")] - async fn fails_cleanly_for_invalid_images() { - let err = load_for_prompt_bytes( - Path::new("in-memory-image"), - b"not an image".to_vec(), - PromptImageMode::ResizeToFit, - ) - .expect_err("invalid image should fail"); - assert!(matches!( - err, - ImageProcessingError::Decode { .. } - | ImageProcessingError::UnsupportedImageFormat { .. } - )); - } - - #[tokio::test(flavor = "multi_thread")] - async fn reprocesses_updated_file_contents() { - { - IMAGE_CACHE.clear(); - } - - let first_image = ImageBuffer::from_pixel(32, 16, Rgba([20u8, 120, 220, 255])); - let first_bytes = image_bytes(&first_image, ImageFormat::Png); - - let first = load_for_prompt_bytes( - Path::new("in-memory-image"), - first_bytes, - PromptImageMode::ResizeToFit, - ) - .expect("process first image"); - - let second_image = ImageBuffer::from_pixel(96, 48, Rgba([50u8, 60, 70, 255])); - let second_bytes = image_bytes(&second_image, ImageFormat::Png); - - let second = load_for_prompt_bytes( - Path::new("in-memory-image"), - second_bytes, - PromptImageMode::ResizeToFit, - ) - .expect("process updated image"); - - assert_eq!(first.width, 32); - assert_eq!(first.height, 16); - assert_eq!(second.width, 96); - assert_eq!(second.height, 48); - assert_ne!(second.bytes, first.bytes); - } -} +#[path = "image_tests.rs"] +mod tests; diff --git a/codex-rs/utils/plugins/Cargo.toml b/codex-rs/utils/plugins/Cargo.toml index 983f4526ea..2c56ab3773 100644 --- a/codex-rs/utils/plugins/Cargo.toml +++ b/codex-rs/utils/plugins/Cargo.toml @@ -17,6 +17,7 @@ workspace = true codex-exec-server = { workspace = true } codex-login = { workspace = true } codex-utils-absolute-path = { workspace = true } +codex-utils-path-uri = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/codex-rs/utils/plugins/src/plugin_namespace.rs b/codex-rs/utils/plugins/src/plugin_namespace.rs index ea5caab52f..d0d8bf685e 100644 --- a/codex-rs/utils/plugins/src/plugin_namespace.rs +++ b/codex-rs/utils/plugins/src/plugin_namespace.rs @@ -2,6 +2,7 @@ use codex_exec_server::ExecutorFileSystem; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; use std::path::Path; use std::path::PathBuf; @@ -29,7 +30,8 @@ async fn plugin_manifest_name( let mut manifest_path = None; for relative_path in DISCOVERABLE_PLUGIN_MANIFEST_PATHS { let candidate = plugin_root.join(relative_path); - match fs.get_metadata(&candidate, /*sandbox*/ None).await { + let candidate_uri = PathUri::from_abs_path(&candidate).ok()?; + match fs.get_metadata(&candidate_uri, /*sandbox*/ None).await { Ok(metadata) if metadata.is_file => { manifest_path = Some(candidate); break; @@ -38,8 +40,9 @@ async fn plugin_manifest_name( } } let manifest_path = manifest_path?; + let manifest_path_uri = PathUri::from_abs_path(&manifest_path).ok()?; let contents = fs - .read_file_text(&manifest_path, /*sandbox*/ None) + .read_file_text(&manifest_path_uri, /*sandbox*/ None) .await .ok()?; let RawPluginManifestName { name: raw_name } = serde_json::from_str(&contents).ok()?; diff --git a/defs.bzl b/defs.bzl index 4af81ecef1..b255160b09 100644 --- a/defs.bzl +++ b/defs.bzl @@ -1,18 +1,8 @@ load("@crates//:data.bzl", "DEP_DATA") load("@crates//:defs.bzl", "all_crate_deps") -load("@rules_platform//platform_data:defs.bzl", "platform_data") load("@rules_rust//cargo/private:cargo_build_script_wrapper.bzl", "cargo_build_script") load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_test") -PLATFORMS = [ - "linux_arm64_musl", - "linux_amd64_musl", - "macos_amd64", - "macos_arm64", - "windows_amd64", - "windows_arm64", -] - # Match Cargo's Windows linker behavior so Bazel-built binaries and tests use # the same stack reserve on both Windows ABIs and resolve UCRT imports on MSVC. WINDOWS_RUSTC_LINK_FLAGS = select({ @@ -53,21 +43,6 @@ MACOS_WEBRTC_RUSTC_LINK_FLAGS = select({ "//conditions:default": [], }) -def multiplatform_binaries(name, platforms = PLATFORMS): - for platform in platforms: - platform_data( - name = name + "_" + platform, - platform = "@llvm//platforms:" + platform, - target = name, - tags = ["manual"], - ) - - native.filegroup( - name = "release_binaries", - srcs = [name + "_" + platform for platform in platforms], - tags = ["manual"], - ) - def _workspace_root_test_impl(ctx): is_windows = ctx.target_platform_has_constraint(ctx.attr._windows_constraint[platform_common.ConstraintValueInfo]) launcher = ctx.actions.declare_file(ctx.label.name + ".bat" if is_windows else ctx.label.name) diff --git a/patches/BUILD.bazel b/patches/BUILD.bazel index 72d6990790..a431c3f3a6 100644 --- a/patches/BUILD.bazel +++ b/patches/BUILD.bazel @@ -5,6 +5,7 @@ exports_files([ "aws-lc-sys_windows_msvc_memcmp_probe.patch", "bzip2_windows_stack_args.patch", "llvm_rusty_v8_custom_libcxx.patch", + "llvm_windows_arm64_powl.patch", "llvm_windows_symlink_extract.patch", "rules_rust_windows_bootstrap_process_wrapper_linker.patch", "rules_rust_windows_build_script_runner_paths.patch", @@ -14,6 +15,7 @@ exports_files([ "rules_rust_windows_msvc_direct_link_args.patch", "rules_rust_windows_gnullvm_build_script.patch", "rules_cc_rusty_v8_custom_libcxx.patch", + "rules_rs_build_script_deps_annotation.patch", "rules_rs_windows_gnullvm_exec.patch", "rules_rs_windows_exec_linker.patch", "rusty_v8_prebuilt_out_dir.patch", diff --git a/patches/llvm_windows_arm64_powl.patch b/patches/llvm_windows_arm64_powl.patch new file mode 100644 index 0000000000..5e12bd7d3b --- /dev/null +++ b/patches/llvm_windows_arm64_powl.patch @@ -0,0 +1,24 @@ +# What: provide and link MinGW's ARM64 long-double `powl` compatibility shim. +# Why: Windows ARM64 uses 64-bit long doubles, but LLVM's Bazel source list +# omits the upstream `powl -> pow` implementation and its default link flags +# omit the MinGW extension archive that owns the shim. + +diff --git a/runtimes/mingw/crt_sources.bzl b/runtimes/mingw/crt_sources.bzl +--- a/runtimes/mingw/crt_sources.bzl ++++ b/runtimes/mingw/crt_sources.bzl +@@ -420,4 +420,5 @@ + MINGWEX_ARM64_SRCS = [ ++ "math/arm-common/powl.c", + "math/sincos.c", + "math/sincosf.c", + "math/arm-common/sincosl.c", +diff --git a/toolchain/args/windows/BUILD.bazel b/toolchain/args/windows/BUILD.bazel +--- a/toolchain/args/windows/BUILD.bazel ++++ b/toolchain/args/windows/BUILD.bazel +@@ -49,5 +49,6 @@ cc_args( ++ "-lmingwex", + # For now, we force on ucrt. + # TODO(cerisier): make this either a constraint or a build setting + "-lucrt", + ], + data = [ diff --git a/patches/rules_rs_build_script_deps_annotation.patch b/patches/rules_rs_build_script_deps_annotation.patch new file mode 100644 index 0000000000..ef1c442bc4 --- /dev/null +++ b/patches/rules_rs_build_script_deps_annotation.patch @@ -0,0 +1,60 @@ +# What: expose the existing unconditional build-script dependency input through +# the `rules_rs` crate annotation API. +# Scope: annotation plumbing only; generated dependency resolution is unchanged. + +diff --git a/rs/extensions.bzl b/rs/extensions.bzl +--- a/rs/extensions.bzl ++++ b/rs/extensions.bzl +@@ -737,7 +737,7 @@ def _generate_hub_and_spokes( + additive_build_file = annotation.additive_build_file, + additive_build_file_content = annotation.additive_build_file_content, + gen_build_script = annotation.gen_build_script, +- build_script_deps = [], ++ build_script_deps = annotation.build_script_deps, + build_script_deps_select = _select(feature_resolutions.build_deps), + build_script_data = annotation.build_script_data, + build_script_data_select = annotation.build_script_data_select, +@@ -1417,9 +1417,9 @@ _annotation = tag_class( + "build_script_data_select": attr.string_list_dict( + doc = "A list of labels to add to a crate's `cargo_build_script::data` attribute. Keys should be the platform triplet. Value should be a list of labels.", + ), +- # "build_script_deps": _relative_label_list( +- # doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute.", +- # ), ++ "build_script_deps": _relative_label_list( ++ doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute.", ++ ), + "build_script_env": attr.string_dict( + doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.", + ), +diff --git a/rs/private/annotations.bzl b/rs/private/annotations.bzl +--- a/rs/private/annotations.bzl ++++ b/rs/private/annotations.bzl +@@ -5,6 +5,7 @@ def _crate_annotation( + gen_build_script = "auto", + build_script_data = [], + build_script_data_select = {}, ++ build_script_deps = [], + build_script_env = {}, + build_script_env_select = {}, + build_script_tools = [], +@@ -28,6 +29,7 @@ def _crate_annotation( + gen_build_script = gen_build_script, + build_script_data = build_script_data, + build_script_data_select = build_script_data_select, ++ build_script_deps = build_script_deps, + build_script_env = build_script_env, + build_script_env_select = build_script_env_select, + build_script_tools = build_script_tools, +diff --git a/rs/private/repository_utils.bzl b/rs/private/repository_utils.bzl +--- a/rs/private/repository_utils.bzl ++++ b/rs/private/repository_utils.bzl +@@ -216,7 +216,7 @@ common_attrs = { + "additive_build_file": attr.label(), + "additive_build_file_content": attr.string(), + "gen_build_script": attr.string(), +- "build_script_deps": attr.label_list(default = []), ++ "build_script_deps": attr.string_list(default = []), + "build_script_deps_select": attr.string_list_dict(), + "build_script_data": attr.label_list(default = []), + "build_script_data_select": attr.string_list_dict(), diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 2329d2ba62..38374f523d 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -101,6 +101,11 @@ def parse_args() -> argparse.Namespace: "--workflow-url", help="Optional workflow URL to reuse for native artifacts.", ) + parser.add_argument( + "--artifacts-dir", + type=Path, + help="Directory containing previously downloaded workflow artifacts.", + ) parser.add_argument( "--output-dir", type=Path, @@ -496,6 +501,7 @@ def main() -> int: vendor_temp_roots: list[Path] = [] vendor_src_by_components: dict[tuple[str, ...], Path] = {} artifacts_temp_root: Path | None = None + remove_artifacts_temp_root = False resolved_head_sha: str | None = None final_messages = [] @@ -506,10 +512,15 @@ def main() -> int: args.release_version, args.workflow_url ) print(f"Using native artifacts from {workflow_url}", flush=True) - artifacts_temp_root = Path( - tempfile.mkdtemp(prefix="npm-native-artifacts-", dir=runner_temp) - ) - print(f"Caching downloaded artifacts in {artifacts_temp_root}", flush=True) + if args.artifacts_dir is not None: + artifacts_temp_root = args.artifacts_dir.resolve() + artifacts_temp_root.mkdir(parents=True, exist_ok=True) + else: + artifacts_temp_root = Path( + tempfile.mkdtemp(prefix="npm-native-artifacts-", dir=runner_temp) + ) + remove_artifacts_temp_root = True + print(f"Using artifact cache at {artifacts_temp_root}", flush=True) for components in native_component_sets: vendor_temp_root = Path( tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp) @@ -570,7 +581,7 @@ def main() -> int: if not args.keep_staging_dirs: for vendor_temp_root in vendor_temp_roots: shutil.rmtree(vendor_temp_root, ignore_errors=True) - if artifacts_temp_root is not None: + if remove_artifacts_temp_root and artifacts_temp_root is not None: shutil.rmtree(artifacts_temp_root, ignore_errors=True) for msg in final_messages: diff --git a/tools/argument-comment-lint/README.md b/tools/argument-comment-lint/README.md index 7270c48867..889bae1df2 100644 --- a/tools/argument-comment-lint/README.md +++ b/tools/argument-comment-lint/README.md @@ -20,6 +20,11 @@ It provides two lints: String and char literals are exempt because they are often already self-descriptive at the callsite. +The sole non-self method argument is also exempt when the method name matches +the resolved parameter name. For example, `.enabled(false)` is already +self-descriptive when it resolves to `fn enabled(self, enabled: bool)`. An +explicit argument comment is still checked for a mismatch. + ## Behavior Given: diff --git a/tools/argument-comment-lint/src/lib.rs b/tools/argument-comment-lint/src/lib.rs index 5586509f24..76b9e17732 100644 --- a/tools/argument-comment-lint/src/lib.rs +++ b/tools/argument-comment-lint/src/lib.rs @@ -88,6 +88,8 @@ rustc_session::declare_lint! { /// /// Requires a `/*param*/` comment before anonymous literal-like /// arguments such as `None`, booleans, and numeric literals. + /// A method's sole non-self argument is exempt when its name matches the + /// method name. /// /// ### Why is this bad? /// @@ -125,6 +127,11 @@ rustc_session::declare_lint! { #[derive(Default)] pub struct ArgumentCommentLint; +enum CallKind { + Function, + Method { name: String }, +} + rustc_session::impl_lint_pass!( ArgumentCommentLint => [ARGUMENT_COMMENT_MISMATCH, UNCOMMENTED_ANONYMOUS_LITERAL_ARGUMENT] ); @@ -137,10 +144,18 @@ impl<'tcx> LateLintPass<'tcx> for ArgumentCommentLint { match expr.kind { ExprKind::Call(callee, args) => { - self.check_call(cx, expr, callee.span, args, 0); + self.check_call(cx, expr, callee.span, args, CallKind::Function); } - ExprKind::MethodCall(_, receiver, args, _) => { - self.check_call(cx, expr, receiver.span, args, 1); + ExprKind::MethodCall(method, receiver, args, _) => { + self.check_call( + cx, + expr, + receiver.span, + args, + CallKind::Method { + name: method.ident.name.to_string(), + }, + ); } _ => {} } @@ -154,7 +169,7 @@ impl ArgumentCommentLint { call: &'tcx Expr<'tcx>, first_gap_anchor: Span, args: &'tcx [Expr<'tcx>], - parameter_offset: usize, + call_kind: CallKind, ) { let Some(def_id) = fn_def_id(cx, call) else { return; @@ -167,6 +182,11 @@ impl ArgumentCommentLint { return; } + // Method parameter lists include `self`, which is not present in `args`. + let (parameter_offset, method_name) = match &call_kind { + CallKind::Function => (0, None), + CallKind::Method { name } => (1, Some(name.as_str())), + }; let parameter_names: Vec<_> = cx.tcx.fn_arg_idents(def_id).iter().copied().collect(); for (index, arg) in args.iter().enumerate() { if arg.span.from_expansion() { @@ -215,6 +235,12 @@ impl ArgumentCommentLint { continue; } + // Don't require a clarifying comment for self-documenting arguments whose names + // match the method. + if args.len() == 1 && method_name == Some(expected_name.as_str()) { + continue; + } + if !is_anonymous_literal_like(cx, arg) { continue; } diff --git a/tools/argument-comment-lint/ui/allow_self_documenting_methods.rs b/tools/argument-comment-lint/ui/allow_self_documenting_methods.rs new file mode 100644 index 0000000000..d57f7c521d --- /dev/null +++ b/tools/argument-comment-lint/ui/allow_self_documenting_methods.rs @@ -0,0 +1,25 @@ +#![warn(argument_comment_mismatch)] +#![warn(uncommented_anonymous_literal_argument)] + +struct Builder; + +impl Builder { + fn enabled(self, enabled: bool) -> Self { + let _ = enabled; + self + } + + fn retry_count(self, retry_count: usize) -> Self { + let _ = retry_count; + self + } + + fn base_url(self, base_url: Option) -> Self { + let _ = base_url; + self + } +} + +fn main() { + let _ = Builder.enabled(false).retry_count(3).base_url(None); +} diff --git a/tools/argument-comment-lint/ui/comment_mismatch.rs b/tools/argument-comment-lint/ui/comment_mismatch.rs index 3eebdd9dc2..f3ad4db754 100644 --- a/tools/argument-comment-lint/ui/comment_mismatch.rs +++ b/tools/argument-comment-lint/ui/comment_mismatch.rs @@ -5,6 +5,16 @@ fn create_openai_url(base_url: Option) -> String { String::new() } +struct Options; + +impl Options { + fn enabled(self, enabled: bool) -> Self { + let _ = enabled; + self + } +} + fn main() { let _ = create_openai_url(/*api_base*/ None); + let _ = Options.enabled(/*value*/ false); } diff --git a/tools/argument-comment-lint/ui/comment_mismatch.stderr b/tools/argument-comment-lint/ui/comment_mismatch.stderr index 6ede656029..058871395a 100644 --- a/tools/argument-comment-lint/ui/comment_mismatch.stderr +++ b/tools/argument-comment-lint/ui/comment_mismatch.stderr @@ -1,5 +1,5 @@ warning: argument comment `/*api_base*/` does not match parameter `base_url` - --> $DIR/comment_mismatch.rs:9:44 + --> $DIR/comment_mismatch.rs:18:44 | LL | let _ = create_openai_url(/*api_base*/ None); | ^^^^ @@ -11,5 +11,13 @@ note: the lint level is defined here LL | #![warn(argument_comment_mismatch)] | ^^^^^^^^^^^^^^^^^^^^^^^^^ -warning: 1 warning emitted +warning: argument comment `/*value*/` does not match parameter `enabled` + --> $DIR/comment_mismatch.rs:19:39 + | +LL | let _ = Options.enabled(/*value*/ false); + | ^^^^^ + | + = help: use `/*enabled*/` + +warning: 2 warnings emitted diff --git a/tools/argument-comment-lint/ui/multiple_method_arguments.rs b/tools/argument-comment-lint/ui/multiple_method_arguments.rs new file mode 100644 index 0000000000..e15f156243 --- /dev/null +++ b/tools/argument-comment-lint/ui/multiple_method_arguments.rs @@ -0,0 +1,14 @@ +#![warn(uncommented_anonymous_literal_argument)] + +struct Options; + +impl Options { + fn enabled(self, enabled: bool, retry_count: usize) -> Self { + let _ = (enabled, retry_count); + self + } +} + +fn main() { + let _ = Options.enabled(false, /*retry_count*/ 3); +} diff --git a/tools/argument-comment-lint/ui/multiple_method_arguments.stderr b/tools/argument-comment-lint/ui/multiple_method_arguments.stderr new file mode 100644 index 0000000000..f9b452e4fc --- /dev/null +++ b/tools/argument-comment-lint/ui/multiple_method_arguments.stderr @@ -0,0 +1,14 @@ +warning: anonymous literal-like argument for parameter `enabled` + --> $DIR/multiple_method_arguments.rs:13:29 + | +LL | let _ = Options.enabled(false, /*retry_count*/ 3); + | ^^^^^ help: prepend the parameter name comment: `/*enabled*/ false` + | +note: the lint level is defined here + --> $DIR/multiple_method_arguments.rs:1:9 + | +LL | #![warn(uncommented_anonymous_literal_argument)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +warning: 1 warning emitted +