From 26d932983398147e4443bd655ce24a6ce6833a1c Mon Sep 17 00:00:00 2001 From: rka-oai Date: Mon, 8 Jun 2026 09:53:04 -0700 Subject: [PATCH 1/8] [codex] Exclude external tool output from memories (#26821) ## Summary - add contains_external_context() to tool output so other tools can be opted out of influencing memory when disable_on_external_context=true - Classify standalone web-search output as external context (to match behavior as hosted web search) - Verify with integration test --- codex-rs/core/src/tools/registry.rs | 11 +++ codex-rs/core/tests/suite/sqlite_state.rs | 88 +++++++++++++++++++++++ codex-rs/ext/web-search/src/output.rs | 6 +- codex-rs/tools/src/tool_output.rs | 10 +++ 4 files changed, 113 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs index 1cc5e67eb5..20e08e9f7d 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -28,6 +28,7 @@ use codex_extension_api::ToolCallOutcome; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseInputItem; use codex_protocol::protocol::EventMsg; +use codex_rollout::state_db; use codex_tools::ToolName; use codex_tools::ToolSearchInfo; use codex_tools::ToolSpec; @@ -696,6 +697,16 @@ async fn handle_any_tool( let call_id = invocation.call_id.clone(); let payload = invocation.payload.clone(); let output = tool.handle(invocation.clone()).await?; + if output.contains_external_context() + && invocation.turn.config.memories.disable_on_external_context + { + state_db::mark_thread_memory_mode_polluted( + invocation.session.services.state_db.as_deref(), + invocation.session.thread_id, + "tool_output", + ) + .await; + } let post_tool_use_payload = CoreToolRuntime::post_tool_use_payload(tool, &invocation, output.as_ref()); Ok(AnyToolResult { diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index 9f8c940a0f..eb9f11a86d 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -1,8 +1,12 @@ use anyhow::Result; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; +use codex_core::config::Config; +use codex_extension_api::ExtensionRegistryBuilder; use codex_features::Feature; +use codex_login::CodexAuth; use codex_protocol::ThreadId; +use codex_protocol::config_types::WebSearchMode; use codex_protocol::dynamic_tools::DynamicToolSpec; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::AskForApproval; @@ -15,6 +19,7 @@ use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::UserMessageEvent; use codex_protocol::user_input::UserInput; +use codex_web_search_extension::install as install_web_search_extension; use core_test_support::responses; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call; @@ -34,9 +39,14 @@ use pretty_assertions::assert_eq; use serde_json::json; use std::collections::HashMap; use std::fs; +use std::sync::Arc; use tokio::time::Duration; use tracing_subscriber::prelude::*; use uuid::Uuid; +use wiremock::Mock; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn new_thread_is_recorded_in_state_db() -> Result<()> { @@ -384,6 +394,84 @@ async fn web_search_marks_thread_memory_mode_polluted_when_configured() -> Resul Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn standalone_web_search_marks_thread_memory_mode_polluted_when_configured() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + Mock::given(method("POST")) + .and(path("/v1/alpha/search")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "output": "Search result", + }))) + .expect(1) + .mount(&server) + .await; + mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + ev_response_created("resp-1"), + responses::ev_function_call_with_namespace( + "web-run-1", + "web", + "run", + &json!({ + "search_query": [{"q": "standalone web search"}], + }) + .to_string(), + ), + ev_completed("resp-1"), + ]), + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ], + ) + .await; + + let auth = CodexAuth::from_api_key("dummy"); + let auth_manager = codex_core::test_support::auth_manager_from_auth(auth.clone()); + let mut extension_builder = ExtensionRegistryBuilder::::new(); + install_web_search_extension(&mut extension_builder, auth_manager); + let mut builder = test_codex() + .with_auth(auth) + .with_extensions(Arc::new(extension_builder.build())) + .with_config(|config| { + config + .features + .enable(Feature::Sqlite) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::StandaloneWebSearch) + .expect("standalone web search should be enabled"); + config.memories.disable_on_external_context = true; + config + .web_search_mode + .set(WebSearchMode::Live) + .expect("web search mode should be accepted"); + }); + let test = builder.build(&server).await?; + let db = test.codex.state_db().expect("state db enabled"); + let thread_id = test.session_configured.thread_id; + + test.submit_turn("search the web").await?; + + let mut memory_mode = None; + for _ in 0..100 { + memory_mode = db.get_thread_memory_mode(thread_id).await?; + if memory_mode.as_deref() == Some("polluted") { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + + assert_eq!(memory_mode.as_deref(), Some("polluted")); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/ext/web-search/src/output.rs b/codex-rs/ext/web-search/src/output.rs index 799897b62d..543afdf7a9 100644 --- a/codex-rs/ext/web-search/src/output.rs +++ b/codex-rs/ext/web-search/src/output.rs @@ -23,9 +23,11 @@ impl ToolOutput for SearchOutput { true } + fn contains_external_context(&self) -> bool { + true + } + fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem { - // TODO: Make standalone search honor memories.disable_on_external_context, - // as hosted web search does. ResponseInputItem::FunctionCallOutput { call_id: call_id.to_string(), output: FunctionCallOutputPayload::from_content_items(vec![ diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs index b61bb80de8..fe57f23ce9 100644 --- a/codex-rs/tools/src/tool_output.rs +++ b/codex-rs/tools/src/tool_output.rs @@ -18,6 +18,12 @@ pub trait ToolOutput: Send { fn success_for_logging(&self) -> bool; + /// Whether this output contains external context that should disable memory generation when + /// `memories.disable_on_external_context` is enabled. + fn contains_external_context(&self) -> bool { + false + } + fn to_response_item(&self, call_id: &str, payload: &ToolPayload) -> ResponseInputItem; /// Returns the tool call id exposed to `PostToolUse` hooks for this output. @@ -58,6 +64,10 @@ where (**self).success_for_logging() } + fn contains_external_context(&self) -> bool { + (**self).contains_external_context() + } + fn to_response_item(&self, call_id: &str, payload: &ToolPayload) -> ResponseInputItem { (**self).to_response_item(call_id, payload) } From 6d0e313e237b6c1dd055fc9b1d7469961e8c02f8 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Mon, 8 Jun 2026 10:16:36 -0700 Subject: [PATCH 2/8] [codex] Restore release symbol artifacts with line tables (#26202) ## Summary - Restore separate release symbol archives for macOS, Linux, and Windows binaries. - Build release binaries with `line-tables-only` debuginfo instead of full debuginfo. - Strip Unix distribution binaries after extracting symbols, preserve Windows PDBs, and keep symbol archives available to the release job. - Strip the packaged Linux `bwrap` binary before hashing it so the embedded digest matches the distributed bytes. ## Root cause The first symbol-artifact implementation enabled `CARGO_PROFILE_RELEASE_DEBUG=full`. In the June 2 release runs, macOS ARM primary builds reached the 90-minute timeout while still inside `Cargo build`. After the symbol changes were reverted, the same primary build completed in about 22 minutes. The archive step itself completed in tens of seconds when reached. Rust's `line-tables-only` debuginfo level preserves function names and source locations for symbolication without emitting the heavier variable and type information from full debuginfo. ## Validation - Ran `just fmt` from `codex-rs`. - Ran `just test-github-scripts` from the repository root: 23 tests passed. - Ran `bash -n` and `shellcheck` on `.github/scripts/archive-release-symbols-and-strip-binaries.sh`. - Parsed both modified workflows as YAML and ran `git diff --check`. - Built a macOS release smoke binary with `line-tables-only`, archived its dSYM through the restored script, stripped the production binary, and verified that `atos` resolves `symbol_smoke_function` to `main.rs:2`. - Ran Linux archive-script control-flow coverage with stubbed `objcopy` and `strip` commands. - Ran Windows PDB archive staging coverage and verified underscore-emitted Rust PDB names are staged under shipped hyphenated binary names. ## Follow-up The release workflow only runs for tags or manual dispatches, so CI cannot dry-run the full release matrix on this PR. The next release run will verify runner time and memory behavior under `line-tables-only`. --- ...hive-release-symbols-and-strip-binaries.sh | 119 ++++++++++++++++++ .github/workflows/rust-release-windows.yml | 33 ++++- .github/workflows/rust-release.yml | 39 +++++- codex-rs/Cargo.toml | 7 +- 4 files changed, 192 insertions(+), 6 deletions(-) create mode 100755 .github/scripts/archive-release-symbols-and-strip-binaries.sh diff --git a/.github/scripts/archive-release-symbols-and-strip-binaries.sh b/.github/scripts/archive-release-symbols-and-strip-binaries.sh new file mode 100755 index 0000000000..3e5894bb99 --- /dev/null +++ b/.github/scripts/archive-release-symbols-and-strip-binaries.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: archive-release-symbols-and-strip-binaries.sh \ + --target \ + --artifact-name \ + --release-dir \ + --archive-dir \ + --binaries "" +EOF +} + +target="" +artifact_name="" +release_dir="" +archive_dir="" +binaries="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --target) + target="${2:?--target requires a value}" + shift 2 + ;; + --artifact-name) + artifact_name="${2:?--artifact-name requires a value}" + shift 2 + ;; + --release-dir) + release_dir="${2:?--release-dir requires a value}" + shift 2 + ;; + --archive-dir) + archive_dir="${2:?--archive-dir requires a value}" + shift 2 + ;; + --binaries) + binaries="${2:?--binaries requires a value}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unexpected argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "$target" || -z "$artifact_name" || -z "$release_dir" || -z "$archive_dir" || -z "$binaries" ]]; then + usage >&2 + exit 1 +fi + +symbols_root="${RUNNER_TEMP:-/tmp}/codex-symbols-${artifact_name}" +symbols_dir="${symbols_root}/codex-symbols-${artifact_name}" +archive_path="${archive_dir%/}/codex-symbols-${artifact_name}.tar.gz" +rm -rf "$symbols_root" +mkdir -p "$symbols_dir" "$archive_dir" +read -r -a binary_names <<< "$binaries" + +case "$target" in + *apple-darwin) + for binary in "${binary_names[@]}"; do + binary_path="${release_dir%/}/${binary}" + dsym_path="${binary_path}.dSYM" + if [[ ! -f "$binary_path" ]]; then + echo "Binary $binary_path not found" >&2 + exit 1 + fi + if [[ ! -d "$dsym_path" ]]; then + echo "dSYM $dsym_path not found" >&2 + exit 1 + fi + + cp -RL "$dsym_path" "${symbols_dir}/${binary}.dSYM" + strip -S -x "$binary_path" + done + ;; + *linux*) + objcopy_bin="${OBJCOPY:-objcopy}" + strip_bin="${STRIP:-strip}" + for binary in "${binary_names[@]}"; do + binary_path="${release_dir%/}/${binary}" + debug_path="${symbols_dir}/${binary}.debug" + if [[ ! -f "$binary_path" ]]; then + echo "Binary $binary_path not found" >&2 + exit 1 + fi + + "$objcopy_bin" --only-keep-debug "$binary_path" "$debug_path" + "$strip_bin" --strip-debug --strip-unneeded "$binary_path" + "$objcopy_bin" --add-gnu-debuglink="$debug_path" "$binary_path" + done + ;; + *windows*) + for binary in "${binary_names[@]}"; do + pdb_path="${release_dir%/}/${binary}.pdb" + if [[ ! -f "$pdb_path" ]]; then + echo "PDB $pdb_path not found" >&2 + exit 1 + fi + + cp "$pdb_path" "${symbols_dir}/${binary}.pdb" + done + ;; + *) + echo "No symbols packaging support for target: $target" >&2 + exit 1 + ;; +esac + +rm -f "$archive_path" +tar -C "$symbols_root" -czf "$archive_path" "codex-symbols-${artifact_name}" diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index 14d38b9db5..67c5c63ac9 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -112,10 +112,22 @@ jobs: - name: Stage Windows binaries shell: bash run: | - output_dir="target/${{ matrix.target }}/release/staged-${{ matrix.bundle }}" + release_dir="target/${{ matrix.target }}/release" + output_dir="$release_dir/staged-${{ matrix.bundle }}" mkdir -p "$output_dir" for binary in ${{ matrix.binaries }}; do - cp "target/${{ matrix.target }}/release/${binary}.exe" "$output_dir/${binary}.exe" + pdb_name="${binary//-/_}" + pdb_path="$release_dir/${pdb_name}.pdb" + if [[ ! -f "$pdb_path" ]]; then + pdb_path="$release_dir/${binary}.pdb" + fi + if [[ ! -f "$pdb_path" ]]; then + echo "PDB for $binary not found at $release_dir/${pdb_name}.pdb or $release_dir/${binary}.pdb" >&2 + exit 1 + fi + + cp "$release_dir/${binary}.exe" "$output_dir/${binary}.exe" + cp "$pdb_path" "$output_dir/${binary}.pdb" done - name: Upload Windows binaries @@ -201,6 +213,23 @@ jobs: account-name: ${{ secrets.AZURE_ARTIFACT_SIGNING_ACCOUNT_NAME }} certificate-profile-name: ${{ secrets.AZURE_ARTIFACT_SIGNING_CERTIFICATE_PROFILE_NAME }} + - name: Build symbols archive + shell: bash + run: | + bash "${GITHUB_WORKSPACE}/.github/scripts/archive-release-symbols-and-strip-binaries.sh" \ + --target "${{ matrix.target }}" \ + --artifact-name "${{ matrix.target }}" \ + --release-dir "target/${{ matrix.target }}/release" \ + --archive-dir "symbols-dist/${{ matrix.target }}" \ + --binaries "${WINDOWS_BINARIES}" + + - name: Upload symbols archive + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ matrix.target }}-symbols + path: codex-rs/symbols-dist/${{ matrix.target }}/* + if-no-files-found: error + - name: Stage artifacts shell: bash run: | diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index f95d1216f8..5dbf1b77eb 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -64,6 +64,8 @@ jobs: run: working-directory: codex-rs env: + # macOS release packages archive packed dSYM bundles before stripping. + CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO: ${{ contains(matrix.target, 'apple-darwin') && 'packed' || 'off' }} # Use the git CLI instead of Cargo's libgit2 path for git dependencies. # macOS release runners have intermittently failed to fetch nested # submodules through SecureTransport/libgit2, especially libwebrtc's @@ -163,7 +165,7 @@ jobs: run: | set -euo pipefail sudo apt-get update -y - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends binutils pkg-config libcap-dev - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 with: targets: ${{ matrix.target }} @@ -222,6 +224,10 @@ jobs: exit 1 fi + # Codex embeds this digest at build time and verifies the bundled + # bwrap resource before use. Strip bwrap before hashing so the digest + # covers the exact bytes that the release packages. + strip --strip-debug --strip-unneeded "$bwrap_path" digest="$(sha256sum "$bwrap_path" | awk '{print $1}')" echo "CODEX_BWRAP_SHA256=${digest}" >> "$GITHUB_ENV" echo "Built bwrap ${bwrap_path} with sha256:${digest}" @@ -235,6 +241,11 @@ jobs: fi build_args=() for binary in ${{ matrix.binaries }}; do + # bwrap was built, finalized, and hashed before this build so + # Codex can embed the digest of the bytes that will be packaged. + if [[ "$binary" == "bwrap" ]]; then + continue + fi build_args+=(--bin "$binary") done cargo build --target "$target" --release --timings "${build_args[@]}" @@ -246,6 +257,32 @@ jobs: path: codex-rs/target/**/cargo-timings/cargo-timing.html if-no-files-found: warn + - name: Build symbols archive and strip binaries + shell: bash + run: | + binaries=() + for binary in ${{ matrix.binaries }}; do + # bwrap is already stripped before hashing. Its symbols are not + # useful enough to justify a separate pre-Codex symbols pass. + if [[ "$binary" == "bwrap" ]]; then + continue + fi + binaries+=("$binary") + done + bash "${GITHUB_WORKSPACE}/.github/scripts/archive-release-symbols-and-strip-binaries.sh" \ + --target "${{ matrix.target }}" \ + --artifact-name "${{ matrix.artifact_name }}" \ + --release-dir "target/${{ matrix.target }}/release" \ + --archive-dir "symbols-dist/${{ matrix.artifact_name }}" \ + --binaries "${binaries[*]}" + + - name: Upload symbols archive + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ matrix.artifact_name }}-symbols + path: codex-rs/symbols-dist/${{ matrix.artifact_name }}/* + if-no-files-found: error + - if: ${{ runner.os == 'macOS' }} name: Stage unsigned macOS artifacts shell: bash diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 0e3aba3288..3bb23ba80a 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -501,10 +501,11 @@ strip = "symbols" [profile.release] lto = "thin" +debug = "line-tables-only" split-debuginfo = "off" -# Because we bundle some of these executables with the TypeScript CLI, we -# remove everything to make the binary as small as possible. -strip = "symbols" +# Keep release binaries symbolicateable until packaging has archived the +# sidecar symbols and stripped the binaries. +strip = false # See https://github.com/openai/codex/issues/1411 for details. codegen-units = 1 From b128da272e640371c75913033bdcc96bcea85ec4 Mon Sep 17 00:00:00 2001 From: Anton Panasenko Date: Mon, 8 Jun 2026 10:20:54 -0700 Subject: [PATCH 3/8] fix(app-server): avoid blocking connection cleanup (#26852) ## Why Remote-control app-server sessions can reconnect every 5-7 seconds when the shared transport-event queue fills. The queue's consumer handled `ConnectionClosed` by awaiting all in-flight RPCs for the disconnected connection. A stuck RPC therefore blocked processing of replacement connection and initialize events until remote-control forwarding hit its five-second timeout and reconnected again. Related issue: N/A (internal remote-control incident investigation). ## What Changed - Split fast RPC admission closure from draining: `ConnectionRpcGate::close()` rejects queued and future RPCs, while `shutdown()` continues waiting for RPCs that already started. - Close a disconnected connection's RPC gate before spawning the existing RPC drain and resource cleanup in a tracked background task, so the transport-event consumer remains available without waiting for active RPCs. - Reap completed cleanup tasks during normal operation, drain them during graceful shutdown, and abort them during forced shutdown. - Add regression coverage for closing with an active RPC, rejecting post-close requests without polling them, and preserving the existing shutdown wait behavior. ## Verification `just test -p codex-app-server --lib connection_rpc_gate` passes all 6 tests, including the new close-versus-drain regression coverage. --- codex-rs/app-server/src/connection_cleanup.rs | 49 +++++++++++++++++++ .../app-server/src/connection_rpc_gate.rs | 43 +++++++++++++--- codex-rs/app-server/src/lib.rs | 21 ++++++-- codex-rs/app-server/src/message_processor.rs | 15 +++++- .../app-server/src/request_serialization.rs | 2 +- 5 files changed, 117 insertions(+), 13 deletions(-) create mode 100644 codex-rs/app-server/src/connection_cleanup.rs diff --git a/codex-rs/app-server/src/connection_cleanup.rs b/codex-rs/app-server/src/connection_cleanup.rs new file mode 100644 index 0000000000..529020fe3f --- /dev/null +++ b/codex-rs/app-server/src/connection_cleanup.rs @@ -0,0 +1,49 @@ +use std::future::Future; +use std::future::pending; + +use tokio::task::JoinError; +use tokio::task::JoinSet; +use tracing::warn; + +pub(crate) struct ConnectionCleanupTasks { + tasks: JoinSet<()>, +} + +impl ConnectionCleanupTasks { + pub(crate) fn new() -> Self { + Self { + tasks: JoinSet::new(), + } + } + + pub(crate) fn spawn(&mut self, future: impl Future + Send + 'static) { + self.tasks.spawn(future); + } + + pub(crate) async fn reap_next(&mut self) { + if self.tasks.is_empty() { + pending::<()>().await; + } + if let Some(result) = self.tasks.join_next().await { + log_cleanup_result(result); + } + } + + pub(crate) async fn drain(&mut self) { + while let Some(result) = self.tasks.join_next().await { + log_cleanup_result(result); + } + } + + pub(crate) fn abort(&mut self) { + self.tasks.abort_all(); + } +} + +fn log_cleanup_result(result: Result<(), JoinError>) { + if let Err(err) = result + && !err.is_cancelled() + { + warn!("connection cleanup task failed: {err}"); + } +} diff --git a/codex-rs/app-server/src/connection_rpc_gate.rs b/codex-rs/app-server/src/connection_rpc_gate.rs index 12fed79b36..fb2aedd352 100644 --- a/codex-rs/app-server/src/connection_rpc_gate.rs +++ b/codex-rs/app-server/src/connection_rpc_gate.rs @@ -38,12 +38,14 @@ impl ConnectionRpcGate { drop(token); } + pub(crate) async fn close(&self) { + let mut accepting = self.accepting.lock().await; + *accepting = false; + self.tasks.close(); + } + pub(crate) async fn shutdown(&self) { - { - let mut accepting = self.accepting.lock().await; - *accepting = false; - self.tasks.close(); - } + self.close().await; self.tasks.wait().await; } @@ -90,9 +92,9 @@ mod tests { } #[tokio::test] - async fn run_drops_future_without_polling_after_shutdown() { + async fn run_drops_future_without_polling_after_close() { let gate = ConnectionRpcGate::new(); - gate.shutdown().await; + gate.close().await; let polled = Arc::new(AtomicBool::new(/*v*/ false)); let polled_clone = Arc::clone(&polled); @@ -105,6 +107,33 @@ mod tests { assert!(!gate.is_accepting().await); } + #[tokio::test] + async fn close_returns_while_started_run_remains_active() { + let gate = Arc::new(ConnectionRpcGate::new()); + let (started_tx, started_rx) = oneshot::channel(); + let (finish_tx, finish_rx) = oneshot::channel(); + let gate_for_run = Arc::clone(&gate); + let run_task = tokio::spawn(async move { + gate_for_run + .run(async move { + started_tx.send(()).expect("receiver should be open"); + let _ = finish_rx.await; + }) + .await; + }); + + started_rx.await.expect("run should start"); + gate.close().await; + assert!(!gate.is_accepting().await); + assert_eq!(gate.inflight_count(), 1); + + finish_tx + .send(()) + .expect("running future should be waiting"); + run_task.await.expect("run task should complete"); + gate.shutdown().await; + } + #[tokio::test] async fn shutdown_waits_for_started_run_to_finish() { let gate = Arc::new(ConnectionRpcGate::new()); diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index 0689a79f00..b6f0b8c859 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -20,6 +20,7 @@ use std::sync::atomic::AtomicBool; use crate::analytics_utils::analytics_events_client_from_config; use crate::config_manager::ConfigManager; +use crate::connection_cleanup::ConnectionCleanupTasks; use crate::message_processor::MessageProcessor; use crate::message_processor::MessageProcessorArgs; use crate::outgoing_message::ConnectionId; @@ -81,6 +82,7 @@ mod command_exec; mod config; mod config_manager; mod config_manager_service; +mod connection_cleanup; mod connection_rpc_gate; mod dynamic_tools; mod error_code; @@ -819,6 +821,7 @@ pub async fn run_main_with_transport_options( let mut thread_created_rx = processor.thread_created_receiver(); let mut running_turn_count_rx = processor.subscribe_running_assistant_turn_count(); let mut connections = HashMap::::new(); + let mut connection_cleanup_tasks = ConnectionCleanupTasks::new(); let mut remote_control_status_rx = remote_control_handle.status_receiver(); let mut remote_control_status = remote_control_status_rx.borrow().clone(); let transport_shutdown_token = transport_shutdown_token.clone(); @@ -906,14 +909,20 @@ pub async fn run_main_with_transport_options( let Some(connection_state) = connections.remove(&connection_id) else { continue; }; - if outbound_control_tx + connection_state.session.rpc_gate.close().await; + let outbound_closed = outbound_control_tx .send(OutboundControlEvent::Closed { connection_id }) .await - .is_err() - { + .is_ok(); + let processor = Arc::clone(&processor); + connection_cleanup_tasks.spawn(async move { + processor + .connection_closed(connection_id, &connection_state.session) + .await; + }); + if !outbound_closed { break; } - processor.connection_closed(connection_id, &connection_state.session).await; if shutdown_when_no_connections && connections.is_empty() { break; } @@ -1010,6 +1019,7 @@ pub async fn run_main_with_transport_options( } } } + _ = connection_cleanup_tasks.reap_next() => {} changed = remote_control_status_rx.changed() => { if changed.is_err() { continue; @@ -1062,8 +1072,11 @@ pub async fn run_main_with_transport_options( .map(|connection_state| connection_state.session.rpc_gate.shutdown()), ) .await; + connection_cleanup_tasks.drain().await; processor.drain_background_tasks().await; processor.shutdown_threads().await; + } else { + connection_cleanup_tasks.abort(); } info!("processor task exited (channel closed)"); } diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index b7e0ff258d..65c57bf266 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -91,6 +91,7 @@ use tokio_util::sync::CancellationToken; use tracing::Instrument; const EXTERNAL_AUTH_REFRESH_TIMEOUT: Duration = Duration::from_secs(10); +const CONNECTION_RPC_DRAIN_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 30); #[derive(Clone)] struct ExternalAuthRefreshBridge { @@ -723,7 +724,19 @@ impl MessageProcessor { connection_id: ConnectionId, session_state: &ConnectionSessionState, ) { - session_state.rpc_gate.shutdown().await; + if timeout( + CONNECTION_RPC_DRAIN_TIMEOUT, + session_state.rpc_gate.shutdown(), + ) + .await + .is_err() + { + tracing::warn!( + ?connection_id, + timeout_seconds = CONNECTION_RPC_DRAIN_TIMEOUT.as_secs(), + "timed out waiting for connection RPCs to drain" + ); + } self.outgoing.connection_closed(connection_id).await; self.fs_processor.connection_closed(connection_id).await; self.command_exec_processor diff --git a/codex-rs/app-server/src/request_serialization.rs b/codex-rs/app-server/src/request_serialization.rs index 0dd167b74d..77ecfc8f56 100644 --- a/codex-rs/app-server/src/request_serialization.rs +++ b/codex-rs/app-server/src/request_serialization.rs @@ -311,7 +311,7 @@ mod tests { let key = RequestSerializationQueueKey::Global("test"); let live_gate = gate(); let closed_gate = gate(); - closed_gate.shutdown().await; + closed_gate.close().await; let (tx, mut rx) = mpsc::unbounded_channel(); let (blocked_tx, blocked_rx) = oneshot::channel::<()>(); From a81531146609a25847ae8d045f30234e6c7eafd7 Mon Sep 17 00:00:00 2001 From: ningyi-oai Date: Mon, 8 Jun 2026 10:49:59 -0700 Subject: [PATCH 4/8] Add HTTP window ID to Responses client metadata (#26923) ## Summary - Keep the existing `x-codex-window-id` HTTP header unchanged. - Also send the same window ID in Responses `client_metadata`, allowing supported backend paths to surface it as `x-client-meta-x-codex-window-id`. - Cover normal HTTP Responses and remote compaction v2 requests without changing window generation or compaction behavior. ## Why In the `2026-06-06T23` production hour, all 28,729 HTTP compaction requests had `window_id` in `x-codex-turn-metadata`, but only 73 retained the direct `x-codex-window-id` header. The request-body `client_metadata` path is already used for installation ID and is preserved through supported Responses API paths. This is additive metadata only. It does not change the direct header, request count, model input, compaction routing, window generation, or user response behavior. Legacy `/v1/responses/compact` is intentionally unchanged. Its current server-side `CompressBody` schema does not accept `client_metadata` and rejects unknown fields, so supporting that path requires a backend schema change before the Codex client can safely send this field. ## Validation - Current head: `219baef3c`, rebased onto `origin/main` at `26d932983`. - The post-rebase diff remains limited to the original five files (`22` insertions, `6` deletions); the legacy experiment remains fully reverted. - `just test -p codex-core responses_stream_includes_subagent_header_on_review`: passed; validates normal HTTP Responses metadata. - `just test -p codex-core remote_compact_v2_reuses_compaction_trigger_for_followups`: passed; validates remote compaction v2. - `just test -p codex-core remote_manual_compact_chatgpt_auth_reuses_service_tier_and_prompt_cache_key`: passed; validates that legacy compact keeps its accepted payload shape. - `just test -p codex-core remote_manual_compact_api_auth_omits_service_tier_and_reuses_prompt_cache_key`: passed; validates the legacy API-key payload as well. - `just fmt`: passed; an unrelated root `justfile` rewrite produced by the formatter was discarded. - `git diff --check origin/main...HEAD`: passed. The focused server pytest could not start in the local monorepo environment because test setup is missing the `dotenv` module. Server source and tests explicitly show that `CompressBody` omits `client_metadata` and `/v1/responses/compact` returns HTTP 400 for unknown body fields. --- codex-rs/core/src/client.rs | 14 ++++++++++---- codex-rs/core/tests/responses_headers.rs | 4 ++++ codex-rs/core/tests/suite/compact_remote.rs | 4 ++++ ...act_api_auth_prompt_cache_key_request_diff.snap | 3 ++- ...service_tier_prompt_cache_key_request_diff.snap | 3 ++- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index dd7b9f400c..02d231c962 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -802,10 +802,16 @@ impl ModelClient { service_tier, prompt_cache_key, text, - client_metadata: Some(HashMap::from([( - X_CODEX_INSTALLATION_ID_HEADER.to_string(), - self.state.installation_id.clone(), - )])), + client_metadata: Some(HashMap::from([ + ( + X_CODEX_INSTALLATION_ID_HEADER.to_string(), + self.state.installation_id.clone(), + ), + ( + X_CODEX_WINDOW_ID_HEADER.to_string(), + self.current_window_id(), + ), + ])), }; Ok(request) } diff --git a/codex-rs/core/tests/responses_headers.rs b/codex-rs/core/tests/responses_headers.rs index 75649fa4bc..1ba93aec10 100644 --- a/codex-rs/core/tests/responses_headers.rs +++ b/codex-rs/core/tests/responses_headers.rs @@ -158,6 +158,10 @@ async fn responses_stream_includes_subagent_header_on_review() { request.body_json()["client_metadata"]["x-codex-installation-id"].as_str(), Some(TEST_INSTALLATION_ID) ); + assert_eq!( + request.body_json()["client_metadata"]["x-codex-window-id"].as_str(), + Some(expected_window_id.as_str()) + ); assert_eq!(request.header("x-codex-sandbox"), None); } diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 2200c7b778..12fd570c56 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -883,6 +883,10 @@ async fn remote_compact_v2_reuses_compaction_trigger_for_followups() -> Result<( compact_metadata["window_id"].as_str(), compact_request.header("x-codex-window-id").as_deref() ); + assert_eq!( + compact_request.body_json()["client_metadata"]["x-codex-window-id"].as_str(), + compact_metadata["window_id"].as_str() + ); assert_eq!( compact_metadata["compaction"], json!({ diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_api_auth_prompt_cache_key_request_diff.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_api_auth_prompt_cache_key_request_diff.snap index 750250f86a..89796cb918 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_api_auth_prompt_cache_key_request_diff.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_api_auth_prompt_cache_key_request_diff.snap @@ -7,7 +7,8 @@ Scenario: After five varied API-key-auth turns, remote manual compaction omits s --- Last Normal /responses Request +++ Remote /responses/compact Request - "client_metadata": { -- "x-codex-installation-id": "" +- "x-codex-installation-id": "", +- "x-codex-window-id": ":0" - }, - "include": [ - "reasoning.encrypted_content" diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_chatgpt_auth_service_tier_prompt_cache_key_request_diff.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_chatgpt_auth_service_tier_prompt_cache_key_request_diff.snap index e28ddd30d1..ab6c9d1b37 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_chatgpt_auth_service_tier_prompt_cache_key_request_diff.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_chatgpt_auth_service_tier_prompt_cache_key_request_diff.snap @@ -7,7 +7,8 @@ Scenario: After five varied ChatGPT-auth turns, remote manual compaction reuses --- Last Normal /responses Request +++ Remote /responses/compact Request - "client_metadata": { -- "x-codex-installation-id": "" +- "x-codex-installation-id": "", +- "x-codex-window-id": ":0" - }, - "include": [ - "reasoning.encrypted_content" From f1c18df9aeacb10bb88f56ff6725be252791c705 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Mon, 8 Jun 2026 10:52:31 -0700 Subject: [PATCH 5/8] [codex-analytics] report compaction analytics details (#26680) ## Why Compaction analytics adds retained image count and compaction summary output tokens for v1.5 specifically. ## What changed - Add nullable `retained_image_count` and `compaction_summary_tokens` fields to `codex_compaction_event`. - Populate them only for `responses_compaction_v2`: retained images come from the retained v2 compacted history, and summary tokens come from `response.completed.token_usage.output_tokens`. - Leave local and legacy remote compaction events as `null` for these detail fields. ## Verification - `just fmt` - `just fix -p codex-core` - `just test -p codex-core build_v2_compacted_history_counts_retained_input_images` - `git diff --check` --- .../analytics/src/analytics_client_tests.rs | 8 ++ codex-rs/analytics/src/events.rs | 4 + codex-rs/analytics/src/facts.rs | 2 + codex-rs/core/src/compact.rs | 22 ++++- codex-rs/core/src/compact_remote.rs | 34 ++++--- codex-rs/core/src/compact_remote_v2.rs | 93 ++++++++++++++----- 6 files changed, 117 insertions(+), 46 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index f4cd18da1e..a3b6dfa949 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -1253,6 +1253,8 @@ fn compaction_event_serializes_expected_shape() { error: None, active_context_tokens_before: 120_000, active_context_tokens_after: 18_000, + retained_image_count: None, + compaction_summary_tokens: None, started_at: 100, completed_at: 106, duration_ms: Some(6543), @@ -1301,6 +1303,8 @@ fn compaction_event_serializes_expected_shape() { "error": null, "active_context_tokens_before": 120000, "active_context_tokens_after": 18000, + "retained_image_count": null, + "compaction_summary_tokens": null, "started_at": 100, "completed_at": 106, "duration_ms": 6543 @@ -1821,6 +1825,8 @@ async fn compaction_event_ingests_custom_fact() { error: Some("context limit exceeded".to_string()), active_context_tokens_before: 131_000, active_context_tokens_after: 131_000, + retained_image_count: None, + compaction_summary_tokens: None, started_at: 100, completed_at: 101, duration_ms: Some(1200), @@ -2743,6 +2749,8 @@ async fn subagent_thread_started_inherits_parent_connection_for_new_thread() { error: None, active_context_tokens_before: 131_000, active_context_tokens_after: 64_000, + retained_image_count: None, + compaction_summary_tokens: None, started_at: 100, completed_at: 101, duration_ms: Some(1200), diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index f0fcddd27f..019d59776a 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -758,6 +758,8 @@ pub(crate) struct CodexCompactionEventParams { pub(crate) error: Option, pub(crate) active_context_tokens_before: i64, pub(crate) active_context_tokens_after: i64, + pub(crate) retained_image_count: Option, + pub(crate) compaction_summary_tokens: Option, pub(crate) started_at: u64, pub(crate) completed_at: u64, pub(crate) duration_ms: Option, @@ -970,6 +972,8 @@ pub(crate) fn codex_compaction_event_params( error: input.error, active_context_tokens_before: input.active_context_tokens_before, active_context_tokens_after: input.active_context_tokens_after, + retained_image_count: input.retained_image_count, + compaction_summary_tokens: input.compaction_summary_tokens, started_at: input.started_at, completed_at: input.completed_at, duration_ms: input.duration_ms, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 38af5ed8b8..c9d245ce90 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -434,6 +434,8 @@ pub struct CodexCompactionEvent { pub error: Option, pub active_context_tokens_before: i64, pub active_context_tokens_after: i64, + pub retained_image_count: Option, + pub compaction_summary_tokens: Option, pub started_at: u64, pub completed_at: u64, pub duration_ms: Option, diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 18b5838292..e2573d6d6d 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -150,7 +150,7 @@ async fn run_compact_task_inner( sess.as_ref(), CompactionStatus::Interrupted, Some(error), - /*active_context_tokens_before*/ None, + CompactionAnalyticsDetails::default(), ) .await; return Err(CodexErr::TurnAborted); @@ -174,7 +174,7 @@ async fn run_compact_task_inner( sess.as_ref(), status, error, - /*active_context_tokens_before*/ None, + CompactionAnalyticsDetails::default(), ) .await; return Err(CodexErr::TurnAborted); @@ -185,7 +185,7 @@ async fn run_compact_task_inner( sess.as_ref(), status, error, - /*active_context_tokens_before*/ None, + CompactionAnalyticsDetails::default(), ) .await; result.map(|_| ()) @@ -335,6 +335,13 @@ pub(crate) struct CompactionAnalyticsAttempt { start_instant: Instant, } +#[derive(Clone, Copy, Default)] +pub(crate) struct CompactionAnalyticsDetails { + pub(crate) active_context_tokens_before: Option, + pub(crate) retained_image_count: Option, + pub(crate) compaction_summary_tokens: Option, +} + impl CompactionAnalyticsAttempt { pub(crate) async fn begin( sess: &Session, @@ -363,8 +370,13 @@ impl CompactionAnalyticsAttempt { sess: &Session, status: CompactionStatus, error: Option, - active_context_tokens_before: Option, + details: CompactionAnalyticsDetails, ) { + let CompactionAnalyticsDetails { + active_context_tokens_before, + retained_image_count, + compaction_summary_tokens, + } = details; let active_context_tokens_before = active_context_tokens_before.unwrap_or(self.active_context_tokens_before); let active_context_tokens_after = sess.get_total_token_usage().await; @@ -382,6 +394,8 @@ impl CompactionAnalyticsAttempt { error, active_context_tokens_before, active_context_tokens_after, + retained_image_count, + compaction_summary_tokens, started_at: self.started_at, completed_at: now_unix_seconds(), duration_ms: Some( diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index c7730ddfb6..5fca191b84 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use crate::Prompt; use crate::client::CompactConversationRequestSettings; use crate::compact::CompactionAnalyticsAttempt; +use crate::compact::CompactionAnalyticsDetails; use crate::compact::InitialContextInjection; use crate::compact::compaction_status_from_result; use crate::compact::insert_initial_context_before_last_real_user_or_summary; @@ -100,7 +101,10 @@ async fn run_remote_compact_task_inner( CompactionImplementation::ResponsesCompact, phase, ); - let mut active_context_tokens_before = sess.get_total_token_usage().await; + let mut analytics_details = CompactionAnalyticsDetails { + active_context_tokens_before: Some(sess.get_total_token_usage().await), + ..Default::default() + }; let attempt = CompactionAnalyticsAttempt::begin( sess.as_ref(), turn_context.as_ref(), @@ -120,7 +124,7 @@ async fn run_remote_compact_task_inner( sess.as_ref(), codex_analytics::CompactionStatus::Interrupted, Some(error), - Some(active_context_tokens_before), + analytics_details, ) .await; return Err(CodexErr::TurnAborted); @@ -131,7 +135,7 @@ async fn run_remote_compact_task_inner( turn_context, initial_context_injection, compaction_metadata, - &mut active_context_tokens_before, + &mut analytics_details, ) .await; let status = compaction_status_from_result(&result); @@ -140,23 +144,13 @@ async fn run_remote_compact_task_inner( let post_compact_outcome = run_post_compact_hooks(sess, turn_context, trigger).await; if let PostCompactHookOutcome::Stopped = post_compact_outcome { attempt - .track( - sess.as_ref(), - status, - error, - Some(active_context_tokens_before), - ) + .track(sess.as_ref(), status, error, analytics_details) .await; return Err(CodexErr::TurnAborted); } } attempt - .track( - sess.as_ref(), - status, - error.clone(), - Some(active_context_tokens_before), - ) + .track(sess.as_ref(), status, error.clone(), analytics_details) .await; if let Err(err) = result { sess.track_turn_codex_error(turn_context, &err); @@ -174,7 +168,7 @@ async fn run_remote_compact_task_inner_impl( turn_context: &Arc, initial_context_injection: InitialContextInjection, compaction_metadata: CompactionTurnMetadata, - active_context_tokens_before: &mut i64, + analytics_details: &mut CompactionAnalyticsDetails, ) -> CodexResult<()> { let context_compaction_item = ContextCompactionItem::new(); // Use the UI compaction item ID as the trace compaction ID so protocol lifecycle events, @@ -208,8 +202,12 @@ async fn run_remote_compact_task_inner_impl( .get_total_token_usage_breakdown() .await .estimated_tokens_of_items_added_since_last_successful_api_response; - *active_context_tokens_before = (*active_context_tokens_before) - .saturating_sub(estimated_deleted_tokens.min(max_local_deleted_tokens)); + analytics_details.active_context_tokens_before = analytics_details + .active_context_tokens_before + .map(|active_context_tokens_before| { + active_context_tokens_before + .saturating_sub(estimated_deleted_tokens.min(max_local_deleted_tokens)) + }); } // This is the history selected for remote compaction, after any output rewriting required to // fit the compact endpoint. The checkpoint below records it separately from the next sampling diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index 5f585862dd..1d4eb8ad5a 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -5,6 +5,7 @@ use crate::ResponseStream; use crate::client::ModelClientSession; use crate::client_common::ResponseEvent; use crate::compact::CompactionAnalyticsAttempt; +use crate::compact::CompactionAnalyticsDetails; use crate::compact::InitialContextInjection; use crate::compact::compaction_status_from_result; use crate::compact_remote::build_compact_request_log_data; @@ -112,7 +113,10 @@ async fn run_remote_compact_task_inner( CompactionImplementation::ResponsesCompactionV2, phase, ); - let mut active_context_tokens_before = sess.get_total_token_usage().await; + let mut analytics_details = CompactionAnalyticsDetails { + active_context_tokens_before: Some(sess.get_total_token_usage().await), + ..Default::default() + }; let attempt = CompactionAnalyticsAttempt::begin( sess.as_ref(), turn_context.as_ref(), @@ -132,7 +136,7 @@ async fn run_remote_compact_task_inner( sess.as_ref(), codex_analytics::CompactionStatus::Interrupted, Some(error), - Some(active_context_tokens_before), + analytics_details, ) .await; return Err(CodexErr::TurnAborted); @@ -144,7 +148,7 @@ async fn run_remote_compact_task_inner( client_session, initial_context_injection, compaction_metadata, - &mut active_context_tokens_before, + &mut analytics_details, ) .await; let status = compaction_status_from_result(&result); @@ -153,23 +157,13 @@ async fn run_remote_compact_task_inner( let post_compact_outcome = run_post_compact_hooks(sess, turn_context, trigger).await; if let PostCompactHookOutcome::Stopped = post_compact_outcome { attempt - .track( - sess.as_ref(), - status, - error, - Some(active_context_tokens_before), - ) + .track(sess.as_ref(), status, error, analytics_details) .await; return Err(CodexErr::TurnAborted); } } attempt - .track( - sess.as_ref(), - status, - error.clone(), - Some(active_context_tokens_before), - ) + .track(sess.as_ref(), status, error.clone(), analytics_details) .await; if let Err(err) = result { sess.track_turn_codex_error(turn_context, &err); @@ -188,7 +182,7 @@ async fn run_remote_compact_task_inner_impl( client_session: Option<&mut ModelClientSession>, initial_context_injection: InitialContextInjection, compaction_metadata: CompactionTurnMetadata, - active_context_tokens_before: &mut i64, + analytics_details: &mut CompactionAnalyticsDetails, ) -> CodexResult<()> { let context_compaction_item = ContextCompactionItem::new(); let compaction_trace = sess.services.rollout_thread_trace.compaction_trace_context( @@ -221,8 +215,12 @@ async fn run_remote_compact_task_inner_impl( .get_total_token_usage_breakdown() .await .estimated_tokens_of_items_added_since_last_successful_api_response; - *active_context_tokens_before = (*active_context_tokens_before) - .saturating_sub(estimated_deleted_tokens.min(max_local_deleted_tokens)); + analytics_details.active_context_tokens_before = analytics_details + .active_context_tokens_before + .map(|active_context_tokens_before| { + active_context_tokens_before + .saturating_sub(estimated_deleted_tokens.min(max_local_deleted_tokens)) + }); } let trace_input_history = history.raw_items().to_vec(); @@ -283,9 +281,12 @@ async fn run_remote_compact_task_inner_impl( token_usage, } = compaction_output_result?; if let Some(token_usage) = token_usage { - *active_context_tokens_before = token_usage.input_tokens; + analytics_details.active_context_tokens_before = Some(token_usage.input_tokens); + analytics_details.compaction_summary_tokens = Some(token_usage.output_tokens); } - let compacted_history = build_v2_compacted_history(&prompt_input, compaction_output); + let (compacted_history, retained_images) = + build_v2_compacted_history(&prompt_input, compaction_output); + analytics_details.retained_image_count = Some(retained_images); let new_history = process_compacted_history( sess.as_ref(), turn_context.as_ref(), @@ -447,7 +448,7 @@ async fn collect_compaction_output( fn build_v2_compacted_history( prompt_input: &[ResponseItem], compaction_output: ResponseItem, -) -> Vec { +) -> (Vec, usize) { let retained = prompt_input .iter() .filter(|item| is_retained_for_remote_compaction_v2(item)) @@ -456,8 +457,12 @@ fn build_v2_compacted_history( .collect::>(); let mut retained = truncate_retained_messages_for_remote_compaction(retained, RETAINED_MESSAGE_TOKEN_BUDGET); + let retained_image_count = retained + .iter() + .map(retained_input_image_count) + .sum::(); retained.push(compaction_output); - retained + (retained, retained_image_count) } fn is_retained_for_remote_compaction_v2(item: &ResponseItem) -> bool { @@ -468,6 +473,17 @@ fn is_retained_for_remote_compaction_v2(item: &ResponseItem) -> bool { matches!(role.as_str(), "user" | "developer" | "system") } +fn retained_input_image_count(item: &ResponseItem) -> usize { + let ResponseItem::Message { content, .. } = item else { + return 0; + }; + + content + .iter() + .filter(|item| matches!(item, ContentItem::InputImage { .. })) + .count() +} + fn truncate_retained_messages_for_remote_compaction( items: Vec, max_tokens: usize, @@ -617,7 +633,7 @@ mod tests { encrypted_content: "new".to_string(), }; - let history = build_v2_compacted_history(&input, output.clone()); + let (history, _) = build_v2_compacted_history(&input, output.clone()); assert_eq!( history, @@ -644,11 +660,40 @@ mod tests { encrypted_content: "new".to_string(), }; - let history = build_v2_compacted_history(&input, output.clone()); + let (history, _) = build_v2_compacted_history(&input, output.clone()); assert_eq!(history, vec![old, new, output]); } + #[test] + fn build_v2_compacted_history_counts_retained_input_images() { + let input = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputText { + text: "user".to_string(), + }, + ContentItem::InputImage { + image_url: "data:image/png;base64,abc".to_string(), + detail: None, + }, + ContentItem::InputImage { + image_url: "data:image/png;base64,def".to_string(), + detail: None, + }, + ], + phase: None, + }]; + let output = ResponseItem::Compaction { + encrypted_content: "new".to_string(), + }; + + let (_, retained_image_count) = build_v2_compacted_history(&input, output); + + assert_eq!(retained_image_count, 2); + } + #[test] fn retained_history_truncation_keeps_newest_messages_first() { let middle = message("user", "middle1234", /*phase*/ None); From 6d8e12ac42508c10bbe1d1769aee7f49d150ba5c Mon Sep 17 00:00:00 2001 From: stefanstokic-oai Date: Mon, 8 Jun 2026 14:16:32 -0400 Subject: [PATCH 6/8] [codex] Speed up external agent session imports (#26637) ## Why Importing large external-agent session histories currently starts a full live Codex thread for every imported session. This initializes unrelated runtime systems and repeats expensive transcript, metadata, hashing, and ledger work. On a 50-session, 238 MiB fixture, the existing path took roughly 70 seconds to complete the import and 77 seconds end to end. ## What changed - Persist imported sessions directly through `ThreadStore` instead of starting full live threads. - Process imports through a bounded five-session pipeline. - Parse, extract, and hash each source file in one pass. - Move blocking source preparation onto the blocking thread pool. - Reuse prepared content hashes and update the import ledger once per batch. - Avoid metadata readback for newly written rollouts. - Preserve imported conversation history and visible thread metadata. - Keep the implementation out of `codex-core` and avoid changes to the public `ThreadStore` trait. ## Performance For the same 50-session, 238 MiB fixture: | Path | Import completion | End to end | | --- | ---: | ---: | | Existing import | 69.61s | 76.62s | | This change | 5.95s | 6.58s | All 50 sessions imported successfully with no warnings or contention signals. ## Validation - `just test -p codex-external-agent-sessions` - `just test -p codex-app-server external_agent_config_import` - Verified imports do not initialize unrelated required MCP servers. - Verified previously imported source versions are skipped and changed sources can be imported again. - Verified imported rollouts remain readable through thread listing and history APIs. --- codex-rs/app-server/src/message_processor.rs | 1 + codex-rs/app-server/src/request_processors.rs | 1 + .../external_agent_config_processor.rs | 143 +--------- .../external_agent_session_import.rs | 260 ++++++++++++++++++ .../tests/suite/v2/external_agent_config.rs | 92 +++++++ .../external-agent-sessions/src/export.rs | 191 +++++++------ .../external-agent-sessions/src/ledger.rs | 67 +++-- .../src/ledger_tests.rs | 85 ++++++ codex-rs/external-agent-sessions/src/lib.rs | 192 +++++-------- .../external-agent-sessions/src/records.rs | 155 +++++++---- 10 files changed, 793 insertions(+), 394 deletions(-) create mode 100644 codex-rs/app-server/src/request_processors/external_agent_session_import.rs create mode 100644 codex-rs/external-agent-sessions/src/ledger_tests.rs diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 65c57bf266..84d3d88dda 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -475,6 +475,7 @@ impl MessageProcessor { let external_agent_config_processor = ExternalAgentConfigRequestProcessor::new( outgoing.clone(), Arc::clone(&thread_manager), + Arc::clone(&thread_store), config_manager.clone(), config_processor.clone(), arg0_paths, diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index 57205798b2..79da49fba4 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -461,6 +461,7 @@ mod command_exec_processor; mod config_processor; mod environment_processor; mod external_agent_config_processor; +mod external_agent_session_import; mod feedback_doctor_report; mod feedback_processor; mod fs_processor; diff --git a/codex-rs/app-server/src/request_processors/external_agent_config_processor.rs b/codex-rs/app-server/src/request_processors/external_agent_config_processor.rs index 87956182d0..5d4c0594eb 100644 --- a/codex-rs/app-server/src/request_processors/external_agent_config_processor.rs +++ b/codex-rs/app-server/src/request_processors/external_agent_config_processor.rs @@ -26,53 +26,47 @@ use codex_app_server_protocol::MigrationDetails; use codex_app_server_protocol::PluginsMigration; use codex_app_server_protocol::ServerNotification; use codex_arg0::Arg0DispatchPaths; -use codex_core::StartThreadOptions; use codex_core::ThreadManager; -use codex_core::config::ConfigOverrides; use codex_external_agent_sessions::ExternalAgentSessionMigration as CoreSessionMigration; -use codex_external_agent_sessions::ImportedExternalAgentSession; -use codex_external_agent_sessions::PendingSessionImport; -use codex_external_agent_sessions::prepare_validated_session_imports; -use codex_external_agent_sessions::record_imported_session; -use codex_protocol::ThreadId; -use codex_protocol::protocol::InitialHistory; -use codex_thread_store::ThreadMetadataPatch; +use codex_thread_store::ThreadStore; use std::collections::HashSet; use std::path::PathBuf; -use tokio::sync::Semaphore; use super::ConfigRequestProcessor; +use super::external_agent_session_import::ExternalAgentSessionImporter; #[derive(Clone)] pub(crate) struct ExternalAgentConfigRequestProcessor { outgoing: Arc, - codex_home: PathBuf, migration_service: ExternalAgentConfigService, - session_import_permits: Arc, + session_importer: ExternalAgentSessionImporter, thread_manager: Arc, - config_manager: ConfigManager, config_processor: ConfigRequestProcessor, - arg0_paths: Arg0DispatchPaths, } impl ExternalAgentConfigRequestProcessor { pub(crate) fn new( outgoing: Arc, thread_manager: Arc, + thread_store: Arc, config_manager: ConfigManager, config_processor: ConfigRequestProcessor, arg0_paths: Arg0DispatchPaths, codex_home: PathBuf, ) -> Self { + let session_importer = ExternalAgentSessionImporter::new( + codex_home.clone(), + Arc::clone(&thread_manager), + thread_store, + config_manager, + arg0_paths, + ); Self { outgoing, - migration_service: ExternalAgentConfigService::new(codex_home.clone()), - codex_home, - session_import_permits: Arc::new(Semaphore::new(1)), + migration_service: ExternalAgentConfigService::new(codex_home), + session_importer, thread_manager, - config_manager, config_processor, - arg0_paths, } } @@ -207,42 +201,12 @@ impl ExternalAgentConfigRequestProcessor { return Ok(()); } - let session_import_permits = Arc::clone(&self.session_import_permits); - let session_processor = self.clone(); + let session_importer = self.session_importer.clone(); let plugin_processor = self.clone(); let outgoing = Arc::clone(&self.outgoing); let thread_manager = Arc::clone(&self.thread_manager); tokio::spawn(async move { - let session_imports = async move { - if !pending_session_imports.is_empty() { - let Ok(_session_import_permit) = session_import_permits.acquire_owned().await - else { - return; - }; - let pending_session_imports = session_processor - .prepare_validated_session_imports(pending_session_imports); - for pending_session_import in pending_session_imports { - match session_processor - .import_external_agent_session(pending_session_import.session) - .await - { - Ok(imported_thread_id) => { - session_processor.record_imported_session( - &pending_session_import.source_path, - imported_thread_id, - ); - } - Err(error) => { - tracing::warn!( - error = %error.message, - path = %pending_session_import.source_path.display(), - "external agent session import failed" - ); - } - } - } - } - }; + let session_imports = session_importer.import_sessions(pending_session_imports); let plugin_imports = async move { for pending_plugin_import in pending_plugin_imports { match plugin_processor @@ -274,65 +238,6 @@ impl ExternalAgentConfigRequestProcessor { Ok(()) } - async fn import_external_agent_session( - &self, - session: ImportedExternalAgentSession, - ) -> Result { - let ImportedExternalAgentSession { - cwd, - title, - rollout_items, - } = session; - let config = self - .config_manager - .load_with_overrides( - /*request_overrides*/ None, - ConfigOverrides { - cwd: Some(PathBuf::from(cwd.to_string_lossy().into_owned())), - codex_linux_sandbox_exe: self.arg0_paths.codex_linux_sandbox_exe.clone(), - main_execve_wrapper_exe: self.arg0_paths.main_execve_wrapper_exe.clone(), - ..Default::default() - }, - ) - .await - .map_err(|err| { - internal_error(format!("failed to load imported session config: {err}")) - })?; - let environments = self - .thread_manager - .default_environment_selections(&config.cwd); - let imported_thread = self - .thread_manager - .start_thread_with_options(StartThreadOptions { - config, - initial_history: InitialHistory::Forked(rollout_items), - session_source: None, - thread_source: None, - dynamic_tools: Vec::new(), - metrics_service_name: None, - parent_trace: None, - environments, - }) - .await - .map_err(|err| internal_error(format!("failed to import session: {err}")))?; - if let Some(title) = title - && let Some(name) = codex_core::util::normalize_thread_name(&title) - { - imported_thread - .thread - .update_thread_metadata( - ThreadMetadataPatch { - name: Some(Some(name)), - ..Default::default() - }, - /*include_archived*/ false, - ) - .await - .map_err(|err| internal_error(format!("failed to name imported session: {err}")))?; - } - Ok(imported_thread.thread_id) - } - fn validate_pending_session_imports( &self, params: &ExternalAgentConfigImportParams, @@ -371,24 +276,6 @@ impl ExternalAgentConfigRequestProcessor { Ok(selected_sessions) } - fn prepare_validated_session_imports( - &self, - sessions: Vec, - ) -> Vec { - prepare_validated_session_imports(&self.codex_home, sessions) - } - - fn record_imported_session(&self, source_path: &std::path::Path, imported_thread_id: ThreadId) { - if let Err(err) = record_imported_session(&self.codex_home, source_path, imported_thread_id) - { - tracing::warn!( - error = %err, - path = %source_path.display(), - "external agent session import ledger update failed" - ); - } - } - async fn import_external_agent_config( &self, params: ExternalAgentConfigImportParams, diff --git a/codex-rs/app-server/src/request_processors/external_agent_session_import.rs b/codex-rs/app-server/src/request_processors/external_agent_session_import.rs new file mode 100644 index 0000000000..163210f164 --- /dev/null +++ b/codex-rs/app-server/src/request_processors/external_agent_session_import.rs @@ -0,0 +1,260 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use chrono::Utc; +use codex_arg0::Arg0DispatchPaths; +use codex_core::ThreadManager; +use codex_core::config::ConfigOverrides; +use codex_external_agent_sessions::CompletedExternalAgentSessionImport; +use codex_external_agent_sessions::ExternalAgentSessionMigration; +use codex_external_agent_sessions::ImportedExternalAgentSession; +use codex_external_agent_sessions::PendingSessionImport; +use codex_external_agent_sessions::prepare_validated_session_import; +use codex_external_agent_sessions::record_completed_session_imports; +use codex_models_manager::manager::RefreshStrategy; +use codex_protocol::ThreadId; +use codex_protocol::models::BaseInstructions; +use codex_protocol::protocol::MultiAgentVersion; +use codex_protocol::protocol::ThreadMemoryMode; +use codex_rollout::is_persisted_rollout_item; +use codex_thread_store::AppendThreadItemsParams; +use codex_thread_store::CreateThreadParams; +use codex_thread_store::ThreadMetadataPatch; +use codex_thread_store::ThreadPersistenceMetadata; +use codex_thread_store::ThreadStore; +use codex_thread_store::UpdateThreadMetadataParams; +use futures::StreamExt; +use tokio::sync::Semaphore; + +use crate::config_manager::ConfigManager; + +const SESSION_IMPORT_CONCURRENCY: usize = 5; + +#[derive(Clone)] +pub(super) struct ExternalAgentSessionImporter { + codex_home: PathBuf, + permits: Arc, + thread_manager: Arc, + thread_store: Arc, + config_manager: ConfigManager, + arg0_paths: Arg0DispatchPaths, +} + +impl ExternalAgentSessionImporter { + pub(super) fn new( + codex_home: PathBuf, + thread_manager: Arc, + thread_store: Arc, + config_manager: ConfigManager, + arg0_paths: Arg0DispatchPaths, + ) -> Self { + Self { + codex_home, + permits: Arc::new(Semaphore::new(1)), + thread_manager, + thread_store, + config_manager, + arg0_paths, + } + } + + pub(super) async fn import_sessions(&self, sessions: Vec) { + if sessions.is_empty() { + return; + } + let Ok(_permit) = self.permits.acquire().await else { + return; + }; + let import_results = futures::stream::iter(sessions) + .map(|session| { + let importer = self.clone(); + async move { importer.import_requested_session(session).await } + }) + .buffer_unordered(SESSION_IMPORT_CONCURRENCY); + futures::pin_mut!(import_results); + + let mut completed_imports = Vec::new(); + while let Some(result) = import_results.next().await { + match result { + Ok(Some(completed_import)) => completed_imports.push(completed_import), + Ok(None) => {} + Err(failure) => { + tracing::warn!( + error = %failure.message, + path = %failure.source_path.display(), + "external agent session import failed" + ); + } + } + } + if let Err(err) = record_completed_session_imports(&self.codex_home, completed_imports) { + tracing::warn!( + error = %err, + "external agent session import ledger update failed" + ); + } + } + + async fn import_requested_session( + &self, + session: ExternalAgentSessionMigration, + ) -> Result, SessionImportFailure> { + let source_path = session.path.clone(); + let Some(pending_import) = + self.prepare_session_import(session) + .await + .map_err(|message| SessionImportFailure { + source_path: source_path.clone(), + message, + })? + else { + return Ok(None); + }; + let imported_thread_id = + self.persist_session(pending_import.session) + .await + .map_err(|message| SessionImportFailure { + source_path: pending_import.source_path.clone(), + message, + })?; + Ok(Some(CompletedExternalAgentSessionImport { + source_path: pending_import.source_path, + source_content_sha256: pending_import.source_content_sha256, + imported_thread_id, + })) + } + + async fn prepare_session_import( + &self, + session: ExternalAgentSessionMigration, + ) -> Result, String> { + let codex_home = self.codex_home.clone(); + tokio::task::spawn_blocking(move || prepare_validated_session_import(&codex_home, session)) + .await + .map_err(|err| format!("external agent session preparation task failed: {err}"))? + .map_err(|err| format!("failed to prepare external agent session: {err}")) + } + + async fn persist_session( + &self, + session: ImportedExternalAgentSession, + ) -> Result { + let ImportedExternalAgentSession { + cwd, + title, + first_user_message, + mut rollout_items, + } = session; + let config = self + .config_manager + .load_with_overrides( + /*request_overrides*/ None, + ConfigOverrides { + cwd: Some(cwd), + codex_linux_sandbox_exe: self.arg0_paths.codex_linux_sandbox_exe.clone(), + main_execve_wrapper_exe: self.arg0_paths.main_execve_wrapper_exe.clone(), + ..Default::default() + }, + ) + .await + .map_err(|err| format!("failed to load imported session config: {err}"))?; + let models_manager = self.thread_manager.get_models_manager(); + let model = models_manager + .get_default_model(&config.model, RefreshStrategy::Offline) + .await; + let model_info = models_manager + .get_model_info(model.as_str(), &config.to_models_manager_config()) + .await; + let thread_id = ThreadId::new(); + let source = self.thread_manager.session_source(); + let cwd = config.cwd.to_path_buf(); + let model_provider = config.model_provider_id.clone(); + let memory_mode = if config.memories.generate_memories { + ThreadMemoryMode::Enabled + } else { + ThreadMemoryMode::Disabled + }; + let now = Utc::now(); + let create_params = CreateThreadParams { + thread_id, + forked_from_id: None, + parent_thread_id: None, + source: source.clone(), + thread_source: None, + base_instructions: BaseInstructions { + text: config + .base_instructions + .clone() + .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), + }, + dynamic_tools: Vec::new(), + multi_agent_version: Some(MultiAgentVersion::V1), + metadata: ThreadPersistenceMetadata { + cwd: Some(cwd.clone()), + model_provider: model_provider.clone(), + memory_mode, + }, + }; + rollout_items.retain(is_persisted_rollout_item); + let title = title + .as_deref() + .and_then(codex_core::util::normalize_thread_name); + let metadata = ThreadMetadataPatch { + title, + preview: first_user_message.clone(), + model_provider: Some(model_provider), + created_at: Some(now), + updated_at: Some(now), + source: Some(source.clone()), + thread_source: Some(None), + agent_nickname: Some(source.get_nickname()), + agent_role: Some(source.get_agent_role()), + agent_path: Some(source.get_agent_path().map(Into::into)), + cwd: Some(cwd), + cli_version: Some(env!("CARGO_PKG_VERSION").to_string()), + first_user_message, + memory_mode: Some(memory_mode), + ..Default::default() + }; + + self.thread_store + .create_thread(create_params) + .await + .map_err(|err| format!("failed to import session: {err}"))?; + if !rollout_items.is_empty() + && let Err(err) = self + .thread_store + .append_items(AppendThreadItemsParams { + thread_id, + items: rollout_items, + }) + .await + { + let _ = self.thread_store.discard_thread(thread_id).await; + return Err(format!("failed to import session: {err}")); + } + + self.thread_store + .update_thread_metadata(UpdateThreadMetadataParams { + thread_id, + patch: metadata, + include_archived: false, + }) + .await + .map_err(|err| format!("failed to update imported session: {err}"))?; + self.thread_store + .persist_thread(thread_id) + .await + .map_err(|err| format!("failed to persist imported session: {err}"))?; + self.thread_store + .shutdown_thread(thread_id) + .await + .map_err(|err| format!("failed to shutdown imported session: {err}"))?; + Ok(thread_id) + } +} + +struct SessionImportFailure { + source_path: PathBuf, + message: String, +} diff --git a/codex-rs/app-server/tests/suite/v2/external_agent_config.rs b/codex-rs/app-server/tests/suite/v2/external_agent_config.rs index fdb996c2f4..991e12dc78 100644 --- a/codex-rs/app-server/tests/suite/v2/external_agent_config.rs +++ b/codex-rs/app-server/tests/suite/v2/external_agent_config.rs @@ -435,6 +435,98 @@ async fn external_agent_config_import_creates_session_rollouts() -> Result<()> { Ok(()) } +#[tokio::test] +async fn external_agent_config_import_does_not_initialize_required_mcp() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("unused").await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + let mut config = std::fs::read_to_string(codex_home.path().join("config.toml"))?; + config.push_str( + r#" +[mcp_servers.required_broken] +command = "this-command-does-not-exist" +required = true +"#, + ); + std::fs::write(codex_home.path().join("config.toml"), config)?; + let project_root = codex_home.path().join("repo"); + let recent_timestamp = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let session_dir = codex_home.path().join(".claude/projects/repo"); + let session_path = session_dir.join("session.jsonl"); + std::fs::create_dir_all(&project_root)?; + std::fs::create_dir_all(&session_dir)?; + std::fs::write( + &session_path, + serde_json::json!({ + "type": "user", + "cwd": &project_root, + "timestamp": &recent_timestamp, + "message": { "content": "first request" }, + }) + .to_string(), + )?; + + let home_dir = codex_home.path().display().to_string(); + let mut mcp = + TestAppServer::new_with_env(codex_home.path(), &[("HOME", Some(home_dir.as_str()))]) + .await?; + timeout(DEFAULT_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_raw_request( + "externalAgentConfig/import", + Some(serde_json::json!({ + "migrationItems": [{ + "itemType": "SESSIONS", + "description": "Migrate recent sessions", + "cwd": null, + "details": { + "sessions": [{ + "path": session_path, + "cwd": project_root, + "title": "first request" + }] + } + }] + })), + ) + .await?; + timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_notification_message("externalAgentConfig/import/completed"), + ) + .await??; + + let request_id = mcp + .send_thread_list_request(ThreadListParams { + cursor: None, + limit: None, + sort_key: None, + sort_direction: None, + model_providers: None, + source_kinds: None, + archived: None, + cwd: None, + use_state_db_only: false, + search_term: None, + }) + .await?; + let response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let response: ThreadListResponse = to_response(response)?; + assert_eq!(response.data.len(), 1); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn external_agent_config_import_accepts_detected_session_payload_after_restart() -> Result<()> { diff --git a/codex-rs/external-agent-sessions/src/export.rs b/codex-rs/external-agent-sessions/src/export.rs index e09805d4d2..05a3004ee9 100644 --- a/codex-rs/external-agent-sessions/src/export.rs +++ b/codex-rs/external-agent-sessions/src/export.rs @@ -1,10 +1,7 @@ use crate::ConversationMessage; use crate::ImportedExternalAgentSession; use crate::MessageRole; -use crate::records::conversation_messages; -use crate::records::project_root_from_records; -use crate::records::read_records; -use crate::records::source_title_from_records; +use crate::records::read_session_import; use crate::summarize_for_label; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; @@ -23,44 +20,55 @@ use std::path::Path; const EXTERNAL_SESSION_IMPORTED_MARKER: &str = ""; -pub fn load_session_for_import(path: &Path) -> io::Result> { - let records = read_records(path)?; - let Some(cwd) = project_root_from_records(&records) else { +#[cfg(test)] +fn load_session_for_import(path: &Path) -> io::Result> { + Ok( + load_session_for_import_with_content_sha256(path)? + .map(|(session, _content_sha256)| session), + ) +} + +pub(crate) fn load_session_for_import_with_content_sha256( + path: &Path, +) -> io::Result> { + let parsed = read_session_import(path)?; + let Some(cwd) = parsed.cwd else { return Ok(None); }; - let messages = conversation_messages(&records); - let rollout_items = rollout_items_from_messages(&messages); + let messages = parsed.messages; + let first_user_message = messages + .iter() + .find(|message| message.role == MessageRole::User) + .map(|message| summarize_for_label(&message.text)); + let title = parsed.source_title.or_else(|| first_user_message.clone()); + let rollout_items = rollout_items_from_messages(messages); if rollout_items.is_empty() { return Ok(None); } - let title = source_title_from_records(&records).or_else(|| { - messages - .iter() - .find(|message| message.role == MessageRole::User) - .map(|message| summarize_for_label(&message.text)) - }); - Ok(Some(ImportedExternalAgentSession { - cwd, - title, - rollout_items, - })) + Ok(Some(( + ImportedExternalAgentSession { + cwd, + title, + first_user_message, + rollout_items, + }, + parsed.content_sha256, + ))) } -fn rollout_items_from_messages(messages: &[ConversationMessage]) -> Vec { +fn rollout_items_from_messages(messages: Vec) -> Vec { let mut items = Vec::new(); - let mut response_items = Vec::new(); - let mut current_turn: Option<(String, Option)> = None; + let mut current_turn = None; + let mut response_item_bytes = 0i64; + let mut last_model_visible_tokens = 0i64; let mut user_turn_count = 0usize; + let completed_at = messages.last().and_then(|message| message.timestamp); for message in messages { match message.role { MessageRole::User => { - if let Some((turn_id, last_agent_message)) = current_turn.take() { - items.push(turn_complete_item( - turn_id, - last_agent_message, - /*completed_at*/ None, - )); + if let Some(turn_id) = current_turn.take() { + items.push(turn_complete_item(turn_id, /*completed_at*/ None)); } user_turn_count += 1; let turn_id = format!("external-import-turn-{user_turn_count}"); @@ -73,28 +81,24 @@ fn rollout_items_from_messages(messages: &[ConversationMessage]) -> Vec { - let Some((_, last_agent_message)) = current_turn.as_mut() else { + if current_turn.is_none() { continue; - }; - let response_item = response_item(message); - response_items.push(response_item.clone()); - items.push(RolloutItem::ResponseItem(response_item)); + } + response_item_bytes = + response_item_bytes.saturating_add(message_byte_count(&message)); + last_model_visible_tokens = approx_tokens_from_byte_count_i64(response_item_bytes); items.push(RolloutItem::EventMsg(EventMsg::AgentMessage( AgentMessageEvent { message: message.text.clone(), @@ -102,20 +106,15 @@ fn rollout_items_from_messages(messages: &[ConversationMessage]) -> Vec RolloutItem { })) } -fn response_item(message: &ConversationMessage) -> ResponseItem { +fn response_item(message: ConversationMessage) -> ResponseItem { let content = match message.role { - MessageRole::Assistant => ContentItem::OutputText { - text: message.text.clone(), - }, - MessageRole::User => ContentItem::InputText { - text: message.text.clone(), - }, + MessageRole::Assistant => ContentItem::OutputText { text: message.text }, + MessageRole::User => ContentItem::InputText { text: message.text }, }; ResponseItem::Message { id: None, @@ -149,13 +144,11 @@ fn response_item(message: &ConversationMessage) -> ResponseItem { } } -fn token_count_item(response_items: &[ResponseItem]) -> RolloutItem { - let last_model_generated = response_items.iter().rposition( - |item| matches!(item, ResponseItem::Message { role, .. } if role == "assistant"), - ); - let last_model_visible_tokens = last_model_generated - .map(|index| estimate_response_items_token_count(&response_items[..=index])) - .unwrap_or_default(); +fn message_byte_count(message: &ConversationMessage) -> i64 { + i64::try_from(message.text.len()).unwrap_or(i64::MAX) +} + +fn token_count_item(last_model_visible_tokens: i64) -> RolloutItem { let usage = TokenUsage { total_tokens: last_model_visible_tokens, ..TokenUsage::default() @@ -170,26 +163,10 @@ fn token_count_item(response_items: &[ResponseItem]) -> RolloutItem { })) } -fn estimate_response_items_token_count(response_items: &[ResponseItem]) -> i64 { - response_items - .iter() - .map(|item| { - serde_json::to_string(item) - .map(|serialized| i64::try_from(serialized.len()).unwrap_or(i64::MAX)) - .map(approx_tokens_from_byte_count_i64) - .unwrap_or_default() - }) - .fold(0i64, i64::saturating_add) -} - -fn turn_complete_item( - turn_id: String, - last_agent_message: Option, - completed_at: Option, -) -> RolloutItem { +fn turn_complete_item(turn_id: String, completed_at: Option) -> RolloutItem { RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent { turn_id, - last_agent_message, + last_agent_message: None, completed_at, duration_ms: None, time_to_first_token_ms: None, @@ -241,7 +218,7 @@ mod tests { } #[test] - fn adds_import_marker_without_replacing_last_agent_message() { + fn adds_import_marker_without_copying_last_agent_message() { let root = TempDir::new().expect("tempdir"); let project_root = root.path().join("repo"); std::fs::create_dir_all(&project_root).expect("project root"); @@ -280,10 +257,54 @@ mod tests { }); assert_eq!( last_turn_complete.and_then(|event| event.last_agent_message.as_deref()), - Some("first answer") + None ); } + #[test] + fn stores_imported_messages_as_response_items_and_visible_events() { + let root = TempDir::new().expect("tempdir"); + let project_root = root.path().join("repo"); + std::fs::create_dir_all(&project_root).expect("project root"); + let path = root.path().join("session.jsonl"); + let request = "r".repeat(1_000); + let answer = "a".repeat(1_000); + std::fs::write( + &path, + jsonl(&[ + record("user", &request, &project_root), + record("assistant", &answer, &project_root), + ]), + ) + .expect("session"); + + let imported = load_session_for_import(&path) + .expect("load") + .expect("session"); + let response_message_count = imported + .rollout_items + .iter() + .filter(|item| { + matches!( + item, + RolloutItem::ResponseItem(ResponseItem::Message { .. }) + ) + }) + .count(); + let visible_message_event_count = imported + .rollout_items + .iter() + .filter(|item| match item { + RolloutItem::EventMsg(EventMsg::UserMessage(event)) => event.message == request, + RolloutItem::EventMsg(EventMsg::AgentMessage(event)) => event.message == answer, + _ => false, + }) + .count(); + + assert_eq!(response_message_count, 2); + assert_eq!(visible_message_event_count, 2); + } + #[test] fn loads_custom_title_for_imported_session() { let root = TempDir::new().expect("tempdir"); diff --git a/codex-rs/external-agent-sessions/src/ledger.rs b/codex-rs/external-agent-sessions/src/ledger.rs index 45ff97bcf0..9a3b2042b8 100644 --- a/codex-rs/external-agent-sessions/src/ledger.rs +++ b/codex-rs/external-agent-sessions/src/ledger.rs @@ -30,6 +30,13 @@ struct ImportedExternalAgentSessionRecord { source_modified_at: Option, } +#[derive(Debug, PartialEq, Eq)] +pub struct CompletedExternalAgentSessionImport { + pub source_path: PathBuf, + pub source_content_sha256: String, + pub imported_thread_id: ThreadId, +} + #[derive(Debug, Clone, Copy)] pub(super) struct ImportedSourceState { pub source_modified_at: Option, @@ -43,29 +50,50 @@ pub fn has_current_session_been_imported( load_import_ledger(codex_home)?.contains_current_source(source_path) } -pub fn record_imported_session( +#[cfg(test)] +pub(crate) fn record_imported_session( codex_home: &Path, source_path: &Path, imported_thread_id: ThreadId, ) -> io::Result<()> { - let mut ledger = load_import_ledger(codex_home)?; let source_path = canonical_source_path(source_path)?; - let content_sha256 = session_content_sha256(&source_path)?; - let source_modified_at = session_modified_at(&source_path)?; - if let Some(index) = ledger.records.iter().rposition(|record| { - record.source_path == source_path && record.content_sha256 == content_sha256 - }) { - let mut record = ledger.records.remove(index); - record.imported_thread_id = imported_thread_id; - record.imported_at = now_unix_seconds(); - record.source_modified_at = source_modified_at; - ledger.records.push(record); - } else { - ledger.records.push(ImportedExternalAgentSessionRecord { + record_completed_session_imports( + codex_home, + vec![CompletedExternalAgentSessionImport { + source_content_sha256: session_content_sha256(&source_path)?, source_path, - content_sha256, imported_thread_id, - imported_at: now_unix_seconds(), + }], + ) +} + +pub fn record_completed_session_imports( + codex_home: &Path, + imports: Vec, +) -> io::Result<()> { + if imports.is_empty() { + return Ok(()); + } + let mut ledger = load_import_ledger(codex_home)?; + let imported_at = now_unix_seconds(); + for import in imports { + let source_modified_at = session_modified_at(&import.source_path).ok().flatten(); + if let Some(index) = ledger.records.iter().rposition(|record| { + record.source_path == import.source_path + && record.content_sha256 == import.source_content_sha256 + }) { + let mut record = ledger.records.remove(index); + record.imported_thread_id = import.imported_thread_id; + record.imported_at = imported_at; + record.source_modified_at = source_modified_at.or(record.source_modified_at); + ledger.records.push(record); + continue; + } + ledger.records.push(ImportedExternalAgentSessionRecord { + source_path: import.source_path, + content_sha256: import.source_content_sha256, + imported_thread_id: import.imported_thread_id, + imported_at, source_modified_at, }); } @@ -88,6 +116,9 @@ impl ImportedExternalAgentSessionLedger { } pub(super) fn contains_current_source(&self, source_path: &Path) -> io::Result { + if self.records.is_empty() { + return Ok(false); + } let source_path = canonical_source_path(source_path)?; if !self .records @@ -188,3 +219,7 @@ fn session_modified_at(path: &Path) -> io::Result> { .ok() .and_then(|duration| i64::try_from(duration.as_nanos()).ok())) } + +#[cfg(test)] +#[path = "ledger_tests.rs"] +mod tests; diff --git a/codex-rs/external-agent-sessions/src/ledger_tests.rs b/codex-rs/external-agent-sessions/src/ledger_tests.rs new file mode 100644 index 0000000000..f4b9dd6f7c --- /dev/null +++ b/codex-rs/external-agent-sessions/src/ledger_tests.rs @@ -0,0 +1,85 @@ +use super::CompletedExternalAgentSessionImport; +use super::ImportedExternalAgentSessionLedger; +use super::record_completed_session_imports; +use codex_protocol::ThreadId; +use sha2::Digest; +use sha2::Sha256; +use tempfile::TempDir; + +#[test] +fn empty_ledger_does_not_read_source() { + let root = TempDir::new().expect("tempdir"); + let missing_source = root.path().join("missing-session.jsonl"); + + assert!( + !ImportedExternalAgentSessionLedger::default() + .contains_current_source(&missing_source) + .expect("empty ledger cannot contain sources") + ); +} + +#[test] +fn completed_imports_do_not_read_source_files() { + let root = TempDir::new().expect("tempdir"); + let codex_home = root.path().join("codex-home"); + let source_path = root.path().join("session.jsonl"); + let contents = b"session contents"; + std::fs::write(&source_path, contents).expect("source"); + let source_path = std::fs::canonicalize(&source_path).expect("canonical source"); + std::fs::remove_file(&source_path).expect("remove source"); + let imported_thread_id = ThreadId::new(); + + record_completed_session_imports( + &codex_home, + vec![CompletedExternalAgentSessionImport { + source_path: source_path.clone(), + source_content_sha256: format!("{:x}", Sha256::digest(contents)), + imported_thread_id, + }], + ) + .expect("record completed imports"); + + let ledger = super::load_import_ledger(&codex_home).expect("ledger"); + assert_eq!(ledger.records.len(), 1); + assert_eq!(ledger.records[0].source_path, source_path); + assert_eq!(ledger.records[0].imported_thread_id, imported_thread_id); + assert_eq!(ledger.records[0].source_modified_at, None); +} + +#[test] +fn completed_import_refreshes_existing_record_metadata() { + let root = TempDir::new().expect("tempdir"); + let codex_home = root.path().join("codex-home"); + let source_path = root.path().join("session.jsonl"); + let contents = b"session contents"; + std::fs::write(&source_path, contents).expect("source"); + let source_path = std::fs::canonicalize(source_path).expect("canonical source"); + let content_sha256 = format!("{:x}", Sha256::digest(contents)); + let first_thread_id = ThreadId::new(); + let second_thread_id = ThreadId::new(); + + record_completed_session_imports( + &codex_home, + vec![CompletedExternalAgentSessionImport { + source_path: source_path.clone(), + source_content_sha256: content_sha256.clone(), + imported_thread_id: first_thread_id, + }], + ) + .expect("record first import"); + record_completed_session_imports( + &codex_home, + vec![CompletedExternalAgentSessionImport { + source_path: source_path.clone(), + source_content_sha256: content_sha256, + imported_thread_id: second_thread_id, + }], + ) + .expect("record replacement import"); + + let ledger = super::load_import_ledger(&codex_home).expect("ledger"); + assert_eq!(ledger.records.len(), 1); + assert_eq!(ledger.records[0].source_path, source_path); + assert_eq!(ledger.records[0].imported_thread_id, second_thread_id); + assert!(ledger.records[0].source_modified_at.is_some()); +} diff --git a/codex-rs/external-agent-sessions/src/lib.rs b/codex-rs/external-agent-sessions/src/lib.rs index fe9699f0c1..0b7a4eb2ba 100644 --- a/codex-rs/external-agent-sessions/src/lib.rs +++ b/codex-rs/external-agent-sessions/src/lib.rs @@ -6,15 +6,15 @@ mod ledger; mod records; use codex_protocol::protocol::RolloutItem; -use std::collections::HashSet; use std::io; use std::path::Path; use std::path::PathBuf; pub use detect::detect_recent_sessions; -pub use export::load_session_for_import; +use export::load_session_for_import_with_content_sha256; +pub use ledger::CompletedExternalAgentSessionImport; pub use ledger::has_current_session_been_imported; -pub use ledger::record_imported_session; +pub use ledger::record_completed_session_imports; pub use records::SessionSummary; pub use records::summarize_session; @@ -31,105 +31,51 @@ pub struct ExternalAgentSessionMigration { pub struct ImportedExternalAgentSession { pub cwd: PathBuf, pub title: Option, + pub first_user_message: Option, pub rollout_items: Vec, } #[derive(Debug, Clone)] pub struct PendingSessionImport { pub source_path: PathBuf, + pub source_content_sha256: String, pub session: ImportedExternalAgentSession, } -#[derive(Debug)] -pub enum PrepareSessionImportsError { - SessionNotDetected(PathBuf), -} - -impl std::fmt::Display for PrepareSessionImportsError { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PrepareSessionImportsError::SessionNotDetected(path) => { - write!( - formatter, - "external agent session was not detected for import: {}", - path.display() - ) - } - } - } -} - -impl std::error::Error for PrepareSessionImportsError {} - -pub fn prepare_pending_session_imports( - codex_home: &Path, - requested_sessions: Vec, - detected_sessions: Vec, -) -> Result, PrepareSessionImportsError> { - let detected_session_paths = detected_sessions - .into_iter() - .map(|session| session.path) - .collect::>(); - let mut pending_session_imports = Vec::new(); - for session in requested_sessions { - let has_been_imported = match has_current_session_been_imported(codex_home, &session.path) { - Ok(has_been_imported) => has_been_imported, - Err(_) => continue, - }; - if !detected_session_paths.contains(&session.path) && !has_been_imported { - return Err(PrepareSessionImportsError::SessionNotDetected(session.path)); - } - if has_been_imported { - continue; - } - let imported_session = match load_importable_session(&session.path) { - Ok(Some(imported_session)) => imported_session, - Ok(None) | Err(_) => continue, - }; - pending_session_imports.push(PendingSessionImport { - source_path: session.path, - session: imported_session, - }); - } - Ok(pending_session_imports) -} - -pub fn prepare_validated_session_imports( - codex_home: &Path, - requested_sessions: Vec, -) -> Vec { - requested_sessions - .into_iter() - .filter_map(|session| pending_session_import(codex_home, session)) - .collect() -} - -fn pending_session_import( +pub fn prepare_validated_session_import( codex_home: &Path, session: ExternalAgentSessionMigration, -) -> Option { - let has_been_imported = match has_current_session_been_imported(codex_home, &session.path) { - Ok(has_been_imported) => has_been_imported, - Err(_) => return None, - }; +) -> io::Result> { + let has_been_imported = has_current_session_been_imported(codex_home, &session.path)?; if has_been_imported { - return None; + return Ok(None); } - let imported_session = match load_importable_session(&session.path) { - Ok(Some(imported_session)) => imported_session, - Ok(None) | Err(_) => return None, - }; - Some(PendingSessionImport { - source_path: session.path, - session: imported_session, - }) -} - -fn load_importable_session(path: &Path) -> io::Result> { - let Some(imported_session) = load_session_for_import(path)? else { + let Some((source_path, imported_session, source_content_sha256)) = + load_importable_session(&session.path)? + else { return Ok(None); }; - Ok(imported_session.cwd.is_dir().then_some(imported_session)) + Ok(Some(PendingSessionImport { + source_path, + source_content_sha256, + session: imported_session, + })) +} + +fn load_importable_session( + path: &Path, +) -> io::Result> { + let source_path = std::fs::canonicalize(path)?; + let Some((imported_session, source_content_sha256)) = + load_session_for_import_with_content_sha256(&source_path)? + else { + return Ok(None); + }; + Ok(imported_session.cwd.is_dir().then_some(( + source_path, + imported_session, + source_content_sha256, + ))) } #[derive(Debug, Clone)] @@ -172,45 +118,59 @@ fn now_unix_seconds() -> i64 { mod tests { use super::*; use codex_protocol::ThreadId; + use sha2::Digest; + use sha2::Sha256; use tempfile::TempDir; - #[test] - fn rejects_session_that_was_not_detected() { - let root = TempDir::new().expect("tempdir"); - let codex_home = root.path().join("codex-home"); - let source_path = root.path().join("session.jsonl"); - std::fs::write(&source_path, "{}\n").expect("session"); - - let err = prepare_pending_session_imports( - &codex_home, - vec![session_migration(&source_path)], - Vec::new(), - ) - .expect_err("undetected session should be rejected"); - - match err { - PrepareSessionImportsError::SessionNotDetected(path) => { - assert_eq!(path, source_path); - } - } - } - #[test] fn skips_session_that_was_already_imported() { let root = TempDir::new().expect("tempdir"); let codex_home = root.path().join("codex-home"); let source_path = root.path().join("session.jsonl"); std::fs::write(&source_path, "{}\n").expect("session"); - record_imported_session(&codex_home, &source_path, ThreadId::new()).expect("record import"); + ledger::record_imported_session(&codex_home, &source_path, ThreadId::new()) + .expect("record import"); - let pending = prepare_pending_session_imports( - &codex_home, - vec![session_migration(&source_path)], - Vec::new(), - ) - .expect("already imported session should be skipped"); + let pending = + prepare_validated_session_import(&codex_home, session_migration(&source_path)) + .expect("already imported session should be skipped"); - assert!(pending.is_empty()); + assert!(pending.is_none()); + } + + #[test] + fn reports_session_preparation_errors() { + let root = TempDir::new().expect("tempdir"); + let source_path = root.path().join("missing-session.jsonl"); + + let err = prepare_validated_session_import(root.path(), session_migration(&source_path)) + .expect_err("missing session should fail preparation"); + + assert_eq!(err.kind(), io::ErrorKind::NotFound); + } + + #[test] + fn prepares_one_validated_session_import_with_content_hash() { + let root = TempDir::new().expect("tempdir"); + let source_path = root.path().join("session.jsonl"); + let contents = serde_json::json!({ + "type": "user", + "cwd": root.path(), + "timestamp": "2026-06-03T12:00:00Z", + "message": { "content": "first request" }, + }) + .to_string(); + std::fs::write(&source_path, &contents).expect("session"); + + let pending = + prepare_validated_session_import(root.path(), session_migration(&source_path)) + .expect("prepare session") + .expect("pending import"); + + assert_eq!( + pending.source_content_sha256, + format!("{:x}", Sha256::digest(contents)) + ); } fn session_migration(path: &Path) -> ExternalAgentSessionMigration { diff --git a/codex-rs/external-agent-sessions/src/records.rs b/codex-rs/external-agent-sessions/src/records.rs index 52f0535452..00307fa1d5 100644 --- a/codex-rs/external-agent-sessions/src/records.rs +++ b/codex-rs/external-agent-sessions/src/records.rs @@ -4,6 +4,8 @@ use crate::MessageRole; use crate::summarize_for_label; use crate::truncate; use serde_json::Value as JsonValue; +use sha2::Digest; +use sha2::Sha256; use std::fs::File; use std::io; use std::io::BufRead; @@ -21,6 +23,13 @@ pub struct SessionSummary { pub migration: ExternalAgentSessionMigration, } +pub(super) struct ParsedSessionImport { + pub cwd: Option, + pub source_title: Option, + pub messages: Vec, + pub content_sha256: String, +} + pub fn summarize_session(path: &Path) -> io::Result> { let file = File::open(path)?; let reader = BufReader::new(file); @@ -37,7 +46,7 @@ pub fn summarize_session(path: &Path) -> io::Result> { if trimmed.is_empty() { continue; } - let Ok(record) = serde_json::from_str::(trimmed) else { + let Ok(mut record) = serde_json::from_str::(trimmed) else { continue; }; if cwd.is_none() { @@ -52,7 +61,7 @@ pub fn summarize_session(path: &Path) -> io::Result> { if let Some(title) = ai_title_from_record(&record) { ai_title = Some(title.to_string()); } - let Some(message) = conversation_message_from_record(&record) else { + let Some(message) = conversation_message_from_owned_record(&mut record) else { continue; }; saw_message = true; @@ -84,54 +93,50 @@ pub fn summarize_session(path: &Path) -> io::Result> { })) } -pub(super) fn source_title_from_records(records: &[JsonValue]) -> Option { - latest_title_from_records(records, custom_title_from_record) - .or_else(|| latest_title_from_records(records, ai_title_from_record)) -} - -pub(super) fn read_records(path: &Path) -> io::Result> { +pub(super) fn read_session_import(path: &Path) -> io::Result { let file = File::open(path)?; - let reader = BufReader::new(file); - let mut records = Vec::new(); - for line in reader.lines() { - let line = line?; + let mut reader = BufReader::new(file); + let mut cwd = None; + let mut custom_title = None; + let mut ai_title = None; + let mut messages = Vec::new(); + let mut line = String::new(); + let mut hasher = Sha256::new(); + loop { + line.clear(); + if reader.read_line(&mut line)? == 0 { + break; + } + hasher.update(line.as_bytes()); let trimmed = line.trim(); if trimmed.is_empty() { continue; } - let Ok(value) = serde_json::from_str::(trimmed) else { + let Ok(mut record) = serde_json::from_str::(trimmed) else { continue; }; - if value.is_object() { - records.push(value); + if cwd.is_none() { + cwd = record + .get("cwd") + .and_then(JsonValue::as_str) + .map(PathBuf::from); + } + if let Some(title) = custom_title_from_record(&record) { + custom_title = Some(title.to_string()); + } + if let Some(title) = ai_title_from_record(&record) { + ai_title = Some(title.to_string()); + } + if let Some(message) = conversation_message_from_owned_record(&mut record) { + messages.push(message); } } - Ok(records) -} - -pub(super) fn project_root_from_records(records: &[JsonValue]) -> Option { - records - .iter() - .find_map(|record| record.get("cwd").and_then(JsonValue::as_str)) - .map(PathBuf::from) -} - -pub(super) fn conversation_messages(records: &[JsonValue]) -> Vec { - records - .iter() - .filter_map(conversation_message_from_record) - .collect() -} - -fn latest_title_from_records<'a>( - records: &'a [JsonValue], - title_from_record: impl Fn(&'a JsonValue) -> Option<&'a str>, -) -> Option { - records - .iter() - .filter_map(title_from_record) - .next_back() - .map(ToOwned::to_owned) + Ok(ParsedSessionImport { + cwd, + source_title: custom_title.or(ai_title), + messages, + content_sha256: format!("{:x}", hasher.finalize()), + }) } fn custom_title_from_record(record: &JsonValue) -> Option<&str> { @@ -150,7 +155,7 @@ fn title_from_record<'a>(record: &'a JsonValue, record_type: &str, field: &str) .filter(|title| !title.is_empty()) } -fn conversation_message_from_record(record: &JsonValue) -> Option { +fn conversation_message_from_owned_record(record: &mut JsonValue) -> Option { let record_type = record.get("type")?.as_str()?; if record_type != "assistant" && record_type != "user" { return None; @@ -161,18 +166,30 @@ fn conversation_message_from_record(record: &JsonValue) -> Option { + if text.trim().is_empty() { + return None; + } + ExtractedMessage { + text, + only_tool_result: false, + } + } + content => extract_message_text(&content)?, + }; Some(ConversationMessage { - role, + role: if is_assistant || extracted.only_tool_result { + MessageRole::Assistant + } else { + MessageRole::User + }, text: extracted.text, timestamp, }) @@ -324,6 +341,46 @@ fn parse_timestamp(timestamp: &str) -> Option { #[cfg(test)] mod tests { use super::*; + use tempfile::TempDir; + + #[test] + fn reads_session_import_in_one_pass() { + let root = TempDir::new().expect("tempdir"); + let path = root.path().join("session.jsonl"); + let contents = [ + serde_json::json!({ + "type": "user", + "cwd": root.path(), + "timestamp": "2026-06-03T12:00:00Z", + "message": { "content": "first request" }, + }) + .to_string(), + "not json".to_string(), + serde_json::json!({ + "type": "ai-title", + "aiTitle": "generated title", + }) + .to_string(), + serde_json::json!({ + "type": "custom-title", + "customTitle": "custom title", + }) + .to_string(), + ] + .join("\n"); + std::fs::write(&path, &contents).expect("session"); + + let parsed = read_session_import(&path).expect("parse session"); + + assert_eq!(parsed.cwd.as_deref(), Some(root.path())); + assert_eq!(parsed.source_title.as_deref(), Some("custom title")); + assert_eq!(parsed.messages.len(), 1); + assert_eq!(parsed.messages[0].text, "first request"); + assert_eq!( + parsed.content_sha256, + format!("{:x}", Sha256::digest(contents)) + ); + } #[test] fn converts_tool_use_blocks_to_bounded_external_agent_tags() { From 0aa9931aeadd9cbb1d6d02f854d1402f8db2bec8 Mon Sep 17 00:00:00 2001 From: mpc-oai Date: Mon, 8 Jun 2026 13:37:55 -0500 Subject: [PATCH 7/8] [plugins] Expose marketplace source in marketplace list JSON (#27009) ## Summary - Follow-up to #26417 and #26631 - Add `marketplaceSource` to `codex plugin marketplace list --json` entries for configured marketplaces - Reuse the existing `marketplaceSource` shape from `codex plugin list --json` - Keep human-readable marketplace list output unchanged - Add CLI coverage for configured local and git marketplace sources Example: ```json { "marketplaces": [ { "name": "debug", "root": "/path/to/.codex/.tmp/marketplaces/debug", "marketplaceSource": { "sourceType": "git", "source": "https://example.com/acme/agent-skills.git" } } ] } ``` ## Validation - `just fmt` - `just fix -p codex-cli` - `just test -p codex-cli marketplace_list` - `just test -p codex-cli` --- codex-rs/cli/src/marketplace_cmd.rs | 51 ++++++++++++- codex-rs/cli/src/plugin_cmd.rs | 4 +- codex-rs/cli/tests/plugin_cli.rs | 108 +++++++++++++++++++++++++++- 3 files changed, 158 insertions(+), 5 deletions(-) diff --git a/codex-rs/cli/src/marketplace_cmd.rs b/codex-rs/cli/src/marketplace_cmd.rs index fc4d9eef62..b392bb53b9 100644 --- a/codex-rs/cli/src/marketplace_cmd.rs +++ b/codex-rs/cli/src/marketplace_cmd.rs @@ -5,7 +5,10 @@ use clap::Parser; use codex_core::config::Config; use codex_core::config::find_codex_home; use codex_core_plugins::PluginMarketplaceUpgradeOutcome; +use codex_core_plugins::PluginsConfigInput; use codex_core_plugins::PluginsManager; +use codex_core_plugins::installed_marketplaces::marketplace_install_root; +use codex_core_plugins::installed_marketplaces::resolve_configured_marketplace_root; use codex_core_plugins::marketplace::marketplace_root_dir; use codex_core_plugins::marketplace_add::MarketplaceAddOutcome; use codex_core_plugins::marketplace_add::MarketplaceAddRequest; @@ -15,9 +18,14 @@ use codex_core_plugins::marketplace_remove::MarketplaceRemoveRequest; use codex_core_plugins::marketplace_remove::remove_marketplace; use codex_utils_cli::CliConfigOverrides; use serde::Serialize; +use std::collections::HashMap; use std::collections::HashSet; +use std::path::Path; +use std::path::PathBuf; +use crate::plugin_cmd::JsonMarketplaceSource; use crate::plugin_cmd::configured_marketplace_snapshot_issues; +use crate::plugin_cmd::configured_marketplace_sources; #[derive(Debug, Parser)] #[command(bin_name = "codex plugin marketplace")] @@ -240,7 +248,10 @@ async fn run_list(overrides: Vec<(String, toml::Value)>, args: ListMarketplaceAr } let marketplaces = marketplace_listing.marketplaces; if args.json { - let output = JsonMarketplaceListOutput::from_marketplaces(marketplaces); + let marketplace_sources = + configured_marketplace_sources_by_root(config.codex_home.as_path(), &plugins_input); + let output = + JsonMarketplaceListOutput::from_marketplaces(marketplaces, &marketplace_sources); println!("{}", serde_json::to_string_pretty(&output)?); return Ok(()); } @@ -288,7 +299,10 @@ struct JsonMarketplaceListOutput { } impl JsonMarketplaceListOutput { - fn from_marketplaces(marketplaces: Vec) -> Self { + fn from_marketplaces( + marketplaces: Vec, + marketplace_sources: &HashMap, + ) -> Self { let mut seen_roots = HashSet::new(); let marketplaces = marketplaces .into_iter() @@ -298,6 +312,7 @@ impl JsonMarketplaceListOutput { return None; } Some(JsonMarketplaceListEntry { + marketplace_source: marketplace_sources.get(root.as_path()).cloned(), name: marketplace.name, root: root.display().to_string(), }) @@ -313,6 +328,38 @@ impl JsonMarketplaceListOutput { struct JsonMarketplaceListEntry { name: String, root: String, + #[serde(skip_serializing_if = "Option::is_none")] + marketplace_source: Option, +} + +fn configured_marketplace_sources_by_root( + codex_home: &Path, + plugins_input: &PluginsConfigInput, +) -> HashMap { + let marketplace_sources = configured_marketplace_sources(plugins_input); + let Some(user_config) = plugins_input.config_layer_stack.effective_user_config() else { + return HashMap::new(); + }; + let Some(marketplaces) = user_config + .get("marketplaces") + .and_then(toml::Value::as_table) + else { + return HashMap::new(); + }; + + let default_install_root = marketplace_install_root(codex_home); + marketplaces + .iter() + .filter_map(|(marketplace_name, marketplace)| { + let marketplace_source = marketplace_sources.get(marketplace_name)?; + let root = resolve_configured_marketplace_root( + marketplace_name, + marketplace, + &default_install_root, + )?; + Some((root, marketplace_source.clone())) + }) + .collect() } async fn run_upgrade( diff --git a/codex-rs/cli/src/plugin_cmd.rs b/codex-rs/cli/src/plugin_cmd.rs index 7def7283c0..94c388066d 100644 --- a/codex-rs/cli/src/plugin_cmd.rs +++ b/codex-rs/cli/src/plugin_cmd.rs @@ -437,12 +437,12 @@ impl JsonPluginSource { #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -struct JsonMarketplaceSource { +pub(crate) struct JsonMarketplaceSource { source_type: String, source: String, } -fn configured_marketplace_sources( +pub(crate) fn configured_marketplace_sources( plugins_input: &PluginsConfigInput, ) -> HashMap { let Some(user_config) = plugins_input.config_layer_stack.effective_user_config() else { diff --git a/codex-rs/cli/tests/plugin_cli.rs b/codex-rs/cli/tests/plugin_cli.rs index cda79cf9a1..c8bd7b5429 100644 --- a/codex-rs/cli/tests/plugin_cli.rs +++ b/codex-rs/cli/tests/plugin_cli.rs @@ -341,6 +341,7 @@ async fn marketplace_list_shows_configured_marketplace_names() -> Result<()> { #[tokio::test] async fn marketplace_list_json_prints_configured_marketplaces() -> Result<()> { let (codex_home, source) = setup_local_marketplace()?; + let source_path = source.path().display().to_string(); let assert = codex_command(codex_home.path())? .args(["plugin", "marketplace", "list", "--json"]) @@ -355,7 +356,112 @@ async fn marketplace_list_json_prints_configured_marketplaces() -> Result<()> { "marketplaces": [ { "name": "debug", - "root": source.path().display().to_string(), + "root": source_path, + "marketplaceSource": { + "sourceType": "local", + "source": source_path, + }, + }, + ], + }) + ); + + Ok(()) +} + +#[tokio::test] +async fn marketplace_list_json_includes_configured_git_marketplace_source() -> Result<()> { + let codex_home = TempDir::new()?; + let marketplace_root = codex_home + .path() + .join(".tmp") + .join("marketplaces") + .join("debug"); + write_plugins_enabled_config(codex_home.path())?; + write_marketplace_source(&marketplace_root)?; + let update = MarketplaceConfigUpdate { + last_updated: "2026-06-04T08:39:49Z", + last_revision: Some("abc123"), + source_type: "git", + source: "https://example.com/acme/agent-skills.git", + ref_name: None, + sparse_paths: &[], + }; + record_user_marketplace(codex_home.path(), "debug", &update)?; + let normalized_root = canonicalize_existing_preserving_symlinks(&marketplace_root)?; + + let assert = codex_command(codex_home.path())? + .args(["plugin", "marketplace", "list", "--json"]) + .assert() + .success(); + let stdout = assert.get_output().stdout.as_slice(); + let actual: serde_json::Value = serde_json::from_slice(stdout)?; + + assert_eq!( + actual, + json!({ + "marketplaces": [ + { + "name": "debug", + "root": normalized_root.display().to_string(), + "marketplaceSource": { + "sourceType": "git", + "source": "https://example.com/acme/agent-skills.git", + }, + }, + ], + }) + ); + + Ok(()) +} + +#[tokio::test] +async fn marketplace_list_json_keys_configured_source_by_root() -> Result<()> { + let codex_home = TempDir::new()?; + let home = TempDir::new()?; + let marketplace_root = codex_home + .path() + .join(".tmp") + .join("marketplaces") + .join("debug"); + write_plugins_enabled_config(codex_home.path())?; + write_marketplace_source(home.path())?; + write_marketplace_source(&marketplace_root)?; + let update = MarketplaceConfigUpdate { + last_updated: "2026-06-04T08:39:49Z", + last_revision: Some("abc123"), + source_type: "git", + source: "https://example.com/acme/agent-skills.git", + ref_name: None, + sparse_paths: &[], + }; + record_user_marketplace(codex_home.path(), "debug", &update)?; + let normalized_root = canonicalize_existing_preserving_symlinks(&marketplace_root)?; + + let assert = codex_command(codex_home.path())? + .env("HOME", home.path()) + .args(["plugin", "marketplace", "list", "--json"]) + .assert() + .success(); + let stdout = assert.get_output().stdout.as_slice(); + let actual: serde_json::Value = serde_json::from_slice(stdout)?; + + assert_eq!( + actual, + json!({ + "marketplaces": [ + { + "name": "debug", + "root": home.path().display().to_string(), + }, + { + "name": "debug", + "root": normalized_root.display().to_string(), + "marketplaceSource": { + "sourceType": "git", + "source": "https://example.com/acme/agent-skills.git", + }, }, ], }) From 2375cb64493367c3f8ff3cb98fcff2b6350ac678 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 8 Jun 2026 11:47:23 -0700 Subject: [PATCH 8/8] ci: template custom runner names by repo (#27024) ## Why These workflows currently hard-code the `codex` runner group and custom runner labels. That makes the same workflow definitions less portable across repository copies or renamed repos, even though the runner fleet follows the repository name scheme. Template the runner identities from the repository name so `openai/codex` still resolves to the existing `codex-*` runners while other repos can use their own `-*` runner names. ## What Changed - Replaced custom runner `group` values such as `codex-runners` with `${{ github.event.repository.name }}-runners`. - Replaced custom runner labels such as `codex-linux-x64` and `codex-windows-arm64` with `${{ github.event.repository.name }}-...`. - Covered direct `runs-on` objects, matrix `runs_on` entries, reusable workflow runner inputs, and release runner labels. ## Verification - Parsed all `.github/workflows/*.yml` files as YAML with Ruby. - Searched `.github/workflows` to confirm no hardcoded runner-field `codex-runners` or `codex-*` labels remain. --- .github/workflows/bazel.yml | 16 ++--- .github/workflows/rust-ci-full.yml | 64 +++++++++---------- .github/workflows/rust-ci.yml | 4 +- .../rust-release-argument-comment-lint.yml | 4 +- .github/workflows/rust-release-windows.yml | 32 +++++----- .github/workflows/rust-release.yml | 8 +-- .github/workflows/sdk.yml | 8 +-- 7 files changed, 68 insertions(+), 68 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 809053069a..ffda87b8a5 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -152,8 +152,8 @@ jobs: - 3 - 4 runs-on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 name: Bazel test on windows-latest for x86_64-pc-windows-gnullvm shard ${{ matrix.shard }}/4 environment: name: bazel @@ -267,8 +267,8 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/main' timeout-minutes: 40 runs-on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 name: Bazel test on windows-latest for x86_64-pc-windows-gnullvm (native main) environment: name: bazel @@ -357,8 +357,8 @@ jobs: - os: windows-latest target: x86_64-pc-windows-gnullvm runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 runs-on: ${{ matrix.runs_on || matrix.os }} name: Bazel clippy on ${{ matrix.os }} for ${{ matrix.target }} environment: @@ -453,8 +453,8 @@ jobs: - os: windows-latest target: x86_64-pc-windows-gnullvm runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 runs-on: ${{ matrix.runs_on || matrix.os }} name: Verify release build on ${{ matrix.os }} for ${{ matrix.target }} environment: diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index d22531baae..7ad1d4b3ad 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -112,8 +112,8 @@ jobs: - name: Windows runner: windows-x64 runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -185,38 +185,38 @@ jobs: target: x86_64-unknown-linux-musl profile: dev runs_on: - group: codex-runners - labels: codex-linux-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-x64 - runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu profile: dev runs_on: - group: codex-runners - labels: codex-linux-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-x64 - runner: ubuntu-24.04-arm target: aarch64-unknown-linux-musl profile: dev runs_on: - group: codex-runners - labels: codex-linux-arm64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-arm64 - runner: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu profile: dev runs_on: - group: codex-runners - labels: codex-linux-arm64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-arm64 - runner: windows-x64 target: x86_64-pc-windows-msvc profile: dev runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 - runner: windows-arm64 target: aarch64-pc-windows-msvc profile: dev runs_on: - group: codex-runners - labels: codex-windows-arm64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-arm64 # Also run representative release builds on Mac and Linux because # there could be release-only build errors we want to catch. @@ -229,26 +229,26 @@ jobs: target: x86_64-unknown-linux-musl profile: release runs_on: - group: codex-runners - labels: codex-linux-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-x64 - runner: ubuntu-24.04-arm target: aarch64-unknown-linux-musl profile: release runs_on: - group: codex-runners - labels: codex-linux-arm64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-arm64 - runner: windows-x64 target: x86_64-pc-windows-msvc profile: release runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 - runner: windows-arm64 target: aarch64-pc-windows-msvc profile: release runs_on: - group: codex-runners - labels: codex-windows-arm64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-arm64 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -479,8 +479,8 @@ jobs: uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: runner: ubuntu-24.04 - runner_group: codex-runners - runner_labels: codex-linux-x64 + runner_group: ${{ github.event.repository.name }}-runners + runner_labels: ${{ github.event.repository.name }}-linux-x64 target: x86_64-unknown-linux-gnu profile: ci-test artifact_id: linux-x64-remote @@ -493,8 +493,8 @@ jobs: uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: runner: ubuntu-24.04-arm - runner_group: codex-runners - runner_labels: codex-linux-arm64 + runner_group: ${{ github.event.repository.name }}-runners + runner_labels: ${{ github.event.repository.name }}-linux-arm64 target: aarch64-unknown-linux-gnu profile: ci-test artifact_id: linux-arm64 @@ -506,8 +506,8 @@ jobs: uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: runner: windows-x64 - runner_group: codex-runners - runner_labels: codex-windows-x64 + runner_group: ${{ github.event.repository.name }}-runners + runner_labels: ${{ github.event.repository.name }}-windows-x64 target: x86_64-pc-windows-msvc profile: ci-test artifact_id: windows-x64 @@ -519,11 +519,11 @@ jobs: uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: runner: windows-arm64 - runner_group: codex-runners - runner_labels: codex-windows-arm64 + runner_group: ${{ github.event.repository.name }}-runners + runner_labels: ${{ github.event.repository.name }}-windows-arm64 archive_runner: windows-x64 - archive_runner_group: codex-runners - archive_runner_labels: codex-windows-x64 + archive_runner_group: ${{ github.event.repository.name }}-runners + archive_runner_labels: ${{ github.event.repository.name }}-windows-x64 target: aarch64-pc-windows-msvc profile: ci-test artifact_id: windows-arm64 diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 4528f4d963..4cf6d6e37a 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -176,8 +176,8 @@ jobs: runner: windows-x64 timeout_minutes: 30 runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 steps: - name: Check whether argument comment lint should run id: argument_comment_lint_gate diff --git a/.github/workflows/rust-release-argument-comment-lint.yml b/.github/workflows/rust-release-argument-comment-lint.yml index 7f5ad01aa5..5bf1121925 100644 --- a/.github/workflows/rust-release-argument-comment-lint.yml +++ b/.github/workflows/rust-release-argument-comment-lint.yml @@ -57,8 +57,8 @@ jobs: runner_binary: argument-comment-lint.exe cargo_dylint_binary: cargo-dylint.exe runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index 67c5c63ac9..e7574d85d2 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -29,43 +29,43 @@ jobs: bundle: primary binaries: "codex codex-responses-api-proxy" runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 - runner: windows-arm64 target: aarch64-pc-windows-msvc bundle: primary binaries: "codex codex-responses-api-proxy" runs_on: - group: codex-runners - labels: codex-windows-arm64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-arm64 - runner: windows-x64 target: x86_64-pc-windows-msvc bundle: helpers binaries: "codex-windows-sandbox-setup codex-command-runner" runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 - runner: windows-arm64 target: aarch64-pc-windows-msvc bundle: helpers binaries: "codex-windows-sandbox-setup codex-command-runner" runs_on: - group: codex-runners - labels: codex-windows-arm64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-arm64 - runner: windows-x64 target: x86_64-pc-windows-msvc bundle: app-server binaries: "codex-app-server" runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 - runner: windows-arm64 target: aarch64-pc-windows-msvc bundle: app-server binaries: "codex-app-server" runs_on: - group: codex-runners - labels: codex-windows-arm64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-arm64 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -162,13 +162,13 @@ jobs: - runner: windows-x64 target: x86_64-pc-windows-msvc runs_on: - group: codex-runners - labels: codex-windows-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 - runner: windows-arm64 target: aarch64-pc-windows-msvc runs_on: - group: codex-runners - labels: codex-windows-arm64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-arm64 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 5dbf1b77eb..b821700b49 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -101,25 +101,25 @@ jobs: binaries: "codex-app-server" build_dmg: "false" # Release artifacts intentionally ship MUSL-linked Linux binaries. - - runner: codex-linux-x64-xl + - runner: ${{ github.event.repository.name }}-linux-x64-xl target: x86_64-unknown-linux-musl bundle: primary artifact_name: x86_64-unknown-linux-musl binaries: "codex codex-responses-api-proxy bwrap" build_dmg: "false" - - runner: codex-linux-x64-xl + - runner: ${{ github.event.repository.name }}-linux-x64-xl target: x86_64-unknown-linux-musl bundle: app-server artifact_name: x86_64-unknown-linux-musl-app-server binaries: "codex-app-server" build_dmg: "false" - - runner: codex-linux-arm64 + - runner: ${{ github.event.repository.name }}-linux-arm64 target: aarch64-unknown-linux-musl bundle: primary artifact_name: aarch64-unknown-linux-musl binaries: "codex codex-responses-api-proxy bwrap" build_dmg: "false" - - runner: codex-linux-arm64 + - runner: ${{ github.event.repository.name }}-linux-arm64 target: aarch64-unknown-linux-musl bundle: app-server artifact_name: aarch64-unknown-linux-musl-app-server diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index 42a44afdb5..3cf6d7c8c3 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -8,8 +8,8 @@ on: jobs: python-sdk: runs-on: - group: codex-runners - labels: codex-linux-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-x64 timeout-minutes: 10 steps: - name: Checkout repository @@ -43,8 +43,8 @@ jobs: sdks: runs-on: - group: codex-runners - labels: codex-linux-x64 + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-x64 timeout-minutes: 10 environment: name: bazel