Move code mode behind an IPC host

Split the code-mode protocol and client from the V8-backed runtime so core and app-server no longer link codex-code-mode in production. ThreadManager now provisions durable code-mode sessions through a shared external host process, while tests can still inject the in-process provider. The IPC protocol uses a persistent stdin/stdout transport. Each frame is a 4-byte big-endian length followed by JSON, with a 16 MiB frame limit. Client requests carry u64 request IDs so create, execute, wait, terminate, and shutdown operations can be multiplexed over one process. Session IDs isolate durable stored values. Execute returns an ExecutionStarted response immediately and an asynchronous InitialResponse when the initial yield or completion is available. Nested tool calls and notifications travel from the host back to the client as delegate requests with their own IDs. Delegate responses, cancellation, and cell-closed lifecycle messages use the same framed channel. Wire operations encode errors as Result values. A dead connection fails pending operations, cancels outstanding delegates, and lets the provider spawn a new host for later sessions. Build codex-code-mode-host with V8 pointer-compression sandbox support and add it to canonical primary and app-server packages, legacy Linux and Windows bundles, signing verification, installers, Python runtime packages, and release CI for macOS, Linux, and Windows. The host is discovered next to the current executable, through CODEX_CODE_MODE_HOST_PATH, or on PATH. OS-level seccomp or seatbelt restrictions remain a follow-up to this cross-platform process split. Benchmarks were run from release builds on Linux x86_64 with the V8 sandbox profile and a text('ok') workload. Cold measurements used 30 samples, warm session provisioning used 200, and warm command execution used 500. Values are mean/p50/p95 in milliseconds: - session startup: in-process 0.002/0.002/0.005, IPC 2.623/2.599/2.894 - fresh-session command: in-process 1.831/1.758/1.915, IPC 7.428/7.252/8.306 - warm session provisioning: in-process 0.002/0.002/0.003, IPC 0.471/0.463/0.581 - warm command: in-process 1.759/1.757/1.940, IPC 2.005/2.001/2.166 The steady-state median command overhead is approximately 0.244 ms. The median fresh host plus first command cost is 7.252 ms. Validation: - 62/62 core code-mode integration tests passed against the external host - focused protocol, client, runtime, host, tools, and trace tests passed - Cargo and Bazel real-process host IPC tests passed - 11/11 package builder tests passed - Bazel lock verification, scoped Clippy fixes, and repository formatting passed
2026-09-10 20:26:47 +00:00 · 2026-06-05 06:23:58 +00:00
parent 9dd4e3c06e
commit ef500f4d23
69 changed files with 2304 additions and 445 deletions
--- a/.github/actions/setup-rusty-v8/action.yml
+++ b/.github/actions/setup-rusty-v8/action.yml
@@ -4,6 +4,10 @@ inputs:
  target:
    description: Rust target triple with Codex-built V8 release artifacts.
    required: true
+  sandbox:
+    description: Use the V8 pointer-compression sandbox artifact profile.
+    required: false
+    default: "false"

 runs:
  using: composite
@@ -11,6 +15,7 @@ runs:
    - name: Configure rusty_v8 artifact overrides and verify checksums
      shell: bash
      env:
+        SANDBOX: ${{ inputs.sandbox }}
        TARGET: ${{ inputs.target }}
      run: |
        set -euo pipefail
@@ -19,14 +24,18 @@ runs:
        release_tag="rusty-v8-v${version}"
        base_url="https://github.com/openai/codex/releases/download/${release_tag}"
        binding_dir="${RUNNER_TEMP}/rusty_v8"
-        archive_path="${binding_dir}/librusty_v8_release_${TARGET}.a.gz"
-        binding_path="${binding_dir}/src_binding_release_${TARGET}.rs"
-        checksums_path="${binding_dir}/rusty_v8_release_${TARGET}.sha256"
+        artifact_profile="release"
+        if [[ "${SANDBOX}" == "true" ]]; then
+          artifact_profile="ptrcomp_sandbox_release"
+        fi
+        archive_path="${binding_dir}/librusty_v8_${artifact_profile}_${TARGET}.a.gz"
+        binding_path="${binding_dir}/src_binding_${artifact_profile}_${TARGET}.rs"
+        checksums_path="${binding_dir}/rusty_v8_${artifact_profile}_${TARGET}.sha256"

        mkdir -p "${binding_dir}"
-        curl -fsSL "${base_url}/librusty_v8_release_${TARGET}.a.gz" -o "${archive_path}"
-        curl -fsSL "${base_url}/src_binding_release_${TARGET}.rs" -o "${binding_path}"
-        curl -fsSL "${base_url}/rusty_v8_release_${TARGET}.sha256" -o "${checksums_path}"
+        curl -fsSL "${base_url}/$(basename "${archive_path}")" -o "${archive_path}"
+        curl -fsSL "${base_url}/$(basename "${binding_path}")" -o "${binding_path}"
+        curl -fsSL "${base_url}/$(basename "${checksums_path}")" -o "${checksums_path}"

        if [[ "$(wc -l < "${checksums_path}")" -ne 2 ]]; then
          echo "Expected exactly two checksums for ${TARGET} in ${checksums_path}" >&2
--- a/.github/scripts/build-codex-package-archive.sh
+++ b/.github/scripts/build-codex-package-archive.sh
@@ -8,6 +8,7 @@ Usage: build-codex-package-archive.sh \
  --bundle <primary|app-server> \
  --entrypoint-dir <dir> \
  --archive-dir <dir> \
+  [--code-mode-host-bin <path>] \
  [--bwrap-bin <path>] \
  [--codex-command-runner-bin <path>] \
  [--codex-windows-sandbox-setup-bin <path>] \
@@ -19,6 +20,7 @@ target=""
 bundle=""
 entrypoint_dir=""
 archive_dir=""
+code_mode_host_bin=""
 target_suffixed_entrypoint="false"
 resource_args=()
 bwrap_bin_provided="false"
@@ -43,6 +45,10 @@ while [[ $# -gt 0 ]]; do
      archive_dir="${2:?--archive-dir requires a value}"
      shift 2
      ;;
+    --code-mode-host-bin)
+      code_mode_host_bin="${2:?--code-mode-host-bin requires a value}"
+      shift 2
+      ;;
    --bwrap-bin)
      resource_args+=(--bwrap-bin "${2:?--bwrap-bin requires a value}")
      bwrap_bin_provided="true"
@@ -110,8 +116,18 @@ case "$target" in
 esac

 entrypoint_name="$entrypoint"
+code_mode_host_name="codex-code-mode-host"
 if [[ "$target_suffixed_entrypoint" == "true" ]]; then
  entrypoint_name="${entrypoint_name}-${target}"
+  code_mode_host_name="${code_mode_host_name}-${target}"
+fi
+
+if [[ -z "$code_mode_host_bin" ]]; then
+  code_mode_host_bin="${entrypoint_dir%/}/${code_mode_host_name}${exe_suffix}"
+fi
+if [[ ! -f "$code_mode_host_bin" ]]; then
+  echo "Code-mode host binary ${code_mode_host_bin} not found" >&2
+  exit 1
 fi

 case "$target" in
@@ -159,6 +175,7 @@ python_args=(
  --target "$target"
  --variant "$variant"
  --entrypoint-bin "${entrypoint_dir%/}/${entrypoint_name}${exe_suffix}"
+  --code-mode-host-bin "$code_mode_host_bin"
  --cargo-profile release
  --package-dir "$package_dir"
  --archive-output "$gzip_archive_path"
--- a/.github/workflows/rust-ci-full.yml
+++ b/.github/workflows/rust-ci-full.yml
@@ -382,6 +382,7 @@ jobs:
        name: Configure rusty_v8 artifact overrides and verify checksums
        uses: ./.github/actions/setup-rusty-v8
        with:
+          sandbox: "true"
          target: ${{ matrix.target }}

      - name: Install cargo-chef
--- a/.github/workflows/rust-release-windows.yml
+++ b/.github/workflows/rust-release-windows.yml
@@ -41,14 +41,14 @@ jobs:
          - runner: windows-x64
            target: x86_64-pc-windows-msvc
            bundle: helpers
-            binaries: "codex-windows-sandbox-setup codex-command-runner"
+            binaries: "codex-windows-sandbox-setup codex-command-runner codex-code-mode-host"
            runs_on:
              group: ${{ github.event.repository.name }}-runners
              labels: ${{ github.event.repository.name }}-windows-x64
          - runner: windows-arm64
            target: aarch64-pc-windows-msvc
            bundle: helpers
-            binaries: "codex-windows-sandbox-setup codex-command-runner"
+            binaries: "codex-windows-sandbox-setup codex-command-runner codex-code-mode-host"
            runs_on:
              group: ${{ github.event.repository.name }}-runners
              labels: ${{ github.event.repository.name }}-windows-arm64
@@ -158,7 +158,7 @@ jobs:
      run:
        working-directory: codex-rs
    env:
-      WINDOWS_BINARIES: "codex codex-responses-api-proxy codex-windows-sandbox-setup codex-command-runner codex-app-server"
+      WINDOWS_BINARIES: "codex codex-responses-api-proxy codex-windows-sandbox-setup codex-command-runner codex-code-mode-host codex-app-server"

    strategy:
      fail-fast: false
@@ -340,16 +340,18 @@ jobs:
              bundle_dir="$(mktemp -d)"
              runner_src="$dest/codex-command-runner-${{ matrix.target }}.exe"
              setup_src="$dest/codex-windows-sandbox-setup-${{ matrix.target }}.exe"
-              if [[ -f "$runner_src" && -f "$setup_src" ]]; then
+              code_mode_host_src="$dest/codex-code-mode-host-${{ matrix.target }}.exe"
+              if [[ -f "$runner_src" && -f "$setup_src" && -f "$code_mode_host_src" ]]; then
                cp "$dest/$base" "$bundle_dir/$base"
                cp "$runner_src" "$bundle_dir/codex-command-runner.exe"
                cp "$setup_src" "$bundle_dir/codex-windows-sandbox-setup.exe"
+                cp "$code_mode_host_src" "$bundle_dir/codex-code-mode-host.exe"
                # Use an absolute path so bundle zips land in the real dist
                # dir even when 7z runs from a temp directory.
                (cd "$bundle_dir" && 7z a "$repo_root/$dest/${base}.zip" .)
              else
-                echo "warning: missing sandbox binaries; falling back to single-binary zip"
-                echo "warning: expected $runner_src and $setup_src"
+                echo "warning: missing bundled binaries; falling back to single-binary zip"
+                echo "warning: expected $runner_src, $setup_src, and $code_mode_host_src"
                (cd "$dest" && 7z a "${base}.zip" "$base")
              fi
              rm -rf "$bundle_dir"
--- a/.github/workflows/rust-release.yml
+++ b/.github/workflows/rust-release.yml
@@ -80,50 +80,50 @@ jobs:
            target: aarch64-apple-darwin
            bundle: primary
            artifact_name: aarch64-apple-darwin
-            binaries: "codex codex-responses-api-proxy"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host"
            build_dmg: "true"
          - runner: macos-15-xlarge
            target: aarch64-apple-darwin
            bundle: app-server
            artifact_name: aarch64-apple-darwin-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"
            build_dmg: "false"
          - runner: macos-15-xlarge
            target: x86_64-apple-darwin
            bundle: primary
            artifact_name: x86_64-apple-darwin
-            binaries: "codex codex-responses-api-proxy"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host"
            build_dmg: "true"
          - runner: macos-15-xlarge
            target: x86_64-apple-darwin
            bundle: app-server
            artifact_name: x86_64-apple-darwin-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"
            build_dmg: "false"
          # Release artifacts intentionally ship MUSL-linked Linux binaries.
          - runner: ${{ github.event.repository.name }}-linux-x64-xl
            target: x86_64-unknown-linux-musl
            bundle: primary
            artifact_name: x86_64-unknown-linux-musl
-            binaries: "codex codex-responses-api-proxy bwrap"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host bwrap"
            build_dmg: "false"
          - runner: ${{ github.event.repository.name }}-linux-x64-xl
            target: x86_64-unknown-linux-musl
            bundle: app-server
            artifact_name: x86_64-unknown-linux-musl-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"
            build_dmg: "false"
          - runner: ${{ github.event.repository.name }}-linux-arm64
            target: aarch64-unknown-linux-musl
            bundle: primary
            artifact_name: aarch64-unknown-linux-musl
-            binaries: "codex codex-responses-api-proxy bwrap"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host bwrap"
            build_dmg: "false"
          - runner: ${{ github.event.repository.name }}-linux-arm64
            target: aarch64-unknown-linux-musl
            bundle: app-server
            artifact_name: aarch64-unknown-linux-musl-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"
            build_dmg: "false"

    steps:
@@ -208,6 +208,7 @@ jobs:
      - name: Configure rusty_v8 artifact overrides and verify checksums
        uses: ./.github/actions/setup-rusty-v8
        with:
+          sandbox: "true"
          target: ${{ matrix.target }}

      - if: ${{ contains(matrix.target, 'linux') }}
@@ -344,9 +345,15 @@ jobs:
            rm -rf "$bundle_root"
            mkdir -p "$bundle_root/codex-resources"
            cp "$dest/codex-${{ matrix.target }}" "$bundle_root/codex"
+            cp "$dest/codex-code-mode-host-${{ matrix.target }}" \
+              "$bundle_root/codex-code-mode-host"
            cp "$dest/bwrap-${{ matrix.target }}" "$bundle_root/codex-resources/bwrap"
-            chmod 0755 "$bundle_root/codex" "$bundle_root/codex-resources/bwrap"
-            tar -C "$bundle_root" -cf - codex codex-resources/bwrap |
+            chmod 0755 \
+              "$bundle_root/codex" \
+              "$bundle_root/codex-code-mode-host" \
+              "$bundle_root/codex-resources/bwrap"
+            tar -C "$bundle_root" -cf - \
+              codex codex-code-mode-host codex-resources/bwrap |
              zstd -T0 -19 -o "$dest/codex-${{ matrix.target }}-bundle.tar.zst"
          fi

@@ -484,19 +491,19 @@ jobs:
          - target: aarch64-apple-darwin
            bundle: primary
            artifact_name: aarch64-apple-darwin
-            binaries: "codex codex-responses-api-proxy"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host"
          - target: aarch64-apple-darwin
            bundle: app-server
            artifact_name: aarch64-apple-darwin-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"
          - target: x86_64-apple-darwin
            bundle: primary
            artifact_name: x86_64-apple-darwin
-            binaries: "codex codex-responses-api-proxy"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host"
          - target: x86_64-apple-darwin
            bundle: app-server
            artifact_name: x86_64-apple-darwin-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -602,22 +609,22 @@ jobs:
          - target: aarch64-apple-darwin
            bundle: primary
            artifact_name: aarch64-apple-darwin
-            binaries: "codex codex-responses-api-proxy"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host"
            build_dmg: "true"
          - target: aarch64-apple-darwin
            bundle: app-server
            artifact_name: aarch64-apple-darwin-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"
            build_dmg: "false"
          - target: x86_64-apple-darwin
            bundle: primary
            artifact_name: x86_64-apple-darwin
-            binaries: "codex codex-responses-api-proxy"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host"
            build_dmg: "true"
          - target: x86_64-apple-darwin
            bundle: app-server
            artifact_name: x86_64-apple-darwin-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"
            build_dmg: "false"

    steps:
@@ -889,22 +896,22 @@ jobs:
          - target: aarch64-apple-darwin
            bundle: primary
            artifact_name: aarch64-apple-darwin
-            binaries: "codex codex-responses-api-proxy"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host"
            verify_dmg: "true"
          - target: aarch64-apple-darwin
            bundle: app-server
            artifact_name: aarch64-apple-darwin-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"
            verify_dmg: "false"
          - target: x86_64-apple-darwin
            bundle: primary
            artifact_name: x86_64-apple-darwin
-            binaries: "codex codex-responses-api-proxy"
+            binaries: "codex codex-responses-api-proxy codex-code-mode-host"
            verify_dmg: "true"
          - target: x86_64-apple-darwin
            bundle: app-server
            artifact_name: x86_64-apple-darwin-app-server
-            binaries: "codex-app-server"
+            binaries: "codex-app-server codex-code-mode-host"
            verify_dmg: "false"

    steps:
@@ -992,6 +999,7 @@ jobs:
          mkdir -p "$package_dir"
          tar -xzf "${packaged_dir}/${package_stem}-${target}.tar.gz" -C "$package_dir"
          verify_signed_binary "${package_dir}/bin/${package_entrypoint}"
+          verify_signed_binary "${package_dir}/bin/codex-code-mode-host"

          if [[ "${{ matrix.verify_dmg }}" != "true" ]]; then
            exit 0