Move code mode behind an IPC host

Split the code-mode protocol and client from the V8-backed runtime so core and app-server no longer link codex-code-mode in production. ThreadManager now provisions durable code-mode sessions through a shared external host process, while tests can still inject the in-process provider. The IPC protocol uses a persistent stdin/stdout transport. Each frame is a 4-byte big-endian length followed by JSON, with a 16 MiB frame limit. Client requests carry u64 request IDs so create, execute, wait, terminate, and shutdown operations can be multiplexed over one process. Session IDs isolate durable stored values. Execute returns an ExecutionStarted response immediately and an asynchronous InitialResponse when the initial yield or completion is available. Nested tool calls and notifications travel from the host back to the client as delegate requests with their own IDs. Delegate responses, cancellation, and cell-closed lifecycle messages use the same framed channel. Wire operations encode errors as Result values. A dead connection fails pending operations, cancels outstanding delegates, and lets the provider spawn a new host for later sessions. Build codex-code-mode-host with V8 pointer-compression sandbox support and add it to canonical primary and app-server packages, legacy Linux and Windows bundles, signing verification, installers, Python runtime packages, and release CI for macOS, Linux, and Windows. The host is discovered next to the current executable, through CODEX_CODE_MODE_HOST_PATH, or on PATH. OS-level seccomp or seatbelt restrictions remain a follow-up to this cross-platform process split. Benchmarks were run from release builds on Linux x86_64 with the V8 sandbox profile and a text('ok') workload. Cold measurements used 30 samples, warm session provisioning used 200, and warm command execution used 500. Values are mean/p50/p95 in milliseconds: - session startup: in-process 0.002/0.002/0.005, IPC 2.623/2.599/2.894 - fresh-session command: in-process 1.831/1.758/1.915, IPC 7.428/7.252/8.306 - warm session provisioning: in-process 0.002/0.002/0.003, IPC 0.471/0.463/0.581 - warm command: in-process 1.759/1.757/1.940, IPC 2.005/2.001/2.166 The steady-state median command overhead is approximately 0.244 ms. The median fresh host plus first command cost is 7.252 ms. Validation: - 62/62 core code-mode integration tests passed against the external host - focused protocol, client, runtime, host, tools, and trace tests passed - Cargo and Bazel real-process host IPC tests passed - 11/11 package builder tests passed - Bazel lock verification, scoped Clippy fixes, and repository formatting passed
2026-09-20 12:47:38 +00:00 · 2026-06-05 06:23:58 +00:00
parent 9dd4e3c06e
commit ef500f4d23
69 changed files with 2304 additions and 445 deletions
--- a/scripts/codex_package/README.md
+++ b/scripts/codex_package/README.md
@@ -10,7 +10,8 @@ The builder creates a canonical Codex package directory:
 .
 ├── codex-package.json
 ├── bin
-│   └── <entrypoint>[.exe]
+│   ├── <entrypoint>[.exe]
+│   └── codex-code-mode-host[.exe]
 ├── codex-resources
 │   ├── bwrap                             # Linux only
 │   ├── zsh/bin/zsh                       # supported Unix targets only
@@ -40,6 +41,7 @@ grouped `cargo build` command per package when they are needed and no prebuilt
 override was provided:

 - all targets: the selected entrypoint, unless `--entrypoint-bin` is provided
+- all targets: `codex-code-mode-host`, unless `--code-mode-host-bin` is provided
 - Linux targets: `bwrap`, unless `--bwrap-bin` is provided
 - Windows targets: `codex-command-runner` and `codex-windows-sandbox-setup`,
  unless the corresponding prebuilt helper flags are provided
--- a/scripts/codex_package/cargo.py
+++ b/scripts/codex_package/cargo.py
@@ -17,6 +17,7 @@ CODEX_RS_ROOT = REPO_ROOT / "codex-rs"
@dataclass(frozen=True)
 class SourceBuildOutputs:
    entrypoint_bin: Path
+    code_mode_host_bin: Path
    bwrap_bin: Path | None
    codex_command_runner_bin: Path | None
    codex_windows_sandbox_setup_bin: Path | None
@@ -29,6 +30,7 @@ def build_source_binaries(
    cargo: str,
    profile: str,
    entrypoint_bin: Path | None,
+    code_mode_host_bin: Path | None,
    bwrap_bin: Path | None,
    codex_command_runner_bin: Path | None,
    codex_windows_sandbox_setup_bin: Path | None,
@@ -43,6 +45,7 @@ def build_source_binaries(
        spec,
        variant,
        build_entrypoint=entrypoint_bin is None,
+        build_code_mode_host=code_mode_host_bin is None,
        build_bwrap=spec.is_linux and bwrap_bin is None,
        build_codex_command_runner=spec.is_windows and codex_command_runner_bin is None,
        build_codex_windows_sandbox_setup=spec.is_windows
@@ -61,7 +64,7 @@ def build_source_binaries(
            cmd.extend(["--bin", binary])

        cargo_env = None
-        if entrypoint_bin is None:
+        if entrypoint_bin is None or code_mode_host_bin is None:
            codex_v8_env = resolve_codex_v8_cargo_env(spec)
            if codex_v8_env:
                cargo_env = {**os.environ, **codex_v8_env}
@@ -80,6 +83,10 @@ def build_source_binaries(
            entrypoint_bin,
            output_dir / variant.entrypoint_name(spec),
        ),
+        code_mode_host_bin=resolve_output_path(
+            code_mode_host_bin,
+            output_dir / f"codex-code-mode-host{spec.exe_suffix}",
+        ),
        bwrap_bin=resolve_output_path(
            bwrap_bin,
            output_dir / "bwrap" if spec.is_linux else None,
@@ -102,6 +109,7 @@ def source_binaries_for_target(
    variant: PackageVariant,
    *,
    build_entrypoint: bool,
+    build_code_mode_host: bool,
    build_bwrap: bool,
    build_codex_command_runner: bool,
    build_codex_windows_sandbox_setup: bool,
@@ -109,6 +117,8 @@ def source_binaries_for_target(
    binaries = []
    if build_entrypoint:
        binaries.append(variant.cargo_bin)
+    if build_code_mode_host:
+        binaries.append("codex-code-mode-host")
    if build_bwrap:
        binaries.append("bwrap")
    if build_codex_command_runner:
@@ -174,6 +184,7 @@ def cargo_profile_dirname(profile: str) -> str:
 def validate_source_outputs(outputs: SourceBuildOutputs) -> None:
    for path in [
        outputs.entrypoint_bin,
+        outputs.code_mode_host_bin,
        outputs.bwrap_bin,
        outputs.codex_command_runner_bin,
        outputs.codex_windows_sandbox_setup_bin,
--- a/scripts/codex_package/cli.py
+++ b/scripts/codex_package/cli.py
@@ -82,6 +82,14 @@ def parse_args() -> argparse.Namespace:
            "variant. If omitted, the entrypoint is built with Cargo."
        ),
    )
+    parser.add_argument(
+        "--code-mode-host-bin",
+        type=Path,
+        help=(
+            "Optional prebuilt codex-code-mode-host executable. If omitted, "
+            "codex-code-mode-host is built with Cargo."
+        ),
+    )
    parser.add_argument(
        "--bwrap-bin",
        type=Path,
@@ -140,6 +148,11 @@ def main() -> int:
            "prebuilt entrypoint executable",
            "--entrypoint-bin",
        ),
+        code_mode_host_bin=resolve_optional_input_path(
+            args.code_mode_host_bin,
+            "prebuilt code-mode host executable",
+            "--code-mode-host-bin",
+        ),
        bwrap_bin=resolve_optional_input_path(
            args.bwrap_bin,
            "prebuilt Linux bwrap executable",
@@ -159,6 +172,7 @@ def main() -> int:
    version = read_workspace_version()
    inputs = PackageInputs(
        entrypoint_bin=source_outputs.entrypoint_bin,
+        code_mode_host_bin=source_outputs.code_mode_host_bin,
        rg_bin=resolve_rg_bin(spec, args.rg_bin),
        zsh_bin=resolve_zsh_bin(spec),
        bwrap_bin=source_outputs.bwrap_bin,
--- a/scripts/codex_package/layout.py
+++ b/scripts/codex_package/layout.py
@@ -51,6 +51,11 @@ def build_package_dir(
        bin_dir / entrypoint_name,
        is_windows=spec.is_windows,
    )
+    copy_executable(
+        inputs.code_mode_host_bin,
+        bin_dir / f"codex-code-mode-host{spec.exe_suffix}",
+        is_windows=spec.is_windows,
+    )
    copy_executable(inputs.rg_bin, path_dir / spec.rg_name, is_windows=spec.is_windows)

    if inputs.zsh_bin is not None:
@@ -83,6 +88,7 @@ def build_package_dir(
        "target": spec.target,
        "variant": variant.name,
        "entrypoint": f"bin/{entrypoint_name}",
+        "codeModeHost": f"bin/codex-code-mode-host{spec.exe_suffix}",
        "resourcesDir": "codex-resources",
        "pathDir": "codex-path",
    }
@@ -118,6 +124,7 @@ def validate_package_dir(
        "target": spec.target,
        "variant": variant.name,
        "entrypoint": f"bin/{variant.entrypoint_name(spec)}",
+        "codeModeHost": f"bin/codex-code-mode-host{spec.exe_suffix}",
        "resourcesDir": "codex-resources",
        "pathDir": "codex-path",
    }
@@ -130,6 +137,7 @@ def validate_package_dir(

    required_files = [
        Path("bin") / variant.entrypoint_name(spec),
+        Path("bin") / f"codex-code-mode-host{spec.exe_suffix}",
        Path("codex-path") / spec.rg_name,
    ]
    executable_files = list(required_files)
--- a/scripts/codex_package/targets.py
+++ b/scripts/codex_package/targets.py
@@ -39,6 +39,7 @@ class PackageVariant:
@dataclass(frozen=True)
 class PackageInputs:
    entrypoint_bin: Path
+    code_mode_host_bin: Path
    rg_bin: Path
    zsh_bin: Path | None
    bwrap_bin: Path | None
--- a/scripts/codex_package/test_cargo.py
+++ b/scripts/codex_package/test_cargo.py
@@ -20,6 +20,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
                TARGET_SPECS["aarch64-apple-darwin"],
                PACKAGE_VARIANTS["codex"],
                build_entrypoint=False,
+                build_code_mode_host=False,
                build_bwrap=False,
                build_codex_command_runner=False,
                build_codex_windows_sandbox_setup=False,
@@ -35,6 +36,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
                TARGET_SPECS["x86_64-unknown-linux-musl"],
                PACKAGE_VARIANTS["codex"],
                build_entrypoint=False,
+                build_code_mode_host=False,
                build_bwrap=False,
                build_codex_command_runner=False,
                build_codex_windows_sandbox_setup=False,
@@ -50,6 +52,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
                TARGET_SPECS["x86_64-pc-windows-msvc"],
                PACKAGE_VARIANTS["codex"],
                build_entrypoint=False,
+                build_code_mode_host=False,
                build_bwrap=False,
                build_codex_command_runner=False,
                build_codex_windows_sandbox_setup=False,
@@ -63,6 +66,7 @@ class SourceBinariesForTargetTest(unittest.TestCase):
                TARGET_SPECS["x86_64-pc-windows-msvc"],
                PACKAGE_VARIANTS["codex"],
                build_entrypoint=False,
+                build_code_mode_host=False,
                build_bwrap=False,
                build_codex_command_runner=True,
                build_codex_windows_sandbox_setup=True,
@@ -70,10 +74,25 @@ class SourceBinariesForTargetTest(unittest.TestCase):
            ["codex-command-runner", "codex-windows-sandbox-setup"],
        )

+    def test_missing_code_mode_host_is_built(self) -> None:
+        self.assertEqual(
+            source_binaries_for_target(
+                TARGET_SPECS["x86_64-unknown-linux-musl"],
+                PACKAGE_VARIANTS["codex"],
+                build_entrypoint=False,
+                build_code_mode_host=True,
+                build_bwrap=False,
+                build_codex_command_runner=False,
+                build_codex_windows_sandbox_setup=False,
+            ),
+            ["codex-code-mode-host"],
+        )
+
    def test_build_uses_prebuilt_windows_helpers_without_running_cargo(self) -> None:
        with tempfile.TemporaryDirectory() as temp_dir:
            root = Path(temp_dir)
            entrypoint = touch_file(root / "codex.exe")
+            code_mode_host = touch_file(root / "codex-code-mode-host.exe")
            command_runner = touch_file(root / "codex-command-runner.exe")
            sandbox_setup = touch_file(root / "codex-windows-sandbox-setup.exe")

@@ -83,12 +102,14 @@ class SourceBinariesForTargetTest(unittest.TestCase):
                cargo=str(root / "cargo-that-should-not-run"),
                profile="release",
                entrypoint_bin=entrypoint,
+                code_mode_host_bin=code_mode_host,
                bwrap_bin=None,
                codex_command_runner_bin=command_runner,
                codex_windows_sandbox_setup_bin=sandbox_setup,
            )

        self.assertEqual(outputs.entrypoint_bin, entrypoint)
+        self.assertEqual(outputs.code_mode_host_bin, code_mode_host)
        self.assertEqual(outputs.codex_command_runner_bin, command_runner)
        self.assertEqual(outputs.codex_windows_sandbox_setup_bin, sandbox_setup)

--- a/scripts/codex_package/test_layout.py
+++ b/scripts/codex_package/test_layout.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+import json
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from codex_package.layout import build_package_dir
+from codex_package.layout import validate_package_dir
+from codex_package.targets import PACKAGE_VARIANTS
+from codex_package.targets import PackageInputs
+from codex_package.targets import TARGET_SPECS
+
+
+class PackageLayoutTest(unittest.TestCase):
+    def test_code_mode_host_is_packaged_and_required(self) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            root = Path(temp_dir)
+            package_dir = root / "package"
+            package_dir.mkdir()
+            entrypoint = touch_file(root / "codex")
+            code_mode_host = touch_file(root / "codex-code-mode-host")
+            rg = touch_file(root / "rg")
+            spec = TARGET_SPECS["aarch64-apple-darwin"]
+            variant = PACKAGE_VARIANTS["codex"]
+
+            build_package_dir(
+                package_dir,
+                "0.0.0-test",
+                variant,
+                spec,
+                PackageInputs(
+                    entrypoint_bin=entrypoint,
+                    code_mode_host_bin=code_mode_host,
+                    rg_bin=rg,
+                    zsh_bin=None,
+                    bwrap_bin=None,
+                    codex_command_runner_bin=None,
+                    codex_windows_sandbox_setup_bin=None,
+                ),
+            )
+
+            packaged_host = package_dir / "bin" / "codex-code-mode-host"
+            self.assertTrue(packaged_host.is_file())
+            self.assertTrue(packaged_host.stat().st_mode & 0o100)
+            metadata = json.loads(
+                (package_dir / "codex-package.json").read_text(encoding="utf-8")
+            )
+            self.assertEqual(metadata["codeModeHost"], "bin/codex-code-mode-host")
+            validate_package_dir(
+                package_dir,
+                variant,
+                spec,
+                include_zsh=False,
+            )
+
+            packaged_host.unlink()
+            with self.assertRaisesRegex(
+                RuntimeError,
+                "Missing package file: bin/codex-code-mode-host",
+            ):
+                validate_package_dir(
+                    package_dir,
+                    variant,
+                    spec,
+                    include_zsh=False,
+                )
+
+
+def touch_file(path: Path) -> Path:
+    path.touch()
+    return path
+
+
+if __name__ == "__main__":
+    unittest.main()
--- a/scripts/codex_package/test_v8.py
+++ b/scripts/codex_package/test_v8.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import hashlib
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from codex_package.targets import TARGET_SPECS
+from codex_package.v8 import fetch_codex_v8_artifacts
+
+
+class RustyV8ArtifactsTest(unittest.TestCase):
+    def test_fetches_pointer_compression_sandbox_artifacts(self) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            payload = b"artifact"
+            digest = hashlib.sha256(payload).hexdigest()
+
+            def download_file(url: str, dest: Path) -> None:
+                if dest.suffix == ".sha256":
+                    target = "x86_64-unknown-linux-musl"
+                    dest.parent.mkdir(parents=True, exist_ok=True)
+                    dest.write_text(
+                        "\n".join(
+                            [
+                                f"{digest}  librusty_v8_ptrcomp_sandbox_release_{target}.a.gz",
+                                f"{digest}  src_binding_ptrcomp_sandbox_release_{target}.rs",
+                            ]
+                        )
+                        + "\n",
+                        encoding="utf-8",
+                    )
+                else:
+                    dest.write_bytes(payload)
+
+            with patch("codex_package.v8.download_file", side_effect=download_file):
+                artifacts = fetch_codex_v8_artifacts(
+                    TARGET_SPECS["x86_64-unknown-linux-musl"],
+                    version="147.4.0",
+                    cache_root=Path(temp_dir),
+                )
+
+            self.assertEqual(
+                artifacts.archive.name,
+                "librusty_v8_ptrcomp_sandbox_release_x86_64-unknown-linux-musl.a.gz",
+            )
+            self.assertEqual(
+                artifacts.binding.name,
+                "src_binding_ptrcomp_sandbox_release_x86_64-unknown-linux-musl.rs",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()
--- a/scripts/codex_package/v8.py
+++ b/scripts/codex_package/v8.py
@@ -16,6 +16,7 @@ from .targets import TargetSpec


 DOWNLOAD_TIMEOUT_SECS = 120
+ARTIFACT_PROFILE = "ptrcomp_sandbox_release"


@dataclass(frozen=True)
@@ -70,9 +71,9 @@ def fetch_codex_v8_artifacts(
    )
    target = spec.target
    cache_dir = (cache_root or default_cache_root()) / f"rusty-v8-{version}-{target}"
-    archive = cache_dir / f"librusty_v8_release_{target}.a.gz"
-    binding = cache_dir / f"src_binding_release_{target}.rs"
-    checksums = cache_dir / f"rusty_v8_release_{target}.sha256"
+    archive = cache_dir / f"librusty_v8_{ARTIFACT_PROFILE}_{target}.a.gz"
+    binding = cache_dir / f"src_binding_{ARTIFACT_PROFILE}_{target}.rs"
+    checksums = cache_dir / f"rusty_v8_{ARTIFACT_PROFILE}_{target}.sha256"

    download_file(f"{release_url}/{checksums.name}", checksums)
    expected_checksums = load_checksums(checksums, {archive.name, binding.name})
--- a/scripts/install/install.ps1
+++ b/scripts/install/install.ps1
@@ -537,6 +537,7 @@ function Test-PackageContentsAreComplete {
    $expectedFiles = @(
        "codex-package.json",
        "bin\codex.exe",
+        "bin\codex-code-mode-host.exe",
        "codex-path\rg.exe",
        "codex-resources\codex-command-runner.exe",
        "codex-resources\codex-windows-sandbox-setup.exe"
--- a/scripts/install/install.sh
+++ b/scripts/install/install.sh
@@ -677,7 +677,10 @@ install_package_release() {
  rm -rf "$stage_release"
  mkdir -p "$stage_release"
  tar -xzf "$archive_path" -C "$stage_release"
-  chmod 0755 "$stage_release/bin/codex" "$stage_release/codex-path/rg"
+  chmod 0755 \
+    "$stage_release/bin/codex" \
+    "$stage_release/bin/codex-code-mode-host" \
+    "$stage_release/codex-path/rg"
  if [ -f "$stage_release/codex-resources/bwrap" ]; then
    chmod 0755 "$stage_release/codex-resources/bwrap"
  fi
@@ -730,6 +733,7 @@ release_dir_is_complete() {
    package)
      [ -f "$release_dir/codex-package.json" ] &&
        [ -x "$release_dir/bin/codex" ] &&
+        [ -x "$release_dir/bin/codex-code-mode-host" ] &&
        [ -x "$release_dir/codex" ] &&
        [ -x "$release_dir/codex-path/rg" ] ||
        return 1