Files
codex/scripts/codex_package/cargo.py
Channing Conger ef500f4d23 Move code mode behind an IPC host
Split the code-mode protocol and client from the V8-backed runtime so core and
app-server no longer link codex-code-mode in production. ThreadManager now
provisions durable code-mode sessions through a shared external host process,
while tests can still inject the in-process provider.

The IPC protocol uses a persistent stdin/stdout transport. Each frame is a
4-byte big-endian length followed by JSON, with a 16 MiB frame limit. Client
requests carry u64 request IDs so create, execute, wait, terminate, and shutdown
operations can be multiplexed over one process. Session IDs isolate durable
stored values. Execute returns an ExecutionStarted response immediately and an
asynchronous InitialResponse when the initial yield or completion is available.

Nested tool calls and notifications travel from the host back to the client as
delegate requests with their own IDs. Delegate responses, cancellation, and
cell-closed lifecycle messages use the same framed channel. Wire operations
encode errors as Result values. A dead connection fails pending operations,
cancels outstanding delegates, and lets the provider spawn a new host for later
sessions.

Build codex-code-mode-host with V8 pointer-compression sandbox support and add
it to canonical primary and app-server packages, legacy Linux and Windows
bundles, signing verification, installers, Python runtime packages, and release
CI for macOS, Linux, and Windows. The host is discovered next to the current
executable, through CODEX_CODE_MODE_HOST_PATH, or on PATH. OS-level seccomp or
seatbelt restrictions remain a follow-up to this cross-platform process split.

Benchmarks were run from release builds on Linux x86_64 with the V8 sandbox
profile and a text('ok') workload. Cold measurements used 30 samples, warm
session provisioning used 200, and warm command execution used 500. Values are
mean/p50/p95 in milliseconds:

- session startup: in-process 0.002/0.002/0.005, IPC 2.623/2.599/2.894
- fresh-session command: in-process 1.831/1.758/1.915, IPC 7.428/7.252/8.306
- warm session provisioning: in-process 0.002/0.002/0.003,
  IPC 0.471/0.463/0.581
- warm command: in-process 1.759/1.757/1.940, IPC 2.005/2.001/2.166

The steady-state median command overhead is approximately 0.244 ms. The median
fresh host plus first command cost is 7.252 ms.

Validation:

- 62/62 core code-mode integration tests passed against the external host
- focused protocol, client, runtime, host, tools, and trace tests passed
- Cargo and Bazel real-process host IPC tests passed
- 11/11 package builder tests passed
- Bazel lock verification, scoped Clippy fixes, and repository formatting passed
2026-06-10 14:38:41 -07:00

194 lines
5.7 KiB
Python

"""Cargo builds for source-built Codex package artifacts."""
import os
import subprocess
from dataclasses import dataclass
from pathlib import Path
from .targets import REPO_ROOT
from .targets import PackageVariant
from .targets import TargetSpec
from .v8 import resolve_codex_v8_cargo_env
CODEX_RS_ROOT = REPO_ROOT / "codex-rs"
@dataclass(frozen=True)
class SourceBuildOutputs:
entrypoint_bin: Path
code_mode_host_bin: Path
bwrap_bin: Path | None
codex_command_runner_bin: Path | None
codex_windows_sandbox_setup_bin: Path | None
def build_source_binaries(
spec: TargetSpec,
variant: PackageVariant,
*,
cargo: str,
profile: str,
entrypoint_bin: Path | None,
code_mode_host_bin: Path | None,
bwrap_bin: Path | None,
codex_command_runner_bin: Path | None,
codex_windows_sandbox_setup_bin: Path | None,
) -> SourceBuildOutputs:
validate_prebuilt_resource_inputs(
spec,
bwrap_bin=bwrap_bin,
codex_command_runner_bin=codex_command_runner_bin,
codex_windows_sandbox_setup_bin=codex_windows_sandbox_setup_bin,
)
binaries = source_binaries_for_target(
spec,
variant,
build_entrypoint=entrypoint_bin is None,
build_code_mode_host=code_mode_host_bin is None,
build_bwrap=spec.is_linux and bwrap_bin is None,
build_codex_command_runner=spec.is_windows and codex_command_runner_bin is None,
build_codex_windows_sandbox_setup=spec.is_windows
and codex_windows_sandbox_setup_bin is None,
)
if binaries:
cmd = [
cargo,
"build",
"--target",
spec.target,
"--profile",
profile,
]
for binary in binaries:
cmd.extend(["--bin", binary])
cargo_env = None
if entrypoint_bin is None or code_mode_host_bin is None:
codex_v8_env = resolve_codex_v8_cargo_env(spec)
if codex_v8_env:
cargo_env = {**os.environ, **codex_v8_env}
print("+", " ".join(cmd))
subprocess.run(
cmd,
cwd=CODEX_RS_ROOT,
check=True,
env=cargo_env,
)
output_dir = cargo_profile_output_dir(spec, profile)
outputs = SourceBuildOutputs(
entrypoint_bin=resolve_output_path(
entrypoint_bin,
output_dir / variant.entrypoint_name(spec),
),
code_mode_host_bin=resolve_output_path(
code_mode_host_bin,
output_dir / f"codex-code-mode-host{spec.exe_suffix}",
),
bwrap_bin=resolve_output_path(
bwrap_bin,
output_dir / "bwrap" if spec.is_linux else None,
),
codex_command_runner_bin=resolve_output_path(
codex_command_runner_bin,
output_dir / "codex-command-runner.exe" if spec.is_windows else None,
),
codex_windows_sandbox_setup_bin=resolve_output_path(
codex_windows_sandbox_setup_bin,
output_dir / "codex-windows-sandbox-setup.exe" if spec.is_windows else None,
),
)
validate_source_outputs(outputs)
return outputs
def source_binaries_for_target(
spec: TargetSpec,
variant: PackageVariant,
*,
build_entrypoint: bool,
build_code_mode_host: bool,
build_bwrap: bool,
build_codex_command_runner: bool,
build_codex_windows_sandbox_setup: bool,
) -> list[str]:
binaries = []
if build_entrypoint:
binaries.append(variant.cargo_bin)
if build_code_mode_host:
binaries.append("codex-code-mode-host")
if build_bwrap:
binaries.append("bwrap")
if build_codex_command_runner:
binaries.append("codex-command-runner")
if build_codex_windows_sandbox_setup:
binaries.append("codex-windows-sandbox-setup")
return binaries
def validate_prebuilt_resource_inputs(
spec: TargetSpec,
*,
bwrap_bin: Path | None,
codex_command_runner_bin: Path | None,
codex_windows_sandbox_setup_bin: Path | None,
) -> None:
if bwrap_bin is not None and not spec.is_linux:
raise RuntimeError("--bwrap-bin is only supported for Linux targets.")
if codex_command_runner_bin is not None and not spec.is_windows:
raise RuntimeError(
"--codex-command-runner-bin is only supported for Windows targets."
)
if codex_windows_sandbox_setup_bin is not None and not spec.is_windows:
raise RuntimeError(
"--codex-windows-sandbox-setup-bin is only supported for Windows targets."
)
def resolve_output_path(
explicit_path: Path | None, default_path: Path | None
) -> Path | None:
if explicit_path is not None:
return explicit_path.resolve()
return default_path
def cargo_profile_output_dir(spec: TargetSpec, profile: str) -> Path:
target_dir = cargo_target_dir()
return target_dir / spec.target / cargo_profile_dirname(profile)
def cargo_target_dir() -> Path:
target_dir = os.environ.get("CARGO_TARGET_DIR")
if target_dir is None:
return CODEX_RS_ROOT / "target"
path = Path(target_dir)
if path.is_absolute():
return path
return CODEX_RS_ROOT / path
def cargo_profile_dirname(profile: str) -> str:
if profile == "dev":
return "debug"
if profile == "release":
return "release"
return profile
def validate_source_outputs(outputs: SourceBuildOutputs) -> None:
for path in [
outputs.entrypoint_bin,
outputs.code_mode_host_bin,
outputs.bwrap_bin,
outputs.codex_command_runner_bin,
outputs.codex_windows_sandbox_setup_bin,
]:
if path is not None and not path.is_file():
raise RuntimeError(f"cargo build did not produce expected binary: {path}")