openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-24 18:18:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

sdk: launch packaged Codex runtimes

Browse Source 
## Why

The Python and TypeScript SDKs launch the native Codex runtime directly, so they need to understand the same package layout that the release artifacts now use. Without this, SDK runtime wheels would keep carrying only a loose `bin/codex` plus ad hoc helper binaries, the TypeScript SDK would keep resolving the legacy optional-dependency layout, and bundled `codex-path` tools such as `rg` would not be prepended when the SDK starts Codex.

## What Changed

- Stage `openai-codex-cli-bin` from a Codex package directory, including `codex-package.json`, `bin/`, `codex-resources/`, and `codex-path/`.
- Make `stage-runtime` package-dir-only; legacy loose-binary wrapping now happens in `_runtime_setup.py` before calling the staging script.
- Expose Python runtime helpers for the bundled package directory and `codex-path`, and prepend that path when `openai_codex` launches the installed runtime.
- Teach the TypeScript SDK to resolve package-layout optional dependencies while keeping the legacy layout fallback.
- Update the Rust release workflows to build Python runtime wheels from the generated Codex package directories instead of loose binaries.

## Test Plan

- `python3 -m py_compile sdk/python/scripts/update_sdk_artifacts.py sdk/python/_runtime_setup.py sdk/python/src/openai_codex/client.py sdk/python-runtime/src/codex_cli_bin/__init__.py`
- `uv run --frozen --project sdk/python --extra dev ruff check sdk/python/scripts/update_sdk_artifacts.py sdk/python/_runtime_setup.py sdk/python/src/openai_codex/client.py sdk/python/tests/test_artifact_workflow_and_binaries.py sdk/python-runtime/src/codex_cli_bin/__init__.py`
- `uv run --frozen --project sdk/python --extra dev pytest sdk/python/tests/test_artifact_workflow_and_binaries.py`
- `pnpm eslint src/exec.ts tests/exec.test.ts`
- `pnpm test --runInBand tests/exec.test.ts`
This commit is contained in:
Michael Bolin
2026-05-20 17:20:00 -07:00
parent 896ee672cc
commit 54ec592d71
10 changed files with 412 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sdk/python/tests/test_artifact_workflow_and_binaries.py
+128 -78
View File
@@ -4,10 +4,11 @@ import ast
import importlib.util
import io
import json
import os
import sys
import tarfile
import urllib.error
from pathlib import Path
from typing import Sequence
import pytest
import tomllib
@@ -39,6 +40,17 @@ def _load_runtime_setup_module():
return module
def _write_fake_codex_package(package_dir: Path, script) -> Path:
(package_dir / "bin").mkdir(parents=True)
(package_dir / "codex-resources").mkdir()
(package_dir / "codex-path").mkdir()
(package_dir / "codex-package.json").write_text('{"variant":"codex"}\n')
(package_dir / "bin" / script.runtime_binary_name()).write_text("fake codex\n")
(package_dir / "codex-resources" / "bwrap").write_text("fake bwrap\n")
(package_dir / "codex-path" / "rg").write_text("fake rg\n")
return package_dir
def test_generation_has_single_maintenance_entrypoint_script() -> None:
"""Keep artifact workflows routed through one script instead of side entrypoints."""
scripts = sorted(p.name for p in (ROOT / "scripts").glob("*.py"))
@@ -276,6 +288,58 @@ def test_runtime_setup_uses_pep440_package_version_and_codex_release_tags() -> N
assert runtime_setup._release_tag("0.116.0a1") == "rust-v0.116.0-alpha.1"
def test_runtime_setup_extracts_package_archive(tmp_path: Path) -> None:
runtime_setup = _load_runtime_setup_module()
package_dir = tmp_path / "codex-package"
(package_dir / "bin").mkdir(parents=True)
(package_dir / "codex-resources").mkdir()
(package_dir / "codex-path").mkdir()
(package_dir / "codex-package.json").write_text('{"variant":"codex"}\n')
(package_dir / "bin" / runtime_setup.runtime_binary_name()).write_text("fake codex\n")
archive_path = tmp_path / "codex-package.tar.gz"
with tarfile.open(archive_path, "w:gz") as archive:
for path in package_dir.rglob("*"):
archive.add(path, arcname=path.relative_to(package_dir))
extracted = runtime_setup._extract_runtime_package_dir(archive_path, tmp_path / "work", "1.2.3")
assert (extracted / "codex-package.json").read_text() == '{"variant":"codex"}\n'
assert (extracted / "bin" / runtime_setup.runtime_binary_name()).read_text() == "fake codex\n"
def test_runtime_setup_wraps_legacy_binary_archive_as_package(tmp_path: Path) -> None:
runtime_setup = _load_runtime_setup_module()
legacy_binary = tmp_path / "codex-x86_64-unknown-linux-musl"
legacy_binary.write_text("fake codex\n")
archive_path = tmp_path / "codex-x86_64-unknown-linux-musl.tar.gz"
with tarfile.open(archive_path, "w:gz") as archive:
archive.add(legacy_binary, arcname=legacy_binary.name)
package_dir = runtime_setup._extract_runtime_package_dir(
archive_path, tmp_path / "work", "1.2.3"
)
assert {
"metadata": json.loads((package_dir / "codex-package.json").read_text()),
"codex": (package_dir / "bin" / runtime_setup.runtime_binary_name()).read_text(),
"resources_dir": (package_dir / "codex-resources").is_dir(),
"path_dir": (package_dir / "codex-path").is_dir(),
} == {
"metadata": {
"layoutVersion": 1,
"version": "1.2.3",
"entrypoint": f"bin/{runtime_setup.runtime_binary_name()}",
"resourcesDir": "codex-resources",
"pathDir": "codex-path",
},
"codex": "fake codex\n",
"resources_dir": True,
"path_dir": True,
}
def test_runtime_package_is_wheel_only_and_builds_platform_specific_wheels() -> None:
pyproject = tomllib.loads((ROOT.parent / "python-runtime" / "pyproject.toml").read_text())
hook_source = (ROOT.parent / "python-runtime" / "hatch_build.py").read_text()
@@ -324,7 +388,12 @@ def test_runtime_package_is_wheel_only_and_builds_platform_specific_wheels() ->
assert pyproject["project"]["name"] == "openai-codex-cli-bin"
assert pyproject["tool"]["hatch"]["build"]["targets"]["wheel"] == {
"packages": ["src/codex_cli_bin"],
"include": ["src/codex_cli_bin/bin/**"],
"include": [
"src/codex_cli_bin/codex-package.json",
"src/codex_cli_bin/bin/**",
"src/codex_cli_bin/codex-resources/**",
"src/codex_cli_bin/codex-path/**",
],
"hooks": {"custom": {}},
}
assert pyproject["tool"]["hatch"]["build"]["targets"]["sdist"] == {
@@ -338,19 +407,30 @@ def test_runtime_package_is_wheel_only_and_builds_platform_specific_wheels() ->
}
def test_stage_runtime_release_copies_binary_and_sets_version(tmp_path: Path) -> None:
def test_stage_runtime_release_copies_package_layout_and_sets_version(
tmp_path: Path,
) -> None:
script = _load_update_script_module()
fake_binary = tmp_path / script.runtime_binary_name()
fake_binary.write_text("fake codex\n")
package_dir = _write_fake_codex_package(tmp_path / "codex-package", script)
staged = script.stage_python_runtime_package(
tmp_path / "runtime-stage",
"1.2.3",
fake_binary,
package_dir,
)
package_root = script.staged_runtime_package_root(staged)
assert staged == tmp_path / "runtime-stage"
assert script.staged_runtime_bin_path(staged).read_text() == "fake codex\n"
assert {
"metadata": (package_root / "codex-package.json").read_text(),
"codex": (package_root / "bin" / script.runtime_binary_name()).read_text(),
"bwrap": (package_root / "codex-resources" / "bwrap").read_text(),
"rg": (package_root / "codex-path" / "rg").read_text(),
} == {
"metadata": '{"variant":"codex"}\n',
"codex": "fake codex\n",
"bwrap": "fake bwrap\n",
"rg": "fake rg\n",
}
assert 'name = "openai-codex-cli-bin"' in (staged / "pyproject.toml").read_text()
assert 'version = "1.2.3"' in (staged / "pyproject.toml").read_text()
@@ -370,30 +450,28 @@ def test_stage_runtime_release_replaces_existing_staging_dir(tmp_path: Path) ->
old_file = staging_dir / "stale.txt"
old_file.parent.mkdir(parents=True)
old_file.write_text("stale")
fake_binary = tmp_path / script.runtime_binary_name()
fake_binary.write_text("fake codex\n")
package_dir = _write_fake_codex_package(tmp_path / "codex-package", script)
staged = script.stage_python_runtime_package(
staging_dir,
"1.2.3",
fake_binary,
package_dir,
)
assert staged == staging_dir
assert not old_file.exists()
assert script.staged_runtime_bin_path(staged).read_text() == "fake codex\n"
package_root = script.staged_runtime_package_root(staged)
assert (package_root / "bin" / script.runtime_binary_name()).read_text() == "fake codex\n"
def test_stage_runtime_release_can_pin_wheel_platform_tag(tmp_path: Path) -> None:
script = _load_update_script_module()
fake_binary = tmp_path / script.runtime_binary_name()
fake_binary.write_text("fake codex\n")
package_dir = _write_fake_codex_package(tmp_path / "codex-package", script)
staged = script.stage_python_runtime_package(
tmp_path / "runtime-stage",
"0.116.0a1",
fake_binary,
package_dir,
platform_tag="musllinux_1_1_x86_64",
)
@@ -401,58 +479,34 @@ def test_stage_runtime_release_can_pin_wheel_platform_tag(tmp_path: Path) -> Non
assert 'platform-tag = "musllinux_1_1_x86_64"' in pyproject
def test_stage_runtime_release_copies_resource_binaries(tmp_path: Path) -> None:
"""Runtime staging should copy every helper binary into the wheel bin dir."""
def test_stage_runtime_release_rejects_incomplete_package_layout(tmp_path: Path) -> None:
script = _load_update_script_module()
fake_binary = tmp_path / script.runtime_binary_name()
helper = tmp_path / "helper"
fallback = tmp_path / "fallback-helper"
fake_binary.write_text("fake codex\n")
helper.write_text("fake helper\n")
fallback.write_text("fake fallback\n")
package_dir = tmp_path / "codex-package"
(package_dir / "bin").mkdir(parents=True)
staged = script.stage_python_runtime_package(
tmp_path / "runtime-stage",
"1.2.3",
fake_binary,
resource_binaries=(helper, fallback),
)
assert {
path.relative_to(staged / "src" / "codex_cli_bin" / "bin").as_posix(): path.read_text()
for path in (staged / "src" / "codex_cli_bin" / "bin").iterdir()
} == {
script.runtime_binary_name(): "fake codex\n",
"fallback-helper": "fake fallback\n",
"helper": "fake helper\n",
}
with pytest.raises(RuntimeError, match="Missing Codex package layout entries"):
script.stage_python_runtime_package(tmp_path / "runtime-stage", "1.2.3", package_dir)
def test_runtime_resource_binaries_are_included_by_wheel_config(
def test_runtime_package_layout_is_included_by_wheel_config(
tmp_path: Path,
) -> None:
"""The runtime wheel config should include helper binaries beside Codex."""
script = _load_update_script_module()
fake_binary = tmp_path / script.runtime_binary_name()
helper = tmp_path / "helper"
fake_binary.write_text("fake codex\n")
helper.write_text("fake helper\n")
package_dir = _write_fake_codex_package(tmp_path / "codex-package", script)
staged = script.stage_python_runtime_package(
tmp_path / "runtime-stage",
"1.2.3",
fake_binary,
resource_binaries=(helper,),
package_dir,
)
pyproject = tomllib.loads((staged / "pyproject.toml").read_text())
assert {
"include": pyproject["tool"]["hatch"]["build"]["targets"]["wheel"]["include"],
"helper": (staged / "src" / "codex_cli_bin" / "bin" / "helper").read_text(),
} == {
"include": ["src/codex_cli_bin/bin/**"],
"helper": "fake helper\n",
}
assert pyproject["tool"]["hatch"]["build"]["targets"]["wheel"]["include"] == [
"src/codex_cli_bin/codex-package.json",
"src/codex_cli_bin/bin/**",
"src/codex_cli_bin/codex-resources/**",
"src/codex_cli_bin/codex-path/**",
]
def test_stage_sdk_release_injects_exact_runtime_pin(tmp_path: Path) -> None:
@@ -492,8 +546,7 @@ def test_stage_sdk_release_replaces_existing_staging_dir(tmp_path: Path) -> None
def test_staged_sdk_and_runtime_versions_match(tmp_path: Path) -> None:
script = _load_update_script_module()
fake_binary = tmp_path / script.runtime_binary_name()
fake_binary.write_text("fake codex\n")
package_dir = _write_fake_codex_package(tmp_path / "codex-package", script)
sdk_stage = script.stage_python_sdk_package(
tmp_path / "sdk-stage",
@@ -502,7 +555,7 @@ def test_staged_sdk_and_runtime_versions_match(tmp_path: Path) -> None:
runtime_stage = script.stage_python_runtime_package(
tmp_path / "runtime-stage",
"rust-v0.116.0-alpha.1",
fake_binary,
package_dir,
)
sdk_pyproject = tomllib.loads((sdk_stage / "pyproject.toml").read_text())
@@ -537,9 +590,8 @@ def test_stage_sdk_runs_type_generation_before_staging(tmp_path: Path) -> None:
def fake_stage_runtime_package(
_staging_dir: Path,
_runtime_version: str,
_runtime_binary: Path,
_package_dir: Path,
_platform_tag: str | None,
_resource_binaries: Sequence[Path],
) -> Path:
raise AssertionError("runtime staging should not run for stage-sdk")
@@ -577,28 +629,19 @@ def test_stage_sdk_rejects_mismatched_legacy_versions(tmp_path: Path) -> None:
script.run_command(args, script.default_cli_ops())
def test_stage_runtime_stages_binary_without_type_generation(tmp_path: Path) -> None:
def test_stage_runtime_stages_package_without_type_generation(tmp_path: Path) -> None:
script = _load_update_script_module()
fake_binary = tmp_path / script.runtime_binary_name()
helper = tmp_path / "helper"
fallback = tmp_path / "fallback-helper"
fake_binary.write_text("fake codex\n")
helper.write_text("fake helper\n")
fallback.write_text("fake fallback\n")
package_dir = _write_fake_codex_package(tmp_path / "codex-package", script)
calls: list[str] = []
args = script.parse_args(
[
"stage-runtime",
str(tmp_path / "runtime-stage"),
str(fake_binary),
str(package_dir),
"--codex-version",
"rust-v0.116.0-alpha.1",
"--platform-tag",
"musllinux_1_1_x86_64",
"--resource-binary",
str(helper),
"--resource-binary",
str(fallback),
]
)
@@ -611,14 +654,10 @@ def test_stage_runtime_stages_binary_without_type_generation(tmp_path: Path) ->
def fake_stage_runtime_package(
_staging_dir: Path,
codex_version: str,
_runtime_binary: Path,
package_dir: Path,
platform_tag: str | None,
resource_binaries: Sequence[Path],
) -> Path:
calls.append(
f"stage_runtime:{codex_version}:{platform_tag}:"
f"{','.join(path.name for path in resource_binaries)}"
)
calls.append(f"stage_runtime:{codex_version}:{platform_tag}:{package_dir.name}")
return tmp_path / "runtime-stage"
def fake_current_sdk_version() -> str:
@@ -633,7 +672,7 @@ def test_stage_runtime_stages_binary_without_type_generation(tmp_path: Path) ->
script.run_command(args, ops)
assert calls == ["stage_runtime:0.116.0a1:musllinux_1_1_x86_64:helper,fallback-helper"]
assert calls == ["stage_runtime:0.116.0a1:musllinux_1_1_x86_64:codex-package"]
def test_default_runtime_is_resolved_from_installed_runtime_package(
@@ -653,6 +692,17 @@ def test_default_runtime_is_resolved_from_installed_runtime_package(
assert client_module.resolve_codex_bin(config, ops) == fake_binary
def test_runtime_path_dir_is_prepended_without_duplicates(tmp_path: Path) -> None:
from openai_codex import client as client_module
path_dir = tmp_path / "codex-path"
env = {"PATH": os.pathsep.join(["/usr/bin", str(path_dir), "/bin"])}
client_module._prepend_path_dirs(env, (path_dir,))
assert env["PATH"] == os.pathsep.join([str(path_dir), "/usr/bin", "/bin"])
def test_explicit_codex_bin_override_takes_priority(tmp_path: Path) -> None:
from openai_codex import client as client_module