From 8975fe66eea7b1cbf15787dea3206fe36e1edda8 Mon Sep 17 00:00:00 2001 From: sdcoffey Date: Fri, 13 Mar 2026 16:55:55 -0700 Subject: [PATCH] sdk/python: split core and bundled packages --- sdk/python/README.md | 35 +++- sdk/python/docs/faq.md | 15 +- sdk/python/pyproject.toml | 4 +- sdk/python/scripts/update_sdk_artifacts.py | 131 +++++++++++--- sdk/python/src/codex_app_server/client.py | 7 +- .../test_artifact_workflow_and_binaries.py | 161 +++++++++++++++++- 6 files changed, 311 insertions(+), 42 deletions(-) diff --git a/sdk/python/README.md b/sdk/python/README.md index ef3abdf630..4fc2df8a91 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -11,9 +11,16 @@ cd sdk/python python -m pip install -e . ``` -Published SDK builds pin an exact `codex-cli-bin` runtime dependency. For local -repo development, pass `AppServerConfig(codex_bin=...)` to point at a local -build explicitly. +This checked-in package is the runtime-free core distribution: +`codex-app-server-sdk-core`. + +Published releases expose two Python package names: + +- `codex-app-server-sdk-core`: the actual Python SDK code, without bundled binaries +- `codex-app-server-sdk`: a bundled metapackage that depends on `codex-app-server-sdk-core` and `codex-cli-bin` + +For local repo development, pass `AppServerConfig(codex_bin=...)` to point at +a local build explicitly. ## Quickstart @@ -48,9 +55,9 @@ python examples/01_quickstart_constructor/async.py The repo no longer checks `codex` binaries into `sdk/python`. -Published SDK builds are pinned to an exact `codex-cli-bin` package version, -and that runtime package carries the platform-specific binary for the target -wheel. +Published `codex-app-server-sdk` builds depend on an exact +`codex-app-server-sdk-core` version plus an exact `codex-cli-bin` version, and +that runtime package carries the platform-specific binary for the target wheel. For local repo development, the checked-in `sdk/python-runtime` package is only a template for staged release artifacts. Editable installs should use an @@ -61,9 +68,16 @@ explicit `codex_bin` override instead. ```bash cd sdk/python python scripts/update_sdk_artifacts.py generate-types +python scripts/update_sdk_artifacts.py \ + stage-sdk-core \ + /tmp/codex-python-release/codex-app-server-sdk-core \ + --skip-generate-types \ + --sdk-version 1.2.3 python scripts/update_sdk_artifacts.py \ stage-sdk \ /tmp/codex-python-release/codex-app-server-sdk \ + --skip-generate-types \ + --sdk-version 1.2.3 \ --runtime-version 1.2.3 python scripts/update_sdk_artifacts.py \ stage-runtime \ @@ -75,17 +89,22 @@ python scripts/update_sdk_artifacts.py \ This supports the CI release flow: - run `generate-types` before packaging -- stage `codex-app-server-sdk` once with an exact `codex-cli-bin==...` dependency +- stage `codex-app-server-sdk-core` with the release tag version as `--sdk-version` +- stage `codex-app-server-sdk` as a bundled metapackage pinned to exact `codex-app-server-sdk-core==...` and `codex-cli-bin==...` - stage `codex-cli-bin` on each supported platform runner with the same pinned runtime version - build and publish `codex-cli-bin` as platform wheels only; do not publish an sdist +- publish `codex-app-server-sdk-core` to PyPI after the runtime wheels land +- publish `codex-app-server-sdk` to PyPI last, using the same release version ## Compatibility and versioning -- Package: `codex-app-server-sdk` +- Core package: `codex-app-server-sdk-core` +- Bundled package: `codex-app-server-sdk` - Runtime package: `codex-cli-bin` - Current SDK version in this repo: `0.2.0` - Python: `>=3.10` - Target protocol: Codex `app-server` JSON-RPC v2 +- Release policy: published Python package versions should match the `rust-v...` release tag - Recommendation: keep SDK and `codex` CLI reasonably up to date together ## Notes diff --git a/sdk/python/docs/faq.md b/sdk/python/docs/faq.md index ebfd2ddad2..a2f1a8404f 100644 --- a/sdk/python/docs/faq.md +++ b/sdk/python/docs/faq.md @@ -38,16 +38,25 @@ Common causes: - local auth/session is missing - incompatible/old app-server -Maintainers stage releases by building the SDK once and the runtime once per -platform with the same pinned runtime version. Publish `codex-cli-bin` as -platform wheels only; do not publish an sdist: +Maintainers stage releases by building the core SDK once, the bundled SDK +metapackage once, and the runtime once per platform with the same pinned +runtime version. Publish `codex-cli-bin` as platform wheels only; do not +publish an sdist. Published Python package versions should match the +`rust-v...` release tag: ```bash cd sdk/python python scripts/update_sdk_artifacts.py generate-types +python scripts/update_sdk_artifacts.py \ + stage-sdk-core \ + /tmp/codex-python-release/codex-app-server-sdk-core \ + --skip-generate-types \ + --sdk-version 1.2.3 python scripts/update_sdk_artifacts.py \ stage-sdk \ /tmp/codex-python-release/codex-app-server-sdk \ + --skip-generate-types \ + --sdk-version 1.2.3 \ --runtime-version 1.2.3 python scripts/update_sdk_artifacts.py \ stage-runtime \ diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index f5129cbf93..6166b12a5b 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -3,9 +3,9 @@ requires = ["hatchling>=1.24.0"] build-backend = "hatchling.build" [project] -name = "codex-app-server-sdk" +name = "codex-app-server-sdk-core" version = "0.2.0" -description = "Python SDK for Codex app-server v2" +description = "Core Python SDK for Codex app-server v2" readme = "README.md" requires-python = ">=3.10" license = { text = "Apache-2.0" } diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py index da4cbceb1a..2c057e2f65 100755 --- a/sdk/python/scripts/update_sdk_artifacts.py +++ b/sdk/python/scripts/update_sdk_artifacts.py @@ -15,8 +15,13 @@ import types import typing from dataclasses import dataclass from pathlib import Path +from textwrap import dedent from typing import Any, Callable, Sequence, get_args, get_origin +CORE_SDK_PKG_NAME = "codex-app-server-sdk-core" +BUNDLED_SDK_PKG_NAME = "codex-app-server-sdk" +RUNTIME_PKG_NAME = "codex-cli-bin" + def repo_root() -> Path: return Path(__file__).resolve().parents[3] @@ -110,23 +115,7 @@ def _rewrite_project_version(pyproject_text: str, version: str) -> str: return updated -def _rewrite_sdk_runtime_dependency(pyproject_text: str, runtime_version: str) -> str: - match = re.search(r"^dependencies = \[(.*?)\]$", pyproject_text, flags=re.MULTILINE) - if match is None: - raise RuntimeError( - "Could not find dependencies array in sdk/python/pyproject.toml" - ) - - raw_items = [item.strip() for item in match.group(1).split(",") if item.strip()] - raw_items = [item for item in raw_items if "codex-cli-bin" not in item] - raw_items.append(f'"codex-cli-bin=={runtime_version}"') - replacement = "dependencies = [\n " + ",\n ".join(raw_items) + ",\n]" - return pyproject_text[: match.start()] + replacement + pyproject_text[match.end() :] - - -def stage_python_sdk_package( - staging_dir: Path, sdk_version: str, runtime_version: str -) -> Path: +def stage_python_core_sdk_package(staging_dir: Path, sdk_version: str) -> Path: _copy_package_tree(sdk_root(), staging_dir) sdk_bin_dir = staging_dir / "src" / "codex_app_server" / "bin" if sdk_bin_dir.exists(): @@ -135,11 +124,73 @@ def stage_python_sdk_package( pyproject_path = staging_dir / "pyproject.toml" pyproject_text = pyproject_path.read_text() pyproject_text = _rewrite_project_version(pyproject_text, sdk_version) - pyproject_text = _rewrite_sdk_runtime_dependency(pyproject_text, runtime_version) pyproject_path.write_text(pyproject_text) return staging_dir +def stage_python_sdk_package( + staging_dir: Path, sdk_version: str, runtime_version: str +) -> Path: + if staging_dir.exists(): + if staging_dir.is_dir(): + shutil.rmtree(staging_dir) + else: + staging_dir.unlink() + + package_dir = staging_dir / "src" / "codex_app_server_sdk_meta" + package_dir.mkdir(parents=True, exist_ok=True) + (package_dir / "__init__.py").write_text( + '"""Bundled Codex app-server SDK package metadata."""\n' + ) + + pyproject = dedent( + f""" + [build-system] + requires = ["hatchling>=1.24.0"] + build-backend = "hatchling.build" + + [project] + name = "{BUNDLED_SDK_PKG_NAME}" + version = "{sdk_version}" + description = "Bundled Python SDK for Codex app-server v2" + readme = "README.md" + requires-python = ">=3.10" + license = {{ text = "Apache-2.0" }} + authors = [{{ name = "OpenClaw Assistant" }}] + dependencies = [ + "{CORE_SDK_PKG_NAME}=={sdk_version}", + "{RUNTIME_PKG_NAME}=={runtime_version}", + ] + + [project.urls] + Homepage = "https://github.com/openai/codex" + Repository = "https://github.com/openai/codex" + Issues = "https://github.com/openai/codex/issues" + + [tool.hatch.build.targets.wheel] + packages = ["src/codex_app_server_sdk_meta"] + + [tool.hatch.build.targets.sdist] + include = ["src/codex_app_server_sdk_meta/**", "README.md", "pyproject.toml"] + """ + ).lstrip() + (staging_dir / "pyproject.toml").write_text(pyproject) + (staging_dir / "README.md").write_text( + "\n".join( + [ + "# Codex App Server Python SDK", + "", + "Bundled metapackage for the Codex app-server Python SDK.", + f"It depends on `{CORE_SDK_PKG_NAME}` and `{RUNTIME_PKG_NAME}`", + "at the same version so a regular install includes both the SDK", + "and the packaged Codex runtime binary.", + "", + ] + ) + ) + return staging_dir + + def stage_python_runtime_package( staging_dir: Path, runtime_version: str, binary_path: Path ) -> Path: @@ -558,6 +609,7 @@ class PublicFieldSpec: @dataclass(frozen=True) class CliOps: generate_types: Callable[[], None] + stage_python_core_sdk_package: Callable[[Path, str], Path] stage_python_sdk_package: Callable[[Path, str, str], Path] stage_python_runtime_package: Callable[[Path, str, Path], Path] current_sdk_version: Callable[[], str] @@ -916,23 +968,47 @@ def build_parser() -> argparse.ArgumentParser: "generate-types", help="Regenerate Python protocol-derived types" ) + stage_sdk_core_parser = subparsers.add_parser( + "stage-sdk-core", + help="Stage a releasable core SDK package without a bundled runtime", + ) + stage_sdk_core_parser.add_argument( + "staging_dir", + type=Path, + help="Output directory for the staged core SDK package", + ) + stage_sdk_core_parser.add_argument( + "--sdk-version", + help="Version to write into the staged core SDK package (defaults to sdk/python current version)", + ) + stage_sdk_core_parser.add_argument( + "--skip-generate-types", + action="store_true", + help="Skip type generation before staging when artifacts were generated earlier in the workflow", + ) + stage_sdk_parser = subparsers.add_parser( "stage-sdk", - help="Stage a releasable SDK package pinned to a runtime version", + help="Stage a releasable bundled SDK metapackage pinned to a runtime version", ) stage_sdk_parser.add_argument( "staging_dir", type=Path, - help="Output directory for the staged SDK package", + help="Output directory for the staged bundled SDK package", ) stage_sdk_parser.add_argument( "--runtime-version", required=True, - help="Pinned codex-cli-bin version for the staged SDK package", + help="Pinned codex-cli-bin version for the staged bundled SDK package", ) stage_sdk_parser.add_argument( "--sdk-version", - help="Version to write into the staged SDK package (defaults to sdk/python current version)", + help="Version to write into the staged bundled SDK package (defaults to sdk/python current version)", + ) + stage_sdk_parser.add_argument( + "--skip-generate-types", + action="store_true", + help="Skip type generation before staging when artifacts were generated earlier in the workflow", ) stage_runtime_parser = subparsers.add_parser( @@ -964,6 +1040,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def default_cli_ops() -> CliOps: return CliOps( generate_types=generate_types, + stage_python_core_sdk_package=stage_python_core_sdk_package, stage_python_sdk_package=stage_python_sdk_package, stage_python_runtime_package=stage_python_runtime_package, current_sdk_version=current_sdk_version, @@ -973,8 +1050,16 @@ def default_cli_ops() -> CliOps: def run_command(args: argparse.Namespace, ops: CliOps) -> None: if args.command == "generate-types": ops.generate_types() + elif args.command == "stage-sdk-core": + if not args.skip_generate_types: + ops.generate_types() + ops.stage_python_core_sdk_package( + args.staging_dir, + args.sdk_version or ops.current_sdk_version(), + ) elif args.command == "stage-sdk": - ops.generate_types() + if not args.skip_generate_types: + ops.generate_types() ops.stage_python_sdk_package( args.staging_dir, args.sdk_version or ops.current_sdk_version(), diff --git a/sdk/python/src/codex_app_server/client.py b/sdk/python/src/codex_app_server/client.py index aa7b574a3f..f496afcd84 100644 --- a/sdk/python/src/codex_app_server/client.py +++ b/sdk/python/src/codex_app_server/client.py @@ -47,6 +47,7 @@ from .retry import retry_on_overload ModelT = TypeVar("ModelT", bound=BaseModel) ApprovalHandler = Callable[[str, JsonObject | None], JsonObject] +BUNDLED_SDK_PKG_NAME = "codex-app-server-sdk" RUNTIME_PKG_NAME = "codex-cli-bin" @@ -82,9 +83,9 @@ def _installed_codex_path() -> Path: from codex_cli_bin import bundled_codex_path except ImportError as exc: raise FileNotFoundError( - "Unable to locate the pinned Codex runtime. Install the published SDK build " - f"with its {RUNTIME_PKG_NAME} dependency, or set AppServerConfig.codex_bin " - "explicitly." + "Unable to locate the Codex runtime. Install the published " + f"{BUNDLED_SDK_PKG_NAME} package, install {RUNTIME_PKG_NAME} " + "alongside the core SDK, or set AppServerConfig.codex_bin explicitly." ) from exc return bundled_codex_path() diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py index 938de05e28..a25a2e2005 100644 --- a/sdk/python/tests/test_artifact_workflow_and_binaries.py +++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py @@ -238,17 +238,31 @@ def test_stage_runtime_release_replaces_existing_staging_dir(tmp_path: Path) -> assert script.staged_runtime_bin_path(staged).read_text() == "fake codex\n" -def test_stage_sdk_release_injects_exact_runtime_pin(tmp_path: Path) -> None: +def test_stage_core_sdk_release_sets_version_without_runtime_pin(tmp_path: Path) -> None: + script = _load_update_script_module() + staged = script.stage_python_core_sdk_package(tmp_path / "sdk-core-stage", "0.2.1") + + pyproject = (staged / "pyproject.toml").read_text() + assert 'version = "0.2.1"' in pyproject + assert 'name = "codex-app-server-sdk-core"' in pyproject + assert "codex-cli-bin" not in pyproject + assert not any((staged / "src" / "codex_app_server").glob("bin/**")) + + +def test_stage_bundled_sdk_release_injects_exact_core_and_runtime_pins( + tmp_path: Path, +) -> None: script = _load_update_script_module() staged = script.stage_python_sdk_package(tmp_path / "sdk-stage", "0.2.1", "1.2.3") pyproject = (staged / "pyproject.toml").read_text() + assert 'name = "codex-app-server-sdk"' in pyproject assert 'version = "0.2.1"' in pyproject + assert '"codex-app-server-sdk-core==0.2.1"' in pyproject assert '"codex-cli-bin==1.2.3"' in pyproject - assert not any((staged / "src" / "codex_app_server").glob("bin/**")) -def test_stage_sdk_release_replaces_existing_staging_dir(tmp_path: Path) -> None: +def test_stage_bundled_sdk_release_replaces_existing_staging_dir(tmp_path: Path) -> None: script = _load_update_script_module() staging_dir = tmp_path / "sdk-stage" old_file = staging_dir / "stale.txt" @@ -261,6 +275,93 @@ def test_stage_sdk_release_replaces_existing_staging_dir(tmp_path: Path) -> None assert not old_file.exists() +def test_stage_sdk_core_runs_type_generation_before_staging(tmp_path: Path) -> None: + script = _load_update_script_module() + calls: list[str] = [] + args = script.parse_args( + [ + "stage-sdk-core", + str(tmp_path / "sdk-core-stage"), + ] + ) + + def fake_generate_types() -> None: + calls.append("generate_types") + + def fake_stage_core_sdk_package(_staging_dir: Path, _sdk_version: str) -> Path: + calls.append("stage_sdk_core") + return tmp_path / "sdk-core-stage" + + def fake_stage_sdk_package( + _staging_dir: Path, _sdk_version: str, _runtime_version: str + ) -> Path: + raise AssertionError("bundled sdk staging should not run for stage-sdk-core") + + def fake_stage_runtime_package( + _staging_dir: Path, _runtime_version: str, _runtime_binary: Path + ) -> Path: + raise AssertionError("runtime staging should not run for stage-sdk-core") + + def fake_current_sdk_version() -> str: + return "0.2.0" + + ops = script.CliOps( + generate_types=fake_generate_types, + stage_python_core_sdk_package=fake_stage_core_sdk_package, + stage_python_sdk_package=fake_stage_sdk_package, + stage_python_runtime_package=fake_stage_runtime_package, + current_sdk_version=fake_current_sdk_version, + ) + + script.run_command(args, ops) + + assert calls == ["generate_types", "stage_sdk_core"] + + +def test_stage_sdk_core_can_skip_type_generation(tmp_path: Path) -> None: + script = _load_update_script_module() + calls: list[str] = [] + args = script.parse_args( + [ + "stage-sdk-core", + str(tmp_path / "sdk-core-stage"), + "--skip-generate-types", + ] + ) + + def fake_generate_types() -> None: + calls.append("generate_types") + + def fake_stage_core_sdk_package(_staging_dir: Path, _sdk_version: str) -> Path: + calls.append("stage_sdk_core") + return tmp_path / "sdk-core-stage" + + def fake_stage_sdk_package( + _staging_dir: Path, _sdk_version: str, _runtime_version: str + ) -> Path: + raise AssertionError("bundled sdk staging should not run for stage-sdk-core") + + def fake_stage_runtime_package( + _staging_dir: Path, _runtime_version: str, _runtime_binary: Path + ) -> Path: + raise AssertionError("runtime staging should not run for stage-sdk-core") + + def fake_current_sdk_version() -> str: + return "0.2.0" + + ops = script.CliOps( + generate_types=fake_generate_types, + stage_python_core_sdk_package=fake_stage_core_sdk_package, + stage_python_sdk_package=fake_stage_sdk_package, + stage_python_runtime_package=fake_stage_runtime_package, + current_sdk_version=fake_current_sdk_version, + ) + + script.run_command(args, ops) + + assert calls == ["stage_sdk_core"] + + def test_stage_sdk_runs_type_generation_before_staging(tmp_path: Path) -> None: script = _load_update_script_module() calls: list[str] = [] @@ -276,6 +377,9 @@ def test_stage_sdk_runs_type_generation_before_staging(tmp_path: Path) -> None: def fake_generate_types() -> None: calls.append("generate_types") + def fake_stage_core_sdk_package(_staging_dir: Path, _sdk_version: str) -> Path: + raise AssertionError("core sdk staging should not run for stage-sdk") + def fake_stage_sdk_package( _staging_dir: Path, _sdk_version: str, _runtime_version: str ) -> Path: @@ -292,6 +396,7 @@ def test_stage_sdk_runs_type_generation_before_staging(tmp_path: Path) -> None: ops = script.CliOps( generate_types=fake_generate_types, + stage_python_core_sdk_package=fake_stage_core_sdk_package, stage_python_sdk_package=fake_stage_sdk_package, stage_python_runtime_package=fake_stage_runtime_package, current_sdk_version=fake_current_sdk_version, @@ -302,6 +407,52 @@ def test_stage_sdk_runs_type_generation_before_staging(tmp_path: Path) -> None: assert calls == ["generate_types", "stage_sdk"] +def test_stage_sdk_can_skip_type_generation(tmp_path: Path) -> None: + script = _load_update_script_module() + calls: list[str] = [] + args = script.parse_args( + [ + "stage-sdk", + str(tmp_path / "sdk-stage"), + "--runtime-version", + "1.2.3", + "--skip-generate-types", + ] + ) + + def fake_generate_types() -> None: + calls.append("generate_types") + + def fake_stage_core_sdk_package(_staging_dir: Path, _sdk_version: str) -> Path: + raise AssertionError("core sdk staging should not run for stage-sdk") + + def fake_stage_sdk_package( + _staging_dir: Path, _sdk_version: str, _runtime_version: str + ) -> Path: + calls.append("stage_sdk") + return tmp_path / "sdk-stage" + + def fake_stage_runtime_package( + _staging_dir: Path, _runtime_version: str, _runtime_binary: Path + ) -> Path: + raise AssertionError("runtime staging should not run for stage-sdk") + + def fake_current_sdk_version() -> str: + return "0.2.0" + + ops = script.CliOps( + generate_types=fake_generate_types, + stage_python_core_sdk_package=fake_stage_core_sdk_package, + stage_python_sdk_package=fake_stage_sdk_package, + stage_python_runtime_package=fake_stage_runtime_package, + current_sdk_version=fake_current_sdk_version, + ) + + script.run_command(args, ops) + + assert calls == ["stage_sdk"] + + def test_stage_runtime_stages_binary_without_type_generation(tmp_path: Path) -> None: script = _load_update_script_module() fake_binary = tmp_path / script.runtime_binary_name() @@ -320,6 +471,9 @@ def test_stage_runtime_stages_binary_without_type_generation(tmp_path: Path) -> def fake_generate_types() -> None: calls.append("generate_types") + def fake_stage_core_sdk_package(_staging_dir: Path, _sdk_version: str) -> Path: + raise AssertionError("core sdk staging should not run for stage-runtime") + def fake_stage_sdk_package( _staging_dir: Path, _sdk_version: str, _runtime_version: str ) -> Path: @@ -336,6 +490,7 @@ def test_stage_runtime_stages_binary_without_type_generation(tmp_path: Path) -> ops = script.CliOps( generate_types=fake_generate_types, + stage_python_core_sdk_package=fake_stage_core_sdk_package, stage_python_sdk_package=fake_stage_sdk_package, stage_python_runtime_package=fake_stage_runtime_package, current_sdk_version=fake_current_sdk_version,