From b0d95427c2443e90998f48065902309187564085 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 14 Sep 2026 21:58:11 +0000 Subject: [PATCH] Stage Python runtime wheels directly from package directories (#45526) ## Why The Windows release workflow builds an extra package archive only to extract it again when staging the Python runtime wheel. Reuse the package directory to avoid this round trip. ## What changed - Allow `stage-runtime` to accept a Codex package directory as well as a `.tar.gz` archive, with the same package layout validation. - Reject overlapping source and staging directories, symlinks, and non-regular directory entries before staging. - Stage Windows runtime wheels from the existing package directory and retain the check that voice resources are absent. ## Testing Add coverage for matching directory and archive output, including file permissions, source preservation, invalid layouts, non-regular entries, overlapping directories, and CLI handling of both source formats. GitOrigin-RevId: bf6d1e05888367d482d8d51a4a8061f92a9cae8b --- .github/workflows/rust-release-windows.yml | 15 ++-- sdk/python/scripts/update_sdk_artifacts.py | 28 +++++-- .../test_artifact_workflow_and_binaries.py | 80 +++++++++++++++++-- 3 files changed, 100 insertions(+), 23 deletions(-) diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index 7b529f8cf7..27e8b96458 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -535,23 +535,18 @@ jobs: # The wheel's Windows support floor predates the native voice # runtime's verified device/VC++ prerequisites. Keep it voice-free. - wheel_archives="${RUNNER_TEMP}/voice-free-wheel/${{ matrix.target }}" - bash "${GITHUB_WORKSPACE}/.github/scripts/build-codex-package-archive.sh" \ - --target "${{ matrix.target }}" --bundle primary \ - --entrypoint-dir "$CARGO_TARGET_DIR/${{ matrix.target }}/release" \ - --archive-dir "$wheel_archives" - python - "$wheel_archives/codex-package-${{ matrix.target }}.tar.gz" <<'PY' + wheel_package="${RUNNER_TEMP}/codex-package-${{ matrix.target }}" + python - "$wheel_package" <<'PY' import sys - import tarfile - with tarfile.open(sys.argv[1]) as archive: - assert not any("codex-resources/voice/" in member.name for member in archive) + from pathlib import Path + assert not (Path(sys.argv[1]) / "codex-resources" / "voice").exists() PY stage_dir="${RUNNER_TEMP}/openai-codex-cli-bin-${{ matrix.target }}" wheel_dir="${GITHUB_WORKSPACE}/python-runtime-dist/${{ matrix.target }}" python "${GITHUB_WORKSPACE}/sdk/python/scripts/update_sdk_artifacts.py" \ stage-runtime \ "$stage_dir" \ - "$wheel_archives/codex-package-${{ matrix.target }}.tar.gz" \ + "$wheel_package" \ --codex-version "${GITHUB_REF_NAME}" \ --platform-tag "$platform_tag" "${RUNNER_TEMP}/python-runtime-build-venv/Scripts/python.exe" -m build --wheel --outdir "$wheel_dir" "$stage_dir" diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py index 0c97d5156e..68234bfc88 100755 --- a/sdk/python/scripts/update_sdk_artifacts.py +++ b/sdk/python/scripts/update_sdk_artifacts.py @@ -188,9 +188,18 @@ def stage_python_sdk_package( def stage_python_runtime_package( staging_dir: Path, codex_version: str, - package_archive: Path, + package_source: Path, platform_tag: str | None = None, ) -> Path: + if package_source.is_dir(): + source = package_source.resolve() + destination = staging_dir.resolve() + if source.is_relative_to(destination) or destination.is_relative_to(source): + raise RuntimeError("Codex package and runtime staging directories must not overlap") + for path in package_source.rglob("*"): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise RuntimeError(f"Expected a regular Codex package entry: {path}") + package_version = normalize_codex_version(codex_version) _copy_package_tree(python_runtime_root(), staging_dir) @@ -202,7 +211,12 @@ def stage_python_runtime_package( pyproject_text = _rewrite_runtime_platform_tag(pyproject_text, platform_tag) pyproject_path.write_text(pyproject_text) - _extract_codex_package_archive(package_archive, staged_runtime_package_root(staging_dir)) + runtime_package_root = staged_runtime_package_root(staging_dir) + if package_source.is_dir(): + shutil.copytree(package_source, runtime_package_root, dirs_exist_ok=True) + _validate_codex_package_layout(runtime_package_root, package_source) + else: + _extract_codex_package_archive(package_source, runtime_package_root) return staging_dir @@ -220,7 +234,7 @@ def _extract_codex_package_archive(package_archive: Path, runtime_package_root: _validate_codex_package_layout(runtime_package_root, package_archive) -def _validate_codex_package_layout(package_dir: Path, package_archive: Path) -> None: +def _validate_codex_package_layout(package_dir: Path, package_source: Path) -> None: missing_entries = [] if not (package_dir / CODEX_PACKAGE_METADATA).is_file(): missing_entries.append(CODEX_PACKAGE_METADATA) @@ -235,7 +249,7 @@ def _validate_codex_package_layout(package_dir: Path, package_archive: Path) -> missing_entries.append(str(Path("bin") / runtime_code_mode_host_name())) if missing_entries: missing = ", ".join(missing_entries) - raise RuntimeError(f"Missing Codex package layout entries in {package_archive}: {missing}") + raise RuntimeError(f"Missing Codex package layout entries in {package_source}: {missing}") def _flatten_string_enum_one_of(definition: dict[str, Any]) -> bool: @@ -1405,9 +1419,9 @@ def build_parser() -> argparse.ArgumentParser: help="Output directory for the staged runtime package", ) stage_runtime_parser.add_argument( - "package_archive", + "package_source", type=Path, - help="Path to a Codex package .tar.gz archive for this platform.", + help="Path to a Codex package directory or .tar.gz archive for this platform.", ) stage_runtime_parser.add_argument( "--codex-version", @@ -1462,7 +1476,7 @@ def run_command(args: argparse.Namespace, ops: CliOps) -> None: ops.stage_python_runtime_package( args.staging_dir, normalize_codex_version(args.codex_version), - args.package_archive.resolve(), + args.package_source.resolve(), args.platform_tag, ) diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py index c89f75d1a1..d9c6617ed7 100644 --- a/sdk/python/tests/test_artifact_workflow_and_binaries.py +++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py @@ -864,7 +864,10 @@ def test_stage_runtime_release_can_pin_wheel_platform_tag(tmp_path: Path) -> Non assert 'platform-tag = "manylinux_2_17_x86_64"' in pyproject -def test_stage_runtime_release_rejects_incomplete_package_layout(tmp_path: Path) -> None: +@pytest.mark.parametrize("source_name", ["codex-package.tar.gz", "codex-package"]) +def test_stage_runtime_release_rejects_incomplete_package_layout( + tmp_path: Path, source_name: str +) -> None: script = _load_update_script_module() package_dir = tmp_path / "codex-package" (package_dir / "bin").mkdir(parents=True) @@ -872,7 +875,69 @@ def test_stage_runtime_release_rejects_incomplete_package_layout(tmp_path: Path) _write_package_archive(package_dir, package_archive) with pytest.raises(RuntimeError, match="Missing Codex package layout entries"): - script.stage_python_runtime_package(tmp_path / "runtime-stage", "1.2.3", package_archive) + script.stage_python_runtime_package( + tmp_path / "runtime-stage", "1.2.3", tmp_path / source_name + ) + + +def test_stage_runtime_directory_matches_archive(tmp_path: Path) -> None: + script = _load_update_script_module() + package_dir = _write_fake_codex_package(tmp_path / "codex-package", script) + (package_dir / "bin" / script.runtime_binary_name()).chmod(0o755) + package_archive = tmp_path / "codex-package.tar.gz" + _write_package_archive(package_dir, package_archive) + + staged_trees = [] + for name, source in [("archive", package_archive), ("directory", package_dir)]: + staged = script.stage_python_runtime_package( + tmp_path / name, "1.2.3", source, platform_tag="win_amd64" + ) + staged_trees.append( + { + path.relative_to(staged): (path.read_bytes(), path.stat().st_mode & 0o777) + for path in staged.rglob("*") + if path.is_file() + } + ) + + assert staged_trees[0] == staged_trees[1] + (script.staged_runtime_package_root(tmp_path / "directory") / "codex-package.json").unlink() + assert (package_dir / "codex-package.json").is_file() + + +@pytest.mark.parametrize("entry_kind", ["file-link", "directory-link", "fifo"]) +def test_stage_runtime_directory_rejects_non_regular_entries( + tmp_path: Path, entry_kind: str +) -> None: + script = _load_update_script_module() + package_dir = _write_fake_codex_package(tmp_path / "codex-package", script) + entry = package_dir / "codex-resources" / "invalid" + if entry_kind == "fifo": + if not hasattr(os, "mkfifo"): + pytest.skip("FIFOs are not available on this platform") + os.mkfifo(entry) + else: + target = package_dir / ("codex-package.json" if entry_kind == "file-link" else "bin") + try: + entry.symlink_to(target, target_is_directory=target.is_dir()) + except OSError: + pytest.skip("Symlinks are not available on this platform") + + with pytest.raises(RuntimeError, match="Expected a regular Codex package entry"): + script.stage_python_runtime_package(tmp_path / "runtime-stage", "1.2.3", package_dir) + + +@pytest.mark.parametrize("staging_path", ["codex-package", "codex-package/stage", "."]) +def test_stage_runtime_directory_rejects_overlapping_staging( + tmp_path: Path, staging_path: str +) -> None: + script = _load_update_script_module() + package_dir = _write_fake_codex_package(tmp_path / "codex-package", script) + + with pytest.raises(RuntimeError, match="directories must not overlap"): + script.stage_python_runtime_package(tmp_path / staging_path, "1.2.3", package_dir) + + assert (package_dir / "codex-package.json").is_file() def test_runtime_package_layout_is_included_by_wheel_config( @@ -1111,15 +1176,18 @@ def test_sdk_beta_can_use_a_supported_runtime( ) -def test_stage_runtime_stages_package_without_type_generation(tmp_path: Path) -> None: +@pytest.mark.parametrize("source_name", ["codex-package.tar.gz", "codex-package"]) +def test_stage_runtime_stages_package_without_type_generation( + tmp_path: Path, source_name: str +) -> None: script = _load_update_script_module() - package_archive = _write_fake_codex_package_archive(tmp_path, script) + _write_fake_codex_package_archive(tmp_path, script) calls: list[str] = [] args = script.parse_args( [ "stage-runtime", str(tmp_path / "runtime-stage"), - str(package_archive), + str(tmp_path / source_name), "--codex-version", "rust-v0.116.0-alpha.1", "--platform-tag", @@ -1152,7 +1220,7 @@ def test_stage_runtime_stages_package_without_type_generation(tmp_path: Path) -> script.run_command(args, ops) - assert calls == ["stage_runtime:0.116.0a1:manylinux_2_17_x86_64:codex-package.tar.gz"] + assert calls == [f"stage_runtime:0.116.0a1:manylinux_2_17_x86_64:{source_name}"] def test_default_runtime_is_resolved_from_installed_runtime_package(