From d4740400cd175704afcdd83cb0fceb62d3ba2d8e Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 21 May 2026 08:20:03 -0700 Subject: [PATCH] npm: remove legacy package artifact synthesis --- .github/workflows/ci.yml | 11 +- codex-cli/scripts/install_native_deps.py | 378 +----------------- codex-cli/scripts/test_install_native_deps.py | 76 ++++ scripts/stage_npm_packages.py | 14 - 4 files changed, 90 insertions(+), 389 deletions(-) create mode 100644 codex-cli/scripts/test_install_native_deps.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25dff134a7..b1ee1395e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,9 +42,6 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - # stage_npm_packages.py requires DotSlash when staging releases. - - uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - - name: Stage npm package id: stage_npm_package env: @@ -55,17 +52,13 @@ jobs: # cross-platform native payload required by the npm package layout. # Passing the workflow URL directly avoids relying on old rust-v* # branches remaining discoverable via `gh run list --branch ...`. - CODEX_VERSION=0.125.0 - WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26131514935" + CODEX_VERSION=0.133.0-alpha.4 + WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26201494185" OUTPUT_DIR="${RUNNER_TEMP}" - # This reused workflow predates codex-package archive artifacts, so - # CI synthesizes the package layout from the older per-binary - # artifacts. Release staging must use real package archives. python3 ./scripts/stage_npm_packages.py \ --release-version "$CODEX_VERSION" \ --workflow-url "$WORKFLOW_URL" \ --package codex \ - --allow-legacy-codex-package \ --output-dir "$OUTPUT_DIR" PACK_OUTPUT="${OUTPUT_DIR}/codex-npm-${CODEX_VERSION}.tgz" echo "pack_output=$PACK_OUTPUT" >> "$GITHUB_OUTPUT" diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py index de157334cd..cd6c906345 100755 --- a/codex-cli/scripts/install_native_deps.py +++ b/codex-cli/scripts/install_native_deps.py @@ -3,27 +3,20 @@ import argparse from contextlib import contextmanager -import json import os import shutil import subprocess import tarfile import tempfile -import zipfile from dataclasses import dataclass from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path -import sys -from typing import Iterable, Sequence -from urllib.parse import urlparse -from urllib.request import urlopen +from typing import Sequence SCRIPT_DIR = Path(__file__).resolve().parent CODEX_CLI_ROOT = SCRIPT_DIR.parent -REPO_ROOT = CODEX_CLI_ROOT.parent -DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26131514935" # rust-v0.132.0 +DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26201494185" # rust-v0.133.0-alpha.4 VENDOR_DIR_NAME = "vendor" -RG_MANIFEST = REPO_ROOT / "scripts" / "codex_package" / "rg" BINARY_TARGETS = ( "x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl", @@ -40,57 +33,16 @@ class BinaryComponent: artifact_prefix: str # matches the artifact filename prefix (e.g. codex-.zst) dest_dir: str # directory under vendor// where the binary is installed binary_basename: str # executable name inside dest_dir (before optional .exe) - targets: tuple[str, ...] | None = None # limit installation to specific targets -WINDOWS_TARGETS = tuple(target for target in BINARY_TARGETS if "windows" in target) -LINUX_TARGETS = tuple(target for target in BINARY_TARGETS if "linux" in target) - BINARY_COMPONENTS = { - "bwrap": BinaryComponent( - artifact_prefix="bwrap", - dest_dir="codex-resources", - binary_basename="bwrap", - targets=LINUX_TARGETS, - ), - "codex": BinaryComponent( - artifact_prefix="codex", - dest_dir="codex", - binary_basename="codex", - ), "codex-responses-api-proxy": BinaryComponent( artifact_prefix="codex-responses-api-proxy", dest_dir="codex-responses-api-proxy", binary_basename="codex-responses-api-proxy", ), - "codex-windows-sandbox-setup": BinaryComponent( - artifact_prefix="codex-windows-sandbox-setup", - dest_dir="codex", - binary_basename="codex-windows-sandbox-setup", - targets=WINDOWS_TARGETS, - ), - "codex-command-runner": BinaryComponent( - artifact_prefix="codex-command-runner", - dest_dir="codex", - binary_basename="codex-command-runner", - targets=WINDOWS_TARGETS, - ), } -RG_TARGET_PLATFORM_PAIRS: list[tuple[str, str]] = [ - ("x86_64-unknown-linux-musl", "linux-x86_64"), - ("aarch64-unknown-linux-musl", "linux-aarch64"), - ("x86_64-apple-darwin", "macos-x86_64"), - ("aarch64-apple-darwin", "macos-aarch64"), - ("x86_64-pc-windows-msvc", "windows-x86_64"), - ("aarch64-pc-windows-msvc", "windows-aarch64"), -] -RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS} -DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS] - -# urllib.request.urlopen() defaults to no timeout (can hang indefinitely), which is painful in CI. -DOWNLOAD_TIMEOUT_SECS = 60 - def _gha_enabled() -> bool: # GitHub Actions supports "workflow commands" (e.g. ::group:: / ::error::) that make logs @@ -141,22 +93,12 @@ def parse_args() -> argparse.Namespace: "--component", dest="components", action="append", - choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS, "rg"]), + choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS]), help=( "Limit installation to the specified components." " May be repeated. Defaults to codex-package and codex-responses-api-proxy." ), ) - parser.add_argument( - "--allow-legacy-codex-package", - action="store_true", - help=( - "Allow codex-package to be synthesized from legacy per-binary artifacts " - "when package archives are missing. Intended for CI compatibility only; " - "release staging should not use this. Automatically enabled for the " - "built-in default workflow." - ), - ) parser.add_argument( "root", nargs="?", @@ -179,7 +121,6 @@ def main() -> int: components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"] workflow_override = (args.workflow_url or "").strip() - use_default_workflow = not workflow_override workflow_url = workflow_override or DEFAULT_WORKFLOW_URL workflow_id = workflow_url.rstrip("/").split("/")[-1] @@ -190,28 +131,13 @@ def main() -> int: artifacts_dir = Path(artifacts_dir_str) _download_artifacts(workflow_id, artifacts_dir) if CODEX_PACKAGE_COMPONENT in components: - try: - install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) - except FileNotFoundError: - if not (args.allow_legacy_codex_package or use_default_workflow): - raise - install_legacy_codex_package_layouts( - artifacts_dir, - vendor_dir, - BINARY_TARGETS, - manifest_path=RG_MANIFEST, - ) + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) install_binary_components( artifacts_dir, vendor_dir, [BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS], ) - if "rg" in components: - with _gha_group("Fetch ripgrep binaries"): - print("Fetching ripgrep binaries...") - fetch_rg(vendor_dir, DEFAULT_RG_TARGETS, manifest_path=RG_MANIFEST) - print(f"Installed native dependencies into {vendor_dir}") return 0 @@ -263,156 +189,6 @@ def _install_single_codex_package_archive( return dest_dir -def install_legacy_codex_package_layouts( - artifacts_dir: Path, - vendor_dir: Path, - targets: Sequence[str], - *, - manifest_path: Path, -) -> None: - targets = list(targets) - print( - "Synthesizing Codex package layouts from legacy artifacts for targets: " - + ", ".join(targets) - ) - with tempfile.TemporaryDirectory(prefix="codex-legacy-package-") as legacy_vendor_dir_str: - legacy_vendor_dir = Path(legacy_vendor_dir_str) - install_binary_components( - artifacts_dir, - legacy_vendor_dir, - [ - BINARY_COMPONENTS["codex"], - BINARY_COMPONENTS["bwrap"], - BINARY_COMPONENTS["codex-windows-sandbox-setup"], - BINARY_COMPONENTS["codex-command-runner"], - ], - ) - fetch_rg(legacy_vendor_dir, targets, manifest_path=manifest_path) - - for target in targets: - dest_dir = vendor_dir / target - if dest_dir.exists(): - shutil.rmtree(dest_dir) - _build_legacy_codex_package_layout(legacy_vendor_dir / target, dest_dir, target) - print(f" synthesized {dest_dir}") - - -def _build_legacy_codex_package_layout( - legacy_target_dir: Path, - package_dir: Path, - target: str, -) -> None: - is_windows = "windows" in target - exe_suffix = ".exe" if is_windows else "" - package_dir.mkdir(parents=True) - - bin_dir = package_dir / "bin" - resources_dir = package_dir / "codex-resources" - path_dir = package_dir / "codex-path" - bin_dir.mkdir() - resources_dir.mkdir() - path_dir.mkdir() - - shutil.copy2( - legacy_target_dir / "codex" / f"codex{exe_suffix}", - bin_dir / f"codex{exe_suffix}", - ) - shutil.copy2( - legacy_target_dir / "path" / f"rg{exe_suffix}", - path_dir / f"rg{exe_suffix}", - ) - - if is_windows: - for helper in [ - "codex-command-runner.exe", - "codex-windows-sandbox-setup.exe", - ]: - shutil.copy2(legacy_target_dir / "codex" / helper, resources_dir / helper) - elif "linux" in target: - shutil.copy2(legacy_target_dir / "codex-resources" / "bwrap", resources_dir / "bwrap") - - write_json( - package_dir / "codex-package.json", - { - "layoutVersion": 1, - "version": "unknown", - "target": target, - "variant": "codex", - "entrypoint": f"bin/codex{exe_suffix}", - "resourcesDir": "codex-resources", - "pathDir": "codex-path", - }, - ) - - -def fetch_rg( - vendor_dir: Path, - targets: Sequence[str] | None = None, - *, - manifest_path: Path, -) -> list[Path]: - """Download ripgrep binaries described by the DotSlash manifest.""" - - if targets is None: - targets = DEFAULT_RG_TARGETS - - if not manifest_path.exists(): - raise FileNotFoundError(f"DotSlash manifest not found: {manifest_path}") - - manifest = _load_manifest(manifest_path) - platforms = manifest.get("platforms", {}) - - vendor_dir.mkdir(parents=True, exist_ok=True) - - targets = list(targets) - if not targets: - return [] - - task_configs: list[tuple[str, str, dict]] = [] - for target in targets: - platform_key = RG_TARGET_TO_PLATFORM.get(target) - if platform_key is None: - raise ValueError(f"Unsupported ripgrep target '{target}'.") - - platform_info = platforms.get(platform_key) - if platform_info is None: - raise RuntimeError(f"Platform '{platform_key}' not found in manifest {manifest_path}.") - - task_configs.append((target, platform_key, platform_info)) - - results: dict[str, Path] = {} - max_workers = min(len(task_configs), max(1, (os.cpu_count() or 1))) - - print("Installing ripgrep binaries for targets: " + ", ".join(targets)) - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - future_map = { - executor.submit( - _fetch_single_rg, - vendor_dir, - target, - platform_key, - platform_info, - manifest_path, - ): target - for target, platform_key, platform_info in task_configs - } - - for future in as_completed(future_map): - target = future_map[future] - try: - results[target] = future.result() - except Exception as exc: - _gha_error( - title="ripgrep install failed", - message=f"target={target} error={exc!r}", - ) - raise RuntimeError(f"Failed to install ripgrep for target {target}.") from exc - print(f" installed ripgrep for {target}") - - return [results[target] for target in targets] - - def _download_artifacts(workflow_id: str, dest_dir: Path) -> None: cmd = [ "gh", @@ -436,7 +212,7 @@ def install_binary_components( return for component in selected_components: - component_targets = list(component.targets or BINARY_TARGETS) + component_targets = list(BINARY_TARGETS) print( f"Installing {component.binary_basename} binaries for targets: " @@ -466,7 +242,7 @@ def _install_single_binary( component: BinaryComponent, ) -> Path: artifact_subdir = artifact_dir_for_target(artifacts_dir, target) - archive_path = legacy_binary_archive_path(artifact_subdir, component.artifact_prefix, target) + archive_path = binary_archive_path(artifact_subdir, component.artifact_prefix, target) dest_dir = vendor_dir / target / component.dest_dir dest_dir.mkdir(parents=True, exist_ok=True) @@ -476,7 +252,7 @@ def _install_single_binary( ) dest = dest_dir / binary_name dest.unlink(missing_ok=True) - extract_archive(archive_path, "zst", None, dest) + extract_zstd_archive(archive_path, dest) if "windows" not in target: dest.chmod(0o755) return dest @@ -488,7 +264,7 @@ def _archive_name_for_target(artifact_prefix: str, target: str) -> str: return f"{artifact_prefix}-{target}.zst" -def legacy_binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: +def binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: archive_names = [_archive_name_for_target(artifact_prefix, target)] if artifact_dir.name == f"{target}-unsigned": archive_names.append(_archive_name_for_target(artifact_prefix, f"{target}-unsigned")) @@ -510,142 +286,12 @@ def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: return artifacts_dir / target -def _fetch_single_rg( - vendor_dir: Path, - target: str, - platform_key: str, - platform_info: dict, - manifest_path: Path, -) -> Path: - providers = platform_info.get("providers", []) - if not providers: - raise RuntimeError(f"No providers listed for platform '{platform_key}' in {manifest_path}.") - - url = providers[0]["url"] - archive_format = platform_info.get("format", "zst") - archive_member = platform_info.get("path") - digest = platform_info.get("digest") - expected_size = platform_info.get("size") - - dest_dir = vendor_dir / target / "path" - dest_dir.mkdir(parents=True, exist_ok=True) - - is_windows = platform_key.startswith("win") - binary_name = "rg.exe" if is_windows else "rg" - dest = dest_dir / binary_name - - with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = Path(tmp_dir_str) - archive_filename = os.path.basename(urlparse(url).path) - download_path = tmp_dir / archive_filename - print( - f" downloading ripgrep for {target} ({platform_key}) from {url}", - flush=True, - ) - try: - _download_file(url, download_path) - except Exception as exc: - _gha_error( - title="ripgrep download failed", - message=f"target={target} platform={platform_key} url={url} error={exc!r}", - ) - raise RuntimeError( - "Failed to download ripgrep " - f"(target={target}, platform={platform_key}, format={archive_format}, " - f"expected_size={expected_size!r}, digest={digest!r}, url={url}, dest={download_path})." - ) from exc - - dest.unlink(missing_ok=True) - try: - extract_archive(download_path, archive_format, archive_member, dest) - except Exception as exc: - raise RuntimeError( - "Failed to extract ripgrep " - f"(target={target}, platform={platform_key}, format={archive_format}, " - f"member={archive_member!r}, url={url}, archive={download_path})." - ) from exc - - if not is_windows: - dest.chmod(0o755) - - return dest - - -def _download_file(url: str, dest: Path) -> None: - dest.parent.mkdir(parents=True, exist_ok=True) - dest.unlink(missing_ok=True) - - with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response, open(dest, "wb") as out: - shutil.copyfileobj(response, out) - - -def extract_archive( - archive_path: Path, - archive_format: str, - archive_member: str | None, - dest: Path, -) -> None: +def extract_zstd_archive(archive_path: Path, dest: Path) -> None: dest.parent.mkdir(parents=True, exist_ok=True) - if archive_format == "zst": - output_path = archive_path.parent / dest.name - subprocess.check_call( - ["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)] - ) - shutil.move(str(output_path), dest) - return - - if archive_format == "tar.gz": - if not archive_member: - raise RuntimeError("Missing 'path' for tar.gz archive in DotSlash manifest.") - with tarfile.open(archive_path, "r:gz") as tar: - try: - member = tar.getmember(archive_member) - except KeyError as exc: - raise RuntimeError( - f"Entry '{archive_member}' not found in archive {archive_path}." - ) from exc - tar.extract(member, path=archive_path.parent, filter="data") - extracted = archive_path.parent / archive_member - shutil.move(str(extracted), dest) - return - - if archive_format == "zip": - if not archive_member: - raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.") - with zipfile.ZipFile(archive_path) as archive: - try: - with archive.open(archive_member) as src, open(dest, "wb") as out: - shutil.copyfileobj(src, out) - except KeyError as exc: - raise RuntimeError( - f"Entry '{archive_member}' not found in archive {archive_path}." - ) from exc - return - - raise RuntimeError(f"Unsupported archive format '{archive_format}'.") - - -def _load_manifest(manifest_path: Path) -> dict: - cmd = ["dotslash", "--", "parse", str(manifest_path)] - stdout = subprocess.check_output(cmd, text=True) - try: - manifest = json.loads(stdout) - except json.JSONDecodeError as exc: - raise RuntimeError(f"Invalid DotSlash manifest output from {manifest_path}.") from exc - - if not isinstance(manifest, dict): - raise RuntimeError( - f"Unexpected DotSlash manifest structure for {manifest_path}: {type(manifest)!r}" - ) - - return manifest - - -def write_json(path: Path, value: object) -> None: - with open(path, "w", encoding="utf-8") as out: - json.dump(value, out, indent=2) - out.write("\n") + output_path = archive_path.parent / dest.name + subprocess.check_call(["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)]) + shutil.move(str(output_path), dest) if __name__ == "__main__": diff --git a/codex-cli/scripts/test_install_native_deps.py b/codex-cli/scripts/test_install_native_deps.py new file mode 100644 index 0000000000..d2131502fe --- /dev/null +++ b/codex-cli/scripts/test_install_native_deps.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +from contextlib import redirect_stdout +import importlib.util +import io +from pathlib import Path +import tarfile +import tempfile +import unittest + + +INSTALL_SCRIPT = Path(__file__).resolve().parent / "install_native_deps.py" +SPEC = importlib.util.spec_from_file_location("install_native_deps", INSTALL_SCRIPT) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"Unable to load module from {INSTALL_SCRIPT}") + +install_native_deps = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(install_native_deps) + + +class InstallCodexPackageArchivesTest(unittest.TestCase): + def test_installs_codex_package_archive(self) -> None: + target = "x86_64-unknown-linux-musl" + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + artifact_dir = root / "artifacts" / target + package_src = root / "package-src" + vendor_dir = root / "vendor" + artifact_dir.mkdir(parents=True) + (package_src / "bin").mkdir(parents=True) + (package_src / "bin" / "codex").write_text("codex\n", encoding="utf-8") + (package_src / "codex-package.json").write_text("{}\n", encoding="utf-8") + + archive_path = artifact_dir / f"codex-package-{target}.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + archive.add(package_src / "bin", arcname="bin") + archive.add(package_src / "codex-package.json", arcname="codex-package.json") + + with redirect_stdout(io.StringIO()): + install_native_deps.install_codex_package_archives( + root / "artifacts", + vendor_dir, + [target], + ) + + self.assertEqual( + sorted( + path.relative_to(vendor_dir / target) + for path in (vendor_dir / target).rglob("*") + ), + [ + Path("bin"), + Path("bin/codex"), + Path("codex-package.json"), + ], + ) + + def test_missing_codex_package_archive_errors(self) -> None: + target = "x86_64-unknown-linux-musl" + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + + with redirect_stdout(io.StringIO()): + with self.assertRaisesRegex( + FileNotFoundError, + "Expected package archive not found", + ): + install_native_deps.install_codex_package_archives( + root / "artifacts", + root / "vendor", + [target], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 4eb69053eb..79120ea2eb 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -66,15 +66,6 @@ def parse_args() -> argparse.Namespace: "Intended for CI compatibility only; release staging should not use this." ), ) - parser.add_argument( - "--allow-legacy-codex-package", - action="store_true", - help=( - "Allow codex-package layouts to be synthesized from legacy per-binary " - "workflow artifacts. Intended for CI compatibility only; release staging " - "should not use this." - ), - ) return parser.parse_args() @@ -131,15 +122,11 @@ def install_native_components( workflow_url: str, components: set[str], vendor_root: Path, - *, - allow_legacy_codex_package: bool, ) -> None: if not components: return cmd = [str(INSTALL_NATIVE_DEPS), "--workflow-url", workflow_url] - if allow_legacy_codex_package: - cmd.append("--allow-legacy-codex-package") for component in sorted(components): cmd.extend(["--component", component]) cmd.append(str(vendor_root)) @@ -187,7 +174,6 @@ def main() -> int: workflow_url, native_components_to_install, vendor_temp_root, - allow_legacy_codex_package=args.allow_legacy_codex_package, ) vendor_src = vendor_temp_root / "vendor"