From 98df71075091496e72a96976e76574153bd96443 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 24 Sep 2025 08:37:32 -0700 Subject: [PATCH] fix: create per-architecture versions of the npm module --- .github/workflows/rust-release.yml | 56 ++++++++++++- codex-cli/scripts/README.md | 11 ++- codex-cli/scripts/build_npm_package.py | 110 +++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index c808216d96..fbfa6dbd5f 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -175,6 +175,7 @@ jobs: tag: ${{ github.ref_name }} should_publish_npm: ${{ steps.npm_publish_settings.outputs.should_publish }} npm_tag: ${{ steps.npm_publish_settings.outputs.npm_tag }} + slice_tags: ${{ steps.slice_tags.outputs.value }} steps: - name: Checkout repository @@ -214,6 +215,25 @@ jobs: echo "npm_tag=" >> "$GITHUB_OUTPUT" fi + - name: Determine slice tags + id: slice_tags + run: | + set -euo pipefail + slice_tags=$(python - <<'PY' +import importlib.util +import sys +from pathlib import Path + +module_path = Path("codex-cli/scripts/build_npm_package.py").resolve() +sys.path.insert(0, str(module_path.parent)) +spec = importlib.util.spec_from_file_location("build_npm_package", module_path) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +print(" ".join(module.DEFAULT_SLICE_TAGS), end="") +PY + ) + echo "value=${slice_tags}" >> "$GITHUB_OUTPUT" + # build_npm_package.py requires DotSlash when staging releases. - uses: facebook/install-dotslash@v2 - name: Stage npm package @@ -222,10 +242,13 @@ jobs: run: | set -euo pipefail TMP_DIR="${RUNNER_TEMP}/npm-stage" + OUTPUT_DIR="${GITHUB_WORKSPACE}/dist/npm" + mkdir -p "${OUTPUT_DIR}" ./codex-cli/scripts/build_npm_package.py \ --release-version "${{ steps.release_name.outputs.name }}" \ --staging-dir "${TMP_DIR}" \ - --pack-output "${GITHUB_WORKSPACE}/dist/npm/codex-npm-${{ steps.release_name.outputs.name }}.tgz" + --pack-output "${OUTPUT_DIR}/codex-npm-${{ steps.release_name.outputs.name }}.tgz" \ + --slice-pack-dir "${OUTPUT_DIR}" - name: Create GitHub Release uses: softprops/action-gh-release@v2 @@ -279,7 +302,7 @@ jobs: mkdir -p dist/npm gh release download "$tag" \ --repo "${GITHUB_REPOSITORY}" \ - --pattern "codex-npm-${version}.tgz" \ + --pattern "codex-npm-${version}*.tgz" \ --dir dist/npm # No NODE_AUTH_TOKEN needed because we use OIDC. @@ -296,6 +319,35 @@ jobs: npm publish "${GITHUB_WORKSPACE}/dist/npm/codex-npm-${VERSION}.tgz" "${tag_args[@]}" + - name: Publish slice npm packages + env: + VERSION: ${{ needs.release.outputs.version }} + NPM_TAG: ${{ needs.release.outputs.npm_tag }} + SLICE_TAGS: ${{ needs.release.outputs.slice_tags }} + run: | + set -euo pipefail + if [[ -z "${SLICE_TAGS}" ]]; then + echo "No slice tags defined; skipping slice publish." >&2 + exit 0 + fi + + IFS=' ' read -r -a slice_tags <<<"${SLICE_TAGS}" + for slice_tag in "${slice_tags[@]}"; do + tarball="${GITHUB_WORKSPACE}/dist/npm/codex-npm-${VERSION}-${slice_tag}.tgz" + if [[ ! -f "${tarball}" ]]; then + echo "Missing slice tarball ${tarball}" >&2 + exit 1 + fi + + publish_tag="${slice_tag}" + if [[ -n "${NPM_TAG}" ]]; then + publish_tag="${NPM_TAG}-${slice_tag}" + fi + + echo "Publishing ${tarball} with npm tag '${publish_tag}'" + npm publish "${tarball}" --tag "${publish_tag}" + done + update-branch: name: Update latest-alpha-cli branch permissions: diff --git a/codex-cli/scripts/README.md b/codex-cli/scripts/README.md index fd2265b299..20d1ba6c43 100644 --- a/codex-cli/scripts/README.md +++ b/codex-cli/scripts/README.md @@ -8,4 +8,13 @@ To build the 0.2.x or later version of the npm module, which runs the Rust versi ./codex-cli/scripts/build_npm_package.py --release-version 0.6.0 ``` -Note this will create `./codex-cli/vendor/` as a side-effect. +To produce per-platform "slice" tarballs in addition to the fat package, supply the +`--slice-pack-dir` flag to write the outputs. For example: + +```bash +./codex-cli/scripts/build_npm_package.py --release-version 0.6.0 --slice-pack-dir dist/npm +``` + +The command above writes the full tarball plus the per-platform archives named with the +VS Code-style identifiers (for example, `codex-npm-0.6.0-darwin-arm64.tgz`). Note this will +create `./codex-cli/vendor/` as a side-effect. diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index 2b35999033..4a8a7bbc4f 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -9,12 +9,35 @@ import subprocess import sys import tempfile from pathlib import Path +from typing import Sequence + +from install_native_deps import CODEX_TARGETS, VENDOR_DIR_NAME SCRIPT_DIR = Path(__file__).resolve().parent CODEX_CLI_ROOT = SCRIPT_DIR.parent REPO_ROOT = CODEX_CLI_ROOT.parent GITHUB_REPO = "openai/codex" +TARGET_TO_SLICE_TAG = { + "x86_64-unknown-linux-musl": "linux-x64", + "aarch64-unknown-linux-musl": "linux-arm64", + "x86_64-apple-darwin": "darwin-x64", + "aarch64-apple-darwin": "darwin-arm64", + "x86_64-pc-windows-msvc": "win32-x64", + "aarch64-pc-windows-msvc": "win32-arm64", +} + +_SLICE_ACCUMULATOR: dict[str, list[str]] = {} +for target in CODEX_TARGETS: + slice_tag = TARGET_TO_SLICE_TAG.get(target) + if slice_tag is None: + raise RuntimeError(f"Missing slice tag mapping for target '{target}'.") + _SLICE_ACCUMULATOR.setdefault(slice_tag, []).append(target) + +SLICE_TAG_TO_TARGETS = {tag: tuple(targets) for tag, targets in _SLICE_ACCUMULATOR.items()} + +DEFAULT_SLICE_TAGS = tuple(SLICE_TAG_TO_TARGETS) + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Build or stage the Codex CLI npm package.") @@ -52,12 +75,29 @@ def parse_args() -> argparse.Namespace: type=Path, help="Path where the generated npm tarball should be written.", ) + parser.add_argument( + "--slice-pack-dir", + type=Path, + help=( + "Directory where per-platform slice npm tarballs should be written. " + "When provided, all known slices are packed unless --slices is given." + ), + ) + parser.add_argument( + "--slices", + nargs="+", + choices=sorted(DEFAULT_SLICE_TAGS), + help="Optional subset of slice tags to pack.", + ) return parser.parse_args() def main() -> int: args = parse_args() + if args.slices and args.slice_pack_dir is None: + raise RuntimeError("--slice-pack-dir is required when specifying --slices.") + version = args.version release_version = args.release_version if release_version: @@ -97,6 +137,16 @@ def main() -> int: install_native_binaries(staging_dir, workflow_url) + slice_outputs: list[tuple[str, Path]] = [] + if args.slice_pack_dir is not None: + slice_tags = tuple(args.slices or DEFAULT_SLICE_TAGS) + slice_outputs = build_slice_packages( + staging_dir, + version, + args.slice_pack_dir, + slice_tags, + ) + if release_version: staging_dir_str = str(staging_dir) print( @@ -111,6 +161,9 @@ def main() -> int: if args.pack_output is not None: output_path = run_npm_pack(staging_dir, args.pack_output) print(f"npm pack output written to {output_path}") + + for slice_tag, output_path in slice_outputs: + print(f"built slice {slice_tag} tarball at {output_path}") finally: if created_temp: # Preserve the staging directory for further inspection. @@ -161,6 +214,63 @@ def install_native_binaries(staging_dir: Path, workflow_url: str | None) -> None subprocess.check_call(cmd, cwd=CODEX_CLI_ROOT) +def build_slice_packages( + base_staging_dir: Path, + version: str, + output_dir: Path, + slice_tags: Sequence[str], +) -> list[tuple[str, Path]]: + if not slice_tags: + return [] + + base_vendor = base_staging_dir / VENDOR_DIR_NAME + if not base_vendor.exists(): + raise RuntimeError( + f"Base staging directory {base_staging_dir} does not include native vendor binaries." + ) + + output_dir = output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + results: list[tuple[str, Path]] = [] + for slice_tag in slice_tags: + targets = SLICE_TAG_TO_TARGETS.get(slice_tag) + if not targets: + raise RuntimeError(f"Unknown slice tag '{slice_tag}'.") + + missing = [target for target in targets if not (base_vendor / target).exists()] + if missing: + missing_label = ", ".join(missing) + raise RuntimeError( + f"Missing native binaries for slice '{slice_tag}': {missing_label} is absent in vendor." + ) + + with tempfile.TemporaryDirectory(prefix=f"codex-npm-slice-{slice_tag}-") as slice_dir_str: + slice_dir = Path(slice_dir_str) + stage_sources(slice_dir, version) + slice_vendor = slice_dir / VENDOR_DIR_NAME + copy_vendor_slice(base_vendor, slice_vendor, targets) + output_path = output_dir / f"codex-npm-{version}-{slice_tag}.tgz" + run_npm_pack(slice_dir, output_path) + results.append((slice_tag, output_path)) + + return results + + +def copy_vendor_slice(base_vendor: Path, dest_vendor: Path, targets: Sequence[str]) -> None: + dest_vendor.parent.mkdir(parents=True, exist_ok=True) + dest_vendor.mkdir(parents=True, exist_ok=True) + + for entry in base_vendor.iterdir(): + if entry.is_file(): + shutil.copy2(entry, dest_vendor / entry.name) + + for target in targets: + src = base_vendor / target + dest = dest_vendor / target + shutil.copytree(src, dest) + + def resolve_latest_alpha_workflow_url() -> str: version = determine_latest_alpha_version() workflow_url = fetch_workflow_url_for_version(version)