From 34cb2225ab116ecbe2994a0c9f1ae12ea842fcd2 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 15 Sep 2025 11:11:27 -0700 Subject: [PATCH] fix: vendor ripgrep in the npm module --- .github/workflows/rust-release.yml | 9 +- codex-cli/bin/codex.js | 25 +-- codex-cli/bin/rg | 79 +++++++++ codex-cli/package-lock.json | 101 ------------ codex-cli/package.json | 3 - codex-cli/scripts/build_npm_package.py | 163 +++++++++++++++++++ codex-cli/scripts/install_native_deps.sh | 4 +- codex-cli/scripts/install_rg.py | 61 +++++++ codex-cli/scripts/rg_utils.py | 197 +++++++++++++++++++++++ codex-cli/scripts/stage_release.sh | 76 ++++----- codex-cli/scripts/stage_rust_release.py | 16 +- 11 files changed, 550 insertions(+), 184 deletions(-) create mode 100755 codex-cli/bin/rg create mode 100755 codex-cli/scripts/build_npm_package.py create mode 100755 codex-cli/scripts/install_rg.py create mode 100644 codex-cli/scripts/rg_utils.py diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 07af62a17c..92b47873d3 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -195,13 +195,8 @@ jobs: TMP_DIR="${RUNNER_TEMP}/npm-stage" python3 codex-cli/scripts/stage_rust_release.py \ --release-version "${{ steps.release_name.outputs.name }}" \ - --tmp "${TMP_DIR}" - mkdir -p dist/npm - # Produce an npm-ready tarball using `npm pack` and store it in dist/npm. - # We then rename it to a stable name used by our publishing script. - (cd "$TMP_DIR" && npm pack --pack-destination "${GITHUB_WORKSPACE}/dist/npm") - mv "${GITHUB_WORKSPACE}"/dist/npm/*.tgz \ - "${GITHUB_WORKSPACE}/dist/npm/codex-npm-${{ steps.release_name.outputs.name }}.tgz" + --tmp "${TMP_DIR}" \ + --pack-output "${GITHUB_WORKSPACE}/dist/npm/codex-npm-${{ steps.release_name.outputs.name }}.tgz" - name: Create GitHub Release uses: softprops/action-gh-release@v2 diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index f24065170d..47b28679b3 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -1,6 +1,7 @@ #!/usr/bin/env node // Unified entry point for the Codex CLI. +import { existsSync } from "fs"; import path from "path"; import { fileURLToPath } from "url"; @@ -66,23 +67,6 @@ const binaryPath = path.join(__dirname, "..", "bin", `codex-${targetTriple}`); // receives a fatal signal, both processes exit in a predictable manner. const { spawn } = await import("child_process"); -async function tryImport(moduleName) { - try { - // eslint-disable-next-line node/no-unsupported-features/es-syntax - return await import(moduleName); - } catch (err) { - return null; - } -} - -async function resolveRgDir() { - const ripgrep = await tryImport("@vscode/ripgrep"); - if (!ripgrep?.rgPath) { - return null; - } - return path.dirname(ripgrep.rgPath); -} - function getUpdatedPath(newDirs) { const pathSep = process.platform === "win32" ? ";" : ":"; const existingPath = process.env.PATH || ""; @@ -94,9 +78,10 @@ function getUpdatedPath(newDirs) { } const additionalDirs = []; -const rgDir = await resolveRgDir(); -if (rgDir) { - additionalDirs.push(rgDir); +const rgBinaryName = `rg-${targetTriple}`; +const rgBinaryPath = path.join(__dirname, "..", "bin", rgBinaryName); +if (existsSync(rgBinaryPath)) { + additionalDirs.push(path.dirname(rgBinaryPath)); } const updatedPath = getUpdatedPath(additionalDirs); diff --git a/codex-cli/bin/rg b/codex-cli/bin/rg new file mode 100755 index 0000000000..5a992570a9 --- /dev/null +++ b/codex-cli/bin/rg @@ -0,0 +1,79 @@ +#!/usr/bin/env dotslash + +{ + "name": "rg", + "platforms": { + "macos-aarch64": { + "size": 1787248, + "hash": "blake3", + "digest": "8d9942032585ea8ee805937634238d9aee7b210069f4703c88fbe568e26fb78a", + "format": "tar.gz", + "path": "ripgrep-14.1.1-aarch64-apple-darwin/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-aarch64-apple-darwin.tar.gz" + } + ] + }, + "linux-aarch64": { + "size": 2047405, + "hash": "blake3", + "digest": "0b670b8fa0a3df2762af2fc82cc4932f684ca4c02dbd1260d4f3133fd4b2a515", + "format": "tar.gz", + "path": "ripgrep-14.1.1-aarch64-unknown-linux-gnu/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-aarch64-unknown-linux-gnu.tar.gz" + } + ] + }, + "macos-x86_64": { + "size": 2082672, + "hash": "blake3", + "digest": "e9b862fc8da3127f92791f0ff6a799504154ca9d36c98bf3e60a81c6b1f7289e", + "format": "tar.gz", + "path": "ripgrep-14.1.1-x86_64-apple-darwin/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-apple-darwin.tar.gz" + } + ] + }, + "linux-x86_64": { + "size": 2566310, + "hash": "blake3", + "digest": "f73cca4e54d78c31f832c7f6e2c0b4db8b04fa3eaa747915727d570893dbee76", + "format": "tar.gz", + "path": "ripgrep-14.1.1-x86_64-unknown-linux-musl/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz" + } + ] + }, + "windows-x86_64": { + "size": 2058893, + "hash": "blake3", + "digest": "a8ce1a6fed4f8093ee997e57f33254e94b2cd18e26358b09db599c89882eadbd", + "format": "zip", + "path": "ripgrep-14.1.1-x86_64-pc-windows-msvc/rg.exe", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-pc-windows-msvc.zip" + } + ] + }, + "windows-aarch64": { + "size": 1667740, + "hash": "blake3", + "digest": "47b971a8c4fca1d23a4e7c19bd4d88465ebc395598458133139406d3bf85f3fa", + "format": "zip", + "path": "rg.exe", + "providers": [ + { + "url": "https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-13/ripgrep-v13.0.0-13-aarch64-pc-windows-msvc.zip" + } + ] + } + } +} diff --git a/codex-cli/package-lock.json b/codex-cli/package-lock.json index a1c840ade0..a28bb9cd9f 100644 --- a/codex-cli/package-lock.json +++ b/codex-cli/package-lock.json @@ -2,118 +2,17 @@ "name": "@openai/codex", "version": "0.0.0-dev", "lockfileVersion": 3, - "requires": true, "packages": { "": { "name": "@openai/codex", "version": "0.0.0-dev", "license": "Apache-2.0", - "dependencies": { - "@vscode/ripgrep": "^1.15.14" - }, "bin": { "codex": "bin/codex.js" }, "engines": { "node": ">=20" } - }, - "node_modules/@vscode/ripgrep": { - "version": "1.15.14", - "resolved": "https://registry.npmjs.org/@vscode/ripgrep/-/ripgrep-1.15.14.tgz", - "integrity": "sha512-/G1UJPYlm+trBWQ6cMO3sv6b8D1+G16WaJH1/DSqw32JOVlzgZbLkDxRyzIpTpv30AcYGMkCf5tUqGlW6HbDWw==", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "https-proxy-agent": "^7.0.2", - "proxy-from-env": "^1.1.0", - "yauzl": "^2.9.2" - } - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "license": "MIT", - "dependencies": { - "pend": "~1.2.0" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", - "license": "MIT" - }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" - }, - "node_modules/yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", - "license": "MIT", - "dependencies": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } } } } diff --git a/codex-cli/package.json b/codex-cli/package.json index 614ca1a832..162773a25d 100644 --- a/codex-cli/package.json +++ b/codex-cli/package.json @@ -17,9 +17,6 @@ "type": "git", "url": "git+https://github.com/openai/codex.git" }, - "dependencies": { - "@vscode/ripgrep": "^1.15.14" - }, "devDependencies": { "prettier": "^3.3.3" } diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py new file mode 100755 index 0000000000..fbf412db49 --- /dev/null +++ b/codex-cli/scripts/build_npm_package.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Stage and optionally package the @openai/codex npm module.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Sequence + +SCRIPT_DIR = Path(__file__).resolve().parent +CODEX_CLI_ROOT = SCRIPT_DIR.parent +REPO_ROOT = CODEX_CLI_ROOT.parent + +sys.path.insert(0, str(SCRIPT_DIR)) + +from rg_utils import DEFAULT_RG_TARGETS, fetch_rg # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the Codex CLI npm package.") + parser.add_argument( + "--version", + required=True, + help="Version number to write to package.json inside the staged package.", + ) + parser.add_argument( + "--workflow-url", + help="Optional GitHub Actions workflow run URL used to download native binaries.", + ) + parser.add_argument( + "--staging-dir", + type=Path, + help=( + "Directory to stage the package contents. Defaults to a new temporary directory " + "if omitted. The directory must be empty when provided." + ), + ) + parser.add_argument( + "--pack-output", + type=Path, + help="Path where the generated npm tarball should be written.", + ) + parser.add_argument( + "--rg-target", + action="append", + dest="rg_targets", + help="Codex target triple for which ripgrep should be fetched. Repeatable.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + staging_dir, created_temp = prepare_staging_dir(args.staging_dir) + + try: + stage_sources(staging_dir, args.version) + install_native_binaries(staging_dir, args.workflow_url) + + rg_targets: Sequence[str] | None = args.rg_targets + if rg_targets is None: + rg_targets = DEFAULT_RG_TARGETS + fetch_rg(staging_dir / "bin", rg_targets) + + print(f"Staged package in {staging_dir}") + + if args.pack_output is not None: + output_path = run_npm_pack(staging_dir, args.pack_output) + print(f"npm pack output written to {output_path}") + finally: + if created_temp: + # Preserve the staging directory for further inspection. + pass + + return 0 + + +def prepare_staging_dir(staging_dir: Path | None) -> tuple[Path, bool]: + if staging_dir is not None: + staging_dir = staging_dir.resolve() + staging_dir.mkdir(parents=True, exist_ok=True) + if any(staging_dir.iterdir()): + raise RuntimeError(f"Staging directory {staging_dir} is not empty.") + return staging_dir, False + + temp_dir = Path(tempfile.mkdtemp(prefix="codex-npm-stage-")) + return temp_dir, True + + +def stage_sources(staging_dir: Path, version: str) -> None: + bin_dir = staging_dir / "bin" + bin_dir.mkdir(parents=True, exist_ok=True) + + shutil.copy2(CODEX_CLI_ROOT / "bin" / "codex.js", bin_dir / "codex.js") + rg_manifest = CODEX_CLI_ROOT / "bin" / "rg" + if rg_manifest.exists(): + shutil.copy2(rg_manifest, bin_dir / "rg") + + readme_src = REPO_ROOT / "README.md" + if readme_src.exists(): + shutil.copy2(readme_src, staging_dir / "README.md") + + with open(CODEX_CLI_ROOT / "package.json", "r", encoding="utf-8") as fh: + package_json = json.load(fh) + package_json["version"] = version + + with open(staging_dir / "package.json", "w", encoding="utf-8") as out: + json.dump(package_json, out, indent=2) + out.write("\n") + + +def install_native_binaries(staging_dir: Path, workflow_url: str | None) -> None: + cmd = ["./scripts/install_native_deps.sh"] + if workflow_url: + cmd.extend(["--workflow-url", workflow_url]) + cmd.append(str(staging_dir)) + subprocess.run(cmd, cwd=CODEX_CLI_ROOT, check=True) + + +def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: + output_path = output_path.resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory(prefix="codex-npm-pack-") as pack_dir_str: + pack_dir = Path(pack_dir_str) + result = subprocess.run( + ["npm", "pack", "--json", "--pack-destination", str(pack_dir)], + cwd=staging_dir, + check=True, + text=True, + capture_output=True, + ) + + try: + pack_output = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError("Failed to parse npm pack output.") from exc + + if not pack_output: + raise RuntimeError("npm pack did not produce an output tarball.") + + tarball_name = pack_output[0].get("filename") or pack_output[0].get("name") + if not tarball_name: + raise RuntimeError("Unable to determine npm pack output filename.") + + tarball_path = pack_dir / tarball_name + if not tarball_path.exists(): + raise RuntimeError(f"Expected npm pack output not found: {tarball_path}") + + shutil.move(str(tarball_path), output_path) + + return output_path + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/codex-cli/scripts/install_native_deps.sh b/codex-cli/scripts/install_native_deps.sh index b9fda2543e..368d89865a 100755 --- a/codex-cli/scripts/install_native_deps.sh +++ b/codex-cli/scripts/install_native_deps.sh @@ -91,4 +91,6 @@ zstd -d "$ARTIFACTS_DIR/x86_64-pc-windows-msvc/codex-x86_64-pc-windows-msvc.exe. zstd -d "$ARTIFACTS_DIR/aarch64-pc-windows-msvc/codex-aarch64-pc-windows-msvc.exe.zst" \ -o "$BIN_DIR/codex-aarch64-pc-windows-msvc.exe" -echo "Installed native dependencies into $BIN_DIR" +python3 "$CODEX_CLI_ROOT/scripts/install_rg.py" --bin-dir "$BIN_DIR" + +echo "Installed native dependencies (including ripgrep) into $BIN_DIR" diff --git a/codex-cli/scripts/install_rg.py b/codex-cli/scripts/install_rg.py new file mode 100755 index 0000000000..5615938da6 --- /dev/null +++ b/codex-cli/scripts/install_rg.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Download ripgrep binaries into the codex-cli bin directory.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIR)) + +from rg_utils import DEFAULT_RG_TARGETS, detect_current_target, fetch_rg # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Install ripgrep binaries.") + parser.add_argument( + "--bin-dir", + type=Path, + required=True, + help="Directory where ripgrep binaries should be written.", + ) + parser.add_argument( + "--target", + action="append", + dest="targets", + help="Codex target triple to fetch. Repeatable. Defaults to all.", + ) + parser.add_argument( + "--current-only", + action="store_true", + help="Download ripgrep only for the current platform.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + if args.current_only and args.targets: + print("Cannot combine --current-only with explicit --target options.", file=sys.stderr) + return 1 + + if args.current_only: + current = detect_current_target() + if current is None: + print("Unable to detect current platform for ripgrep download.", file=sys.stderr) + return 1 + targets = [current] + elif args.targets: + targets = args.targets + else: + targets = DEFAULT_RG_TARGETS + + fetch_rg(args.bin_dir, targets) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/codex-cli/scripts/rg_utils.py b/codex-cli/scripts/rg_utils.py new file mode 100644 index 0000000000..217f9b4d9e --- /dev/null +++ b/codex-cli/scripts/rg_utils.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Shared helpers for staging ripgrep binaries.""" + +from __future__ import annotations + +import json +import os +import platform +import shutil +import subprocess +import tarfile +import tempfile +import zipfile +from pathlib import Path +from typing import Iterable, Sequence +from urllib.parse import urlparse +from urllib.request import urlopen + +SCRIPT_DIR = Path(__file__).resolve().parent +CODEX_CLI_ROOT = SCRIPT_DIR.parent +RG_MANIFEST = CODEX_CLI_ROOT / "bin" / "rg" + +RG_TARGET_PLATFORM_PAIRS: list[tuple[str, str]] = [ + ("x86_64-unknown-linux-musl", "linux-x86_64"), + ("aarch64-unknown-linux-musl", "linux-aarch64"), + ("x86_64-apple-darwin", "macos-x86_64"), + ("aarch64-apple-darwin", "macos-aarch64"), + ("x86_64-pc-windows-msvc.exe", "windows-x86_64"), + ("aarch64-pc-windows-msvc.exe", "windows-aarch64"), +] +RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS} +DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS] + + +def detect_current_target() -> str | None: + """Return the Codex target triple for the current platform, if known.""" + + sys_platform = platform.system().lower() + machine = platform.machine().lower() + + if sys_platform in {"linux", "linux2"}: + if machine in {"x86_64", "amd64"}: + return "x86_64-unknown-linux-musl" + if machine in {"arm64", "aarch64"}: + return "aarch64-unknown-linux-musl" + elif sys_platform == "darwin": + if machine == "x86_64": + return "x86_64-apple-darwin" + if machine in {"arm64", "aarch64"}: + return "aarch64-apple-darwin" + elif sys_platform.startswith("win"): + if machine in {"x86_64", "amd64"}: + return "x86_64-pc-windows-msvc.exe" + if machine in {"arm64", "aarch64"}: + return "aarch64-pc-windows-msvc.exe" + return None + + +def fetch_rg( + bin_dir: Path, + targets: Sequence[str] | None = None, + *, + manifest_path: Path = RG_MANIFEST, +) -> list[Path]: + """Download ripgrep binaries described by the DotSlash manifest. + + Args: + bin_dir: Destination directory where the binaries will be written. + targets: Optional iterable of Codex target triples. When omitted, all + supported targets are fetched. + manifest_path: Path to the DotSlash manifest describing providers. + + Returns: + List of Paths to the downloaded binaries. + """ + + if targets is None: + targets = DEFAULT_RG_TARGETS + + if not manifest_path.exists(): + raise FileNotFoundError(f"DotSlash manifest not found: {manifest_path}") + + manifest = json.loads(manifest_path.read_text()) + platforms = manifest.get("platforms", {}) + + bin_dir.mkdir(parents=True, exist_ok=True) + + results: list[Path] = [] + for target in targets: + platform_key = RG_TARGET_TO_PLATFORM.get(target) + if platform_key is None: + raise ValueError(f"Unsupported ripgrep target '{target}'.") + + platform_info = platforms.get(platform_key) + if platform_info is None: + raise RuntimeError( + f"Platform '{platform_key}' not found in manifest {manifest_path}." + ) + + providers = platform_info.get("providers", []) + if not providers: + raise RuntimeError( + f"No providers listed for platform '{platform_key}' in {manifest_path}." + ) + + url = providers[0]["url"] + archive_format = platform_info.get("format", "zst") + archive_path = platform_info.get("path") + + dest_name = f"rg-{target}" + dest = bin_dir / dest_name + if platform_key.startswith("win") and dest.suffix != ".exe": + dest = dest.with_suffix(".exe") + + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = Path(tmp_dir_str) + archive_filename = os.path.basename(urlparse(url).path) + download_path = tmp_dir / archive_filename + _download_file(url, download_path) + + dest.parent.mkdir(parents=True, exist_ok=True) + dest.unlink(missing_ok=True) + _extract_archive(download_path, archive_format, archive_path, dest) + + if not platform_key.startswith("win"): + dest.chmod(0o755) + + results.append(dest) + + return results + + +def _download_file(url: str, dest: Path) -> None: + """Download the content at url and write it to dest.""" + + dest.parent.mkdir(parents=True, exist_ok=True) + with urlopen(url) as response, open(dest, "wb") as out: + shutil.copyfileobj(response, out) + + +def _extract_archive( + archive_path: Path, + archive_format: str, + archive_member: str | None, + dest: Path, +) -> None: + """Extract `archive_member` from `archive_path` into `dest`.""" + + dest.parent.mkdir(parents=True, exist_ok=True) + + if archive_format == "zst": + output_path = archive_path.parent / dest.name + subprocess.run( + ["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)], + check=True, + ) + shutil.move(str(output_path), dest) + return + + if archive_format == "tar.gz": + if not archive_member: + raise RuntimeError("Missing 'path' for tar.gz archive in DotSlash manifest.") + with tarfile.open(archive_path, "r:gz") as tar: + try: + member = tar.getmember(archive_member) + except KeyError as exc: + raise RuntimeError( + f"Entry '{archive_member}' not found in archive {archive_path}." + ) from exc + tar.extract(member, path=archive_path.parent) + extracted = archive_path.parent / archive_member + shutil.move(str(extracted), dest) + return + + if archive_format == "zip": + if not archive_member: + raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.") + with zipfile.ZipFile(archive_path) as archive: + try: + with archive.open(archive_member) as src, open(dest, "wb") as out: + shutil.copyfileobj(src, out) + except KeyError as exc: + raise RuntimeError( + f"Entry '{archive_member}' not found in archive {archive_path}." + ) from exc + return + + raise RuntimeError(f"Unsupported archive format '{archive_format}'.") + + +__all__ = [ + "DEFAULT_RG_TARGETS", + "RG_TARGET_PLATFORM_PAIRS", + "RG_TARGET_TO_PLATFORM", + "detect_current_target", + "fetch_rg", +] diff --git a/codex-cli/scripts/stage_release.sh b/codex-cli/scripts/stage_release.sh index 96236fc53c..02461e9018 100755 --- a/codex-cli/scripts/stage_release.sh +++ b/codex-cli/scripts/stage_release.sh @@ -7,22 +7,23 @@ # Usage: # # --tmp : Use instead of a freshly created temp directory. +# --version : Version string to write into package.json. +# --workflow-url : Workflow run that produced the native binaries. # -h|--help : Print usage. # # ----------------------------------------------------------------------------- set -euo pipefail -# Helper - usage / flag parsing - usage() { cat <&2 - usage 1 + if [[ -z "$TMPDIR" ]]; then + TMPDIR="$1" + else + echo "Unexpected extra argument: $1" >&2 + usage 1 + fi ;; esac shift done -# Fallback when the caller did not specify a directory. -# If no directory was specified create a fresh temporary one. if [[ -z "$TMPDIR" ]]; then TMPDIR="$(mktemp -d)" fi -# Ensure the directory exists, then resolve to an absolute path. mkdir -p "$TMPDIR" TMPDIR="$(cd "$TMPDIR" && pwd)" -# Main build logic - -echo "Staging release in $TMPDIR" - -# The script lives in codex-cli/scripts/ - change into codex-cli root so that -# relative paths keep working. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CODEX_CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -pushd "$CODEX_CLI_ROOT" >/dev/null +BUILD_ARGS=( + --version "$VERSION" + --staging-dir "$TMPDIR" +) +if [[ -n "$WORKFLOW_URL" ]]; then + BUILD_ARGS+=(--workflow-url "$WORKFLOW_URL") +fi -# 1. Build the JS artifacts --------------------------------------------------- +python3 "$CODEX_CLI_ROOT/scripts/build_npm_package.py" "${BUILD_ARGS[@]}" -# Paths inside the staged package -mkdir -p "$TMPDIR/bin" +cat < "$TMPDIR/package.json" - -# 2. Native runtime deps (sandbox plus optional Rust binaries) - -./scripts/install_native_deps.sh --workflow-url "$WORKFLOW_URL" "$TMPDIR" - -popd >/dev/null - -echo "Staged version $VERSION for release in $TMPDIR" - -echo "Verify the CLI:" -echo " node ${TMPDIR}/bin/codex.js --version" -echo " node ${TMPDIR}/bin/codex.js --help" - -# Print final hint for convenience -echo "Next: cd \"$TMPDIR\" && npm publish" +Next: cd "$TMPDIR" && npm publish +EOF diff --git a/codex-cli/scripts/stage_rust_release.py b/codex-cli/scripts/stage_rust_release.py index 9a554b77d0..e1142648c3 100755 --- a/codex-cli/scripts/stage_rust_release.py +++ b/codex-cli/scripts/stage_rust_release.py @@ -23,7 +23,11 @@ forwarded to stage_release.sh. ) parser.add_argument( "--tmp", - help="Optional path to stage the npm package; forwarded to stage_release.sh", + help="Optional path to stage the npm package; forwarded to the builder", + ) + parser.add_argument( + "--pack-output", + help="Optional path to write the generated npm tarball.", ) args = parser.parse_args() version = args.release_version @@ -51,17 +55,19 @@ forwarded to stage_release.sh. current_dir = Path(__file__).parent.resolve() cmd = [ - str(current_dir / "stage_release.sh"), + sys.executable, + str(current_dir / "build_npm_package.py"), "--version", version, "--workflow-url", workflow["url"], ] if args.tmp: - cmd.extend(["--tmp", args.tmp]) + cmd.extend(["--staging-dir", args.tmp]) + if args.pack_output: + cmd.extend(["--pack-output", args.pack_output]) - stage_release = subprocess.run(cmd) - stage_release.check_returncode() + subprocess.run(cmd, check=True) return 0