From c84a22202a3db44f828c3dd16356d2719a255783 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 23 Sep 2025 10:51:11 -0700 Subject: [PATCH] fix: vendor ripgrep in the npm module --- .github/workflows/rust-release.yml | 9 +- codex-cli/.gitignore | 14 +- codex-cli/bin/codex.js | 32 +-- codex-cli/bin/rg | 79 +++++++ codex-cli/package-lock.json | 101 --------- codex-cli/package.json | 3 - codex-cli/scripts/build_npm_package.py | 259 +++++++++++++++++++++++ codex-cli/scripts/install_native_deps.py | 147 +++++++++++++ codex-cli/scripts/install_native_deps.sh | 94 -------- codex-cli/scripts/rg_utils.py | 221 +++++++++++++++++++ codex-cli/scripts/stage_release.sh | 76 +++---- codex-cli/scripts/stage_rust_release.py | 16 +- 12 files changed, 764 insertions(+), 287 deletions(-) create mode 100755 codex-cli/bin/rg create mode 100755 codex-cli/scripts/build_npm_package.py create mode 100755 codex-cli/scripts/install_native_deps.py delete mode 100755 codex-cli/scripts/install_native_deps.sh create mode 100644 codex-cli/scripts/rg_utils.py diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index e3776f149c..5875d19c6d 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -201,13 +201,8 @@ jobs: TMP_DIR="${RUNNER_TEMP}/npm-stage" python3 codex-cli/scripts/stage_rust_release.py \ --release-version "${{ steps.release_name.outputs.name }}" \ - --tmp "${TMP_DIR}" - mkdir -p dist/npm - # Produce an npm-ready tarball using `npm pack` and store it in dist/npm. - # We then rename it to a stable name used by our publishing script. - (cd "$TMP_DIR" && npm pack --pack-destination "${GITHUB_WORKSPACE}/dist/npm") - mv "${GITHUB_WORKSPACE}"/dist/npm/*.tgz \ - "${GITHUB_WORKSPACE}/dist/npm/codex-npm-${{ steps.release_name.outputs.name }}.tgz" + --tmp "${TMP_DIR}" \ + --pack-output "${GITHUB_WORKSPACE}/dist/npm/codex-npm-${{ steps.release_name.outputs.name }}.tgz" - name: Create GitHub Release uses: softprops/action-gh-release@v2 diff --git a/codex-cli/.gitignore b/codex-cli/.gitignore index f886e64f46..a6dc92a909 100644 --- a/codex-cli/.gitignore +++ b/codex-cli/.gitignore @@ -1,7 +1,7 @@ -# Added by ./scripts/install_native_deps.sh -/bin/codex-aarch64-apple-darwin -/bin/codex-aarch64-unknown-linux-musl -/bin/codex-linux-sandbox-arm64 -/bin/codex-linux-sandbox-x64 -/bin/codex-x86_64-apple-darwin -/bin/codex-x86_64-unknown-linux-musl +# Needed by ./scripts/install_native_deps.py +/bin/aarch64-apple-darwin +/bin/aarch64-unknown-linux-musl +/bin/aarch64-pc-windows-msvc +/bin/x86_64-apple-darwin +/bin/x86_64-unknown-linux-musl +/bin/x86_64-pc-windows-msvc diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index f24065170d..27234f0548 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -1,6 +1,7 @@ #!/usr/bin/env node // Unified entry point for the Codex CLI. +import { existsSync } from "fs"; import path from "path"; import { fileURLToPath } from "url"; @@ -40,10 +41,10 @@ switch (platform) { case "win32": switch (arch) { case "x64": - targetTriple = "x86_64-pc-windows-msvc.exe"; + targetTriple = "x86_64-pc-windows-msvc"; break; case "arm64": - targetTriple = "aarch64-pc-windows-msvc.exe"; + targetTriple = "aarch64-pc-windows-msvc"; break; default: break; @@ -57,7 +58,9 @@ if (!targetTriple) { throw new Error(`Unsupported platform: ${platform} (${arch})`); } -const binaryPath = path.join(__dirname, "..", "bin", `codex-${targetTriple}`); +const archRoot = path.join(__dirname, "..", "bin", targetTriple); +const codexBinaryName = process.platform === "win32" ? "codex.exe" : "codex"; +const binaryPath = path.join(archRoot, "codex", codexBinaryName); // Use an asynchronous spawn instead of spawnSync so that Node is able to // respond to signals (e.g. Ctrl-C / SIGINT) while the native binary is @@ -66,23 +69,6 @@ const binaryPath = path.join(__dirname, "..", "bin", `codex-${targetTriple}`); // receives a fatal signal, both processes exit in a predictable manner. const { spawn } = await import("child_process"); -async function tryImport(moduleName) { - try { - // eslint-disable-next-line node/no-unsupported-features/es-syntax - return await import(moduleName); - } catch (err) { - return null; - } -} - -async function resolveRgDir() { - const ripgrep = await tryImport("@vscode/ripgrep"); - if (!ripgrep?.rgPath) { - return null; - } - return path.dirname(ripgrep.rgPath); -} - function getUpdatedPath(newDirs) { const pathSep = process.platform === "win32" ? ";" : ":"; const existingPath = process.env.PATH || ""; @@ -94,9 +80,9 @@ function getUpdatedPath(newDirs) { } const additionalDirs = []; -const rgDir = await resolveRgDir(); -if (rgDir) { - additionalDirs.push(rgDir); +const pathDir = path.join(archRoot, "path"); +if (existsSync(pathDir)) { + additionalDirs.push(pathDir); } const updatedPath = getUpdatedPath(additionalDirs); diff --git a/codex-cli/bin/rg b/codex-cli/bin/rg new file mode 100755 index 0000000000..5a992570a9 --- /dev/null +++ b/codex-cli/bin/rg @@ -0,0 +1,79 @@ +#!/usr/bin/env dotslash + +{ + "name": "rg", + "platforms": { + "macos-aarch64": { + "size": 1787248, + "hash": "blake3", + "digest": "8d9942032585ea8ee805937634238d9aee7b210069f4703c88fbe568e26fb78a", + "format": "tar.gz", + "path": "ripgrep-14.1.1-aarch64-apple-darwin/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-aarch64-apple-darwin.tar.gz" + } + ] + }, + "linux-aarch64": { + "size": 2047405, + "hash": "blake3", + "digest": "0b670b8fa0a3df2762af2fc82cc4932f684ca4c02dbd1260d4f3133fd4b2a515", + "format": "tar.gz", + "path": "ripgrep-14.1.1-aarch64-unknown-linux-gnu/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-aarch64-unknown-linux-gnu.tar.gz" + } + ] + }, + "macos-x86_64": { + "size": 2082672, + "hash": "blake3", + "digest": "e9b862fc8da3127f92791f0ff6a799504154ca9d36c98bf3e60a81c6b1f7289e", + "format": "tar.gz", + "path": "ripgrep-14.1.1-x86_64-apple-darwin/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-apple-darwin.tar.gz" + } + ] + }, + "linux-x86_64": { + "size": 2566310, + "hash": "blake3", + "digest": "f73cca4e54d78c31f832c7f6e2c0b4db8b04fa3eaa747915727d570893dbee76", + "format": "tar.gz", + "path": "ripgrep-14.1.1-x86_64-unknown-linux-musl/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz" + } + ] + }, + "windows-x86_64": { + "size": 2058893, + "hash": "blake3", + "digest": "a8ce1a6fed4f8093ee997e57f33254e94b2cd18e26358b09db599c89882eadbd", + "format": "zip", + "path": "ripgrep-14.1.1-x86_64-pc-windows-msvc/rg.exe", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-pc-windows-msvc.zip" + } + ] + }, + "windows-aarch64": { + "size": 1667740, + "hash": "blake3", + "digest": "47b971a8c4fca1d23a4e7c19bd4d88465ebc395598458133139406d3bf85f3fa", + "format": "zip", + "path": "rg.exe", + "providers": [ + { + "url": "https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-13/ripgrep-v13.0.0-13-aarch64-pc-windows-msvc.zip" + } + ] + } + } +} diff --git a/codex-cli/package-lock.json b/codex-cli/package-lock.json index a1c840ade0..a28bb9cd9f 100644 --- a/codex-cli/package-lock.json +++ b/codex-cli/package-lock.json @@ -2,118 +2,17 @@ "name": "@openai/codex", "version": "0.0.0-dev", "lockfileVersion": 3, - "requires": true, "packages": { "": { "name": "@openai/codex", "version": "0.0.0-dev", "license": "Apache-2.0", - "dependencies": { - "@vscode/ripgrep": "^1.15.14" - }, "bin": { "codex": "bin/codex.js" }, "engines": { "node": ">=20" } - }, - "node_modules/@vscode/ripgrep": { - "version": "1.15.14", - "resolved": "https://registry.npmjs.org/@vscode/ripgrep/-/ripgrep-1.15.14.tgz", - "integrity": "sha512-/G1UJPYlm+trBWQ6cMO3sv6b8D1+G16WaJH1/DSqw32JOVlzgZbLkDxRyzIpTpv30AcYGMkCf5tUqGlW6HbDWw==", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "https-proxy-agent": "^7.0.2", - "proxy-from-env": "^1.1.0", - "yauzl": "^2.9.2" - } - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "license": "MIT", - "dependencies": { - "pend": "~1.2.0" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", - "license": "MIT" - }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" - }, - "node_modules/yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", - "license": "MIT", - "dependencies": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } } } } diff --git a/codex-cli/package.json b/codex-cli/package.json index 02124f3257..03fbba5577 100644 --- a/codex-cli/package.json +++ b/codex-cli/package.json @@ -18,9 +18,6 @@ "url": "git+https://github.com/openai/codex.git", "directory": "codex-cli" }, - "dependencies": { - "@vscode/ripgrep": "^1.15.14" - }, "devDependencies": { "prettier": "^3.3.3" } diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py new file mode 100755 index 0000000000..d8e75b13aa --- /dev/null +++ b/codex-cli/scripts/build_npm_package.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Stage and optionally package the @openai/codex npm module.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Sequence + +SCRIPT_DIR = Path(__file__).resolve().parent +CODEX_CLI_ROOT = SCRIPT_DIR.parent +REPO_ROOT = CODEX_CLI_ROOT.parent +GITHUB_REPO = "openai/codex" + + +def _load_rg_utils(): + spec = importlib.util.spec_from_file_location("rg_utils", SCRIPT_DIR / "rg_utils.py") + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load rg_utils module") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +rg_utils = _load_rg_utils() +DEFAULT_RG_TARGETS = rg_utils.DEFAULT_RG_TARGETS +fetch_rg = rg_utils.fetch_rg + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the Codex CLI npm package.") + parser.add_argument( + "--version", + required=True, + help="Version number to write to package.json inside the staged package.", + ) + parser.add_argument( + "--workflow-url", + help="Optional GitHub Actions workflow run URL used to download native binaries.", + ) + parser.add_argument( + "--staging-dir", + type=Path, + help=( + "Directory to stage the package contents. Defaults to a new temporary directory " + "if omitted. The directory must be empty when provided." + ), + ) + parser.add_argument( + "--pack-output", + type=Path, + help="Path where the generated npm tarball should be written.", + ) + parser.add_argument( + "--rg-target", + action="append", + dest="rg_targets", + help="Codex target triple for which ripgrep should be fetched. Repeatable.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + staging_dir, created_temp = prepare_staging_dir(args.staging_dir) + + try: + stage_sources(staging_dir, args.version) + workflow_url = args.workflow_url or resolve_latest_alpha_workflow_url() + install_native_binaries(staging_dir, workflow_url) + + rg_targets: Sequence[str] | None = args.rg_targets + if rg_targets is None: + rg_targets = DEFAULT_RG_TARGETS + fetch_rg(staging_dir / "bin", rg_targets) + + print(f"Staged package in {staging_dir}") + + if args.pack_output is not None: + output_path = run_npm_pack(staging_dir, args.pack_output) + print(f"npm pack output written to {output_path}") + finally: + if created_temp: + # Preserve the staging directory for further inspection. + pass + + return 0 + + +def prepare_staging_dir(staging_dir: Path | None) -> tuple[Path, bool]: + if staging_dir is not None: + staging_dir = staging_dir.resolve() + staging_dir.mkdir(parents=True, exist_ok=True) + if any(staging_dir.iterdir()): + raise RuntimeError(f"Staging directory {staging_dir} is not empty.") + return staging_dir, False + + temp_dir = Path(tempfile.mkdtemp(prefix="codex-npm-stage-")) + return temp_dir, True + + +def stage_sources(staging_dir: Path, version: str) -> None: + bin_dir = staging_dir / "bin" + bin_dir.mkdir(parents=True, exist_ok=True) + + shutil.copy2(CODEX_CLI_ROOT / "bin" / "codex.js", bin_dir / "codex.js") + rg_manifest = CODEX_CLI_ROOT / "bin" / "rg" + if rg_manifest.exists(): + shutil.copy2(rg_manifest, bin_dir / "rg") + + readme_src = REPO_ROOT / "README.md" + if readme_src.exists(): + shutil.copy2(readme_src, staging_dir / "README.md") + + with open(CODEX_CLI_ROOT / "package.json", "r", encoding="utf-8") as fh: + package_json = json.load(fh) + package_json["version"] = version + + with open(staging_dir / "package.json", "w", encoding="utf-8") as out: + json.dump(package_json, out, indent=2) + out.write("\n") + + +def install_native_binaries(staging_dir: Path, workflow_url: str | None) -> None: + cmd = [sys.executable, "./scripts/install_native_deps.py"] + if workflow_url: + cmd.extend(["--workflow-url", workflow_url]) + cmd.append(str(staging_dir)) + subprocess.run(cmd, cwd=CODEX_CLI_ROOT, check=True) + + +def resolve_latest_alpha_workflow_url() -> str: + version = determine_latest_alpha_version() + workflow_url = fetch_workflow_url_for_version(version) + if not workflow_url: + raise RuntimeError(f"Unable to locate workflow for version {version}.") + return workflow_url + + +def determine_latest_alpha_version() -> str: + releases = list_releases() + best_key: tuple[int, int, int, int] | None = None + best_version: str | None = None + pattern = re.compile(r"^rust-v(\d+)\.(\d+)\.(\d+)-alpha\.(\d+)$") + for release in releases: + tag = release.get("tag_name", "") + match = pattern.match(tag) + if not match: + continue + key = tuple(int(match.group(i)) for i in range(1, 5)) + if best_key is None or key > best_key: + best_key = key + best_version = f"{match.group(1)}.{match.group(2)}.{match.group(3)}-alpha.{match.group(4)}" + + if best_version is None: + raise RuntimeError("No alpha releases found when resolving workflow URL.") + return best_version + + +def list_releases() -> list[dict]: + result = subprocess.run( + ["gh", "api", f"/repos/{GITHUB_REPO}/releases?per_page=100"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to list releases: {result.stderr.strip() or result.stdout.strip()}" + ) + try: + releases = json.loads(result.stdout or "[]") + except json.JSONDecodeError as exc: + raise RuntimeError("Unable to parse releases JSON.") from exc + if not isinstance(releases, list): + raise RuntimeError("Unexpected response when listing releases.") + return releases + + +def fetch_workflow_url_for_version(version: str) -> str | None: + ref = f"rust-v{version}" + result = subprocess.run( + [ + "gh", + "run", + "list", + "--branch", + ref, + "--limit", + "20", + "--json", + "workflowName,url", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to list workflow runs for {ref}: " + f"{result.stderr.strip() or result.stdout.strip()}" + ) + + try: + runs = json.loads(result.stdout or "[]") + except json.JSONDecodeError as exc: + raise RuntimeError("Unable to parse workflow run listing.") from exc + + for run in runs: + if run.get("workflowName") == "rust-release": + url = run.get("url") + if url: + return url + return None + + +def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: + output_path = output_path.resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory(prefix="codex-npm-pack-") as pack_dir_str: + pack_dir = Path(pack_dir_str) + result = subprocess.run( + ["npm", "pack", "--json", "--pack-destination", str(pack_dir)], + cwd=staging_dir, + check=True, + text=True, + capture_output=True, + ) + + try: + pack_output = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError("Failed to parse npm pack output.") from exc + + if not pack_output: + raise RuntimeError("npm pack did not produce an output tarball.") + + tarball_name = pack_output[0].get("filename") or pack_output[0].get("name") + if not tarball_name: + raise RuntimeError("Unable to determine npm pack output filename.") + + tarball_path = pack_dir / tarball_name + if not tarball_path.exists(): + raise RuntimeError(f"Expected npm pack output not found: {tarball_path}") + + shutil.move(str(tarball_path), output_path) + + return output_path + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py new file mode 100755 index 0000000000..fad7050b96 --- /dev/null +++ b/codex-cli/scripts/install_native_deps.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Install Codex native binaries (Rust CLI plus ripgrep helpers).""" + +from __future__ import annotations + +import argparse +import importlib.util +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Iterable + +SCRIPT_DIR = Path(__file__).resolve().parent +CODEX_CLI_ROOT = SCRIPT_DIR.parent +DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/17952349351" # rust-v0.40.0 +CODEX_TARGETS = ( + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc", +) + + +def _load_rg_utils(): + spec = importlib.util.spec_from_file_location("rg_utils", SCRIPT_DIR / "rg_utils.py") + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load rg_utils module") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +rg_utils = _load_rg_utils() +fetch_rg = rg_utils.fetch_rg +extract_archive = rg_utils.extract_archive +DEFAULT_RG_TARGETS = rg_utils.DEFAULT_RG_TARGETS + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Install native Codex binaries.") + parser.add_argument( + "--workflow-url", + help=( + "GitHub Actions workflow URL that produced the artifacts. Defaults to a " + "known good run when omitted." + ), + ) + parser.add_argument( + "root", + nargs="?", + type=Path, + help=( + "Directory containing package.json for the staged package. If omitted, the " + "repository checkout is used." + ), + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + codex_cli_root = (args.root or CODEX_CLI_ROOT).resolve() + bin_dir = codex_cli_root / "bin" + bin_dir.mkdir(parents=True, exist_ok=True) + + workflow_url = (args.workflow_url or DEFAULT_WORKFLOW_URL).strip() + if not workflow_url: + workflow_url = DEFAULT_WORKFLOW_URL + + _prepare_bin_layout(bin_dir, CODEX_TARGETS) + + workflow_id = workflow_url.rstrip("/").split("/")[-1] + + with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: + artifacts_dir = Path(artifacts_dir_str) + _download_artifacts(workflow_id, artifacts_dir) + install_codex_binaries(artifacts_dir, bin_dir, CODEX_TARGETS) + + fetch_rg(bin_dir, DEFAULT_RG_TARGETS) + + print(f"Installed native dependencies into {bin_dir}") + return 0 + + +def _prepare_bin_layout(bin_dir: Path, targets: Iterable[str]) -> None: + for legacy in bin_dir.glob("codex-*"): + if legacy.is_file(): + legacy.unlink() + for legacy in bin_dir.glob("rg-*"): + if legacy.is_file(): + legacy.unlink() + + for target in targets: + shutil.rmtree(bin_dir / target, ignore_errors=True) + + +def _download_artifacts(workflow_id: str, dest_dir: Path) -> None: + cmd = [ + "gh", + "run", + "download", + "--dir", + str(dest_dir), + "--repo", + "openai/codex", + workflow_id, + ] + subprocess.run(cmd, check=True) + + +def install_codex_binaries( + artifacts_dir: Path, bin_dir: Path, targets: Iterable[str] +) -> list[Path]: + installed: list[Path] = [] + for target in targets: + artifact_subdir = artifacts_dir / target + archive_name = _archive_name_for_target(target) + archive_path = artifact_subdir / archive_name + if not archive_path.exists(): + raise FileNotFoundError(f"Expected artifact not found: {archive_path}") + + dest_dir = bin_dir / target / "codex" + dest_dir.mkdir(parents=True, exist_ok=True) + + binary_name = "codex.exe" if "windows" in target else "codex" + dest = dest_dir / binary_name + dest.unlink(missing_ok=True) + extract_archive(archive_path, "zst", None, dest) + if "windows" not in target: + dest.chmod(0o755) + installed.append(dest) + return installed + + +def _archive_name_for_target(target: str) -> str: + if "windows" in target: + return f"codex-{target}.exe.zst" + return f"codex-{target}.zst" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/codex-cli/scripts/install_native_deps.sh b/codex-cli/scripts/install_native_deps.sh deleted file mode 100755 index b9fda2543e..0000000000 --- a/codex-cli/scripts/install_native_deps.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -# Install native runtime dependencies for codex-cli. -# -# Usage -# install_native_deps.sh [--workflow-url URL] [CODEX_CLI_ROOT] -# -# The optional RELEASE_ROOT is the path that contains package.json. Omitting -# it installs the binaries into the repository's own bin/ folder to support -# local development. - -set -euo pipefail - -# ------------------ -# Parse arguments -# ------------------ - -CODEX_CLI_ROOT="" - -# Until we start publishing stable GitHub releases, we have to grab the binaries -# from the GitHub Action that created them. Update the URL below to point to the -# appropriate workflow run: -WORKFLOW_URL="https://github.com/openai/codex/actions/runs/17417194663" # rust-v0.28.0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --workflow-url) - shift || { echo "--workflow-url requires an argument"; exit 1; } - if [ -n "$1" ]; then - WORKFLOW_URL="$1" - fi - ;; - *) - if [[ -z "$CODEX_CLI_ROOT" ]]; then - CODEX_CLI_ROOT="$1" - else - echo "Unexpected argument: $1" >&2 - exit 1 - fi - ;; - esac - shift -done - -# ---------------------------------------------------------------------------- -# Determine where the binaries should be installed. -# ---------------------------------------------------------------------------- - -if [ -n "$CODEX_CLI_ROOT" ]; then - # The caller supplied a release root directory. - BIN_DIR="$CODEX_CLI_ROOT/bin" -else - # No argument; fall back to the repo’s own bin directory. - # Resolve the path of this script, then walk up to the repo root. - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - CODEX_CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - BIN_DIR="$CODEX_CLI_ROOT/bin" -fi - -# Make sure the destination directory exists. -mkdir -p "$BIN_DIR" - -# ---------------------------------------------------------------------------- -# Download and decompress the artifacts from the GitHub Actions workflow. -# ---------------------------------------------------------------------------- - -WORKFLOW_ID="${WORKFLOW_URL##*/}" - -ARTIFACTS_DIR="$(mktemp -d)" -trap 'rm -rf "$ARTIFACTS_DIR"' EXIT - -# NB: The GitHub CLI `gh` must be installed and authenticated. -gh run download --dir "$ARTIFACTS_DIR" --repo openai/codex "$WORKFLOW_ID" - -# x64 Linux -zstd -d "$ARTIFACTS_DIR/x86_64-unknown-linux-musl/codex-x86_64-unknown-linux-musl.zst" \ - -o "$BIN_DIR/codex-x86_64-unknown-linux-musl" -# ARM64 Linux -zstd -d "$ARTIFACTS_DIR/aarch64-unknown-linux-musl/codex-aarch64-unknown-linux-musl.zst" \ - -o "$BIN_DIR/codex-aarch64-unknown-linux-musl" -# x64 macOS -zstd -d "$ARTIFACTS_DIR/x86_64-apple-darwin/codex-x86_64-apple-darwin.zst" \ - -o "$BIN_DIR/codex-x86_64-apple-darwin" -# ARM64 macOS -zstd -d "$ARTIFACTS_DIR/aarch64-apple-darwin/codex-aarch64-apple-darwin.zst" \ - -o "$BIN_DIR/codex-aarch64-apple-darwin" -# x64 Windows -zstd -d "$ARTIFACTS_DIR/x86_64-pc-windows-msvc/codex-x86_64-pc-windows-msvc.exe.zst" \ - -o "$BIN_DIR/codex-x86_64-pc-windows-msvc.exe" -# ARM64 Windows -zstd -d "$ARTIFACTS_DIR/aarch64-pc-windows-msvc/codex-aarch64-pc-windows-msvc.exe.zst" \ - -o "$BIN_DIR/codex-aarch64-pc-windows-msvc.exe" - -echo "Installed native dependencies into $BIN_DIR" diff --git a/codex-cli/scripts/rg_utils.py b/codex-cli/scripts/rg_utils.py new file mode 100644 index 0000000000..ea5ac29e12 --- /dev/null +++ b/codex-cli/scripts/rg_utils.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Shared helpers for staging ripgrep binaries.""" + +from __future__ import annotations + +import json +import os +import platform +import shutil +import subprocess +import tarfile +import tempfile +import zipfile +from pathlib import Path +from typing import Sequence +from urllib.parse import urlparse +from urllib.request import urlopen + +SCRIPT_DIR = Path(__file__).resolve().parent +CODEX_CLI_ROOT = SCRIPT_DIR.parent +RG_MANIFEST = CODEX_CLI_ROOT / "bin" / "rg" + +RG_TARGET_PLATFORM_PAIRS: list[tuple[str, str]] = [ + ("x86_64-unknown-linux-musl", "linux-x86_64"), + ("aarch64-unknown-linux-musl", "linux-aarch64"), + ("x86_64-apple-darwin", "macos-x86_64"), + ("aarch64-apple-darwin", "macos-aarch64"), + ("x86_64-pc-windows-msvc", "windows-x86_64"), + ("aarch64-pc-windows-msvc", "windows-aarch64"), +] +RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS} +DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS] + + +def detect_current_target() -> str | None: + """Return the Codex target triple for the current platform, if known.""" + + sys_platform = platform.system().lower() + machine = platform.machine().lower() + + if sys_platform in {"linux", "linux2"}: + if machine in {"x86_64", "amd64"}: + return "x86_64-unknown-linux-musl" + if machine in {"arm64", "aarch64"}: + return "aarch64-unknown-linux-musl" + elif sys_platform == "darwin": + if machine == "x86_64": + return "x86_64-apple-darwin" + if machine in {"arm64", "aarch64"}: + return "aarch64-apple-darwin" + elif sys_platform.startswith("win"): + if machine in {"x86_64", "amd64"}: + return "x86_64-pc-windows-msvc" + if machine in {"arm64", "aarch64"}: + return "aarch64-pc-windows-msvc" + return None + + +def fetch_rg( + bin_dir: Path, + targets: Sequence[str] | None = None, + *, + manifest_path: Path = RG_MANIFEST, +) -> list[Path]: + """Download ripgrep binaries described by the DotSlash manifest. + + Args: + bin_dir: Destination directory where the binaries will be written. + targets: Optional iterable of Codex target triples. When omitted, all + supported targets are fetched. + manifest_path: Path to the DotSlash manifest describing providers. + + Returns: + List of Paths to the downloaded binaries. + """ + + if targets is None: + targets = DEFAULT_RG_TARGETS + + if not manifest_path.exists(): + raise FileNotFoundError(f"DotSlash manifest not found: {manifest_path}") + + manifest = _load_manifest(manifest_path) + platforms = manifest.get("platforms", {}) + + bin_dir.mkdir(parents=True, exist_ok=True) + + results: list[Path] = [] + for target in targets: + platform_key = RG_TARGET_TO_PLATFORM.get(target) + if platform_key is None: + raise ValueError(f"Unsupported ripgrep target '{target}'.") + + platform_info = platforms.get(platform_key) + if platform_info is None: + raise RuntimeError(f"Platform '{platform_key}' not found in manifest {manifest_path}.") + + providers = platform_info.get("providers", []) + if not providers: + raise RuntimeError( + f"No providers listed for platform '{platform_key}' in {manifest_path}." + ) + + url = providers[0]["url"] + archive_format = platform_info.get("format", "zst") + archive_path = platform_info.get("path") + + dest_dir = bin_dir / target / "path" + dest_dir.mkdir(parents=True, exist_ok=True) + + is_windows = platform_key.startswith("win") + binary_name = "rg.exe" if is_windows else "rg" + dest = dest_dir / binary_name + + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = Path(tmp_dir_str) + archive_filename = os.path.basename(urlparse(url).path) + download_path = tmp_dir / archive_filename + _download_file(url, download_path) + + dest.unlink(missing_ok=True) + extract_archive(download_path, archive_format, archive_path, dest) + + if not is_windows: + dest.chmod(0o755) + + results.append(dest) + + return results + + +def _download_file(url: str, dest: Path) -> None: + """Download the content at url and write it to dest.""" + + dest.parent.mkdir(parents=True, exist_ok=True) + with urlopen(url) as response, open(dest, "wb") as out: + shutil.copyfileobj(response, out) + + +def extract_archive( + archive_path: Path, + archive_format: str, + archive_member: str | None, + dest: Path, +) -> None: + """Extract `archive_member` from `archive_path` into `dest`.""" + + dest.parent.mkdir(parents=True, exist_ok=True) + + if archive_format == "zst": + output_path = archive_path.parent / dest.name + subprocess.run( + ["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)], + check=True, + ) + shutil.move(str(output_path), dest) + return + + if archive_format == "tar.gz": + if not archive_member: + raise RuntimeError("Missing 'path' for tar.gz archive in DotSlash manifest.") + with tarfile.open(archive_path, "r:gz") as tar: + try: + member = tar.getmember(archive_member) + except KeyError as exc: + raise RuntimeError( + f"Entry '{archive_member}' not found in archive {archive_path}." + ) from exc + tar.extract(member, path=archive_path.parent) + extracted = archive_path.parent / archive_member + shutil.move(str(extracted), dest) + return + + if archive_format == "zip": + if not archive_member: + raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.") + with zipfile.ZipFile(archive_path) as archive: + try: + with archive.open(archive_member) as src, open(dest, "wb") as out: + shutil.copyfileobj(src, out) + except KeyError as exc: + raise RuntimeError( + f"Entry '{archive_member}' not found in archive {archive_path}." + ) from exc + return + + raise RuntimeError(f"Unsupported archive format '{archive_format}'.") + + +def _load_manifest(manifest_path: Path) -> dict: + """Parse a DotSlash manifest using the DotSlash CLI.""" + + cmd = ["dotslash", "--", "parse", str(manifest_path)] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + details = result.stderr.strip() or result.stdout.strip() + raise RuntimeError( + f"Failed to parse DotSlash manifest {manifest_path}: {details or 'unknown error.'}" + ) + + try: + manifest = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Invalid DotSlash manifest output from {manifest_path}.") from exc + + if not isinstance(manifest, dict): + raise RuntimeError( + f"Unexpected DotSlash manifest structure for {manifest_path}: {type(manifest)!r}" + ) + + return manifest + + +__all__ = [ + "DEFAULT_RG_TARGETS", + "RG_TARGET_PLATFORM_PAIRS", + "RG_TARGET_TO_PLATFORM", + "detect_current_target", + "fetch_rg", + "extract_archive", +] diff --git a/codex-cli/scripts/stage_release.sh b/codex-cli/scripts/stage_release.sh index 96236fc53c..02461e9018 100755 --- a/codex-cli/scripts/stage_release.sh +++ b/codex-cli/scripts/stage_release.sh @@ -7,22 +7,23 @@ # Usage: # # --tmp : Use instead of a freshly created temp directory. +# --version : Version string to write into package.json. +# --workflow-url : Workflow run that produced the native binaries. # -h|--help : Print usage. # # ----------------------------------------------------------------------------- set -euo pipefail -# Helper - usage / flag parsing - usage() { cat <&2 - usage 1 + if [[ -z "$TMPDIR" ]]; then + TMPDIR="$1" + else + echo "Unexpected extra argument: $1" >&2 + usage 1 + fi ;; esac shift done -# Fallback when the caller did not specify a directory. -# If no directory was specified create a fresh temporary one. if [[ -z "$TMPDIR" ]]; then TMPDIR="$(mktemp -d)" fi -# Ensure the directory exists, then resolve to an absolute path. mkdir -p "$TMPDIR" TMPDIR="$(cd "$TMPDIR" && pwd)" -# Main build logic - -echo "Staging release in $TMPDIR" - -# The script lives in codex-cli/scripts/ - change into codex-cli root so that -# relative paths keep working. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CODEX_CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -pushd "$CODEX_CLI_ROOT" >/dev/null +BUILD_ARGS=( + --version "$VERSION" + --staging-dir "$TMPDIR" +) +if [[ -n "$WORKFLOW_URL" ]]; then + BUILD_ARGS+=(--workflow-url "$WORKFLOW_URL") +fi -# 1. Build the JS artifacts --------------------------------------------------- +python3 "$CODEX_CLI_ROOT/scripts/build_npm_package.py" "${BUILD_ARGS[@]}" -# Paths inside the staged package -mkdir -p "$TMPDIR/bin" +cat < "$TMPDIR/package.json" - -# 2. Native runtime deps (sandbox plus optional Rust binaries) - -./scripts/install_native_deps.sh --workflow-url "$WORKFLOW_URL" "$TMPDIR" - -popd >/dev/null - -echo "Staged version $VERSION for release in $TMPDIR" - -echo "Verify the CLI:" -echo " node ${TMPDIR}/bin/codex.js --version" -echo " node ${TMPDIR}/bin/codex.js --help" - -# Print final hint for convenience -echo "Next: cd \"$TMPDIR\" && npm publish" +Next: cd "$TMPDIR" && npm publish +EOF diff --git a/codex-cli/scripts/stage_rust_release.py b/codex-cli/scripts/stage_rust_release.py index 9a554b77d0..e1142648c3 100755 --- a/codex-cli/scripts/stage_rust_release.py +++ b/codex-cli/scripts/stage_rust_release.py @@ -23,7 +23,11 @@ forwarded to stage_release.sh. ) parser.add_argument( "--tmp", - help="Optional path to stage the npm package; forwarded to stage_release.sh", + help="Optional path to stage the npm package; forwarded to the builder", + ) + parser.add_argument( + "--pack-output", + help="Optional path to write the generated npm tarball.", ) args = parser.parse_args() version = args.release_version @@ -51,17 +55,19 @@ forwarded to stage_release.sh. current_dir = Path(__file__).parent.resolve() cmd = [ - str(current_dir / "stage_release.sh"), + sys.executable, + str(current_dir / "build_npm_package.py"), "--version", version, "--workflow-url", workflow["url"], ] if args.tmp: - cmd.extend(["--tmp", args.tmp]) + cmd.extend(["--staging-dir", args.tmp]) + if args.pack_output: + cmd.extend(["--pack-output", args.pack_output]) - stage_release = subprocess.run(cmd) - stage_release.check_returncode() + subprocess.run(cmd, check=True) return 0