diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dfd7596d1..f7f5a75c3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,12 +27,13 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - # Run all tasks using workspace filters + # stage_npm_release.py requires DotSlash. + - uses: facebook/install-dotslash@v2 - name: Ensure staging a release works. env: GH_TOKEN: ${{ github.token }} - run: ./codex-cli/scripts/stage_release.sh + run: python3 ./codex-cli/scripts/stage_npm_release.py --release-version 0.40.0 - name: Ensure root README.md contains only ASCII and certain Unicode code points run: ./scripts/asciicheck.py README.md diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index e3776f149c..a3690f48d5 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -193,21 +193,18 @@ jobs: version="${GITHUB_REF_NAME#rust-v}" echo "name=${version}" >> $GITHUB_OUTPUT + # stage_npm_release.py requires DotSlash. + - uses: facebook/install-dotslash@v2 - name: Stage npm package env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail TMP_DIR="${RUNNER_TEMP}/npm-stage" - python3 codex-cli/scripts/stage_rust_release.py \ + python3 codex-cli/scripts/stage_npm_release.py \ --release-version "${{ steps.release_name.outputs.name }}" \ - --tmp "${TMP_DIR}" - mkdir -p dist/npm - # Produce an npm-ready tarball using `npm pack` and store it in dist/npm. - # We then rename it to a stable name used by our publishing script. - (cd "$TMP_DIR" && npm pack --pack-destination "${GITHUB_WORKSPACE}/dist/npm") - mv "${GITHUB_WORKSPACE}"/dist/npm/*.tgz \ - "${GITHUB_WORKSPACE}/dist/npm/codex-npm-${{ steps.release_name.outputs.name }}.tgz" + --tmp "${TMP_DIR}" \ + --pack-output "${GITHUB_WORKSPACE}/dist/npm/codex-npm-${{ steps.release_name.outputs.name }}.tgz" - name: Create GitHub Release uses: softprops/action-gh-release@v2 diff --git a/codex-cli/.gitignore b/codex-cli/.gitignore index f886e64f46..57872d0f1e 100644 --- a/codex-cli/.gitignore +++ b/codex-cli/.gitignore @@ -1,7 +1 @@ -# Added by ./scripts/install_native_deps.sh -/bin/codex-aarch64-apple-darwin -/bin/codex-aarch64-unknown-linux-musl -/bin/codex-linux-sandbox-arm64 -/bin/codex-linux-sandbox-x64 -/bin/codex-x86_64-apple-darwin -/bin/codex-x86_64-unknown-linux-musl +/vendor/ diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index f24065170d..3621f48c12 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -1,6 +1,7 @@ #!/usr/bin/env node // Unified entry point for the Codex CLI. +import { existsSync } from "fs"; import path from "path"; import { fileURLToPath } from "url"; @@ -40,10 +41,10 @@ switch (platform) { case "win32": switch (arch) { case "x64": - targetTriple = "x86_64-pc-windows-msvc.exe"; + targetTriple = "x86_64-pc-windows-msvc"; break; case "arm64": - targetTriple = "aarch64-pc-windows-msvc.exe"; + targetTriple = "aarch64-pc-windows-msvc"; break; default: break; @@ -57,7 +58,10 @@ if (!targetTriple) { throw new Error(`Unsupported platform: ${platform} (${arch})`); } -const binaryPath = path.join(__dirname, "..", "bin", `codex-${targetTriple}`); +const vendorRoot = path.join(__dirname, "..", "vendor"); +const archRoot = path.join(vendorRoot, targetTriple); +const codexBinaryName = process.platform === "win32" ? "codex.exe" : "codex"; +const binaryPath = path.join(archRoot, "codex", codexBinaryName); // Use an asynchronous spawn instead of spawnSync so that Node is able to // respond to signals (e.g. Ctrl-C / SIGINT) while the native binary is @@ -66,23 +70,6 @@ const binaryPath = path.join(__dirname, "..", "bin", `codex-${targetTriple}`); // receives a fatal signal, both processes exit in a predictable manner. const { spawn } = await import("child_process"); -async function tryImport(moduleName) { - try { - // eslint-disable-next-line node/no-unsupported-features/es-syntax - return await import(moduleName); - } catch (err) { - return null; - } -} - -async function resolveRgDir() { - const ripgrep = await tryImport("@vscode/ripgrep"); - if (!ripgrep?.rgPath) { - return null; - } - return path.dirname(ripgrep.rgPath); -} - function getUpdatedPath(newDirs) { const pathSep = process.platform === "win32" ? ";" : ":"; const existingPath = process.env.PATH || ""; @@ -94,9 +81,9 @@ function getUpdatedPath(newDirs) { } const additionalDirs = []; -const rgDir = await resolveRgDir(); -if (rgDir) { - additionalDirs.push(rgDir); +const pathDir = path.join(archRoot, "path"); +if (existsSync(pathDir)) { + additionalDirs.push(pathDir); } const updatedPath = getUpdatedPath(additionalDirs); diff --git a/codex-cli/bin/rg b/codex-cli/bin/rg new file mode 100755 index 0000000000..5a992570a9 --- /dev/null +++ b/codex-cli/bin/rg @@ -0,0 +1,79 @@ +#!/usr/bin/env dotslash + +{ + "name": "rg", + "platforms": { + "macos-aarch64": { + "size": 1787248, + "hash": "blake3", + "digest": "8d9942032585ea8ee805937634238d9aee7b210069f4703c88fbe568e26fb78a", + "format": "tar.gz", + "path": "ripgrep-14.1.1-aarch64-apple-darwin/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-aarch64-apple-darwin.tar.gz" + } + ] + }, + "linux-aarch64": { + "size": 2047405, + "hash": "blake3", + "digest": "0b670b8fa0a3df2762af2fc82cc4932f684ca4c02dbd1260d4f3133fd4b2a515", + "format": "tar.gz", + "path": "ripgrep-14.1.1-aarch64-unknown-linux-gnu/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-aarch64-unknown-linux-gnu.tar.gz" + } + ] + }, + "macos-x86_64": { + "size": 2082672, + "hash": "blake3", + "digest": "e9b862fc8da3127f92791f0ff6a799504154ca9d36c98bf3e60a81c6b1f7289e", + "format": "tar.gz", + "path": "ripgrep-14.1.1-x86_64-apple-darwin/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-apple-darwin.tar.gz" + } + ] + }, + "linux-x86_64": { + "size": 2566310, + "hash": "blake3", + "digest": "f73cca4e54d78c31f832c7f6e2c0b4db8b04fa3eaa747915727d570893dbee76", + "format": "tar.gz", + "path": "ripgrep-14.1.1-x86_64-unknown-linux-musl/rg", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz" + } + ] + }, + "windows-x86_64": { + "size": 2058893, + "hash": "blake3", + "digest": "a8ce1a6fed4f8093ee997e57f33254e94b2cd18e26358b09db599c89882eadbd", + "format": "zip", + "path": "ripgrep-14.1.1-x86_64-pc-windows-msvc/rg.exe", + "providers": [ + { + "url": "https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-pc-windows-msvc.zip" + } + ] + }, + "windows-aarch64": { + "size": 1667740, + "hash": "blake3", + "digest": "47b971a8c4fca1d23a4e7c19bd4d88465ebc395598458133139406d3bf85f3fa", + "format": "zip", + "path": "rg.exe", + "providers": [ + { + "url": "https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-13/ripgrep-v13.0.0-13-aarch64-pc-windows-msvc.zip" + } + ] + } + } +} diff --git a/codex-cli/package-lock.json b/codex-cli/package-lock.json index a1c840ade0..a28bb9cd9f 100644 --- a/codex-cli/package-lock.json +++ b/codex-cli/package-lock.json @@ -2,118 +2,17 @@ "name": "@openai/codex", "version": "0.0.0-dev", "lockfileVersion": 3, - "requires": true, "packages": { "": { "name": "@openai/codex", "version": "0.0.0-dev", "license": "Apache-2.0", - "dependencies": { - "@vscode/ripgrep": "^1.15.14" - }, "bin": { "codex": "bin/codex.js" }, "engines": { "node": ">=20" } - }, - "node_modules/@vscode/ripgrep": { - "version": "1.15.14", - "resolved": "https://registry.npmjs.org/@vscode/ripgrep/-/ripgrep-1.15.14.tgz", - "integrity": "sha512-/G1UJPYlm+trBWQ6cMO3sv6b8D1+G16WaJH1/DSqw32JOVlzgZbLkDxRyzIpTpv30AcYGMkCf5tUqGlW6HbDWw==", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "https-proxy-agent": "^7.0.2", - "proxy-from-env": "^1.1.0", - "yauzl": "^2.9.2" - } - }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "license": "MIT", - "dependencies": { - "pend": "~1.2.0" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", - "license": "MIT" - }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" - }, - "node_modules/yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", - "license": "MIT", - "dependencies": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } } } } diff --git a/codex-cli/package.json b/codex-cli/package.json index 02124f3257..f688418226 100644 --- a/codex-cli/package.json +++ b/codex-cli/package.json @@ -11,16 +11,13 @@ }, "files": [ "bin", - "dist" + "vendor" ], "repository": { "type": "git", "url": "git+https://github.com/openai/codex.git", "directory": "codex-cli" }, - "dependencies": { - "@vscode/ripgrep": "^1.15.14" - }, "devDependencies": { "prettier": "^3.3.3" } diff --git a/codex-cli/scripts/README.md b/codex-cli/scripts/README.md index 21e4f3e883..4532cd94c4 100644 --- a/codex-cli/scripts/README.md +++ b/codex-cli/scripts/README.md @@ -5,5 +5,5 @@ Run the following: To build the 0.2.x or later version of the npm module, which runs the Rust version of the CLI, build it as follows: ```bash -./codex-cli/scripts/stage_rust_release.py --release-version 0.6.0 +./codex-cli/scripts/stage_npm_release.py --release-version 0.6.0 ``` diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py new file mode 100755 index 0000000000..05c87db29f --- /dev/null +++ b/codex-cli/scripts/build_npm_package.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Stage and optionally package the @openai/codex npm module.""" + +import argparse +import importlib.util +import json +import re +import shutil +import subprocess +import tempfile +from pathlib import Path +import sys +from typing import Sequence + +SCRIPT_DIR = Path(__file__).resolve().parent +CODEX_CLI_ROOT = SCRIPT_DIR.parent +REPO_ROOT = CODEX_CLI_ROOT.parent +GITHUB_REPO = "openai/codex" + + +def _load_rg_utils(): + spec = importlib.util.spec_from_file_location("rg_utils", SCRIPT_DIR / "rg_utils.py") + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load rg_utils module") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +rg_utils = _load_rg_utils() +DEFAULT_RG_TARGETS = rg_utils.DEFAULT_RG_TARGETS +fetch_rg = rg_utils.fetch_rg + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the Codex CLI npm package.") + parser.add_argument( + "--version", + required=True, + help="Version number to write to package.json inside the staged package.", + ) + parser.add_argument( + "--workflow-url", + help="Optional GitHub Actions workflow run URL used to download native binaries.", + ) + parser.add_argument( + "--staging-dir", + type=Path, + help=( + "Directory to stage the package contents. Defaults to a new temporary directory " + "if omitted. The directory must be empty when provided." + ), + ) + parser.add_argument( + "--pack-output", + type=Path, + help="Path where the generated npm tarball should be written.", + ) + parser.add_argument( + "--rg-target", + action="append", + dest="rg_targets", + help="Codex target triple for which ripgrep should be fetched. Repeatable.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + staging_dir, created_temp = prepare_staging_dir(args.staging_dir) + + try: + stage_sources(staging_dir, args.version) + workflow_url = args.workflow_url or resolve_latest_alpha_workflow_url() + install_native_binaries(staging_dir, workflow_url) + + rg_targets: Sequence[str] | None = args.rg_targets + if rg_targets is None: + rg_targets = DEFAULT_RG_TARGETS + fetch_rg(staging_dir / "vendor", rg_targets) + + print(f"Staged package in {staging_dir}") + + if args.pack_output is not None: + output_path = run_npm_pack(staging_dir, args.pack_output) + print(f"npm pack output written to {output_path}") + finally: + if created_temp: + # Preserve the staging directory for further inspection. + pass + + return 0 + + +def prepare_staging_dir(staging_dir: Path | None) -> tuple[Path, bool]: + if staging_dir is not None: + staging_dir = staging_dir.resolve() + staging_dir.mkdir(parents=True, exist_ok=True) + if any(staging_dir.iterdir()): + raise RuntimeError(f"Staging directory {staging_dir} is not empty.") + return staging_dir, False + + temp_dir = Path(tempfile.mkdtemp(prefix="codex-npm-stage-")) + return temp_dir, True + + +def stage_sources(staging_dir: Path, version: str) -> None: + bin_dir = staging_dir / "bin" + bin_dir.mkdir(parents=True, exist_ok=True) + + shutil.copy2(CODEX_CLI_ROOT / "bin" / "codex.js", bin_dir / "codex.js") + rg_manifest = CODEX_CLI_ROOT / "bin" / "rg" + if rg_manifest.exists(): + shutil.copy2(rg_manifest, bin_dir / "rg") + + readme_src = REPO_ROOT / "README.md" + if readme_src.exists(): + shutil.copy2(readme_src, staging_dir / "README.md") + + with open(CODEX_CLI_ROOT / "package.json", "r", encoding="utf-8") as fh: + package_json = json.load(fh) + package_json["version"] = version + + with open(staging_dir / "package.json", "w", encoding="utf-8") as out: + json.dump(package_json, out, indent=2) + out.write("\n") + + +def install_native_binaries(staging_dir: Path, workflow_url: str | None) -> None: + cmd = [sys.executable, "./scripts/install_native_deps.py"] + if workflow_url: + cmd.extend(["--workflow-url", workflow_url]) + cmd.append(str(staging_dir)) + subprocess.run(cmd, cwd=CODEX_CLI_ROOT, check=True) + + +def resolve_latest_alpha_workflow_url() -> str: + version = determine_latest_alpha_version() + workflow_url = fetch_workflow_url_for_version(version) + if not workflow_url: + raise RuntimeError(f"Unable to locate workflow for version {version}.") + return workflow_url + + +def determine_latest_alpha_version() -> str: + releases = list_releases() + best_key: tuple[int, int, int, int] | None = None + best_version: str | None = None + pattern = re.compile(r"^rust-v(\d+)\.(\d+)\.(\d+)-alpha\.(\d+)$") + for release in releases: + tag = release.get("tag_name", "") + match = pattern.match(tag) + if not match: + continue + key = tuple(int(match.group(i)) for i in range(1, 5)) + if best_key is None or key > best_key: + best_key = key + best_version = ( + f"{match.group(1)}.{match.group(2)}.{match.group(3)}-alpha.{match.group(4)}" + ) + + if best_version is None: + raise RuntimeError("No alpha releases found when resolving workflow URL.") + return best_version + + +def list_releases() -> list[dict]: + result = subprocess.run( + ["gh", "api", f"/repos/{GITHUB_REPO}/releases?per_page=100"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to list releases: {result.stderr.strip() or result.stdout.strip()}" + ) + try: + releases = json.loads(result.stdout or "[]") + except json.JSONDecodeError as exc: + raise RuntimeError("Unable to parse releases JSON.") from exc + if not isinstance(releases, list): + raise RuntimeError("Unexpected response when listing releases.") + return releases + + +def fetch_workflow_url_for_version(version: str) -> str | None: + ref = f"rust-v{version}" + result = subprocess.run( + [ + "gh", + "run", + "list", + "--branch", + ref, + "--limit", + "20", + "--json", + "workflowName,url", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to list workflow runs for {ref}: " + f"{result.stderr.strip() or result.stdout.strip()}" + ) + + try: + runs = json.loads(result.stdout or "[]") + except json.JSONDecodeError as exc: + raise RuntimeError("Unable to parse workflow run listing.") from exc + + for run in runs: + if run.get("workflowName") == "rust-release": + url = run.get("url") + if url: + return url + return None + + +def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: + output_path = output_path.resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory(prefix="codex-npm-pack-") as pack_dir_str: + pack_dir = Path(pack_dir_str) + result = subprocess.run( + ["npm", "pack", "--json", "--pack-destination", str(pack_dir)], + cwd=staging_dir, + check=True, + text=True, + capture_output=True, + ) + + try: + pack_output = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError("Failed to parse npm pack output.") from exc + + if not pack_output: + raise RuntimeError("npm pack did not produce an output tarball.") + + tarball_name = pack_output[0].get("filename") or pack_output[0].get("name") + if not tarball_name: + raise RuntimeError("Unable to determine npm pack output filename.") + + tarball_path = pack_dir / tarball_name + if not tarball_path.exists(): + raise RuntimeError(f"Expected npm pack output not found: {tarball_path}") + + shutil.move(str(tarball_path), output_path) + + return output_path + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py new file mode 100755 index 0000000000..c5673a4a45 --- /dev/null +++ b/codex-cli/scripts/install_native_deps.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Install Codex native binaries (Rust CLI plus ripgrep helpers).""" + +from __future__ import annotations + +import argparse +import importlib.util +import os +import shutil +import subprocess +import tempfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Iterable + +SCRIPT_DIR = Path(__file__).resolve().parent +CODEX_CLI_ROOT = SCRIPT_DIR.parent +DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/17952349351" # rust-v0.40.0 +VENDOR_DIR_NAME = "vendor" +CODEX_TARGETS = ( + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc", +) + + +def _load_rg_utils(): + spec = importlib.util.spec_from_file_location("rg_utils", SCRIPT_DIR / "rg_utils.py") + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load rg_utils module") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +rg_utils = _load_rg_utils() +fetch_rg = rg_utils.fetch_rg +extract_archive = rg_utils.extract_archive +DEFAULT_RG_TARGETS = rg_utils.DEFAULT_RG_TARGETS + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Install native Codex binaries.") + parser.add_argument( + "--workflow-url", + help=( + "GitHub Actions workflow URL that produced the artifacts. Defaults to a " + "known good run when omitted." + ), + ) + parser.add_argument( + "root", + nargs="?", + type=Path, + help=( + "Directory containing package.json for the staged package. If omitted, the " + "repository checkout is used." + ), + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + + codex_cli_root = (args.root or CODEX_CLI_ROOT).resolve() + vendor_dir = codex_cli_root / VENDOR_DIR_NAME + vendor_dir.mkdir(parents=True, exist_ok=True) + + workflow_url = (args.workflow_url or DEFAULT_WORKFLOW_URL).strip() + if not workflow_url: + workflow_url = DEFAULT_WORKFLOW_URL + + _prepare_vendor_layout(vendor_dir, CODEX_TARGETS) + + workflow_id = workflow_url.rstrip("/").split("/")[-1] + + with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: + artifacts_dir = Path(artifacts_dir_str) + _download_artifacts(workflow_id, artifacts_dir) + install_codex_binaries(artifacts_dir, vendor_dir, CODEX_TARGETS) + + fetch_rg(vendor_dir, DEFAULT_RG_TARGETS) + + print(f"Installed native dependencies into {vendor_dir}") + return 0 + + +def _prepare_vendor_layout(vendor_dir: Path, targets: Iterable[str]) -> None: + # Clear legacy file layout living directly in bin/. + bin_dir = vendor_dir.parent / "bin" + for legacy in bin_dir.glob("codex-*"): + if legacy.is_file(): + try: + legacy.unlink() + except OSError: + pass + for legacy in bin_dir.glob("rg-*"): + if legacy.is_file(): + try: + legacy.unlink() + except OSError: + pass + for target in targets: + shutil.rmtree(bin_dir / target, ignore_errors=True) + + shutil.rmtree(vendor_dir, ignore_errors=True) + vendor_dir.mkdir(parents=True, exist_ok=True) + + +def _download_artifacts(workflow_id: str, dest_dir: Path) -> None: + cmd = [ + "gh", + "run", + "download", + "--dir", + str(dest_dir), + "--repo", + "openai/codex", + workflow_id, + ] + subprocess.run(cmd, check=True) + + +def install_codex_binaries( + artifacts_dir: Path, vendor_dir: Path, targets: Iterable[str] +) -> list[Path]: + targets = list(targets) + if not targets: + return [] + + results: dict[str, Path] = {} + max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_map = { + executor.submit( + _install_single_codex_binary, artifacts_dir, vendor_dir, target + ): target + for target in targets + } + + for future in as_completed(future_map): + target = future_map[future] + results[target] = future.result() + + return [results[target] for target in targets] + + +def _install_single_codex_binary( + artifacts_dir: Path, vendor_dir: Path, target: str +) -> Path: + artifact_subdir = artifacts_dir / target + archive_name = _archive_name_for_target(target) + archive_path = artifact_subdir / archive_name + if not archive_path.exists(): + raise FileNotFoundError(f"Expected artifact not found: {archive_path}") + + dest_dir = vendor_dir / target / "codex" + dest_dir.mkdir(parents=True, exist_ok=True) + + binary_name = "codex.exe" if "windows" in target else "codex" + dest = dest_dir / binary_name + dest.unlink(missing_ok=True) + extract_archive(archive_path, "zst", None, dest) + if "windows" not in target: + dest.chmod(0o755) + return dest + + +def _archive_name_for_target(target: str) -> str: + if "windows" in target: + return f"codex-{target}.exe.zst" + return f"codex-{target}.zst" + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/codex-cli/scripts/install_native_deps.sh b/codex-cli/scripts/install_native_deps.sh deleted file mode 100755 index b9fda2543e..0000000000 --- a/codex-cli/scripts/install_native_deps.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -# Install native runtime dependencies for codex-cli. -# -# Usage -# install_native_deps.sh [--workflow-url URL] [CODEX_CLI_ROOT] -# -# The optional RELEASE_ROOT is the path that contains package.json. Omitting -# it installs the binaries into the repository's own bin/ folder to support -# local development. - -set -euo pipefail - -# ------------------ -# Parse arguments -# ------------------ - -CODEX_CLI_ROOT="" - -# Until we start publishing stable GitHub releases, we have to grab the binaries -# from the GitHub Action that created them. Update the URL below to point to the -# appropriate workflow run: -WORKFLOW_URL="https://github.com/openai/codex/actions/runs/17417194663" # rust-v0.28.0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --workflow-url) - shift || { echo "--workflow-url requires an argument"; exit 1; } - if [ -n "$1" ]; then - WORKFLOW_URL="$1" - fi - ;; - *) - if [[ -z "$CODEX_CLI_ROOT" ]]; then - CODEX_CLI_ROOT="$1" - else - echo "Unexpected argument: $1" >&2 - exit 1 - fi - ;; - esac - shift -done - -# ---------------------------------------------------------------------------- -# Determine where the binaries should be installed. -# ---------------------------------------------------------------------------- - -if [ -n "$CODEX_CLI_ROOT" ]; then - # The caller supplied a release root directory. - BIN_DIR="$CODEX_CLI_ROOT/bin" -else - # No argument; fall back to the repo’s own bin directory. - # Resolve the path of this script, then walk up to the repo root. - SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - CODEX_CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - BIN_DIR="$CODEX_CLI_ROOT/bin" -fi - -# Make sure the destination directory exists. -mkdir -p "$BIN_DIR" - -# ---------------------------------------------------------------------------- -# Download and decompress the artifacts from the GitHub Actions workflow. -# ---------------------------------------------------------------------------- - -WORKFLOW_ID="${WORKFLOW_URL##*/}" - -ARTIFACTS_DIR="$(mktemp -d)" -trap 'rm -rf "$ARTIFACTS_DIR"' EXIT - -# NB: The GitHub CLI `gh` must be installed and authenticated. -gh run download --dir "$ARTIFACTS_DIR" --repo openai/codex "$WORKFLOW_ID" - -# x64 Linux -zstd -d "$ARTIFACTS_DIR/x86_64-unknown-linux-musl/codex-x86_64-unknown-linux-musl.zst" \ - -o "$BIN_DIR/codex-x86_64-unknown-linux-musl" -# ARM64 Linux -zstd -d "$ARTIFACTS_DIR/aarch64-unknown-linux-musl/codex-aarch64-unknown-linux-musl.zst" \ - -o "$BIN_DIR/codex-aarch64-unknown-linux-musl" -# x64 macOS -zstd -d "$ARTIFACTS_DIR/x86_64-apple-darwin/codex-x86_64-apple-darwin.zst" \ - -o "$BIN_DIR/codex-x86_64-apple-darwin" -# ARM64 macOS -zstd -d "$ARTIFACTS_DIR/aarch64-apple-darwin/codex-aarch64-apple-darwin.zst" \ - -o "$BIN_DIR/codex-aarch64-apple-darwin" -# x64 Windows -zstd -d "$ARTIFACTS_DIR/x86_64-pc-windows-msvc/codex-x86_64-pc-windows-msvc.exe.zst" \ - -o "$BIN_DIR/codex-x86_64-pc-windows-msvc.exe" -# ARM64 Windows -zstd -d "$ARTIFACTS_DIR/aarch64-pc-windows-msvc/codex-aarch64-pc-windows-msvc.exe.zst" \ - -o "$BIN_DIR/codex-aarch64-pc-windows-msvc.exe" - -echo "Installed native dependencies into $BIN_DIR" diff --git a/codex-cli/scripts/rg_utils.py b/codex-cli/scripts/rg_utils.py new file mode 100644 index 0000000000..88a4999f2e --- /dev/null +++ b/codex-cli/scripts/rg_utils.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Shared helpers for staging ripgrep binaries.""" + +import json +import os +import platform +import shutil +import subprocess +import tarfile +import tempfile +import zipfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Sequence +from urllib.parse import urlparse +from urllib.request import urlopen + +SCRIPT_DIR = Path(__file__).resolve().parent +CODEX_CLI_ROOT = SCRIPT_DIR.parent +RG_MANIFEST = CODEX_CLI_ROOT / "bin" / "rg" + +RG_TARGET_PLATFORM_PAIRS: list[tuple[str, str]] = [ + ("x86_64-unknown-linux-musl", "linux-x86_64"), + ("aarch64-unknown-linux-musl", "linux-aarch64"), + ("x86_64-apple-darwin", "macos-x86_64"), + ("aarch64-apple-darwin", "macos-aarch64"), + ("x86_64-pc-windows-msvc", "windows-x86_64"), + ("aarch64-pc-windows-msvc", "windows-aarch64"), +] +RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS} +DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS] + + +def detect_current_target() -> str | None: + """Return the Codex target triple for the current platform, if known.""" + + sys_platform = platform.system().lower() + machine = platform.machine().lower() + + if sys_platform in {"linux", "linux2"}: + if machine in {"x86_64", "amd64"}: + return "x86_64-unknown-linux-musl" + if machine in {"arm64", "aarch64"}: + return "aarch64-unknown-linux-musl" + elif sys_platform == "darwin": + if machine == "x86_64": + return "x86_64-apple-darwin" + if machine in {"arm64", "aarch64"}: + return "aarch64-apple-darwin" + elif sys_platform.startswith("win"): + if machine in {"x86_64", "amd64"}: + return "x86_64-pc-windows-msvc" + if machine in {"arm64", "aarch64"}: + return "aarch64-pc-windows-msvc" + return None + + +def fetch_rg( + bin_dir: Path, + targets: Sequence[str] | None = None, + *, + manifest_path: Path = RG_MANIFEST, +) -> list[Path]: + """Download ripgrep binaries described by the DotSlash manifest. + + Args: + bin_dir: Destination directory where the binaries will be written. + targets: Optional iterable of Codex target triples. When omitted, all + supported targets are fetched. + manifest_path: Path to the DotSlash manifest describing providers. + + Returns: + List of Paths to the downloaded binaries. + """ + + if targets is None: + targets = DEFAULT_RG_TARGETS + + if not manifest_path.exists(): + raise FileNotFoundError(f"DotSlash manifest not found: {manifest_path}") + + manifest = _load_manifest(manifest_path) + platforms = manifest.get("platforms", {}) + + bin_dir.mkdir(parents=True, exist_ok=True) + + targets = list(targets) + if not targets: + return [] + + task_configs: list[tuple[str, str, dict]] = [] + for target in targets: + platform_key = RG_TARGET_TO_PLATFORM.get(target) + if platform_key is None: + raise ValueError(f"Unsupported ripgrep target '{target}'.") + + platform_info = platforms.get(platform_key) + if platform_info is None: + raise RuntimeError(f"Platform '{platform_key}' not found in manifest {manifest_path}.") + + task_configs.append((target, platform_key, platform_info)) + + results: dict[str, Path] = {} + max_workers = min(len(task_configs), max(1, (os.cpu_count() or 1))) + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_map = { + executor.submit( + _fetch_single_rg, + bin_dir, + target, + platform_key, + platform_info, + manifest_path, + ): target + for target, platform_key, platform_info in task_configs + } + + for future in as_completed(future_map): + target = future_map[future] + results[target] = future.result() + + return [results[target] for target in targets] + + +def _fetch_single_rg( + bin_dir: Path, + target: str, + platform_key: str, + platform_info: dict, + manifest_path: Path, +) -> Path: + providers = platform_info.get("providers", []) + if not providers: + raise RuntimeError( + f"No providers listed for platform '{platform_key}' in {manifest_path}." + ) + + url = providers[0]["url"] + archive_format = platform_info.get("format", "zst") + archive_path = platform_info.get("path") + + dest_dir = bin_dir / target / "path" + dest_dir.mkdir(parents=True, exist_ok=True) + + is_windows = platform_key.startswith("win") + binary_name = "rg.exe" if is_windows else "rg" + dest = dest_dir / binary_name + + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = Path(tmp_dir_str) + archive_filename = os.path.basename(urlparse(url).path) + download_path = tmp_dir / archive_filename + _download_file(url, download_path) + + dest.unlink(missing_ok=True) + extract_archive(download_path, archive_format, archive_path, dest) + + if not is_windows: + dest.chmod(0o755) + + return dest + + +def _download_file(url: str, dest: Path) -> None: + """Download the content at url and write it to dest.""" + + dest.parent.mkdir(parents=True, exist_ok=True) + with urlopen(url) as response, open(dest, "wb") as out: + shutil.copyfileobj(response, out) + + +def extract_archive( + archive_path: Path, + archive_format: str, + archive_member: str | None, + dest: Path, +) -> None: + """Extract `archive_member` from `archive_path` into `dest`.""" + + dest.parent.mkdir(parents=True, exist_ok=True) + + if archive_format == "zst": + output_path = archive_path.parent / dest.name + subprocess.run( + ["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)], + check=True, + ) + shutil.move(str(output_path), dest) + return + + if archive_format == "tar.gz": + if not archive_member: + raise RuntimeError("Missing 'path' for tar.gz archive in DotSlash manifest.") + with tarfile.open(archive_path, "r:gz") as tar: + try: + member = tar.getmember(archive_member) + except KeyError as exc: + raise RuntimeError( + f"Entry '{archive_member}' not found in archive {archive_path}." + ) from exc + tar.extract(member, path=archive_path.parent) + extracted = archive_path.parent / archive_member + shutil.move(str(extracted), dest) + return + + if archive_format == "zip": + if not archive_member: + raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.") + with zipfile.ZipFile(archive_path) as archive: + try: + with archive.open(archive_member) as src, open(dest, "wb") as out: + shutil.copyfileobj(src, out) + except KeyError as exc: + raise RuntimeError( + f"Entry '{archive_member}' not found in archive {archive_path}." + ) from exc + return + + raise RuntimeError(f"Unsupported archive format '{archive_format}'.") + + +def _load_manifest(manifest_path: Path) -> dict: + """Parse a DotSlash manifest using the DotSlash CLI.""" + + cmd = ["dotslash", "--", "parse", str(manifest_path)] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + details = result.stderr.strip() or result.stdout.strip() + raise RuntimeError( + f"Failed to parse DotSlash manifest {manifest_path}: {details or 'unknown error.'}" + ) + + try: + manifest = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Invalid DotSlash manifest output from {manifest_path}.") from exc + + if not isinstance(manifest, dict): + raise RuntimeError( + f"Unexpected DotSlash manifest structure for {manifest_path}: {type(manifest)!r}" + ) + + return manifest + + +__all__ = [ + "DEFAULT_RG_TARGETS", + "RG_TARGET_PLATFORM_PAIRS", + "RG_TARGET_TO_PLATFORM", + "detect_current_target", + "fetch_rg", + "extract_archive", +] diff --git a/codex-cli/scripts/stage_npm_release.py b/codex-cli/scripts/stage_npm_release.py new file mode 100644 index 0000000000..fe74a3a648 --- /dev/null +++ b/codex-cli/scripts/stage_npm_release.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +import argparse +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.dont_write_bytecode = True + + +def main() -> int: + parser = argparse.ArgumentParser( + description="""Stage an npm release for the Codex CLI. + +Run this after the corresponding GitHub Release has been created and use +`--release-version` to specify the version to publish. + +Optionally pass `--tmp` to control the temporary staging directory that will be +forwarded to the builder. +""" + ) + parser.add_argument("--release-version", required=True, help="Version to release, e.g., 0.3.0") + parser.add_argument( + "--tmp", + help="Optional path to stage the npm package; forwarded to the builder", + ) + parser.add_argument( + "--pack-output", + help="Optional path to write the generated npm tarball.", + ) + parser.add_argument( + "--workflow-url", + help=( + "Optional GitHub Actions workflow run URL that produced the native binaries. " + "When omitted, the script resolves the latest rust-release run for the given version." + ), + ) + args = parser.parse_args() + version = args.release_version + + staging_dir, created_temp = prepare_staging_dir(args.tmp) + + workflow_url = args.workflow_url + head_sha: str | None = None + if not workflow_url: + workflow = resolve_release_workflow(version) + workflow_url = workflow["url"] + head_sha = workflow.get("headSha") + + if head_sha: + print(f"should `git checkout {head_sha}`") + + current_dir = Path(__file__).parent.resolve() + cmd = [ + sys.executable, + str(current_dir / "build_npm_package.py"), + "--version", + version, + ] + if workflow_url: + cmd.extend(["--workflow-url", workflow_url]) + cmd.extend(["--staging-dir", str(staging_dir)]) + if args.pack_output: + cmd.extend(["--pack-output", args.pack_output]) + + subprocess.run(cmd, check=True) + + staging_dir_str = str(staging_dir) + print( + f"Staged version {version} for release in {staging_dir_str}\n\n" + "Verify the CLI:\n" + f" node {staging_dir_str}/bin/codex.js --version\n" + f" node {staging_dir_str}/bin/codex.js --help\n\n" + f'Next: cd "{staging_dir_str}" && npm publish' + ) + + if created_temp: + print("(Temporary staging directory preserved for inspection.)") + + return 0 + + +def prepare_staging_dir(tmp_path: str | None) -> tuple[Path, bool]: + if tmp_path: + staging_dir = Path(tmp_path).expanduser().resolve() + staging_dir.mkdir(parents=True, exist_ok=True) + return staging_dir, False + + temp_dir = Path(tempfile.mkdtemp(prefix="codex-npm-stage-")) + return temp_dir, True + + +def resolve_release_workflow(version: str) -> dict: + gh_run = subprocess.run( + [ + "gh", + "run", + "list", + "--branch", + f"rust-v{version}", + "--json", + "workflowName,url,headSha", + "--jq", + 'first(.[] | select(.workflowName == "rust-release"))', + ], + stdout=subprocess.PIPE, + check=True, + ) + gh_run.check_returncode() + workflow = json.loads(gh_run.stdout) + if not workflow: + raise RuntimeError(f"Unable to find rust-release workflow for version {version}.") + return workflow + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/codex-cli/scripts/stage_release.sh b/codex-cli/scripts/stage_release.sh deleted file mode 100755 index 96236fc53c..0000000000 --- a/codex-cli/scripts/stage_release.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# ----------------------------------------------------------------------------- -# stage_release.sh -# ----------------------------------------------------------------------------- -# Stages an npm release for @openai/codex. -# -# Usage: -# -# --tmp : Use instead of a freshly created temp directory. -# -h|--help : Print usage. -# -# ----------------------------------------------------------------------------- - -set -euo pipefail - -# Helper - usage / flag parsing - -usage() { - cat <&2 - usage 1 - ;; - *) - echo "Unexpected extra argument: $1" >&2 - usage 1 - ;; - esac - shift -done - -# Fallback when the caller did not specify a directory. -# If no directory was specified create a fresh temporary one. -if [[ -z "$TMPDIR" ]]; then - TMPDIR="$(mktemp -d)" -fi - -# Ensure the directory exists, then resolve to an absolute path. -mkdir -p "$TMPDIR" -TMPDIR="$(cd "$TMPDIR" && pwd)" - -# Main build logic - -echo "Staging release in $TMPDIR" - -# The script lives in codex-cli/scripts/ - change into codex-cli root so that -# relative paths keep working. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CODEX_CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - -pushd "$CODEX_CLI_ROOT" >/dev/null - -# 1. Build the JS artifacts --------------------------------------------------- - -# Paths inside the staged package -mkdir -p "$TMPDIR/bin" - -cp -r bin/codex.js "$TMPDIR/bin/codex.js" -cp ../README.md "$TMPDIR" || true # README is one level up - ignore if missing - -# Modify package.json - bump version and optionally add the native directory to -# the files array so that the binaries are published to npm. - -jq --arg version "$VERSION" \ - '.version = $version' \ - package.json > "$TMPDIR/package.json" - -# 2. Native runtime deps (sandbox plus optional Rust binaries) - -./scripts/install_native_deps.sh --workflow-url "$WORKFLOW_URL" "$TMPDIR" - -popd >/dev/null - -echo "Staged version $VERSION for release in $TMPDIR" - -echo "Verify the CLI:" -echo " node ${TMPDIR}/bin/codex.js --version" -echo " node ${TMPDIR}/bin/codex.js --help" - -# Print final hint for convenience -echo "Next: cd \"$TMPDIR\" && npm publish" diff --git a/codex-cli/scripts/stage_rust_release.py b/codex-cli/scripts/stage_rust_release.py deleted file mode 100755 index 9a554b77d0..0000000000 --- a/codex-cli/scripts/stage_rust_release.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 - -import json -import subprocess -import sys -import argparse -from pathlib import Path - - -def main() -> int: - parser = argparse.ArgumentParser( - description="""Stage a release for the npm module. - -Run this after the GitHub Release has been created and use -`--release-version` to specify the version to release. - -Optionally pass `--tmp` to control the temporary staging directory that will be -forwarded to stage_release.sh. -""" - ) - parser.add_argument( - "--release-version", required=True, help="Version to release, e.g., 0.3.0" - ) - parser.add_argument( - "--tmp", - help="Optional path to stage the npm package; forwarded to stage_release.sh", - ) - args = parser.parse_args() - version = args.release_version - - gh_run = subprocess.run( - [ - "gh", - "run", - "list", - "--branch", - f"rust-v{version}", - "--json", - "workflowName,url,headSha", - "--jq", - 'first(.[] | select(.workflowName == "rust-release"))', - ], - stdout=subprocess.PIPE, - check=True, - ) - gh_run.check_returncode() - workflow = json.loads(gh_run.stdout) - sha = workflow["headSha"] - - print(f"should `git checkout {sha}`") - - current_dir = Path(__file__).parent.resolve() - cmd = [ - str(current_dir / "stage_release.sh"), - "--version", - version, - "--workflow-url", - workflow["url"], - ] - if args.tmp: - cmd.extend(["--tmp", args.tmp]) - - stage_release = subprocess.run(cmd) - stage_release.check_returncode() - - return 0 - - -if __name__ == "__main__": - sys.exit(main())