openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-02 19:43:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: vendor ripgrep in the npm module

Browse Source
This commit is contained in:
Michael Bolin
2025-09-15 11:02:50 -07:00
parent 8c2fd35453
commit 9a01542d1d
11 changed files with 550 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-cli/scripts/build_npm_package.py
Executable
+163
View File
@@ -0,0 +1,163 @@
#!/usr/bin/env python3
"""Stage and optionally package the @openai/codex npm module."""
from __future__ import annotations
import argparse
import json
import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Sequence
SCRIPT_DIR = Path(__file__).resolve().parent
CODEX_CLI_ROOT = SCRIPT_DIR.parent
REPO_ROOT = CODEX_CLI_ROOT.parent
sys.path.insert(0, str(SCRIPT_DIR))
from rg_utils import DEFAULT_RG_TARGETS, fetch_rg # noqa: E402
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build the Codex CLI npm package.")
parser.add_argument(
"--version",
required=True,
help="Version number to write to package.json inside the staged package.",
)
parser.add_argument(
"--workflow-url",
help="Optional GitHub Actions workflow run URL used to download native binaries.",
)
parser.add_argument(
"--staging-dir",
type=Path,
help=(
"Directory to stage the package contents. Defaults to a new temporary directory "
"if omitted. The directory must be empty when provided."
),
)
parser.add_argument(
"--pack-output",
type=Path,
help="Path where the generated npm tarball should be written.",
)
parser.add_argument(
"--rg-target",
action="append",
dest="rg_targets",
help="Codex target triple for which ripgrep should be fetched. Repeatable.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
staging_dir, created_temp = prepare_staging_dir(args.staging_dir)
try:
stage_sources(staging_dir, args.version)
install_native_binaries(staging_dir, args.workflow_url)
rg_targets: Sequence[str] | None = args.rg_targets
if rg_targets is None:
rg_targets = DEFAULT_RG_TARGETS
fetch_rg(staging_dir / "bin", rg_targets)
print(f"Staged package in {staging_dir}")
if args.pack_output is not None:
output_path = run_npm_pack(staging_dir, args.pack_output)
print(f"npm pack output written to {output_path}")
finally:
if created_temp:
# Preserve the staging directory for further inspection.
pass
return 0
def prepare_staging_dir(staging_dir: Path | None) -> tuple[Path, bool]:
if staging_dir is not None:
staging_dir = staging_dir.resolve()
staging_dir.mkdir(parents=True, exist_ok=True)
if any(staging_dir.iterdir()):
raise RuntimeError(f"Staging directory {staging_dir} is not empty.")
return staging_dir, False
temp_dir = Path(tempfile.mkdtemp(prefix="codex-npm-stage-"))
return temp_dir, True
def stage_sources(staging_dir: Path, version: str) -> None:
bin_dir = staging_dir / "bin"
bin_dir.mkdir(parents=True, exist_ok=True)
shutil.copy2(CODEX_CLI_ROOT / "bin" / "codex.js", bin_dir / "codex.js")
rg_manifest = CODEX_CLI_ROOT / "bin" / "rg"
if rg_manifest.exists():
shutil.copy2(rg_manifest, bin_dir / "rg")
readme_src = REPO_ROOT / "README.md"
if readme_src.exists():
shutil.copy2(readme_src, staging_dir / "README.md")
with open(CODEX_CLI_ROOT / "package.json", "r", encoding="utf-8") as fh:
package_json = json.load(fh)
package_json["version"] = version
with open(staging_dir / "package.json", "w", encoding="utf-8") as out:
json.dump(package_json, out, indent=2)
out.write("\n")
def install_native_binaries(staging_dir: Path, workflow_url: str | None) -> None:
cmd = ["./scripts/install_native_deps.sh"]
if workflow_url:
cmd.extend(["--workflow-url", workflow_url])
cmd.append(str(staging_dir))
subprocess.run(cmd, cwd=CODEX_CLI_ROOT, check=True)
def run_npm_pack(staging_dir: Path, output_path: Path) -> Path:
output_path = output_path.resolve()
output_path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="codex-npm-pack-") as pack_dir_str:
pack_dir = Path(pack_dir_str)
result = subprocess.run(
["npm", "pack", "--json", "--pack-destination", str(pack_dir)],
cwd=staging_dir,
check=True,
text=True,
capture_output=True,
)
try:
pack_output = json.loads(result.stdout)
except json.JSONDecodeError as exc:
raise RuntimeError("Failed to parse npm pack output.") from exc
if not pack_output:
raise RuntimeError("npm pack did not produce an output tarball.")
tarball_name = pack_output[0].get("filename") or pack_output[0].get("name")
if not tarball_name:
raise RuntimeError("Unable to determine npm pack output filename.")
tarball_path = pack_dir / tarball_name
if not tarball_path.exists():
raise RuntimeError(f"Expected npm pack output not found: {tarball_path}")
shutil.move(str(tarball_path), output_path)
return output_path
if __name__ == "__main__":
sys.exit(main())
codex-cli/scripts/install_native_deps.sh
+3 -1
View File
@@ -91,4 +91,6 @@ zstd -d "$ARTIFACTS_DIR/x86_64-pc-windows-msvc/codex-x86_64-pc-windows-msvc.exe.
zstd -d "$ARTIFACTS_DIR/aarch64-pc-windows-msvc/codex-aarch64-pc-windows-msvc.exe.zst" \
-o "$BIN_DIR/codex-aarch64-pc-windows-msvc.exe"
echo "Installed native dependencies into $BIN_DIR"
python3 "$CODEX_CLI_ROOT/scripts/install_rg.py" --bin-dir "$BIN_DIR"
echo "Installed native dependencies (including ripgrep) into $BIN_DIR"
codex-cli/scripts/install_rg.py
Executable
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env python3
"""Download ripgrep binaries into the codex-cli bin directory."""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))
from rg_utils import DEFAULT_RG_TARGETS, detect_current_target, fetch_rg # noqa: E402
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Install ripgrep binaries.")
parser.add_argument(
"--bin-dir",
type=Path,
required=True,
help="Directory where ripgrep binaries should be written.",
)
parser.add_argument(
"--target",
action="append",
dest="targets",
help="Codex target triple to fetch. Repeatable. Defaults to all.",
)
parser.add_argument(
"--current-only",
action="store_true",
help="Download ripgrep only for the current platform.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
if args.current_only and args.targets:
print("Cannot combine --current-only with explicit --target options.", file=sys.stderr)
return 1
if args.current_only:
current = detect_current_target()
if current is None:
print("Unable to detect current platform for ripgrep download.", file=sys.stderr)
return 1
targets = [current]
elif args.targets:
targets = args.targets
else:
targets = DEFAULT_RG_TARGETS
fetch_rg(args.bin_dir, targets)
return 0
if __name__ == "__main__":
sys.exit(main())
codex-cli/scripts/rg_utils.py
+197
View File
@@ -0,0 +1,197 @@
#!/usr/bin/env python3
"""Shared helpers for staging ripgrep binaries."""
from __future__ import annotations
import json
import os
import platform
import shutil
import subprocess
import tarfile
import tempfile
import zipfile
from pathlib import Path
from typing import Iterable, Sequence
from urllib.parse import urlparse
from urllib.request import urlopen
SCRIPT_DIR = Path(__file__).resolve().parent
CODEX_CLI_ROOT = SCRIPT_DIR.parent
RG_MANIFEST = CODEX_CLI_ROOT / "bin" / "rg"
RG_TARGET_PLATFORM_PAIRS: list[tuple[str, str]] = [
("x86_64-unknown-linux-musl", "linux-x86_64"),
("aarch64-unknown-linux-musl", "linux-aarch64"),
("x86_64-apple-darwin", "macos-x86_64"),
("aarch64-apple-darwin", "macos-aarch64"),
("x86_64-pc-windows-msvc.exe", "windows-x86_64"),
("aarch64-pc-windows-msvc.exe", "windows-aarch64"),
]
RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS}
DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS]
def detect_current_target() -> str | None:
"""Return the Codex target triple for the current platform, if known."""
sys_platform = platform.system().lower()
machine = platform.machine().lower()
if sys_platform in {"linux", "linux2"}:
if machine in {"x86_64", "amd64"}:
return "x86_64-unknown-linux-musl"
if machine in {"arm64", "aarch64"}:
return "aarch64-unknown-linux-musl"
elif sys_platform == "darwin":
if machine == "x86_64":
return "x86_64-apple-darwin"
if machine in {"arm64", "aarch64"}:
return "aarch64-apple-darwin"
elif sys_platform.startswith("win"):
if machine in {"x86_64", "amd64"}:
return "x86_64-pc-windows-msvc.exe"
if machine in {"arm64", "aarch64"}:
return "aarch64-pc-windows-msvc.exe"
return None
def fetch_rg(
bin_dir: Path,
targets: Sequence[str] | None = None,
*,
manifest_path: Path = RG_MANIFEST,
) -> list[Path]:
"""Download ripgrep binaries described by the DotSlash manifest.
Args:
bin_dir: Destination directory where the binaries will be written.
targets: Optional iterable of Codex target triples. When omitted, all
supported targets are fetched.
manifest_path: Path to the DotSlash manifest describing providers.
Returns:
List of Paths to the downloaded binaries.
"""
if targets is None:
targets = DEFAULT_RG_TARGETS
if not manifest_path.exists():
raise FileNotFoundError(f"DotSlash manifest not found: {manifest_path}")
manifest = json.loads(manifest_path.read_text())
platforms = manifest.get("platforms", {})
bin_dir.mkdir(parents=True, exist_ok=True)
results: list[Path] = []
for target in targets:
platform_key = RG_TARGET_TO_PLATFORM.get(target)
if platform_key is None:
raise ValueError(f"Unsupported ripgrep target '{target}'.")
platform_info = platforms.get(platform_key)
if platform_info is None:
raise RuntimeError(
f"Platform '{platform_key}' not found in manifest {manifest_path}."
)
providers = platform_info.get("providers", [])
if not providers:
raise RuntimeError(
f"No providers listed for platform '{platform_key}' in {manifest_path}."
)
url = providers[0]["url"]
archive_format = platform_info.get("format", "zst")
archive_path = platform_info.get("path")
dest_name = f"rg-{target}"
dest = bin_dir / dest_name
if platform_key.startswith("win") and dest.suffix != ".exe":
dest = dest.with_suffix(".exe")
with tempfile.TemporaryDirectory() as tmp_dir_str:
tmp_dir = Path(tmp_dir_str)
archive_filename = os.path.basename(urlparse(url).path)
download_path = tmp_dir / archive_filename
_download_file(url, download_path)
dest.parent.mkdir(parents=True, exist_ok=True)
dest.unlink(missing_ok=True)
_extract_archive(download_path, archive_format, archive_path, dest)
if not platform_key.startswith("win"):
dest.chmod(0o755)
results.append(dest)
return results
def _download_file(url: str, dest: Path) -> None:
"""Download the content at url and write it to dest."""
dest.parent.mkdir(parents=True, exist_ok=True)
with urlopen(url) as response, open(dest, "wb") as out:
shutil.copyfileobj(response, out)
def _extract_archive(
archive_path: Path,
archive_format: str,
archive_member: str | None,
dest: Path,
) -> None:
"""Extract `archive_member` from `archive_path` into `dest`."""
dest.parent.mkdir(parents=True, exist_ok=True)
if archive_format == "zst":
output_path = archive_path.parent / dest.name
subprocess.run(
["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)],
check=True,
)
shutil.move(str(output_path), dest)
return
if archive_format == "tar.gz":
if not archive_member:
raise RuntimeError("Missing 'path' for tar.gz archive in DotSlash manifest.")
with tarfile.open(archive_path, "r:gz") as tar:
try:
member = tar.getmember(archive_member)
except KeyError as exc:
raise RuntimeError(
f"Entry '{archive_member}' not found in archive {archive_path}."
) from exc
tar.extract(member, path=archive_path.parent)
extracted = archive_path.parent / archive_member
shutil.move(str(extracted), dest)
return
if archive_format == "zip":
if not archive_member:
raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.")
with zipfile.ZipFile(archive_path) as archive:
try:
with archive.open(archive_member) as src, open(dest, "wb") as out:
shutil.copyfileobj(src, out)
except KeyError as exc:
raise RuntimeError(
f"Entry '{archive_member}' not found in archive {archive_path}."
) from exc
return
raise RuntimeError(f"Unsupported archive format '{archive_format}'.")
__all__ = [
"DEFAULT_RG_TARGETS",
"RG_TARGET_PLATFORM_PAIRS",
"RG_TARGET_TO_PLATFORM",
"detect_current_target",
"fetch_rg",
]
codex-cli/scripts/stage_release.sh
+29 -47
View File
@@ -7,22 +7,23 @@
# Usage:
#
# --tmp <dir> : Use <dir> instead of a freshly created temp directory.
# --version : Version string to write into package.json.
# --workflow-url <url>: Workflow run that produced the native binaries.
# -h|--help : Print usage.
#
# -----------------------------------------------------------------------------
set -euo pipefail
# Helper - usage / flag parsing
usage() {
cat <<EOF
Usage: $(basename "$0") [--tmp DIR] [--version VERSION]
Usage: $(basename "$0") [--tmp DIR] [--version VERSION] [--workflow-url URL]
Options
--tmp DIR Use DIR to stage the release (defaults to a fresh mktemp dir)
--version Specify the version to release (defaults to a timestamp-based version)
-h, --help Show this help
--tmp DIR Use DIR to stage the release (defaults to a fresh mktemp dir)
--version VERSION Set the npm package version (defaults to timestamp scheme)
--workflow-url URL Workflow run URL that produced the native binaries
-h, --help Show this help
Legacy positional argument: the first non-flag argument is still interpreted
as the temporary directory (for backwards compatibility) but is deprecated.
@@ -31,11 +32,9 @@ EOF
}
TMPDIR=""
# Default to a timestamp-based version (keep same scheme as before)
VERSION="$(printf '0.1.%d' "$(date +%y%m%d%H%M)")"
WORKFLOW_URL=""
# Manual flag parser - Bash getopts does not handle GNU long options well.
while [[ $# -gt 0 ]]; do
case "$1" in
--tmp)
@@ -50,7 +49,7 @@ while [[ $# -gt 0 ]]; do
VERSION="$1"
;;
--workflow-url)
shift || { echo "--workflow-url requires an argument"; exit 1; }
shift || { echo "--workflow-url requires an argument"; usage 1; }
WORKFLOW_URL="$1"
;;
-h|--help)
@@ -61,60 +60,43 @@ while [[ $# -gt 0 ]]; do
usage 1
;;
*)
echo "Unexpected extra argument: $1" >&2
usage 1
if [[ -z "$TMPDIR" ]]; then
TMPDIR="$1"
else
echo "Unexpected extra argument: $1" >&2
usage 1
fi
;;
esac
shift
done
# Fallback when the caller did not specify a directory.
# If no directory was specified create a fresh temporary one.
if [[ -z "$TMPDIR" ]]; then
TMPDIR="$(mktemp -d)"
fi
# Ensure the directory exists, then resolve to an absolute path.
mkdir -p "$TMPDIR"
TMPDIR="$(cd "$TMPDIR" && pwd)"
# Main build logic
echo "Staging release in $TMPDIR"
# The script lives in codex-cli/scripts/ - change into codex-cli root so that
# relative paths keep working.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CODEX_CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
pushd "$CODEX_CLI_ROOT" >/dev/null
BUILD_ARGS=(
--version "$VERSION"
--staging-dir "$TMPDIR"
)
if [[ -n "$WORKFLOW_URL" ]]; then
BUILD_ARGS+=(--workflow-url "$WORKFLOW_URL")
fi
# 1. Build the JS artifacts ---------------------------------------------------
python3 "$CODEX_CLI_ROOT/scripts/build_npm_package.py" "${BUILD_ARGS[@]}"
# Paths inside the staged package
mkdir -p "$TMPDIR/bin"
cat <<EOF
Staged version $VERSION for release in $TMPDIR
cp -r bin/codex.js "$TMPDIR/bin/codex.js"
cp ../README.md "$TMPDIR" || true # README is one level up - ignore if missing
Verify the CLI:
node ${TMPDIR}/bin/codex.js --version
node ${TMPDIR}/bin/codex.js --help
# Modify package.json - bump version and optionally add the native directory to
# the files array so that the binaries are published to npm.
jq --arg version "$VERSION" \
'.version = $version' \
package.json > "$TMPDIR/package.json"
# 2. Native runtime deps (sandbox plus optional Rust binaries)
./scripts/install_native_deps.sh --workflow-url "$WORKFLOW_URL" "$TMPDIR"
popd >/dev/null
echo "Staged version $VERSION for release in $TMPDIR"
echo "Verify the CLI:"
echo " node ${TMPDIR}/bin/codex.js --version"
echo " node ${TMPDIR}/bin/codex.js --help"
# Print final hint for convenience
echo "Next: cd \"$TMPDIR\" && npm publish"
Next: cd "$TMPDIR" && npm publish
EOF
codex-cli/scripts/stage_rust_release.py
+11 -5
View File
@@ -23,7 +23,11 @@ forwarded to stage_release.sh.
)
parser.add_argument(
"--tmp",
help="Optional path to stage the npm package; forwarded to stage_release.sh",
help="Optional path to stage the npm package; forwarded to the builder",
)
parser.add_argument(
"--pack-output",
help="Optional path to write the generated npm tarball.",
)
args = parser.parse_args()
version = args.release_version
@@ -51,17 +55,19 @@ forwarded to stage_release.sh.
current_dir = Path(__file__).parent.resolve()
cmd = [
str(current_dir / "stage_release.sh"),
sys.executable,
str(current_dir / "build_npm_package.py"),
"--version",
version,
"--workflow-url",
workflow["url"],
]
if args.tmp:
cmd.extend(["--tmp", args.tmp])
cmd.extend(["--staging-dir", args.tmp])
if args.pack_output:
cmd.extend(["--pack-output", args.pack_output])
stage_release = subprocess.run(cmd)
stage_release.check_returncode()
subprocess.run(cmd, check=True)
return 0