openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-09-06 15:29:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

npm: remove legacy package artifact synthesis

Browse Source
This commit is contained in:
Michael Bolin
2026-05-21 08:20:03 -07:00
parent 32fe95ccbc
commit 5b5e95ce00
4 changed files with 90 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/ci.yml vendored
View File

@@ -42,9 +42,6 @@ jobs:
- name: Install dependencies
run: pnpm install --frozen-lockfile
# stage_npm_packages.py requires DotSlash when staging releases.
- uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2
- name: Stage npm package
id: stage_npm_package
env:
@@ -55,17 +52,13 @@ jobs:
# cross-platform native payload required by the npm package layout.
# Passing the workflow URL directly avoids relying on old rust-v*
# branches remaining discoverable via `gh run list --branch ...`.
CODEX_VERSION=0.125.0
WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26131514935"
CODEX_VERSION=0.133.0-alpha.4
WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26201494185"
OUTPUT_DIR="${RUNNER_TEMP}"
# This reused workflow predates codex-package archive artifacts, so
# CI synthesizes the package layout from the older per-binary
# artifacts. Release staging must use real package archives.
python3 ./scripts/stage_npm_packages.py \
--release-version "$CODEX_VERSION" \
--workflow-url "$WORKFLOW_URL" \
--package codex \
--allow-legacy-codex-package \
--output-dir "$OUTPUT_DIR"
PACK_OUTPUT="${OUTPUT_DIR}/codex-npm-${CODEX_VERSION}.tgz"
echo "pack_output=$PACK_OUTPUT" >> "$GITHUB_OUTPUT"

378
codex-cli/scripts/install_native_deps.py
View File

@@ -3,27 +3,20 @@
import argparse
from contextlib import contextmanager
import json
import os
import shutil
import subprocess
import tarfile
import tempfile
import zipfile
from dataclasses import dataclass
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
import sys
from typing import Iterable, Sequence
from urllib.parse import urlparse
from urllib.request import urlopen
from typing import Sequence
SCRIPT_DIR = Path(__file__).resolve().parent
CODEX_CLI_ROOT = SCRIPT_DIR.parent
REPO_ROOT = CODEX_CLI_ROOT.parent
DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26131514935" # rust-v0.132.0
DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26201494185" # rust-v0.133.0-alpha.4
VENDOR_DIR_NAME = "vendor"
RG_MANIFEST = REPO_ROOT / "scripts" / "codex_package" / "rg"
BINARY_TARGETS = (
"x86_64-unknown-linux-musl",
"aarch64-unknown-linux-musl",
@@ -40,57 +33,16 @@ class BinaryComponent:
artifact_prefix: str # matches the artifact filename prefix (e.g. codex-<target>.zst)
dest_dir: str # directory under vendor/<target>/ where the binary is installed
binary_basename: str # executable name inside dest_dir (before optional .exe)
targets: tuple[str, ...] | None = None # limit installation to specific targets
WINDOWS_TARGETS = tuple(target for target in BINARY_TARGETS if "windows" in target)
LINUX_TARGETS = tuple(target for target in BINARY_TARGETS if "linux" in target)
BINARY_COMPONENTS = {
"bwrap": BinaryComponent(
artifact_prefix="bwrap",
dest_dir="codex-resources",
binary_basename="bwrap",
targets=LINUX_TARGETS,
),
"codex": BinaryComponent(
artifact_prefix="codex",
dest_dir="codex",
binary_basename="codex",
),
"codex-responses-api-proxy": BinaryComponent(
artifact_prefix="codex-responses-api-proxy",
dest_dir="codex-responses-api-proxy",
binary_basename="codex-responses-api-proxy",
),
"codex-windows-sandbox-setup": BinaryComponent(
artifact_prefix="codex-windows-sandbox-setup",
dest_dir="codex",
binary_basename="codex-windows-sandbox-setup",
targets=WINDOWS_TARGETS,
),
"codex-command-runner": BinaryComponent(
artifact_prefix="codex-command-runner",
dest_dir="codex",
binary_basename="codex-command-runner",
targets=WINDOWS_TARGETS,
),
}
RG_TARGET_PLATFORM_PAIRS: list[tuple[str, str]] = [
("x86_64-unknown-linux-musl", "linux-x86_64"),
("aarch64-unknown-linux-musl", "linux-aarch64"),
("x86_64-apple-darwin", "macos-x86_64"),
("aarch64-apple-darwin", "macos-aarch64"),
("x86_64-pc-windows-msvc", "windows-x86_64"),
("aarch64-pc-windows-msvc", "windows-aarch64"),
]
RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS}
DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS]
# urllib.request.urlopen() defaults to no timeout (can hang indefinitely), which is painful in CI.
DOWNLOAD_TIMEOUT_SECS = 60
def _gha_enabled() -> bool:
# GitHub Actions supports "workflow commands" (e.g. ::group:: / ::error::) that make logs
@@ -141,22 +93,12 @@ def parse_args() -> argparse.Namespace:
"--component",
dest="components",
action="append",
choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS, "rg"]),
choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS]),
help=(
"Limit installation to the specified components."
" May be repeated. Defaults to codex-package and codex-responses-api-proxy."
),
)
parser.add_argument(
"--allow-legacy-codex-package",
action="store_true",
help=(
"Allow codex-package to be synthesized from legacy per-binary artifacts "
"when package archives are missing. Intended for CI compatibility only; "
"release staging should not use this. Automatically enabled for the "
"built-in default workflow."
),
)
parser.add_argument(
"root",
nargs="?",
@@ -179,7 +121,6 @@ def main() -> int:
components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"]
workflow_override = (args.workflow_url or "").strip()
use_default_workflow = not workflow_override
workflow_url = workflow_override or DEFAULT_WORKFLOW_URL
workflow_id = workflow_url.rstrip("/").split("/")[-1]
@@ -190,28 +131,13 @@ def main() -> int:
artifacts_dir = Path(artifacts_dir_str)
_download_artifacts(workflow_id, artifacts_dir)
if CODEX_PACKAGE_COMPONENT in components:
try:
install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS)
except FileNotFoundError:
if not (args.allow_legacy_codex_package or use_default_workflow):
raise
install_legacy_codex_package_layouts(
artifacts_dir,
vendor_dir,
BINARY_TARGETS,
manifest_path=RG_MANIFEST,
)
install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS)
install_binary_components(
artifacts_dir,
vendor_dir,
[BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS],
)
if "rg" in components:
with _gha_group("Fetch ripgrep binaries"):
print("Fetching ripgrep binaries...")
fetch_rg(vendor_dir, DEFAULT_RG_TARGETS, manifest_path=RG_MANIFEST)
print(f"Installed native dependencies into {vendor_dir}")
return 0
@@ -263,156 +189,6 @@ def _install_single_codex_package_archive(
return dest_dir
def install_legacy_codex_package_layouts(
artifacts_dir: Path,
vendor_dir: Path,
targets: Sequence[str],
*,
manifest_path: Path,
) -> None:
targets = list(targets)
print(
"Synthesizing Codex package layouts from legacy artifacts for targets: "
+ ", ".join(targets)
)
with tempfile.TemporaryDirectory(prefix="codex-legacy-package-") as legacy_vendor_dir_str:
legacy_vendor_dir = Path(legacy_vendor_dir_str)
install_binary_components(
artifacts_dir,
legacy_vendor_dir,
[
BINARY_COMPONENTS["codex"],
BINARY_COMPONENTS["bwrap"],
BINARY_COMPONENTS["codex-windows-sandbox-setup"],
BINARY_COMPONENTS["codex-command-runner"],
],
)
fetch_rg(legacy_vendor_dir, targets, manifest_path=manifest_path)
for target in targets:
dest_dir = vendor_dir / target
if dest_dir.exists():
shutil.rmtree(dest_dir)
_build_legacy_codex_package_layout(legacy_vendor_dir / target, dest_dir, target)
print(f" synthesized {dest_dir}")
def _build_legacy_codex_package_layout(
legacy_target_dir: Path,
package_dir: Path,
target: str,
) -> None:
is_windows = "windows" in target
exe_suffix = ".exe" if is_windows else ""
package_dir.mkdir(parents=True)
bin_dir = package_dir / "bin"
resources_dir = package_dir / "codex-resources"
path_dir = package_dir / "codex-path"
bin_dir.mkdir()
resources_dir.mkdir()
path_dir.mkdir()
shutil.copy2(
legacy_target_dir / "codex" / f"codex{exe_suffix}",
bin_dir / f"codex{exe_suffix}",
)
shutil.copy2(
legacy_target_dir / "path" / f"rg{exe_suffix}",
path_dir / f"rg{exe_suffix}",
)
if is_windows:
for helper in [
"codex-command-runner.exe",
"codex-windows-sandbox-setup.exe",
]:
shutil.copy2(legacy_target_dir / "codex" / helper, resources_dir / helper)
elif "linux" in target:
shutil.copy2(legacy_target_dir / "codex-resources" / "bwrap", resources_dir / "bwrap")
write_json(
package_dir / "codex-package.json",
{
"layoutVersion": 1,
"version": "unknown",
"target": target,
"variant": "codex",
"entrypoint": f"bin/codex{exe_suffix}",
"resourcesDir": "codex-resources",
"pathDir": "codex-path",
},
)
def fetch_rg(
vendor_dir: Path,
targets: Sequence[str] | None = None,
*,
manifest_path: Path,
) -> list[Path]:
"""Download ripgrep binaries described by the DotSlash manifest."""
if targets is None:
targets = DEFAULT_RG_TARGETS
if not manifest_path.exists():
raise FileNotFoundError(f"DotSlash manifest not found: {manifest_path}")
manifest = _load_manifest(manifest_path)
platforms = manifest.get("platforms", {})
vendor_dir.mkdir(parents=True, exist_ok=True)
targets = list(targets)
if not targets:
return []
task_configs: list[tuple[str, str, dict]] = []
for target in targets:
platform_key = RG_TARGET_TO_PLATFORM.get(target)
if platform_key is None:
raise ValueError(f"Unsupported ripgrep target '{target}'.")
platform_info = platforms.get(platform_key)
if platform_info is None:
raise RuntimeError(f"Platform '{platform_key}' not found in manifest {manifest_path}.")
task_configs.append((target, platform_key, platform_info))
results: dict[str, Path] = {}
max_workers = min(len(task_configs), max(1, (os.cpu_count() or 1)))
print("Installing ripgrep binaries for targets: " + ", ".join(targets))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
future_map = {
executor.submit(
_fetch_single_rg,
vendor_dir,
target,
platform_key,
platform_info,
manifest_path,
): target
for target, platform_key, platform_info in task_configs
}
for future in as_completed(future_map):
target = future_map[future]
try:
results[target] = future.result()
except Exception as exc:
_gha_error(
title="ripgrep install failed",
message=f"target={target} error={exc!r}",
)
raise RuntimeError(f"Failed to install ripgrep for target {target}.") from exc
print(f" installed ripgrep for {target}")
return [results[target] for target in targets]
def _download_artifacts(workflow_id: str, dest_dir: Path) -> None:
cmd = [
"gh",
@@ -436,7 +212,7 @@ def install_binary_components(
return
for component in selected_components:
component_targets = list(component.targets or BINARY_TARGETS)
component_targets = list(BINARY_TARGETS)
print(
f"Installing {component.binary_basename} binaries for targets: "
@@ -466,7 +242,7 @@ def _install_single_binary(
component: BinaryComponent,
) -> Path:
artifact_subdir = artifact_dir_for_target(artifacts_dir, target)
archive_path = legacy_binary_archive_path(artifact_subdir, component.artifact_prefix, target)
archive_path = binary_archive_path(artifact_subdir, component.artifact_prefix, target)
dest_dir = vendor_dir / target / component.dest_dir
dest_dir.mkdir(parents=True, exist_ok=True)
@@ -476,7 +252,7 @@ def _install_single_binary(
)
dest = dest_dir / binary_name
dest.unlink(missing_ok=True)
extract_archive(archive_path, "zst", None, dest)
extract_zstd_archive(archive_path, dest)
if "windows" not in target:
dest.chmod(0o755)
return dest
@@ -488,7 +264,7 @@ def _archive_name_for_target(artifact_prefix: str, target: str) -> str:
return f"{artifact_prefix}-{target}.zst"
def legacy_binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path:
def binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path:
archive_names = [_archive_name_for_target(artifact_prefix, target)]
if artifact_dir.name == f"{target}-unsigned":
archive_names.append(_archive_name_for_target(artifact_prefix, f"{target}-unsigned"))
@@ -510,142 +286,12 @@ def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path:
return artifacts_dir / target
def _fetch_single_rg(
vendor_dir: Path,
target: str,
platform_key: str,
platform_info: dict,
manifest_path: Path,
) -> Path:
providers = platform_info.get("providers", [])
if not providers:
raise RuntimeError(f"No providers listed for platform '{platform_key}' in {manifest_path}.")
url = providers[0]["url"]
archive_format = platform_info.get("format", "zst")
archive_member = platform_info.get("path")
digest = platform_info.get("digest")
expected_size = platform_info.get("size")
dest_dir = vendor_dir / target / "path"
dest_dir.mkdir(parents=True, exist_ok=True)
is_windows = platform_key.startswith("win")
binary_name = "rg.exe" if is_windows else "rg"
dest = dest_dir / binary_name
with tempfile.TemporaryDirectory() as tmp_dir_str:
tmp_dir = Path(tmp_dir_str)
archive_filename = os.path.basename(urlparse(url).path)
download_path = tmp_dir / archive_filename
print(
f" downloading ripgrep for {target} ({platform_key}) from {url}",
flush=True,
)
try:
_download_file(url, download_path)
except Exception as exc:
_gha_error(
title="ripgrep download failed",
message=f"target={target} platform={platform_key} url={url} error={exc!r}",
)
raise RuntimeError(
"Failed to download ripgrep "
f"(target={target}, platform={platform_key}, format={archive_format}, "
f"expected_size={expected_size!r}, digest={digest!r}, url={url}, dest={download_path})."
) from exc
dest.unlink(missing_ok=True)
try:
extract_archive(download_path, archive_format, archive_member, dest)
except Exception as exc:
raise RuntimeError(
"Failed to extract ripgrep "
f"(target={target}, platform={platform_key}, format={archive_format}, "
f"member={archive_member!r}, url={url}, archive={download_path})."
) from exc
if not is_windows:
dest.chmod(0o755)
return dest
def _download_file(url: str, dest: Path) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
dest.unlink(missing_ok=True)
with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response, open(dest, "wb") as out:
shutil.copyfileobj(response, out)
def extract_archive(
archive_path: Path,
archive_format: str,
archive_member: str | None,
dest: Path,
) -> None:
def extract_zstd_archive(archive_path: Path, dest: Path) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
if archive_format == "zst":
output_path = archive_path.parent / dest.name
subprocess.check_call(
["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)]
)
shutil.move(str(output_path), dest)
return
if archive_format == "tar.gz":
if not archive_member:
raise RuntimeError("Missing 'path' for tar.gz archive in DotSlash manifest.")
with tarfile.open(archive_path, "r:gz") as tar:
try:
member = tar.getmember(archive_member)
except KeyError as exc:
raise RuntimeError(
f"Entry '{archive_member}' not found in archive {archive_path}."
) from exc
tar.extract(member, path=archive_path.parent, filter="data")
extracted = archive_path.parent / archive_member
shutil.move(str(extracted), dest)
return
if archive_format == "zip":
if not archive_member:
raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.")
with zipfile.ZipFile(archive_path) as archive:
try:
with archive.open(archive_member) as src, open(dest, "wb") as out:
shutil.copyfileobj(src, out)
except KeyError as exc:
raise RuntimeError(
f"Entry '{archive_member}' not found in archive {archive_path}."
) from exc
return
raise RuntimeError(f"Unsupported archive format '{archive_format}'.")
def _load_manifest(manifest_path: Path) -> dict:
cmd = ["dotslash", "--", "parse", str(manifest_path)]
stdout = subprocess.check_output(cmd, text=True)
try:
manifest = json.loads(stdout)
except json.JSONDecodeError as exc:
raise RuntimeError(f"Invalid DotSlash manifest output from {manifest_path}.") from exc
if not isinstance(manifest, dict):
raise RuntimeError(
f"Unexpected DotSlash manifest structure for {manifest_path}: {type(manifest)!r}"
)
return manifest
def write_json(path: Path, value: object) -> None:
with open(path, "w", encoding="utf-8") as out:
json.dump(value, out, indent=2)
out.write("\n")
output_path = archive_path.parent / dest.name
subprocess.check_call(["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)])
shutil.move(str(output_path), dest)
if __name__ == "__main__":

76
codex-cli/scripts/test_install_native_deps.py Normal file
View File

@@ -0,0 +1,76 @@
#!/usr/bin/env python3
from contextlib import redirect_stdout
import importlib.util
import io
from pathlib import Path
import tarfile
import tempfile
import unittest
INSTALL_SCRIPT = Path(__file__).resolve().parent / "install_native_deps.py"
SPEC = importlib.util.spec_from_file_location("install_native_deps", INSTALL_SCRIPT)
if SPEC is None or SPEC.loader is None:
raise RuntimeError(f"Unable to load module from {INSTALL_SCRIPT}")
install_native_deps = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(install_native_deps)
class InstallCodexPackageArchivesTest(unittest.TestCase):
def test_installs_codex_package_archive(self) -> None:
target = "x86_64-unknown-linux-musl"
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
artifact_dir = root / "artifacts" / target
package_src = root / "package-src"
vendor_dir = root / "vendor"
artifact_dir.mkdir(parents=True)
(package_src / "bin").mkdir(parents=True)
(package_src / "bin" / "codex").write_text("codex\n", encoding="utf-8")
(package_src / "codex-package.json").write_text("{}\n", encoding="utf-8")
archive_path = artifact_dir / f"codex-package-{target}.tar.gz"
with tarfile.open(archive_path, "w:gz") as archive:
archive.add(package_src / "bin", arcname="bin")
archive.add(package_src / "codex-package.json", arcname="codex-package.json")
with redirect_stdout(io.StringIO()):
install_native_deps.install_codex_package_archives(
root / "artifacts",
vendor_dir,
[target],
)
self.assertEqual(
sorted(
path.relative_to(vendor_dir / target)
for path in (vendor_dir / target).rglob("*")
),
[
Path("bin"),
Path("bin/codex"),
Path("codex-package.json"),
],
)
def test_missing_codex_package_archive_errors(self) -> None:
target = "x86_64-unknown-linux-musl"
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
with redirect_stdout(io.StringIO()):
with self.assertRaisesRegex(
FileNotFoundError,
"Expected package archive not found",
):
install_native_deps.install_codex_package_archives(
root / "artifacts",
root / "vendor",
[target],
)
if __name__ == "__main__":
unittest.main()

14
scripts/stage_npm_packages.py
View File

@@ -66,15 +66,6 @@ def parse_args() -> argparse.Namespace:
"Intended for CI compatibility only; release staging should not use this."
),
)
parser.add_argument(
"--allow-legacy-codex-package",
action="store_true",
help=(
"Allow codex-package layouts to be synthesized from legacy per-binary "
"workflow artifacts. Intended for CI compatibility only; release staging "
"should not use this."
),
)
return parser.parse_args()
@@ -131,15 +122,11 @@ def install_native_components(
workflow_url: str,
components: set[str],
vendor_root: Path,
*,
allow_legacy_codex_package: bool,
) -> None:
if not components:
return
cmd = [str(INSTALL_NATIVE_DEPS), "--workflow-url", workflow_url]
if allow_legacy_codex_package:
cmd.append("--allow-legacy-codex-package")
for component in sorted(components):
cmd.extend(["--component", component])
cmd.append(str(vendor_root))
@@ -187,7 +174,6 @@ def main() -> int:
workflow_url,
native_components_to_install,
vendor_temp_root,
allow_legacy_codex_package=args.allow_legacy_codex_package,
)
vendor_src = vendor_temp_root / "vendor"