diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 93e23b8fd7..7268a24ad8 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -1208,8 +1208,6 @@ jobs: if: ${{ env.SIGN_MACOS == 'true' }} run: pnpm install --frozen-lockfile - # stage_npm_packages.py requires DotSlash when staging releases. - - uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - name: Stage npm packages if: ${{ env.SIGN_MACOS == 'true' }} env: diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index 475239549a..1a43ce7e1f 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -77,33 +77,43 @@ if (!platformPackage) { const codexBinaryName = process.platform === "win32" ? "codex.exe" : "codex"; const localVendorRoot = path.join(__dirname, "..", "vendor"); -const localBinaryPath = path.join( - localVendorRoot, - targetTriple, - "codex", - codexBinaryName, -); +const packageBinaryPath = (vendorRoot) => + path.join(vendorRoot, targetTriple, "bin", codexBinaryName); +const legacyBinaryPath = (vendorRoot) => + path.join(vendorRoot, targetTriple, "codex", codexBinaryName); -let vendorRoot; -try { - const packageJsonPath = require.resolve(`${platformPackage}/package.json`); - vendorRoot = path.join(path.dirname(packageJsonPath), "vendor"); -} catch { - if (existsSync(localBinaryPath)) { - vendorRoot = localVendorRoot; - } else { - const packageManager = detectPackageManager(); - const updateCommand = - packageManager === "bun" - ? "bun install -g @openai/codex@latest" - : "npm install -g @openai/codex@latest"; - throw new Error( - `Missing optional dependency ${platformPackage}. Reinstall Codex: ${updateCommand}`, - ); +function resolveNativePackage(vendorRoot) { + const packageRoot = path.join(vendorRoot, targetTriple); + const binaryPath = packageBinaryPath(vendorRoot); + if (existsSync(binaryPath)) { + return { + binaryPath, + pathDir: path.join(packageRoot, "codex-path"), + }; } + + const legacyPath = legacyBinaryPath(vendorRoot); + if (existsSync(legacyPath)) { + return { + binaryPath: legacyPath, + pathDir: path.join(packageRoot, "path"), + }; + } + + return null; } -if (!vendorRoot) { +let nativePackage; +try { + const packageJsonPath = require.resolve(`${platformPackage}/package.json`); + nativePackage = resolveNativePackage( + path.join(path.dirname(packageJsonPath), "vendor"), + ); +} catch { + nativePackage = resolveNativePackage(localVendorRoot); +} + +if (!nativePackage) { const packageManager = detectPackageManager(); const updateCommand = packageManager === "bun" @@ -114,8 +124,7 @@ if (!vendorRoot) { ); } -const archRoot = path.join(vendorRoot, targetTriple); -const binaryPath = path.join(archRoot, "codex", codexBinaryName); +const { binaryPath, pathDir } = nativePackage; // Use an asynchronous spawn instead of spawnSync so that Node is able to // respond to signals (e.g. Ctrl-C / SIGINT) while the native binary is @@ -159,7 +168,6 @@ function detectPackageManager() { } const additionalDirs = []; -const pathDir = path.join(archRoot, "path"); if (existsSync(pathDir)) { additionalDirs.push(pathDir); } diff --git a/codex-cli/scripts/README.md b/codex-cli/scripts/README.md index ca0d54b544..ce58097db5 100644 --- a/codex-cli/scripts/README.md +++ b/codex-cli/scripts/README.md @@ -11,13 +11,13 @@ example, to stage the CLI, responses proxy, and SDK packages for version `0.6.0` --package codex-sdk ``` -This downloads the native artifacts once, hydrates `vendor/` for each package, and writes -tarballs to `dist/npm/`. +This downloads the native package archive artifacts once, hydrates `vendor/` for each +package, and writes tarballs to `dist/npm/`. When `--package codex` is provided, the staging helper builds the lightweight `@openai/codex` meta package plus all platform-native `@openai/codex` variants that are later published under platform-specific dist-tags. If you need to invoke `build_npm_package.py` directly, run -`codex-cli/scripts/install_native_deps.py` first and pass `--vendor-src` pointing to the -directory that contains the populated `vendor/` tree. +`codex-cli/scripts/install_native_deps.py --component codex-package` first and pass +`--vendor-src` pointing to the directory that contains the populated `vendor/` tree. diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index c38fed68d9..5d4bc99c4e 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -15,6 +15,8 @@ REPO_ROOT = CODEX_CLI_ROOT.parent RESPONSES_API_PROXY_NPM_ROOT = REPO_ROOT / "codex-rs" / "responses-api-proxy" / "npm" CODEX_SDK_ROOT = REPO_ROOT / "sdk" / "typescript" CODEX_NPM_NAME = "@openai/codex" +CODEX_PACKAGE_COMPONENT = "codex-package" +CODEX_PACKAGE_ENTRIES = ("codex-package.json", "bin", "codex-resources", "codex-path") # `npm_name` is the local optional-dependency alias consumed by `bin/codex.js`. # The underlying package published to npm is always `@openai/codex`. @@ -69,12 +71,12 @@ PACKAGE_EXPANSIONS: dict[str, list[str]] = { PACKAGE_NATIVE_COMPONENTS: dict[str, list[str]] = { "codex": [], - "codex-linux-x64": ["bwrap", "codex", "rg"], - "codex-linux-arm64": ["bwrap", "codex", "rg"], - "codex-darwin-x64": ["codex", "rg"], - "codex-darwin-arm64": ["codex", "rg"], - "codex-win32-x64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"], - "codex-win32-arm64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"], + "codex-linux-x64": [CODEX_PACKAGE_COMPONENT], + "codex-linux-arm64": [CODEX_PACKAGE_COMPONENT], + "codex-darwin-x64": [CODEX_PACKAGE_COMPONENT], + "codex-darwin-arm64": [CODEX_PACKAGE_COMPONENT], + "codex-win32-x64": [CODEX_PACKAGE_COMPONENT], + "codex-win32-arm64": [CODEX_PACKAGE_COMPONENT], "codex-responses-api-proxy": ["codex-responses-api-proxy"], "codex-sdk": [], } @@ -383,7 +385,11 @@ def copy_native_binaries( if not vendor_src.exists(): raise RuntimeError(f"Vendor source directory not found: {vendor_src}") - components_set = {component for component in components if component in COMPONENT_DEST_DIR} + components_set = { + component + for component in components + if component == CODEX_PACKAGE_COMPONENT or component in COMPONENT_DEST_DIR + } allow_missing_components = allow_missing_components or set() if not components_set: return @@ -402,11 +408,26 @@ def copy_native_binaries( if target_filter is not None and target_dir.name not in target_filter: continue - dest_target_dir = vendor_dest / target_dir.name - dest_target_dir.mkdir(parents=True, exist_ok=True) copied_targets.add(target_dir.name) - for component in components_set: + dest_target_dir = vendor_dest / target_dir.name + + if CODEX_PACKAGE_COMPONENT in components_set: + validate_codex_package_dir(target_dir) + if dest_target_dir.exists(): + shutil.rmtree(dest_target_dir) + dest_target_dir.mkdir(parents=True, exist_ok=True) + for entry in CODEX_PACKAGE_ENTRIES: + src = target_dir / entry + dest = dest_target_dir / entry + if src.is_dir(): + shutil.copytree(src, dest) + else: + shutil.copy2(src, dest) + else: + dest_target_dir.mkdir(parents=True, exist_ok=True) + + for component in components_set - {CODEX_PACKAGE_COMPONENT}: dest_dir_name = COMPONENT_DEST_DIR.get(component) if dest_dir_name is None: continue @@ -431,6 +452,35 @@ def copy_native_binaries( raise RuntimeError(f"Missing target directories in vendor source: {missing_list}") +def validate_codex_package_dir(package_dir: Path) -> None: + is_windows = "windows" in package_dir.name + required_files = [ + Path("codex-package.json"), + Path("bin") / ("codex.exe" if is_windows else "codex"), + Path("codex-path") / ("rg.exe" if is_windows else "rg"), + ] + + if "linux" in package_dir.name: + required_files.append(Path("codex-resources") / "bwrap") + + if is_windows: + required_files.extend( + [ + Path("codex-resources") / "codex-command-runner.exe", + Path("codex-resources") / "codex-windows-sandbox-setup.exe", + ] + ) + + missing_files = [ + str(relative_path) + for relative_path in required_files + if not (package_dir / relative_path).is_file() + ] + if missing_files: + missing = ", ".join(missing_files) + raise RuntimeError(f"Missing files in Codex package directory {package_dir}: {missing}") + + def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: output_path = output_path.resolve() output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py index 79f596fbaf..b2b6ba37fa 100755 --- a/codex-cli/scripts/install_native_deps.py +++ b/codex-cli/scripts/install_native_deps.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Install Codex native binaries (Rust CLI, bwrap, and ripgrep helpers).""" +"""Install Codex package archives and native helper binaries.""" import argparse from contextlib import contextmanager @@ -31,6 +31,7 @@ BINARY_TARGETS = ( "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc", ) +CODEX_PACKAGE_COMPONENT = "codex-package" @dataclass(frozen=True) @@ -139,11 +140,10 @@ def parse_args() -> argparse.Namespace: "--component", dest="components", action="append", - choices=tuple(list(BINARY_COMPONENTS) + ["rg"]), + choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS, "rg"]), help=( "Limit installation to the specified components." - " May be repeated. Defaults to bwrap, codex, codex-windows-sandbox-setup," - " codex-command-runner, and rg." + " May be repeated. Defaults to codex-package and codex-responses-api-proxy." ), ) parser.add_argument( @@ -165,13 +165,7 @@ def main() -> int: vendor_dir = codex_cli_root / VENDOR_DIR_NAME vendor_dir.mkdir(parents=True, exist_ok=True) - components = args.components or [ - "bwrap", - "codex", - "codex-windows-sandbox-setup", - "codex-command-runner", - "rg", - ] + components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"] workflow_url = (args.workflow_url or DEFAULT_WORKFLOW_URL).strip() if not workflow_url: @@ -184,6 +178,8 @@ def main() -> int: with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: artifacts_dir = Path(artifacts_dir_str) _download_artifacts(workflow_id, artifacts_dir) + if CODEX_PACKAGE_COMPONENT in components: + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) install_binary_components( artifacts_dir, vendor_dir, @@ -199,6 +195,53 @@ def main() -> int: return 0 +def install_codex_package_archives( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], +) -> None: + targets = list(targets) + if not targets: + return + + print("Installing Codex package archives for targets: " + ", ".join(targets)) + max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + _install_single_codex_package_archive, + artifacts_dir, + vendor_dir, + target, + ): target + for target in targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}") + + +def _install_single_codex_package_archive( + artifacts_dir: Path, + vendor_dir: Path, + target: str, +) -> Path: + artifact_subdir = artifacts_dir / target + archive_path = artifact_subdir / f"codex-package-{target}.tar.gz" + if not archive_path.exists(): + raise FileNotFoundError(f"Expected package archive not found: {archive_path}") + + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(archive_path, "r:gz") as archive: + archive.extractall(dest_dir, filter="data") + + return dest_dir + + def fetch_rg( vendor_dir: Path, targets: Sequence[str] | None = None, diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 2a0cd0028d..da46d4eefa 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """Stage one or more Codex npm packages for release.""" -from __future__ import annotations - import argparse import importlib.util import json