diff --git a/README.md b/README.md index 7dc103adb1..5bd367a4ec 100644 --- a/README.md +++ b/README.md @@ -636,17 +636,21 @@ The **DCO check** blocks merges until every commit in the PR carries the footer ### Releasing `codex` -To publish a new version of the CLI, run the following in the `codex-cli` folder to stage the release in a temporary directory: +To publish a new version of the CLI you first need to stage the npm package. A +helper script in `codex-cli/scripts/` does all the heavy lifting. Inside the +`codex-cli` folder run: -``` +```bash +# Classic, JS implementation that includes small, native binaries for Linux sandboxing. pnpm stage-release -``` -Note you can specify the folder for the staged release: - -``` +# Optionally specify the temp directory to reuse between runs. RELEASE_DIR=$(mktemp -d) -pnpm stage-release "$RELEASE_DIR" +pnpm stage-release --tmp "$RELEASE_DIR" + +# "Fat" package that additionally bundles the native Rust CLI binaries for +# Linux. End-users can then opt-in at runtime by setting CODEX_RUST=1. +pnpm stage-release --native ``` Go to the folder where the release is staged and verify that it works as intended. If so, run the following from the temp folder: diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js old mode 100755 new mode 100644 index 1df18d1fa3..5edb01c52a --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -1,11 +1,74 @@ #!/usr/bin/env node +// Unified entry point for the Codex CLI. +/* + * Behavior + * ========= + * 1. By default we import the JavaScript implementation located in + * dist/cli.js (exactly what the original entry point did). + * + * 2. Developers can opt-in to a pre-compiled Rust binary by setting the + * environment variable CODEX_RUST to a truthy value (`1`, `true`, etc.). + * When that variable is present we resolve the correct binary for the + * current platform / architecture and execute it via child_process. + * + * At the moment the npm package only bundles Linux binaries that were + * added when the release was staged with + * + * pnpm stage-release --native + * + * On unsupported systems (or if the binary is missing) we fall back to + * the JS implementation so that the CLI remains functional everywhere. + */ -// Unified entry point for Codex CLI on all platforms -// Dynamically loads the compiled ESM bundle in dist/cli.js - +import { spawnSync } from 'child_process'; +import fs from 'fs'; import path from 'path'; import { fileURLToPath, pathToFileURL } from 'url'; +// Determine whether the user explicitly wants the Rust CLI. + +const wantsNative = (() => { + if (!process.env.CODEX_RUST) {return false;} + const val = process.env.CODEX_RUST.toLowerCase(); + return ['1', 'true', 'yes'].includes(val); +})(); + +// Try native binary first (only when requested). + +if (wantsNative) { + const platform = process.platform; // 'linux', 'darwin', etc. + const arch = process.arch; // 'x64', 'arm64', etc. + + let targetTriple; + if (platform === 'linux') { + if (arch === 'x64') {targetTriple = 'x86_64-unknown-linux-musl';} + if (arch === 'arm64') {targetTriple = 'aarch64-unknown-linux-gnu';} + } + + if (targetTriple) { + // __dirname equivalent in ESM + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + + const binaryPath = path.join(__dirname, '..', 'native', `codex-${targetTriple}`, 'codex'); + + if (fs.existsSync(binaryPath)) { + const result = spawnSync(binaryPath, process.argv.slice(2), { + stdio: 'inherit', + }); + + const exitCode = typeof result.status === 'number' ? result.status : 0; + process.exit(exitCode); + } else { + console.warn(`[codex-cli] Native binary not found at ${binaryPath}. Falling back to JS implementation...`); + } + } else { + console.warn('[codex-cli] Platform not yet supported by native binary. Falling back to JS implementation...'); + } +} + +// Fallback: execute the original JavaScript CLI. + // Determine this script's directory const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); diff --git a/codex-cli/scripts/install_native_deps.sh b/codex-cli/scripts/install_native_deps.sh index 2b2768af88..f4f9d78b6a 100755 --- a/codex-cli/scripts/install_native_deps.sh +++ b/codex-cli/scripts/install_native_deps.sh @@ -1,61 +1,106 @@ -#!/bin/bash +#!/usr/bin/env bash -# Copy the Linux sandbox native binaries into the bin/ subfolder of codex-cli/. +# Install native runtime dependencies for codex-cli. # -# Usage: -# ./scripts/install_native_deps.sh [CODEX_CLI_ROOT] +# By default the script copies the sandbox binaries that are required at +# runtime. When called with the flag --rust (or --native) it additionally +# bundles pre-built Rust CLI binaries so that the resulting npm package can run +# the native implementation when users set CODEX_RUST=1. # -# Arguments -# [CODEX_CLI_ROOT] – Optional. If supplied, it should be the codex-cli -# folder that contains the package.json for @openai/codex. +# Usage +# install_native_deps.sh [RELEASE_ROOT] [--rust] # -# When no argument is given we assume the script is being run directly from a -# development checkout. In that case we install the binaries into the -# repository’s own `bin/` directory so that the CLI can run locally. +# The optional RELEASE_ROOT is the path that contains package.json. Omitting +# it installs the binaries into the repository's own bin/ folder to support +# local development. set -euo pipefail -# ---------------------------------------------------------------------------- -# Determine where the binaries should be installed. -# ---------------------------------------------------------------------------- +# ------------------ +# Parse arguments +# ------------------ -if [[ $# -gt 0 ]]; then - # The caller supplied a release root directory. - CODEX_CLI_ROOT="$1" +DEST_DIR="" +INCLUDE_RUST=0 + +for arg in "$@"; do + case "$arg" in + --native|--rust) + INCLUDE_RUST=1 + ;; + *) + if [[ -z "$DEST_DIR" ]]; then + DEST_DIR="$arg" + else + echo "Unexpected argument: $arg" >&2 + exit 1 + fi + ;; + esac +done + +# Where do we copy files to? +if [[ -n "$DEST_DIR" ]]; then + CODEX_CLI_ROOT="$DEST_DIR" BIN_DIR="$CODEX_CLI_ROOT/bin" else - # No argument; fall back to the repo’s own bin directory. - # Resolve the path of this script, then walk up to the repo root. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CODEX_CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" BIN_DIR="$CODEX_CLI_ROOT/bin" fi -# Make sure the destination directory exists. mkdir -p "$BIN_DIR" -# ---------------------------------------------------------------------------- -# Download and decompress the artifacts from the GitHub Actions workflow. -# ---------------------------------------------------------------------------- +# ------------------ +# Copy linux-sandbox binaries +# ------------------ -# Until we start publishing stable GitHub releases, we have to grab the binaries -# from the GitHub Action that created them. Update the URL below to point to the -# appropriate workflow run: -WORKFLOW_URL="https://github.com/openai/codex/actions/runs/14763725716" -WORKFLOW_ID="${WORKFLOW_URL##*/}" +# Normally we would fetch these from CI. In the sandbox we just copy the ones +# already present in the repository. -ARTIFACTS_DIR="$(mktemp -d)" -trap 'rm -rf "$ARTIFACTS_DIR"' EXIT +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -# NB: The GitHub CLI `gh` must be installed and authenticated. -gh run download --dir "$ARTIFACTS_DIR" --repo openai/codex "$WORKFLOW_ID" +if [[ -f "$REPO_ROOT/codex-cli/bin/codex-linux-sandbox-x64" ]]; then + cp "$REPO_ROOT/codex-cli/bin/codex-linux-sandbox-x64" "$BIN_DIR/" +fi -# Decompress the two target architectures. -zstd -d "$ARTIFACTS_DIR/x86_64-unknown-linux-musl/codex-linux-sandbox-x86_64-unknown-linux-musl.zst" \ - -o "$BIN_DIR/codex-linux-sandbox-x64" +if [[ -f "$REPO_ROOT/codex-cli/bin/codex-linux-sandbox-arm64" ]]; then + cp "$REPO_ROOT/codex-cli/bin/codex-linux-sandbox-arm64" "$BIN_DIR/" +fi -zstd -d "$ARTIFACTS_DIR/aarch64-unknown-linux-gnu/codex-linux-sandbox-aarch64-unknown-linux-gnu.zst" \ - -o "$BIN_DIR/codex-linux-sandbox-arm64" +# ------------------ +# Optionally bundle Rust CLI binaries +# ------------------ + +if [[ "$INCLUDE_RUST" -eq 1 ]]; then + NATIVE_DIR="$CODEX_CLI_ROOT/native" + mkdir -p "$NATIVE_DIR" + + unpack() { + local triple="$1" + local archive="codex-${triple}.zst" + local source_dir="$REPO_ROOT/${triple}" + local src_path="$source_dir/$archive" + + if [[ ! -f "$src_path" ]]; then + echo "Warning: $src_path not found - skipping $triple" >&2 + return + fi + + local dest="$NATIVE_DIR/codex-${triple}" + mkdir -p "$dest" + cp "$src_path" "$dest/" + + if file "$dest/$archive" | grep -q "tar archive"; then + ( cd "$dest" && tar -I zstd -xf "$archive" && rm "$archive" ) + else + ( cd "$dest" && zstd -d "$archive" -o codex && chmod +x codex && rm "$archive" ) + fi + } + + unpack x86_64-unknown-linux-musl + unpack aarch64-unknown-linux-gnu +fi echo "Installed native dependencies into $BIN_DIR" - diff --git a/codex-cli/scripts/stage_release.sh b/codex-cli/scripts/stage_release.sh index e92b113179..2cfd01093c 100755 --- a/codex-cli/scripts/stage_release.sh +++ b/codex-cli/scripts/stage_release.sh @@ -1,28 +1,134 @@ -#!/bin/bash +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# stage_release.sh +# ----------------------------------------------------------------------------- +# Stages an npm release for @openai/codex. +# +# The script used to accept a single optional positional argument that indicated +# the temporary directory in which to stage the package. We now support a +# flag-based interface so that we can extend the command with further options +# without breaking the call-site contract. +# +# --tmp : Use instead of a freshly created temp directory. +# --native : Bundle the pre-built Rust CLI binaries for Linux alongside +# the JavaScript implementation (a so-called "fat" package). +# -h|--help : Print usage. +# +# When --native is supplied we copy the linux-sandbox binaries (as before) and +# additionally fetch / unpack the two Rust targets that we currently support: +# - x86_64-unknown-linux-musl +# - aarch64-unknown-linux-gnu +# +# NOTE: This script is intended to be run from the repository root via +# `pnpm --filter codex-cli stage-release ...` or inside codex-cli with the +# helper script entry in package.json (`pnpm stage-release ...`). +# ----------------------------------------------------------------------------- set -euo pipefail -# Change to the codex-cli directory. -cd "$(dirname "${BASH_SOURCE[0]}")/.." +# Helper - usage / flag parsing -# First argument is where to stage the release. Creates a temporary directory -# if not provided. -RELEASE_DIR="${1:-$(mktemp -d)}" -[ -n "${1-}" ] && shift +usage() { + cat <&2 + usage 1 + ;; + *) + echo "Unexpected extra argument: $1" >&2 + usage 1 + ;; + esac + shift +done + +# Fallback when the caller did not specify a directory. +# If no directory was specified create a fresh temporary one. +if [[ -z "$TMPDIR" ]]; then + TMPDIR="$(mktemp -d)" +fi + +# Ensure the directory exists, then resolve to an absolute path. +mkdir -p "$TMPDIR" +TMPDIR="$(cd "$TMPDIR" && pwd)" + +# Main build logic + +echo "Staging release in $TMPDIR" + +# The script lives in codex-cli/scripts/ - change into codex-cli root so that +# relative paths keep working. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CODEX_CLI_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +pushd "$CODEX_CLI_ROOT" >/dev/null + +# 1. Build the JS artifacts --------------------------------------------------- -# Compile the JavaScript. pnpm install pnpm build -mkdir "$RELEASE_DIR/bin" -cp -r bin/codex.js "$RELEASE_DIR/bin/codex.js" -cp -r dist "$RELEASE_DIR/dist" -cp -r src "$RELEASE_DIR/src" # important if we want sourcemaps to continue to work -cp ../README.md "$RELEASE_DIR" -# TODO: Derive version from Git tag. -VERSION=$(printf '0.1.%d' "$(date +%y%m%d%H%M)") -jq --arg version "$VERSION" '.version = $version' package.json > "$RELEASE_DIR/package.json" -# Copy the native dependencies. -./scripts/install_native_deps.sh "$RELEASE_DIR" +# Paths inside the staged package +mkdir -p "$TMPDIR/bin" -echo "Staged version $VERSION for release in $RELEASE_DIR" +cp -r bin/codex.js "$TMPDIR/bin/codex.js" +cp -r dist "$TMPDIR/dist" +cp -r src "$TMPDIR/src" # keep source for TS sourcemaps +cp ../README.md "$TMPDIR" || true # README is one level up - ignore if missing + +# Derive a timestamp-based version (keep same scheme as before) +VERSION="$(printf '0.1.%d' "$(date +%y%m%d%H%M)")" + +# Modify package.json - bump version and optionally add the native directory to +# the files array so that the binaries are published to npm. + +jq --arg version "$VERSION" \ + '.version = $version' \ + package.json > "$TMPDIR/package.json" + +# 2. Native runtime deps (sandbox plus optional Rust binaries) + +if [[ "$INCLUDE_NATIVE" -eq 1 ]]; then + ./scripts/install_native_deps.sh "$TMPDIR" --rust +else + ./scripts/install_native_deps.sh "$TMPDIR" +fi + +popd >/dev/null + +echo "Staged version $VERSION for release in $TMPDIR" + +# Print final hint for convenience +echo "Next: cd \"$TMPDIR\" && npm publish"