From 2a174f40257b337b608b8aab946ba0a783c6f21d Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 08:48:10 -0700 Subject: [PATCH 1/3] install: consume Codex package archives ## Summary Standalone installs should exercise the same canonical package archive layout that release builds produce, rather than unpacking npm platform packages and reconstructing a parallel install tree. This updates `install.sh` and `install.ps1` to prefer `codex-package-.tar.gz` plus `codex-package_SHA256SUMS`, authenticate the checksum manifest against GitHub release metadata, verify the selected package archive against the authenticated manifest, and install the package archive directly. Package installs now also leave a compatibility command at `current/codex` for managed daemon flows while visible command shims point at `bin/codex`. Recent releases that predate package archives still publish per-platform npm artifacts, so both installers keep a legacy platform npm fallback for those versions and verify those archives against release metadata directly. Releases old enough to publish only the single root `codex-npm-.tgz` archive are intentionally out of scope; the installers fail clearly when neither package archives nor per-platform npm archives are present. ## Test plan - `sh -n scripts/install/install.sh` - `bash -n scripts/install/install.sh` - `pwsh -NoProfile -Command '$tokens=$null; $errors=$null; $null = [System.Management.Automation.Language.Parser]::ParseFile("scripts/install/install.ps1", [ref]$tokens, [ref]$errors); if ($errors.Count) { $errors | Format-List *; exit 1 }'` - `HOME="$home_dir" CODEX_HOME="$tmp_dir/codex-home" CODEX_INSTALL_DIR="$bin_dir" PATH="$bin_dir:$PATH" sh scripts/install/install.sh --release 0.125.0` - Verified the 0.125.0 isolated install leaves the visible command pointed at `current/codex` and includes the legacy `codex-resources/rg` payload. --- scripts/install/install.ps1 | 181 ++++++++++++++++++++++++++------ scripts/install/install.sh | 199 ++++++++++++++++++++++++++++++------ 2 files changed, 318 insertions(+), 62 deletions(-) diff --git a/scripts/install/install.ps1 b/scripts/install/install.ps1 index ed4a3e1a32..8ebc4fbb82 100644 --- a/scripts/install/install.ps1 +++ b/scripts/install/install.ps1 @@ -55,7 +55,7 @@ function Normalize-Version { return $RawVersion } -function Get-ReleaseAssetMetadata { +function Find-ReleaseAssetMetadata { param( [string]$AssetName, [string]$ResolvedVersion @@ -64,7 +64,7 @@ function Get-ReleaseAssetMetadata { $release = Invoke-RestMethod -Uri "https://api.github.com/repos/openai/codex/releases/tags/rust-v$ResolvedVersion" $asset = $release.assets | Where-Object { $_.name -eq $AssetName } | Select-Object -First 1 if ($null -eq $asset) { - throw "Could not find release asset $AssetName for Codex $ResolvedVersion." + return $null } $digestMatch = [regex]::Match([string]$asset.digest, "^sha256:([0-9a-fA-F]{64})$") @@ -78,6 +78,20 @@ function Get-ReleaseAssetMetadata { } } +function Get-ReleaseAssetMetadata { + param( + [string]$AssetName, + [string]$ResolvedVersion + ) + + $metadata = Find-ReleaseAssetMetadata -AssetName $AssetName -ResolvedVersion $ResolvedVersion + if ($null -eq $metadata) { + throw "Could not find release asset $AssetName for Codex $ResolvedVersion." + } + + return $metadata +} + function Test-ArchiveDigest { param( [string]$ArchivePath, @@ -86,10 +100,27 @@ function Test-ArchiveDigest { $actualDigest = (Get-FileHash -LiteralPath $ArchivePath -Algorithm SHA256).Hash.ToLowerInvariant() if ($actualDigest -ne $ExpectedDigest) { - throw "Downloaded Codex archive checksum did not match release metadata. Expected $ExpectedDigest but got $actualDigest." + throw "Downloaded Codex archive checksum did not match expected digest. Expected $ExpectedDigest but got $actualDigest." } } +function Get-PackageArchiveDigest { + param( + [string]$ManifestPath, + [string]$AssetName + ) + + $escapedAssetName = [regex]::Escape($AssetName) + foreach ($line in Get-Content -LiteralPath $ManifestPath) { + $match = [regex]::Match($line, "^\s*([0-9a-fA-F]{64})\s+$escapedAssetName\s*$") + if ($match.Success) { + return $match.Groups[1].Value.ToLowerInvariant() + } + } + + throw "Could not find SHA-256 digest for $AssetName in codex-package_SHA256SUMS." +} + function Path-Contains { param( [string]$PathValue, @@ -190,6 +221,11 @@ function Get-CurrentInstalledVersion { [string]$StandaloneCurrentDir ) + $standaloneVersion = Get-VersionFromBinary -CodexPath (Join-Path $StandaloneCurrentDir "bin\codex.exe") + if (-not [string]::IsNullOrWhiteSpace($standaloneVersion)) { + return $standaloneVersion + } + $standaloneVersion = Get-VersionFromBinary -CodexPath (Join-Path $StandaloneCurrentDir "codex.exe") if (-not [string]::IsNullOrWhiteSpace($standaloneVersion)) { return $standaloneVersion @@ -449,14 +485,37 @@ function Ensure-Junction { throw "Refusing to replace file at $LinkPath with a junction." } -function Test-ReleaseIsComplete { +function Test-PackageContentsAreComplete { param( - [string]$ReleaseDir, - [string]$ExpectedVersion, - [string]$ExpectedTarget + [string]$PackageDir ) - if (-not (Test-Path -LiteralPath $ReleaseDir -PathType Container)) { + if (-not (Test-Path -LiteralPath $PackageDir -PathType Container)) { + return $false + } + + $expectedFiles = @( + "codex-package.json", + "bin\codex.exe", + "codex-path\rg.exe", + "codex-resources\codex-command-runner.exe", + "codex-resources\codex-windows-sandbox-setup.exe" + ) + foreach ($name in $expectedFiles) { + if (-not (Test-Path -LiteralPath (Join-Path $PackageDir $name) -PathType Leaf)) { + return $false + } + } + + return $true +} + +function Test-LegacyPlatformNpmContentsAreComplete { + param( + [string]$PackageDir + ) + + if (-not (Test-Path -LiteralPath $PackageDir -PathType Container)) { return $false } @@ -467,11 +526,38 @@ function Test-ReleaseIsComplete { "codex-resources\rg.exe" ) foreach ($name in $expectedFiles) { - if (-not (Test-Path -LiteralPath (Join-Path $ReleaseDir $name) -PathType Leaf)) { + if (-not (Test-Path -LiteralPath (Join-Path $PackageDir $name) -PathType Leaf)) { return $false } } + return $true +} + +function Test-ReleaseIsComplete { + param( + [string]$ReleaseDir, + [string]$ExpectedVersion, + [string]$ExpectedTarget, + [string]$Layout + ) + + switch ($Layout) { + "Package" { + if (-not (Test-PackageContentsAreComplete -PackageDir $ReleaseDir)) { + return $false + } + } + "LegacyPlatformNpm" { + if (-not (Test-LegacyPlatformNpmContentsAreComplete -PackageDir $ReleaseDir)) { + return $false + } + } + default { + throw "Unknown Codex installer layout: $Layout" + } + } + return (Split-Path -Leaf $ReleaseDir) -eq "$ExpectedVersion-$ExpectedTarget" } @@ -637,7 +723,21 @@ Write-Step "Resolved version: $resolvedVersion" $conflictingInstall = Get-ConflictingInstall -VisibleBinDir $visibleBinDir $oldStandaloneBackup = $null -$packageAsset = "codex-npm-$npmTag-$resolvedVersion.tgz" +$packageAsset = "codex-package-$target.tar.gz" +$checksumAsset = "codex-package_SHA256SUMS" +$packageMetadata = Find-ReleaseAssetMetadata -AssetName $packageAsset -ResolvedVersion $resolvedVersion +$checksumMetadata = Find-ReleaseAssetMetadata -AssetName $checksumAsset -ResolvedVersion $resolvedVersion +$installLayout = "Package" +if ($null -eq $packageMetadata -or $null -eq $checksumMetadata) { + $packageAsset = "codex-npm-$npmTag-$resolvedVersion.tgz" + $packageMetadata = Find-ReleaseAssetMetadata -AssetName $packageAsset -ResolvedVersion $resolvedVersion + if ($null -ne $packageMetadata) { + $installLayout = "LegacyPlatformNpm" + } else { + throw "Could not find Codex package or platform npm release assets for Codex $resolvedVersion." + } + $checksumMetadata = $null +} $tempDir = Join-Path ([System.IO.Path]::GetTempPath()) ("codex-install-" + [System.Guid]::NewGuid().ToString("N")) New-Item -ItemType Directory -Force -Path $tempDir | Out-Null @@ -645,40 +745,58 @@ try { Invoke-WithInstallLock -LockPath $lockPath -Script { Remove-StaleInstallArtifacts -ReleasesDir $releasesDir - if (-not (Test-ReleaseIsComplete -ReleaseDir $releaseDir -ExpectedVersion $resolvedVersion -ExpectedTarget $target)) { + if (-not (Test-ReleaseIsComplete -ReleaseDir $releaseDir -ExpectedVersion $resolvedVersion -ExpectedTarget $target -Layout $installLayout)) { if (Test-Path -LiteralPath $releaseDir) { Write-WarningStep "Found incomplete existing release at $releaseDir. Reinstalling." } $archivePath = Join-Path $tempDir $packageAsset - $extractDir = Join-Path $tempDir "extract" + $checksumPath = Join-Path $tempDir $checksumAsset $stagingDir = Join-Path $releasesDir ".staging.$releaseName.$PID" - $assetMetadata = Get-ReleaseAssetMetadata -AssetName $packageAsset -ResolvedVersion $resolvedVersion Write-Step "Downloading Codex CLI" - Invoke-WebRequest -Uri $assetMetadata.Url -OutFile $archivePath - Test-ArchiveDigest -ArchivePath $archivePath -ExpectedDigest $assetMetadata.Sha256 + if ($installLayout -eq "Package") { + Invoke-WebRequest -Uri $checksumMetadata.Url -OutFile $checksumPath + Test-ArchiveDigest -ArchivePath $checksumPath -ExpectedDigest $checksumMetadata.Sha256 + $expectedPackageDigest = Get-PackageArchiveDigest -ManifestPath $checksumPath -AssetName $packageAsset + } else { + $expectedPackageDigest = $packageMetadata.Sha256 + } + Invoke-WebRequest -Uri $packageMetadata.Url -OutFile $archivePath + Test-ArchiveDigest -ArchivePath $archivePath -ExpectedDigest $expectedPackageDigest - New-Item -ItemType Directory -Force -Path $extractDir | Out-Null New-Item -ItemType Directory -Force -Path $releasesDir | Out-Null if (Test-Path -LiteralPath $stagingDir) { Remove-Item -LiteralPath $stagingDir -Recurse -Force } New-Item -ItemType Directory -Force -Path $stagingDir | Out-Null - tar -xzf $archivePath -C $extractDir + if ($installLayout -eq "Package") { + tar -xzf $archivePath -C $stagingDir + if (-not (Test-PackageContentsAreComplete -PackageDir $stagingDir)) { + throw "Downloaded Codex package archive did not contain the expected package layout." + } + } else { + $extractDir = Join-Path $tempDir "extract" + New-Item -ItemType Directory -Force -Path $extractDir | Out-Null + tar -xzf $archivePath -C $extractDir - $vendorRoot = Join-Path $extractDir "package/vendor/$target" - $resourcesDir = Join-Path $stagingDir "codex-resources" - New-Item -ItemType Directory -Force -Path $resourcesDir | Out-Null - $copyMap = @{ - "codex/codex.exe" = "codex.exe" - "codex/codex-command-runner.exe" = "codex-resources\codex-command-runner.exe" - "codex/codex-windows-sandbox-setup.exe" = "codex-resources\codex-windows-sandbox-setup.exe" - "path/rg.exe" = "codex-resources\rg.exe" - } + $vendorRoot = Join-Path $extractDir "package/vendor/$target" + $resourcesDir = Join-Path $stagingDir "codex-resources" + New-Item -ItemType Directory -Force -Path $resourcesDir | Out-Null + $copyMap = @{ + "codex/codex.exe" = "codex.exe" + "codex/codex-command-runner.exe" = "codex-resources\codex-command-runner.exe" + "codex/codex-windows-sandbox-setup.exe" = "codex-resources\codex-windows-sandbox-setup.exe" + "path/rg.exe" = "codex-resources\rg.exe" + } - foreach ($relativeSource in $copyMap.Keys) { - Copy-Item -LiteralPath (Join-Path $vendorRoot $relativeSource) -Destination (Join-Path $stagingDir $copyMap[$relativeSource]) + foreach ($relativeSource in $copyMap.Keys) { + Copy-Item -LiteralPath (Join-Path $vendorRoot $relativeSource) -Destination (Join-Path $stagingDir $copyMap[$relativeSource]) + } + + if (-not (Test-LegacyPlatformNpmContentsAreComplete -PackageDir $stagingDir)) { + throw "Downloaded Codex npm archive did not contain the expected legacy platform package layout." + } } if (Test-Path -LiteralPath $releaseDir) { @@ -691,10 +809,15 @@ try { Ensure-Junction -LinkPath $currentDir -TargetPath $releaseDir -InstallerOwnedTargetPrefix $releasesDir $visibleParent = Split-Path -Parent $visibleBinDir + $currentBinDir = if ($installLayout -eq "Package") { + Join-Path $currentDir "bin" + } else { + $currentDir + } New-Item -ItemType Directory -Force -Path $visibleParent | Out-Null $oldStandaloneBackup = Move-OldStandaloneBinIfApproved -VisibleBinDir $visibleBinDir -DefaultVisibleBinDir $defaultVisibleBinDir try { - Ensure-Junction -LinkPath $visibleBinDir -TargetPath $currentDir -InstallerOwnedTargetPrefix $standaloneRoot + Ensure-Junction -LinkPath $visibleBinDir -TargetPath $currentBinDir -InstallerOwnedTargetPrefix $standaloneRoot Test-VisibleCodexCommand -VisibleBinDir $visibleBinDir } catch { if ($null -ne $oldStandaloneBackup -and (Test-Path -LiteralPath $oldStandaloneBackup)) { diff --git a/scripts/install/install.sh b/scripts/install/install.sh index 2fc585d7e9..cc4fc91345 100755 --- a/scripts/install/install.sh +++ b/scripts/install/install.sh @@ -120,24 +120,29 @@ release_metadata_url() { printf 'https://api.github.com/repos/openai/codex/releases/tags/rust-v%s\n' "$resolved_version" } -release_asset_digest() { +release_asset_digest_or_empty() { asset="$1" resolved_version="$2" release_json="$(download_text "$(release_metadata_url "$resolved_version")")" digest="$(printf '%s\n' "$release_json" | awk -v asset="$asset" ' - { - if ($0 ~ "\"name\":[[:space:]]*\"" asset "\"") { + /"name":[[:space:]]*"[^"]+"/ { + name = $0 + sub(/^.*"name":[[:space:]]*"/, "", name) + sub(/".*$/, "", name) + if (name == asset) { in_asset = 1 asset_depth = depth } + } - if (in_asset && /"digest":[[:space:]]*"[^"]+"/) { - sub(/^.*"digest":[[:space:]]*"/, "") - sub(/".*$/, "") - digest = $0 - } + in_asset && /"digest":[[:space:]]*"[^"]+"/ { + digest = $0 + sub(/^.*"digest":[[:space:]]*"/, "", digest) + sub(/".*$/, "", digest) + } + { line = $0 opens = gsub(/\{/, "{", line) closes = gsub(/\}/, "}", line) @@ -147,6 +152,7 @@ release_asset_digest() { in_asset = 0 } } + END { if (digest != "") { print digest @@ -159,12 +165,56 @@ release_asset_digest() { printf '%s\n' "${digest#sha256:}" ;; *) - echo "Could not find SHA-256 digest for release asset $asset." >&2 - exit 1 + return 1 ;; esac } +release_asset_exists() { + asset="$1" + resolved_version="$2" + + release_asset_digest_or_empty "$asset" "$resolved_version" >/dev/null 2>&1 +} + +release_asset_digest() { + asset="$1" + resolved_version="$2" + + digest="$(release_asset_digest_or_empty "$asset" "$resolved_version" || true)" + if [ -z "$digest" ]; then + echo "Could not find SHA-256 digest for release asset $asset." >&2 + exit 1 + fi + + printf '%s\n' "$digest" +} + +package_archive_digest() { + asset="$1" + manifest_path="$2" + + digest="$(awk -v asset="$asset" ' + $2 == asset && $1 ~ /^[0-9a-fA-F]{64}$/ { + print tolower($1) + found = 1 + exit + } + END { + if (!found) { + exit 1 + } + } + ' "$manifest_path" 2>/dev/null || true)" + + if [ -z "$digest" ]; then + echo "Could not find SHA-256 digest for $asset in codex-package_SHA256SUMS." >&2 + exit 1 + fi + + printf '%s\n' "$digest" +} + file_sha256() { path="$1" @@ -193,7 +243,7 @@ verify_archive_digest() { actual_digest="$(file_sha256 "$archive_path")" if [ "$actual_digest" != "$expected_digest" ]; then - echo "Downloaded Codex archive checksum did not match release metadata." >&2 + echo "Downloaded Codex archive checksum did not match expected digest." >&2 echo "expected: $expected_digest" >&2 echo "actual: $actual_digest" >&2 exit 1 @@ -441,6 +491,12 @@ version_from_binary() { } current_installed_version() { + version="$(version_from_binary "$CURRENT_LINK/bin/codex" || true)" + if [ -n "$version" ]; then + printf '%s\n' "$version" + return 0 + fi + version="$(version_from_binary "$CURRENT_LINK/codex" || true)" if [ -n "$version" ]; then printf '%s\n' "$version" @@ -584,18 +640,43 @@ handle_conflicting_install() { fi } -install_release() { +install_package_release() { release_dir="$1" - vendor_root="$2" + archive_path="$2" stage_release="$RELEASES_DIR/.staging.$(basename "$release_dir").$$" mkdir -p "$RELEASES_DIR" rm -rf "$stage_release" - mkdir -p "$stage_release/codex-resources" + mkdir -p "$stage_release" + tar -xzf "$archive_path" -C "$stage_release" + chmod 0755 "$stage_release/bin/codex" "$stage_release/codex-path/rg" + if [ -f "$stage_release/codex-resources/bwrap" ]; then + chmod 0755 "$stage_release/codex-resources/bwrap" + fi + ln -sf "bin/codex" "$stage_release/codex" + + if [ -e "$release_dir" ] || [ -L "$release_dir" ]; then + rm -rf "$release_dir" + fi + mv "$stage_release" "$release_dir" +} + +install_legacy_platform_npm_release() { + release_dir="$1" + archive_path="$2" + target="$3" + stage_release="$RELEASES_DIR/.staging.$(basename "$release_dir").$$" + extract_dir="$tmp_dir/extract" + vendor_root="$extract_dir/package/vendor/$target" + + mkdir -p "$RELEASES_DIR" + rm -rf "$stage_release" "$extract_dir" + mkdir -p "$stage_release/codex-resources" "$extract_dir" + tar -xzf "$archive_path" -C "$extract_dir" + cp "$vendor_root/codex/codex" "$stage_release/codex" cp "$vendor_root/path/rg" "$stage_release/codex-resources/rg" - chmod 0755 "$stage_release/codex" - chmod 0755 "$stage_release/codex-resources/rg" + chmod 0755 "$stage_release/codex" "$stage_release/codex-resources/rg" if [ -f "$vendor_root/codex-resources/bwrap" ]; then cp "$vendor_root/codex-resources/bwrap" "$stage_release/codex-resources/bwrap" chmod 0755 "$stage_release/codex-resources/bwrap" @@ -611,15 +692,34 @@ release_dir_is_complete() { release_dir="$1" expected_version="$2" expected_target="$3" + layout="$4" [ -d "$release_dir" ] && - [ -x "$release_dir/codex" ] && - [ -x "$release_dir/codex-resources/rg" ] && - [ "$(basename "$release_dir")" = "$expected_version-$expected_target" ] && - case "$expected_target" in - *linux*) [ -x "$release_dir/codex-resources/bwrap" ] ;; - *) true ;; - esac + [ "$(basename "$release_dir")" = "$expected_version-$expected_target" ] || + return 1 + + case "$layout" in + package) + [ -f "$release_dir/codex-package.json" ] && + [ -x "$release_dir/bin/codex" ] && + [ -x "$release_dir/codex" ] && + [ -x "$release_dir/codex-path/rg" ] || + return 1 + ;; + legacy-platform-npm) + [ -x "$release_dir/codex" ] && + [ -x "$release_dir/codex-resources/rg" ] || + return 1 + ;; + *) + return 1 + ;; + esac + + case "$layout:$expected_target" in + package:*linux* | legacy-platform-npm:*linux*) [ -x "$release_dir/codex-resources/bwrap" ] ;; + *) true ;; + esac } update_current_link() { @@ -629,11 +729,23 @@ update_current_link() { replace_path_with_symlink "$CURRENT_LINK" "$release_dir" "$tmp_link" } +release_codex_relative_path() { + release_dir="$1" + + if [ -x "$release_dir/bin/codex" ]; then + printf 'bin/codex\n' + else + printf 'codex\n' + fi +} + update_visible_command() { + release_dir="$1" mkdir -p "$BIN_DIR" tmp_link="$BIN_DIR/.codex.$$" + codex_relative_path="$(release_codex_relative_path "$release_dir")" - replace_path_with_symlink "$BIN_PATH" "$CURRENT_LINK/codex" "$tmp_link" + replace_path_with_symlink "$BIN_PATH" "$CURRENT_LINK/$codex_relative_path" "$tmp_link" } verify_visible_command() { @@ -700,8 +812,21 @@ else fi resolved_version="$(resolve_version)" -asset="codex-npm-$npm_tag-$resolved_version.tgz" +package_asset="codex-package-$vendor_target.tar.gz" +checksum_asset="codex-package_SHA256SUMS" +if release_asset_exists "$package_asset" "$resolved_version" && + release_asset_exists "$checksum_asset" "$resolved_version"; then + install_layout="package" + asset="$package_asset" +elif release_asset_exists "codex-npm-$npm_tag-$resolved_version.tgz" "$resolved_version"; then + install_layout="legacy-platform-npm" + asset="codex-npm-$npm_tag-$resolved_version.tgz" +else + echo "Could not find Codex package or platform npm release assets for Codex $resolved_version." >&2 + exit 1 +fi download_url="$(release_url_for_asset "$asset" "$resolved_version")" +checksum_url="$(release_url_for_asset "$checksum_asset" "$resolved_version")" release_name="$resolved_version-$vendor_target" release_dir="$RELEASES_DIR/$release_name" current_version="$(current_installed_version)" @@ -730,27 +855,35 @@ trap cleanup EXIT INT TERM acquire_install_lock cleanup_stale_install_artifacts -if ! release_dir_is_complete "$release_dir" "$resolved_version" "$vendor_target"; then +if ! release_dir_is_complete "$release_dir" "$resolved_version" "$vendor_target" "$install_layout"; then if [ -e "$release_dir" ] || [ -L "$release_dir" ]; then warn "Found incomplete existing release at $release_dir; reinstalling." fi archive_path="$tmp_dir/$asset" - extract_dir="$tmp_dir/extract" + checksum_path="$tmp_dir/$checksum_asset" step "Downloading Codex CLI" - expected_digest="$(release_asset_digest "$asset" "$resolved_version")" + if [ "$install_layout" = "package" ]; then + checksum_digest="$(release_asset_digest "$checksum_asset" "$resolved_version")" + download_file "$checksum_url" "$checksum_path" + verify_archive_digest "$checksum_path" "$checksum_digest" + expected_digest="$(package_archive_digest "$asset" "$checksum_path")" + else + expected_digest="$(release_asset_digest "$asset" "$resolved_version")" + fi download_file "$download_url" "$archive_path" verify_archive_digest "$archive_path" "$expected_digest" - mkdir -p "$extract_dir" - tar -xzf "$archive_path" -C "$extract_dir" - step "Installing standalone package to $release_dir" - install_release "$release_dir" "$extract_dir/package/vendor/$vendor_target" + if [ "$install_layout" = "package" ]; then + install_package_release "$release_dir" "$archive_path" + else + install_legacy_platform_npm_release "$release_dir" "$archive_path" "$vendor_target" + fi fi update_current_link "$release_dir" -update_visible_command +update_visible_command "$release_dir" add_to_path verify_visible_command release_install_lock From 74b0b65324fe6076b58ffc5f27749410e90ba02b Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 08:48:10 -0700 Subject: [PATCH 2/3] npm: ship platform packages in Codex package layout ## Summary The npm platform packages should stop carrying a bespoke native layout now that the release workflow builds canonical Codex package archives. Keeping npm on the same `bin/`, `codex-resources/`, and `codex-path/` structure lets the Rust package-layout detection behave consistently across standalone, npm, and future DotSlash installs. This changes platform npm packages to stage the `codex-package` artifact for each target under `vendor/`. The Node launcher now resolves `bin/codex` and prepends `codex-path`, while retaining legacy `vendor//codex` and `vendor//path` fallback support for local development and migration. The npm staging helper downloads `codex-package` archives instead of rebuilding the CLI payload from individual `codex`, `rg`, `bwrap`, and sandbox helper artifacts. CI still needs to stage npm packages from historical rust-release workflow artifacts that predate package archives, so the staging scripts now expose an explicit `--allow-legacy-codex-package` fallback. That fallback synthesizes the canonical package layout from legacy per-binary artifacts and is wired only into the CI smoke path; release staging remains strict and continues to require real package archives. ## Test plan - `python3 -m py_compile codex-cli/scripts/build_npm_package.py codex-cli/scripts/install_native_deps.py scripts/stage_npm_packages.py scripts/codex_package/cli.py` - `node --check codex-cli/bin/codex.js` - `ruby -e 'require "yaml"; YAML.load_file(".github/workflows/rust-release.yml"); YAML.load_file(".github/workflows/ci.yml"); puts "ok"'` - Staged a synthetic `codex-linux-x64` platform package from a canonical vendor tree and verified it copied only `bin/`, `codex-path/`, `codex-resources/`, and `codex-package.json`. - Imported `install_native_deps.py` and extracted a synthetic `codex-package-x86_64-unknown-linux-musl.tar.gz` into `vendor/`. - Ran legacy-layout conversion smokes for Linux, Windows, and unsigned macOS artifact naming. - `NPM_CONFIG_CACHE="$tmp_dir/npm-cache" python3 ./scripts/stage_npm_packages.py --release-version 0.125.0 --workflow-url https://github.com/openai/codex/actions/runs/26131514935 --package codex --allow-legacy-codex-package --output-dir "$tmp_dir"` - `node codex-cli/bin/codex.js --version` --- .github/workflows/ci.yml | 8 +- .github/workflows/rust-release.yml | 2 - codex-cli/bin/codex.js | 60 ++++--- codex-cli/scripts/README.md | 8 +- codex-cli/scripts/build_npm_package.py | 70 ++++++-- codex-cli/scripts/install_native_deps.py | 201 +++++++++++++++++++++-- scripts/stage_npm_packages.py | 22 ++- 7 files changed, 307 insertions(+), 64 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1c60acc26..97591d4c19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,14 +53,16 @@ jobs: # Passing the workflow URL directly avoids relying on old rust-v* # branches remaining discoverable via `gh run list --branch ...`. CODEX_VERSION=0.125.0 - WORKFLOW_URL="https://github.com/openai/codex/actions/runs/24901475298" + WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26131514935" OUTPUT_DIR="${RUNNER_TEMP}" - # This reused workflow predates the standalone bwrap artifact. + # This reused workflow predates codex-package archive artifacts, so + # CI synthesizes the package layout from the older per-binary + # artifacts. Release staging must use real package archives. python3 ./scripts/stage_npm_packages.py \ --release-version "$CODEX_VERSION" \ --workflow-url "$WORKFLOW_URL" \ --package codex \ - --allow-missing-native-component bwrap \ + --allow-legacy-codex-package \ --output-dir "$OUTPUT_DIR" PACK_OUTPUT="${OUTPUT_DIR}/codex-npm-${CODEX_VERSION}.tgz" echo "pack_output=$PACK_OUTPUT" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 93e23b8fd7..7268a24ad8 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -1208,8 +1208,6 @@ jobs: if: ${{ env.SIGN_MACOS == 'true' }} run: pnpm install --frozen-lockfile - # stage_npm_packages.py requires DotSlash when staging releases. - - uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - name: Stage npm packages if: ${{ env.SIGN_MACOS == 'true' }} env: diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index 475239549a..1a43ce7e1f 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -77,33 +77,43 @@ if (!platformPackage) { const codexBinaryName = process.platform === "win32" ? "codex.exe" : "codex"; const localVendorRoot = path.join(__dirname, "..", "vendor"); -const localBinaryPath = path.join( - localVendorRoot, - targetTriple, - "codex", - codexBinaryName, -); +const packageBinaryPath = (vendorRoot) => + path.join(vendorRoot, targetTriple, "bin", codexBinaryName); +const legacyBinaryPath = (vendorRoot) => + path.join(vendorRoot, targetTriple, "codex", codexBinaryName); -let vendorRoot; -try { - const packageJsonPath = require.resolve(`${platformPackage}/package.json`); - vendorRoot = path.join(path.dirname(packageJsonPath), "vendor"); -} catch { - if (existsSync(localBinaryPath)) { - vendorRoot = localVendorRoot; - } else { - const packageManager = detectPackageManager(); - const updateCommand = - packageManager === "bun" - ? "bun install -g @openai/codex@latest" - : "npm install -g @openai/codex@latest"; - throw new Error( - `Missing optional dependency ${platformPackage}. Reinstall Codex: ${updateCommand}`, - ); +function resolveNativePackage(vendorRoot) { + const packageRoot = path.join(vendorRoot, targetTriple); + const binaryPath = packageBinaryPath(vendorRoot); + if (existsSync(binaryPath)) { + return { + binaryPath, + pathDir: path.join(packageRoot, "codex-path"), + }; } + + const legacyPath = legacyBinaryPath(vendorRoot); + if (existsSync(legacyPath)) { + return { + binaryPath: legacyPath, + pathDir: path.join(packageRoot, "path"), + }; + } + + return null; } -if (!vendorRoot) { +let nativePackage; +try { + const packageJsonPath = require.resolve(`${platformPackage}/package.json`); + nativePackage = resolveNativePackage( + path.join(path.dirname(packageJsonPath), "vendor"), + ); +} catch { + nativePackage = resolveNativePackage(localVendorRoot); +} + +if (!nativePackage) { const packageManager = detectPackageManager(); const updateCommand = packageManager === "bun" @@ -114,8 +124,7 @@ if (!vendorRoot) { ); } -const archRoot = path.join(vendorRoot, targetTriple); -const binaryPath = path.join(archRoot, "codex", codexBinaryName); +const { binaryPath, pathDir } = nativePackage; // Use an asynchronous spawn instead of spawnSync so that Node is able to // respond to signals (e.g. Ctrl-C / SIGINT) while the native binary is @@ -159,7 +168,6 @@ function detectPackageManager() { } const additionalDirs = []; -const pathDir = path.join(archRoot, "path"); if (existsSync(pathDir)) { additionalDirs.push(pathDir); } diff --git a/codex-cli/scripts/README.md b/codex-cli/scripts/README.md index ca0d54b544..ce58097db5 100644 --- a/codex-cli/scripts/README.md +++ b/codex-cli/scripts/README.md @@ -11,13 +11,13 @@ example, to stage the CLI, responses proxy, and SDK packages for version `0.6.0` --package codex-sdk ``` -This downloads the native artifacts once, hydrates `vendor/` for each package, and writes -tarballs to `dist/npm/`. +This downloads the native package archive artifacts once, hydrates `vendor/` for each +package, and writes tarballs to `dist/npm/`. When `--package codex` is provided, the staging helper builds the lightweight `@openai/codex` meta package plus all platform-native `@openai/codex` variants that are later published under platform-specific dist-tags. If you need to invoke `build_npm_package.py` directly, run -`codex-cli/scripts/install_native_deps.py` first and pass `--vendor-src` pointing to the -directory that contains the populated `vendor/` tree. +`codex-cli/scripts/install_native_deps.py --component codex-package` first and pass +`--vendor-src` pointing to the directory that contains the populated `vendor/` tree. diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index c38fed68d9..5d4bc99c4e 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -15,6 +15,8 @@ REPO_ROOT = CODEX_CLI_ROOT.parent RESPONSES_API_PROXY_NPM_ROOT = REPO_ROOT / "codex-rs" / "responses-api-proxy" / "npm" CODEX_SDK_ROOT = REPO_ROOT / "sdk" / "typescript" CODEX_NPM_NAME = "@openai/codex" +CODEX_PACKAGE_COMPONENT = "codex-package" +CODEX_PACKAGE_ENTRIES = ("codex-package.json", "bin", "codex-resources", "codex-path") # `npm_name` is the local optional-dependency alias consumed by `bin/codex.js`. # The underlying package published to npm is always `@openai/codex`. @@ -69,12 +71,12 @@ PACKAGE_EXPANSIONS: dict[str, list[str]] = { PACKAGE_NATIVE_COMPONENTS: dict[str, list[str]] = { "codex": [], - "codex-linux-x64": ["bwrap", "codex", "rg"], - "codex-linux-arm64": ["bwrap", "codex", "rg"], - "codex-darwin-x64": ["codex", "rg"], - "codex-darwin-arm64": ["codex", "rg"], - "codex-win32-x64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"], - "codex-win32-arm64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"], + "codex-linux-x64": [CODEX_PACKAGE_COMPONENT], + "codex-linux-arm64": [CODEX_PACKAGE_COMPONENT], + "codex-darwin-x64": [CODEX_PACKAGE_COMPONENT], + "codex-darwin-arm64": [CODEX_PACKAGE_COMPONENT], + "codex-win32-x64": [CODEX_PACKAGE_COMPONENT], + "codex-win32-arm64": [CODEX_PACKAGE_COMPONENT], "codex-responses-api-proxy": ["codex-responses-api-proxy"], "codex-sdk": [], } @@ -383,7 +385,11 @@ def copy_native_binaries( if not vendor_src.exists(): raise RuntimeError(f"Vendor source directory not found: {vendor_src}") - components_set = {component for component in components if component in COMPONENT_DEST_DIR} + components_set = { + component + for component in components + if component == CODEX_PACKAGE_COMPONENT or component in COMPONENT_DEST_DIR + } allow_missing_components = allow_missing_components or set() if not components_set: return @@ -402,11 +408,26 @@ def copy_native_binaries( if target_filter is not None and target_dir.name not in target_filter: continue - dest_target_dir = vendor_dest / target_dir.name - dest_target_dir.mkdir(parents=True, exist_ok=True) copied_targets.add(target_dir.name) - for component in components_set: + dest_target_dir = vendor_dest / target_dir.name + + if CODEX_PACKAGE_COMPONENT in components_set: + validate_codex_package_dir(target_dir) + if dest_target_dir.exists(): + shutil.rmtree(dest_target_dir) + dest_target_dir.mkdir(parents=True, exist_ok=True) + for entry in CODEX_PACKAGE_ENTRIES: + src = target_dir / entry + dest = dest_target_dir / entry + if src.is_dir(): + shutil.copytree(src, dest) + else: + shutil.copy2(src, dest) + else: + dest_target_dir.mkdir(parents=True, exist_ok=True) + + for component in components_set - {CODEX_PACKAGE_COMPONENT}: dest_dir_name = COMPONENT_DEST_DIR.get(component) if dest_dir_name is None: continue @@ -431,6 +452,35 @@ def copy_native_binaries( raise RuntimeError(f"Missing target directories in vendor source: {missing_list}") +def validate_codex_package_dir(package_dir: Path) -> None: + is_windows = "windows" in package_dir.name + required_files = [ + Path("codex-package.json"), + Path("bin") / ("codex.exe" if is_windows else "codex"), + Path("codex-path") / ("rg.exe" if is_windows else "rg"), + ] + + if "linux" in package_dir.name: + required_files.append(Path("codex-resources") / "bwrap") + + if is_windows: + required_files.extend( + [ + Path("codex-resources") / "codex-command-runner.exe", + Path("codex-resources") / "codex-windows-sandbox-setup.exe", + ] + ) + + missing_files = [ + str(relative_path) + for relative_path in required_files + if not (package_dir / relative_path).is_file() + ] + if missing_files: + missing = ", ".join(missing_files) + raise RuntimeError(f"Missing files in Codex package directory {package_dir}: {missing}") + + def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: output_path = output_path.resolve() output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py index 79f596fbaf..73f5134f09 100755 --- a/codex-cli/scripts/install_native_deps.py +++ b/codex-cli/scripts/install_native_deps.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Install Codex native binaries (Rust CLI, bwrap, and ripgrep helpers).""" +"""Install Codex package archives and native helper binaries.""" import argparse from contextlib import contextmanager @@ -31,6 +31,7 @@ BINARY_TARGETS = ( "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc", ) +CODEX_PACKAGE_COMPONENT = "codex-package" @dataclass(frozen=True) @@ -139,11 +140,19 @@ def parse_args() -> argparse.Namespace: "--component", dest="components", action="append", - choices=tuple(list(BINARY_COMPONENTS) + ["rg"]), + choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS, "rg"]), help=( "Limit installation to the specified components." - " May be repeated. Defaults to bwrap, codex, codex-windows-sandbox-setup," - " codex-command-runner, and rg." + " May be repeated. Defaults to codex-package and codex-responses-api-proxy." + ), + ) + parser.add_argument( + "--allow-legacy-codex-package", + action="store_true", + help=( + "Allow codex-package to be synthesized from legacy per-binary artifacts " + "when package archives are missing. Intended for CI compatibility only; " + "release staging should not use this." ), ) parser.add_argument( @@ -165,13 +174,7 @@ def main() -> int: vendor_dir = codex_cli_root / VENDOR_DIR_NAME vendor_dir.mkdir(parents=True, exist_ok=True) - components = args.components or [ - "bwrap", - "codex", - "codex-windows-sandbox-setup", - "codex-command-runner", - "rg", - ] + components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"] workflow_url = (args.workflow_url or DEFAULT_WORKFLOW_URL).strip() if not workflow_url: @@ -184,6 +187,18 @@ def main() -> int: with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: artifacts_dir = Path(artifacts_dir_str) _download_artifacts(workflow_id, artifacts_dir) + if CODEX_PACKAGE_COMPONENT in components: + try: + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) + except FileNotFoundError: + if not args.allow_legacy_codex_package: + raise + install_legacy_codex_package_layouts( + artifacts_dir, + vendor_dir, + BINARY_TARGETS, + manifest_path=RG_MANIFEST, + ) install_binary_components( artifacts_dir, vendor_dir, @@ -199,6 +214,135 @@ def main() -> int: return 0 +def install_codex_package_archives( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], +) -> None: + targets = list(targets) + if not targets: + return + + print("Installing Codex package archives for targets: " + ", ".join(targets)) + max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + _install_single_codex_package_archive, + artifacts_dir, + vendor_dir, + target, + ): target + for target in targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}") + + +def _install_single_codex_package_archive( + artifacts_dir: Path, + vendor_dir: Path, + target: str, +) -> Path: + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = artifact_subdir / f"codex-package-{target}.tar.gz" + if not archive_path.exists(): + raise FileNotFoundError(f"Expected package archive not found: {archive_path}") + + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(archive_path, "r:gz") as archive: + archive.extractall(dest_dir, filter="data") + + return dest_dir + + +def install_legacy_codex_package_layouts( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], + *, + manifest_path: Path, +) -> None: + targets = list(targets) + print( + "Synthesizing Codex package layouts from legacy artifacts for targets: " + + ", ".join(targets) + ) + with tempfile.TemporaryDirectory(prefix="codex-legacy-package-") as legacy_vendor_dir_str: + legacy_vendor_dir = Path(legacy_vendor_dir_str) + install_binary_components( + artifacts_dir, + legacy_vendor_dir, + [ + BINARY_COMPONENTS["codex"], + BINARY_COMPONENTS["bwrap"], + BINARY_COMPONENTS["codex-windows-sandbox-setup"], + BINARY_COMPONENTS["codex-command-runner"], + ], + ) + fetch_rg(legacy_vendor_dir, targets, manifest_path=manifest_path) + + for target in targets: + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + _build_legacy_codex_package_layout(legacy_vendor_dir / target, dest_dir, target) + print(f" synthesized {dest_dir}") + + +def _build_legacy_codex_package_layout( + legacy_target_dir: Path, + package_dir: Path, + target: str, +) -> None: + is_windows = "windows" in target + exe_suffix = ".exe" if is_windows else "" + package_dir.mkdir(parents=True) + + bin_dir = package_dir / "bin" + resources_dir = package_dir / "codex-resources" + path_dir = package_dir / "codex-path" + bin_dir.mkdir() + resources_dir.mkdir() + path_dir.mkdir() + + shutil.copy2( + legacy_target_dir / "codex" / f"codex{exe_suffix}", + bin_dir / f"codex{exe_suffix}", + ) + shutil.copy2( + legacy_target_dir / "path" / f"rg{exe_suffix}", + path_dir / f"rg{exe_suffix}", + ) + + if is_windows: + for helper in [ + "codex-command-runner.exe", + "codex-windows-sandbox-setup.exe", + ]: + shutil.copy2(legacy_target_dir / "codex" / helper, resources_dir / helper) + elif "linux" in target: + shutil.copy2(legacy_target_dir / "codex-resources" / "bwrap", resources_dir / "bwrap") + + write_json( + package_dir / "codex-package.json", + { + "layoutVersion": 1, + "version": "unknown", + "target": target, + "variant": "codex", + "entrypoint": f"bin/codex{exe_suffix}", + "resourcesDir": "codex-resources", + "pathDir": "codex-path", + }, + ) + + def fetch_rg( vendor_dir: Path, targets: Sequence[str] | None = None, @@ -319,11 +463,8 @@ def _install_single_binary( target: str, component: BinaryComponent, ) -> Path: - artifact_subdir = artifacts_dir / target - archive_name = _archive_name_for_target(component.artifact_prefix, target) - archive_path = artifact_subdir / archive_name - if not archive_path.exists(): - raise FileNotFoundError(f"Expected artifact not found: {archive_path}") + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = legacy_binary_archive_path(artifact_subdir, component.artifact_prefix, target) dest_dir = vendor_dir / target / component.dest_dir dest_dir.mkdir(parents=True, exist_ok=True) @@ -345,6 +486,28 @@ def _archive_name_for_target(artifact_prefix: str, target: str) -> str: return f"{artifact_prefix}-{target}.zst" +def legacy_binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: + archive_names = [_archive_name_for_target(artifact_prefix, target)] + if artifact_dir.name == f"{target}-unsigned": + archive_names.append(_archive_name_for_target(artifact_prefix, f"{target}-unsigned")) + + for archive_name in archive_names: + archive_path = artifact_dir / archive_name + if archive_path.exists(): + return archive_path + + raise FileNotFoundError(f"Expected artifact not found: {artifact_dir / archive_names[0]}") + + +def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: + for artifact_name in [target, f"{target}-unsigned"]: + artifact_dir = artifacts_dir / artifact_name + if artifact_dir.is_dir(): + return artifact_dir + + return artifacts_dir / target + + def _fetch_single_rg( vendor_dir: Path, target: str, @@ -477,6 +640,12 @@ def _load_manifest(manifest_path: Path) -> dict: return manifest +def write_json(path: Path, value: object) -> None: + with open(path, "w", encoding="utf-8") as out: + json.dump(value, out, indent=2) + out.write("\n") + + if __name__ == "__main__": import sys diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 2a0cd0028d..4eb69053eb 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """Stage one or more Codex npm packages for release.""" -from __future__ import annotations - import argparse import importlib.util import json @@ -68,6 +66,15 @@ def parse_args() -> argparse.Namespace: "Intended for CI compatibility only; release staging should not use this." ), ) + parser.add_argument( + "--allow-legacy-codex-package", + action="store_true", + help=( + "Allow codex-package layouts to be synthesized from legacy per-binary " + "workflow artifacts. Intended for CI compatibility only; release staging " + "should not use this." + ), + ) return parser.parse_args() @@ -124,11 +131,15 @@ def install_native_components( workflow_url: str, components: set[str], vendor_root: Path, + *, + allow_legacy_codex_package: bool, ) -> None: if not components: return cmd = [str(INSTALL_NATIVE_DEPS), "--workflow-url", workflow_url] + if allow_legacy_codex_package: + cmd.append("--allow-legacy-codex-package") for component in sorted(components): cmd.extend(["--component", component]) cmd.append(str(vendor_root)) @@ -172,7 +183,12 @@ def main() -> int: args.release_version, args.workflow_url ) vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) - install_native_components(workflow_url, native_components_to_install, vendor_temp_root) + install_native_components( + workflow_url, + native_components_to_install, + vendor_temp_root, + allow_legacy_codex_package=args.allow_legacy_codex_package, + ) vendor_src = vendor_temp_root / "vendor" if resolved_head_sha: From 6c3f501c1d045b8ed45befcabc2172b8c7cf5c1a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 08:48:10 -0700 Subject: [PATCH 3/3] dotslash: publish Codex entrypoints from package archives ## Summary DotSlash should resolve the same canonical package layout used by standalone installers and npm platform packages, but it should not take the size regression of downloading gzip-compressed package archives. Installers continue to use `.tar.gz` because that format is broadly available, while DotSlash can use zstd-compressed tar archives. This updates the package archive release helper to emit both `*.tar.gz` and `*.tar.zst` package archives from the same package directory. The Codex CLI and `codex-app-server` DotSlash release config entries now match `codex-package-.tar.zst` and `codex-app-server-package-.tar.zst`, with paths that select `bin/codex` or `bin/codex-app-server` inside the extracted package. The Windows release compression loop also skips prebuilt `.tar.zst` archives so it does not try to recompress them as ordinary files. The other helper outputs stay on their existing per-binary artifacts for now. ## Test plan - `python3 -m json.tool .github/dotslash-config.json > /dev/null` - `bash -n .github/scripts/build-codex-package-archive.sh` - `python3 -m py_compile scripts/codex_package/cli.py` - `ruby -e 'require "yaml"; YAML.load_file(".github/workflows/rust-release-windows.yml"); puts "ok"'` - Ran a Python regex smoke test that checked every updated `codex` and `codex-app-server` platform entry against the `.tar.zst` archive names emitted by `.github/scripts/build-codex-package-archive.sh`. - Ran `scripts/build_codex_package.py` with synthetic macOS entrypoint and `rg` binaries and repeated `--archive-output` values for `.tar.gz` and `.tar.zst`, verifying both archives were written. --- .github/dotslash-config.json | 48 +++++++++---------- .../scripts/build-codex-package-archive.sh | 10 +++- .github/workflows/rust-release-windows.yml | 2 +- scripts/codex_package/cli.py | 9 ++-- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/.github/dotslash-config.json b/.github/dotslash-config.json index a0297c269a..78ea6b37e6 100644 --- a/.github/dotslash-config.json +++ b/.github/dotslash-config.json @@ -3,56 +3,56 @@ "codex": { "platforms": { "macos-aarch64": { - "regex": "^codex-aarch64-apple-darwin\\.zst$", - "path": "codex" + "regex": "^codex-package-aarch64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex" }, "macos-x86_64": { - "regex": "^codex-x86_64-apple-darwin\\.zst$", - "path": "codex" + "regex": "^codex-package-x86_64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex" }, "linux-x86_64": { - "regex": "^codex-x86_64-unknown-linux-musl-bundle\\.tar\\.zst$", - "path": "codex" + "regex": "^codex-package-x86_64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex" }, "linux-aarch64": { - "regex": "^codex-aarch64-unknown-linux-musl-bundle\\.tar\\.zst$", - "path": "codex" + "regex": "^codex-package-aarch64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex" }, "windows-x86_64": { - "regex": "^codex-x86_64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex.exe" + "regex": "^codex-package-x86_64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex.exe" }, "windows-aarch64": { - "regex": "^codex-aarch64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex.exe" + "regex": "^codex-package-aarch64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex.exe" } } }, "codex-app-server": { "platforms": { "macos-aarch64": { - "regex": "^codex-app-server-aarch64-apple-darwin\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-aarch64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "macos-x86_64": { - "regex": "^codex-app-server-x86_64-apple-darwin\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-x86_64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "linux-x86_64": { - "regex": "^codex-app-server-x86_64-unknown-linux-musl\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-x86_64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "linux-aarch64": { - "regex": "^codex-app-server-aarch64-unknown-linux-musl\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-aarch64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "windows-x86_64": { - "regex": "^codex-app-server-x86_64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex-app-server.exe" + "regex": "^codex-app-server-package-x86_64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex-app-server.exe" }, "windows-aarch64": { - "regex": "^codex-app-server-aarch64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex-app-server.exe" + "regex": "^codex-app-server-package-aarch64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex-app-server.exe" } } }, diff --git a/.github/scripts/build-codex-package-archive.sh b/.github/scripts/build-codex-package-archive.sh index 90eae12ef0..c475ffe952 100644 --- a/.github/scripts/build-codex-package-archive.sh +++ b/.github/scripts/build-codex-package-archive.sh @@ -97,9 +97,14 @@ else python_bin="python" fi +if ! command -v zstd >/dev/null 2>&1 && [[ -x "${repo_root}/.github/workflows/zstd" ]]; then + export PATH="${repo_root}/.github/workflows:${PATH}" +fi + mkdir -p "$archive_dir" package_dir="${RUNNER_TEMP:-/tmp}/${archive_stem}-${target}" -archive_path="${archive_dir}/${archive_stem}-${target}.tar.gz" +gzip_archive_path="${archive_dir}/${archive_stem}-${target}.tar.gz" +zstd_archive_path="${archive_dir}/${archive_stem}-${target}.tar.zst" rm -rf "$package_dir" "$python_bin" "${repo_root}/scripts/build_codex_package.py" \ @@ -108,5 +113,6 @@ rm -rf "$package_dir" --entrypoint-bin "${entrypoint_dir%/}/${entrypoint_name}${exe_suffix}" \ --cargo-profile release \ --package-dir "$package_dir" \ - --archive-output "$archive_path" \ + --archive-output "$gzip_archive_path" \ + --archive-output "$zstd_archive_path" \ --force diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index ac28b7855a..81b5a1e51b 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -295,7 +295,7 @@ jobs: base="$(basename "$f")" # Skip files that are already archives (shouldn't happen, but be # safe). - if [[ "$base" == *.tar.gz || "$base" == *.zip || "$base" == *.dmg ]]; then + if [[ "$base" == *.tar.gz || "$base" == *.tar.zst || "$base" == *.zip || "$base" == *.dmg ]]; then continue fi diff --git a/scripts/codex_package/cli.py b/scripts/codex_package/cli.py index 16324cf44d..0ca9f5d35e 100644 --- a/scripts/codex_package/cli.py +++ b/scripts/codex_package/cli.py @@ -50,9 +50,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--archive-output", type=Path, + action="append", + default=[], help=( - "Optional archive output path. Supported suffixes: .tar.gz, .tgz, " - ".tar.zst, .zip." + "Optional archive output path. May be repeated. Supported suffixes: " + ".tar.gz, .tgz, .tar.zst, .zip." ), ) parser.add_argument( @@ -130,8 +132,7 @@ def main() -> int: build_package_dir(package_dir, version, variant, spec, inputs) validate_package_dir(package_dir, variant, spec) - archive_output = args.archive_output - if archive_output is not None: + for archive_output in args.archive_output: archive_path = archive_output.resolve() write_archive(package_dir, archive_path, force=args.force) print(f"Built Codex package archive at {archive_path}")