openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-08-23 13:09:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: parse compact release metadata in installer (#31667)

Browse Source 
# Summary

GitHub's latest-release endpoint can return compact, single-line JSON.
The standalone installer treated release metadata as line-oriented text,
so those responses could make asset lookup fail even though the
requested assets were present.

The regression was introduced by
[#31056](https://github.com/openai/codex/pull/31056). That change reused
the `/releases/latest` metadata response for both version resolution and
asset lookup, exposing the existing formatting-sensitive asset parser to
compact responses from that endpoint.

This change parses the release metadata once with a one-pass POSIX awk
scanner. The scanner tracks JSON strings and nesting, extracts the root
release tag plus direct asset name/digest pairs, and produces the same
result regardless of whitespace or object field order. It uses POSIX
`fold` to bound awk record sizes so compact responses stay fast across
awk implementations.

Fixes #31520.

## Changes

- replace line-oriented release metadata matching with structure-aware
parsing
- reuse the parsed metadata for latest-version and asset-digest lookup
- add regression coverage for compact JSON, reordered fields, nested
decoys, and JSON-looking release text

## Design decisions

- Keep the installer dependency-free by using standard POSIX tools
already required by the shell installer.
- Parse only the GitHub release fields the installer consumes, in one
pass, instead of vendoring a general JSON library.
- Preserve asset-object boundaries so nested or string-encoded `name`
and `digest` fields cannot be mistaken for release assets.

## Testing

- Tests: focused installer suite locally and on Linux.
- Smoke tests: real pretty and compact GitHub release metadata,
latest-release resolution, and checksum-asset selection.
- Portability: macOS awk plus Linux gawk, mawk, and nawk.
- Stress coverage: randomized formatting and field order, adversarial
nested/string content, and a synthetic 2,000-asset compact response.
This commit is contained in:
efrazer-oai
2026-07-08 22:08:00 -07:00
committed by GitHub
parent 3380969a29
commit dc23c7bcc8
2 changed files with 200 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
scripts/install/install.sh
View File

@@ -125,6 +125,113 @@ download_text() {
exit 1
}
parse_release_metadata() {
# Bound awk's record size so compact, single-line JSON stays fast on every
# supported awk implementation. JSON strings cannot contain literal newlines,
# so the record boundaries inserted by fold do not change the document.
LC_ALL=C fold -b -w 4096 | LC_ALL=C awk '
function finish_string(value) {
if (object_depth == 1 && key == "tag_name") {
print "tag_name\t" value
} else if (object_depth == asset_object_depth) {
if (key == "name") {
asset_name = value
} else if (key == "digest") {
asset_digest = value
}
}
expecting_value = 0
key = ""
}
{
for (i = 1; i <= length($0); i++) {
char = substr($0, i, 1)
if (in_string) {
if (escaped) {
token = token "\\" char
escaped = 0
} else if (char == "\\") {
escaped = 1
} else if (char == "\"") {
in_string = 0
if (string_is_value) {
finish_string(token)
} else {
pending_key = token
}
} else {
token = token char
}
continue
}
if (char == "\"") {
in_string = 1
token = ""
escaped = 0
string_is_value = expecting_value
} else if (char == ":" && pending_key != "") {
key = pending_key
pending_key = ""
expecting_value = 1
} else if (char == "{") {
object_depth++
if (assets_array_depth != 0 &&
array_depth == assets_array_depth &&
asset_object_depth == 0) {
asset_object_depth = object_depth
asset_name = ""
asset_digest = ""
}
expecting_value = 0
key = ""
} else if (char == "}") {
if (object_depth == asset_object_depth) {
if (asset_name != "" && asset_digest != "") {
print "asset\t" asset_name "\t" asset_digest
}
asset_object_depth = 0
asset_name = ""
asset_digest = ""
}
object_depth--
expecting_value = 0
key = ""
pending_key = ""
} else if (char == "[") {
array_depth++
if (expecting_value && key == "assets" && object_depth == 1) {
assets_array_depth = array_depth
}
expecting_value = 0
key = ""
} else if (char == "]") {
if (array_depth == assets_array_depth) {
assets_array_depth = 0
}
array_depth--
expecting_value = 0
key = ""
pending_key = ""
} else if (char == ",") {
expecting_value = 0
key = ""
pending_key = ""
}
}
}
END {
if (in_string || object_depth != 0 || array_depth != 0) {
exit 1
}
}
'
}
release_url_for_asset() {
asset="$1"
resolved_version="$2"
@@ -156,8 +263,17 @@ resolve_release() {
exit 1
fi
if ! release_metadata="$(printf '%s\n' "$release_json" | parse_release_metadata)"; then
echo "Could not parse GitHub release metadata for Codex $requested_release." >&2
exit 1
fi
if [ "$normalized_version" = "latest" ]; then
resolved_version="$(printf '%s\n' "$release_json" | sed -n 's/.*"tag_name":[[:space:]]*"rust-v\([^"]*\)".*/\1/p' | head -n 1)"
release_tag="$(printf '%s\n' "$release_metadata" | awk -F '\t' '$1 == "tag_name" { print $2; exit }')"
case "$release_tag" in
rust-v*) resolved_version="${release_tag#rust-v}" ;;
*) resolved_version="" ;;
esac
if [ -z "$resolved_version" ]; then
echo "Failed to resolve the latest Codex release version." >&2
exit 1
@@ -169,38 +285,10 @@ resolve_release() {
release_asset_digest_or_empty() {
asset="$1"
digest="$(printf '%s\n' "$release_json" | awk -v asset="$asset" '
/"name":[[:space:]]*"[^"]+"/ {
name = $0
sub(/^.*"name":[[:space:]]*"/, "", name)
sub(/".*$/, "", name)
if (name == asset) {
in_asset = 1
asset_depth = depth
}
}
in_asset && /"digest":[[:space:]]*"[^"]+"/ {
digest = $0
sub(/^.*"digest":[[:space:]]*"/, "", digest)
sub(/".*$/, "", digest)
}
{
line = $0
opens = gsub(/\{/, "{", line)
closes = gsub(/\}/, "}", line)
depth += opens - closes
if (in_asset && depth < asset_depth) {
in_asset = 0
}
}
END {
if (digest != "") {
print digest
}
digest="$(printf '%s\n' "$release_metadata" | awk -F '\t' -v asset="$asset" '
$1 == "asset" && $2 == asset {
print $3
exit
}
')"

92
scripts/install/test_install_sh.py
View File

@@ -60,9 +60,38 @@ class InstallShTest(unittest.TestCase):
)
self.assertIn(f"Resolved version: {VERSION}", result.stdout)
def test_compact_metadata_is_independent_of_field_order(self) -> None:
result, requests = run_installer(
"latest", metadata_json=release_metadata(compact=True, reorder=True)
)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(
requests,
[
"https://api.github.com/repos/openai/codex/releases/latest",
"https://github.com/openai/codex/releases/download/"
f"rust-v{VERSION}/codex-package_SHA256SUMS",
],
)
self.assertIn(f"Resolved version: {VERSION}", result.stdout)
def test_json_like_strings_and_nested_fields_do_not_define_assets(self) -> None:
result, requests = run_installer(
VERSION, metadata_json=legacy_release_metadata_with_decoys()
)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(len(requests), 2)
self.assertIn("/codex-npm-", requests[1])
self.assertNotIn("codex-package_SHA256SUMS", requests[1])
def run_installer(
release: str, *, metadata_failure: bool = False
release: str,
*,
metadata_failure: bool = False,
metadata_json: str | None = None,
) -> tuple[subprocess.CompletedProcess[str], list[str]]:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
@@ -108,7 +137,9 @@ def run_installer(
"CODEX_NON_INTERACTIVE": "1",
"CODEX_RELEASE": release,
"CODEX_TEST_METADATA_FAILURE": "1" if metadata_failure else "0",
"CODEX_TEST_METADATA_JSON": release_metadata(),
"CODEX_TEST_METADATA_JSON": (
metadata_json if metadata_json is not None else release_metadata()
),
"CODEX_TEST_REQUEST_LOG": str(request_log),
"HOME": str(root / "home"),
"PATH": f"{bin_dir}:/usr/bin:/bin",
@@ -130,12 +161,13 @@ def run_installer(
return result, requests
def release_metadata() -> str:
def release_metadata(*, compact: bool = False, reorder: bool = False) -> str:
assets = [
{
"name": f"codex-package-{target}.tar.gz",
"digest": f"sha256:{'a' * 64}",
}
asset_metadata(
f"codex-package-{target}.tar.gz",
f"sha256:{'a' * 64}",
reorder=reorder,
)
for target in (
"aarch64-apple-darwin",
"x86_64-apple-darwin",
@@ -144,14 +176,48 @@ def release_metadata() -> str:
)
]
assets.append(
{
"name": "codex-package_SHA256SUMS",
"digest": f"sha256:{'b' * 64}",
}
asset_metadata(
"codex-package_SHA256SUMS",
f"sha256:{'b' * 64}",
reorder=reorder,
)
)
separators = (",", ":") if compact else None
return json.dumps(
{"tag_name": f"rust-v{VERSION}", "assets": assets},
indent=2,
{"assets": assets, "body": "braces: { } [ ]", "tag_name": f"rust-v{VERSION}"},
indent=None if compact else 2,
separators=separators,
)
def asset_metadata(name: str, digest: str, *, reorder: bool) -> dict[str, str]:
if reorder:
return {"digest": digest, "name": name}
return {"name": name, "digest": digest}
def legacy_release_metadata_with_decoys() -> str:
fake_digest = f"sha256:{'0' * 64}"
assets = [
{
"metadata": {
"name": "codex-package-x86_64-unknown-linux-musl.tar.gz",
"digest": fake_digest,
},
"digest": f"sha256:{'c' * 64}",
"name": f"codex-npm-{target}-{VERSION}.tgz",
}
for target in ("darwin-arm64", "darwin-x64", "linux-arm64", "linux-x64")
]
return json.dumps(
{
"body": (
f'fake: {{"name":"codex-package_SHA256SUMS","digest":"{fake_digest}"}}'
),
"assets": assets,
"tag_name": f"rust-v{VERSION}",
},
separators=(",", ":"),
)