diff --git a/scripts/install/install.sh b/scripts/install/install.sh index 12b66f958a..adc79a8a27 100755 --- a/scripts/install/install.sh +++ b/scripts/install/install.sh @@ -125,6 +125,113 @@ download_text() { exit 1 } +parse_release_metadata() { + # Bound awk's record size so compact, single-line JSON stays fast on every + # supported awk implementation. JSON strings cannot contain literal newlines, + # so the record boundaries inserted by fold do not change the document. + LC_ALL=C fold -b -w 4096 | LC_ALL=C awk ' + function finish_string(value) { + if (object_depth == 1 && key == "tag_name") { + print "tag_name\t" value + } else if (object_depth == asset_object_depth) { + if (key == "name") { + asset_name = value + } else if (key == "digest") { + asset_digest = value + } + } + + expecting_value = 0 + key = "" + } + + { + for (i = 1; i <= length($0); i++) { + char = substr($0, i, 1) + + if (in_string) { + if (escaped) { + token = token "\\" char + escaped = 0 + } else if (char == "\\") { + escaped = 1 + } else if (char == "\"") { + in_string = 0 + if (string_is_value) { + finish_string(token) + } else { + pending_key = token + } + } else { + token = token char + } + continue + } + + if (char == "\"") { + in_string = 1 + token = "" + escaped = 0 + string_is_value = expecting_value + } else if (char == ":" && pending_key != "") { + key = pending_key + pending_key = "" + expecting_value = 1 + } else if (char == "{") { + object_depth++ + if (assets_array_depth != 0 && + array_depth == assets_array_depth && + asset_object_depth == 0) { + asset_object_depth = object_depth + asset_name = "" + asset_digest = "" + } + expecting_value = 0 + key = "" + } else if (char == "}") { + if (object_depth == asset_object_depth) { + if (asset_name != "" && asset_digest != "") { + print "asset\t" asset_name "\t" asset_digest + } + asset_object_depth = 0 + asset_name = "" + asset_digest = "" + } + object_depth-- + expecting_value = 0 + key = "" + pending_key = "" + } else if (char == "[") { + array_depth++ + if (expecting_value && key == "assets" && object_depth == 1) { + assets_array_depth = array_depth + } + expecting_value = 0 + key = "" + } else if (char == "]") { + if (array_depth == assets_array_depth) { + assets_array_depth = 0 + } + array_depth-- + expecting_value = 0 + key = "" + pending_key = "" + } else if (char == ",") { + expecting_value = 0 + key = "" + pending_key = "" + } + } + } + + END { + if (in_string || object_depth != 0 || array_depth != 0) { + exit 1 + } + } + ' +} + release_url_for_asset() { asset="$1" resolved_version="$2" @@ -156,8 +263,17 @@ resolve_release() { exit 1 fi + if ! release_metadata="$(printf '%s\n' "$release_json" | parse_release_metadata)"; then + echo "Could not parse GitHub release metadata for Codex $requested_release." >&2 + exit 1 + fi + if [ "$normalized_version" = "latest" ]; then - resolved_version="$(printf '%s\n' "$release_json" | sed -n 's/.*"tag_name":[[:space:]]*"rust-v\([^"]*\)".*/\1/p' | head -n 1)" + release_tag="$(printf '%s\n' "$release_metadata" | awk -F '\t' '$1 == "tag_name" { print $2; exit }')" + case "$release_tag" in + rust-v*) resolved_version="${release_tag#rust-v}" ;; + *) resolved_version="" ;; + esac if [ -z "$resolved_version" ]; then echo "Failed to resolve the latest Codex release version." >&2 exit 1 @@ -169,38 +285,10 @@ resolve_release() { release_asset_digest_or_empty() { asset="$1" - digest="$(printf '%s\n' "$release_json" | awk -v asset="$asset" ' - /"name":[[:space:]]*"[^"]+"/ { - name = $0 - sub(/^.*"name":[[:space:]]*"/, "", name) - sub(/".*$/, "", name) - if (name == asset) { - in_asset = 1 - asset_depth = depth - } - } - - in_asset && /"digest":[[:space:]]*"[^"]+"/ { - digest = $0 - sub(/^.*"digest":[[:space:]]*"/, "", digest) - sub(/".*$/, "", digest) - } - - { - line = $0 - opens = gsub(/\{/, "{", line) - closes = gsub(/\}/, "}", line) - depth += opens - closes - - if (in_asset && depth < asset_depth) { - in_asset = 0 - } - } - - END { - if (digest != "") { - print digest - } + digest="$(printf '%s\n' "$release_metadata" | awk -F '\t' -v asset="$asset" ' + $1 == "asset" && $2 == asset { + print $3 + exit } ')" diff --git a/scripts/install/test_install_sh.py b/scripts/install/test_install_sh.py index 556c6213bb..c77a0b0dac 100644 --- a/scripts/install/test_install_sh.py +++ b/scripts/install/test_install_sh.py @@ -60,9 +60,38 @@ class InstallShTest(unittest.TestCase): ) self.assertIn(f"Resolved version: {VERSION}", result.stdout) + def test_compact_metadata_is_independent_of_field_order(self) -> None: + result, requests = run_installer( + "latest", metadata_json=release_metadata(compact=True, reorder=True) + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual( + requests, + [ + "https://api.github.com/repos/openai/codex/releases/latest", + "https://github.com/openai/codex/releases/download/" + f"rust-v{VERSION}/codex-package_SHA256SUMS", + ], + ) + self.assertIn(f"Resolved version: {VERSION}", result.stdout) + + def test_json_like_strings_and_nested_fields_do_not_define_assets(self) -> None: + result, requests = run_installer( + VERSION, metadata_json=legacy_release_metadata_with_decoys() + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(requests), 2) + self.assertIn("/codex-npm-", requests[1]) + self.assertNotIn("codex-package_SHA256SUMS", requests[1]) + def run_installer( - release: str, *, metadata_failure: bool = False + release: str, + *, + metadata_failure: bool = False, + metadata_json: str | None = None, ) -> tuple[subprocess.CompletedProcess[str], list[str]]: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) @@ -108,7 +137,9 @@ def run_installer( "CODEX_NON_INTERACTIVE": "1", "CODEX_RELEASE": release, "CODEX_TEST_METADATA_FAILURE": "1" if metadata_failure else "0", - "CODEX_TEST_METADATA_JSON": release_metadata(), + "CODEX_TEST_METADATA_JSON": ( + metadata_json if metadata_json is not None else release_metadata() + ), "CODEX_TEST_REQUEST_LOG": str(request_log), "HOME": str(root / "home"), "PATH": f"{bin_dir}:/usr/bin:/bin", @@ -130,12 +161,13 @@ def run_installer( return result, requests -def release_metadata() -> str: +def release_metadata(*, compact: bool = False, reorder: bool = False) -> str: assets = [ - { - "name": f"codex-package-{target}.tar.gz", - "digest": f"sha256:{'a' * 64}", - } + asset_metadata( + f"codex-package-{target}.tar.gz", + f"sha256:{'a' * 64}", + reorder=reorder, + ) for target in ( "aarch64-apple-darwin", "x86_64-apple-darwin", @@ -144,14 +176,48 @@ def release_metadata() -> str: ) ] assets.append( - { - "name": "codex-package_SHA256SUMS", - "digest": f"sha256:{'b' * 64}", - } + asset_metadata( + "codex-package_SHA256SUMS", + f"sha256:{'b' * 64}", + reorder=reorder, + ) ) + separators = (",", ":") if compact else None return json.dumps( - {"tag_name": f"rust-v{VERSION}", "assets": assets}, - indent=2, + {"assets": assets, "body": "braces: { } [ ]", "tag_name": f"rust-v{VERSION}"}, + indent=None if compact else 2, + separators=separators, + ) + + +def asset_metadata(name: str, digest: str, *, reorder: bool) -> dict[str, str]: + if reorder: + return {"digest": digest, "name": name} + return {"name": name, "digest": digest} + + +def legacy_release_metadata_with_decoys() -> str: + fake_digest = f"sha256:{'0' * 64}" + assets = [ + { + "metadata": { + "name": "codex-package-x86_64-unknown-linux-musl.tar.gz", + "digest": fake_digest, + }, + "digest": f"sha256:{'c' * 64}", + "name": f"codex-npm-{target}-{VERSION}.tgz", + } + for target in ("darwin-arm64", "darwin-x64", "linux-arm64", "linux-x64") + ] + return json.dumps( + { + "body": ( + f'fake: {{"name":"codex-package_SHA256SUMS","digest":"{fake_digest}"}}' + ), + "assets": assets, + "tag_name": f"rust-v{VERSION}", + }, + separators=(",", ":"), )