Resolve stable releases dynamically for exec-server compatibility tests (#54623)

## What changed

Replace fixed release pins for app-server/exec-server compatibility tests with the latest stable version and the stable release published closest to 60 days ago. Exclude drafts and prereleases, and prefer the higher version when publication dates are equally close.

Resolve release versions and SHA-256 asset digests before Linux Bazel CI runs, passing them through `CODEX_COMPAT_RELEASES` to the release archive repositories. Reuse the argument comment lint action so it also prepares compatibility releases. Add bounded retries for transient GitHub release reads.

## Testing

Add unit coverage for stable release filtering, version ordering, nearest publication dates, tie-breaking, and deduplication when both selections resolve to the same release.

GitOrigin-RevId: 5a8cf8bdabd77ab3fa89bbb711367b5d9ce8044b

.github/actions/run-argument-comment-lint/action.yml

@@ -24,6 +24,15 @@ runs:
         sudo DEBIAN_FRONTEND=noninteractive apt-get update
         sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev
 
+    - name: Resolve app-server/exec-server compatibility releases
+      if: ${{ runner.os == 'Linux' }}
+      env:
+        GH_TOKEN: ${{ github.token }}
+      shell: bash
+      run: |
+        releases="$(python3 .github/scripts/resolve_exec_server_releases.py)"
+        echo "CODEX_COMPAT_RELEASES=$releases" >> "$GITHUB_ENV"
+
     - name: Run argument comment lint on codex-rs via Bazel
       if: ${{ runner.os != 'Windows' }}
       env:

.github/scripts/contract_releases.py

@@ -0,0 +1,126 @@
+"""Discover and download published Codex packages for contract runners."""
+
+import json
+import re
+import subprocess
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+ALPHA_TAG = re.compile(r"rust-v[0-9]+\.[0-9]+\.[0-9]+-alpha(?:\.[0-9]+){0,2}")
+RELEASE_TAG = re.compile(r"rust-v[0-9]+\.[0-9]+\.[0-9]+(?:-alpha(?:\.[0-9]+){0,2})?")
+STABLE_TAG = re.compile(r"rust-v[0-9]+\.[0-9]+\.[0-9]+")
+PACKAGE = "codex-package-x86_64-unknown-linux-musl.tar.gz"
+
+
+def github_read(args: list[str]) -> str:
+    """Retry release reads unless the error is known permanent; never retry verification."""
+    for attempt, delay in enumerate((2, 5, 0), start=1):
+        try:
+            return subprocess.check_output(
+                args, text=True, stderr=subprocess.PIPE, timeout=120
+            )
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
+            reason = "timeout"
+            exit_code = None
+            http_status = "unknown"
+            retryable = True
+            if isinstance(error, subprocess.CalledProcessError):
+                message = error.stderr or ""
+                status = re.search(r"\bHTTP (\d{3})\b", message)
+                reason = "command_failed"
+                exit_code = error.returncode
+                http_status = status[1] if status else "unknown"
+                # gh uses 1 for general failures, 2 for cancellation and 4 for auth.
+                # Unknown transport/protocol wording should still get bounded retries.
+                retryable = (
+                    error.returncode == 1
+                    and (status is None or status[1] in {"502", "503", "504"})
+                    and not any(
+                        marker in message.lower()
+                        for marker in (
+                            "release not found",
+                            "no assets match",
+                            "no assets to download",
+                            "unknown flag",
+                            "unknown command",
+                        )
+                    )
+                )
+            # Only emit classified fields; stderr can contain unknown credential formats.
+            will_retry = retryable and bool(delay)
+            print(
+                f"GitHub release read failed: attempt={attempt} reason={reason} "
+                f"exit_code={exit_code} http_status={http_status} "
+                f"will_retry={str(will_retry).lower()}",
+                file=sys.stderr,
+            )
+            if not will_retry:
+                raise
+            print(f"Retrying GitHub release read in {delay}s.", file=sys.stderr)
+            time.sleep(delay)
+
+
+def list_releases(*, exclude_prereleases: bool = False) -> list[dict]:
+    # GraphQL-backed listing avoids fetching assets and release notes.
+    args = [
+        "gh",
+        "release",
+        "list",
+        "--repo",
+        "openai/codex",
+        "--limit",
+        "100",
+        "--json",
+        "tagName,publishedAt,isDraft,isPrerelease",
+    ]
+    if exclude_prereleases:
+        args.append("--exclude-pre-releases")
+    return json.loads(github_read(args))
+
+
+def download_release(tag: str, directory: Path) -> Path:
+    github_read(
+        [
+            "gh",
+            "release",
+            "download",
+            tag,
+            "--repo",
+            "openai/codex",
+            "--pattern",
+            PACKAGE,
+            "--dir",
+            str(directory),
+            # A failed download can leave a partial file in this temp directory.
+            "--clobber",
+        ]
+    )
+    return directory / PACKAGE
+
+
+def select_stables(releases: list[dict], target_date: datetime) -> list[str]:
+    stable = [
+        release
+        for release in releases
+        if not release["isDraft"]
+        and not release["isPrerelease"]
+        and STABLE_TAG.fullmatch(release["tagName"])
+    ]
+    stable.sort(
+        key=lambda release: tuple(map(int, release["tagName"][6:].split("."))),
+        reverse=True,
+    )
+    if not stable:
+        raise ValueError("No stable releases available")
+    # The descending version order breaks equal-distance ties deterministically.
+    baseline = min(
+        stable,
+        key=lambda release: abs(
+            datetime.fromisoformat(release["publishedAt"].replace("Z", "+00:00"))
+            - target_date
+        ),
+    )
+    # A long release pause can make latest and historical resolve to one build.
+    return list(dict.fromkeys([stable[0]["tagName"], baseline["tagName"]]))

.github/scripts/resolve_exec_server_releases.py

@@ -0,0 +1,39 @@
+"""Print CODEX_COMPAT_RELEASES JSON for the existing Bazel compatibility tests.
+
+For local runs, export CODEX_COMPAT_RELEASES="$(python3 <path-to-this-script>)"
+before invoking Bazel. CI resolves it once before building the test targets.
+"""
+
+import json
+import re
+import sys
+from datetime import datetime, timedelta, timezone
+
+from contract_releases import PACKAGE, github_read, list_releases, select_stables
+
+STABLE_LOOKBACK_DAYS = 60
+
+
+def main() -> None:
+    target_date = datetime.now(timezone.utc) - timedelta(days=STABLE_LOOKBACK_DAYS)
+    tags = select_stables(list_releases(exclude_prereleases=True), target_date)
+    releases = {}
+    for tag in tags:
+        metadata = json.loads(
+            github_read(["gh", "api", f"repos/openai/codex/releases/tags/{tag}"])
+        )
+        asset = next(asset for asset in metadata["assets"] if asset["name"] == PACKAGE)
+        digest = asset.get("digest") or ""
+        if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
+            raise ValueError(f"Missing SHA-256 digest for {tag}/{PACKAGE}")
+        releases[tag] = {"version": tag.removeprefix("rust-v"), "sha256": digest[7:]}
+    print(
+        f"Compatibility releases: latest={tags[0]}, "
+        f"closest to {target_date.isoformat()}={tags[-1]}",
+        file=sys.stderr,
+    )
+    print(json.dumps({"latest": releases[tags[0]], "historical": releases[tags[-1]]}))
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/test_contract_releases.py

@@ -0,0 +1,52 @@
+import unittest
+from datetime import datetime, timezone
+
+from contract_releases import select_stables
+
+
+class StableReleaseSelectionTests(unittest.TestCase):
+    def test_selects_latest_and_closest_stable_by_publication_date(self):
+        releases = [
+            {
+                "tagName": tag,
+                "publishedAt": date,
+                "isPrerelease": False,
+                "isDraft": False,
+            }
+            for tag, date in (
+                ("rust-v0.9.9", "2026-02-08T00:00:00Z"),
+                ("rust-v0.10.2", "2026-01-10T00:00:00Z"),
+                ("rust-v0.10.10", "2026-01-13T00:00:00Z"),
+                ("rust-v0.10.11", "2026-01-16T00:00:00Z"),
+                ("rust-v1.0.1", "2026-01-20T00:00:00Z"),
+                ("rust-v1.1.0", "2026-02-06T00:00:00Z"),
+                ("rust-v2.0.0-beta.1", "2026-02-06T00:00:00Z"),
+            )
+        ]
+        releases.extend(
+            [
+                {"tagName": "rust-v3.0.0", "isPrerelease": True, "isDraft": False},
+                {"tagName": "rust-v4.0.0", "isPrerelease": False, "isDraft": True},
+            ]
+        )
+        for target_date, expected in (
+            # One day newer than the target beats two days older (59 vs. 62).
+            (datetime(2026, 1, 15, tzinfo=timezone.utc), "rust-v0.10.11"),
+            # The older release wins when it is closer.
+            (datetime(2026, 1, 14, tzinfo=timezone.utc), "rust-v0.10.10"),
+            # Equal distances prefer the higher version.
+            (datetime(2026, 1, 14, 12, tzinfo=timezone.utc), "rust-v0.10.11"),
+        ):
+            with self.subTest(target_date=target_date):
+                self.assertEqual(
+                    select_stables(releases, target_date),
+                    ["rust-v1.1.0", expected],
+                )
+        self.assertEqual(
+            select_stables(releases, datetime(2026, 2, 6, tzinfo=timezone.utc)),
+            ["rust-v1.1.0"],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/bazel.yml

@@ -75,6 +75,14 @@ jobs:
         shell: bash
         run: ./scripts/check-module-bazel-lock.sh
 
+      - name: Resolve app-server/exec-server compatibility releases
+        if: runner.os == 'Linux'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          releases="$(python3 .github/scripts/resolve_exec_server_releases.py)"
+          echo "CODEX_COMPAT_RELEASES=$releases" >> "$GITHUB_ENV"
+
       - name: bazel test //...
         env:
           BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
@@ -367,6 +375,14 @@ jobs:
           target: ${{ matrix.target }}
           cache-scope: bazel-${{ github.job }}
 
+      - name: Resolve app-server/exec-server compatibility releases
+        if: runner.os == 'Linux'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          releases="$(python3 .github/scripts/resolve_exec_server_releases.py)"
+          echo "CODEX_COMPAT_RELEASES=$releases" >> "$GITHUB_ENV"
+
       - name: bazel build --config=clippy lint targets
         env:
           BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
@@ -466,6 +482,14 @@ jobs:
           target: ${{ matrix.target }}
           cache-scope: bazel-${{ github.job }}
 
+      - name: Resolve app-server/exec-server compatibility releases
+        if: runner.os == 'Linux'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          releases="$(python3 .github/scripts/resolve_exec_server_releases.py)"
+          echo "CODEX_COMPAT_RELEASES=$releases" >> "$GITHUB_ENV"
+
       - name: bazel build verify-release-build targets
         env:
           BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}

.github/workflows/rust-ci-full.yml

@@ -113,41 +113,11 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
-      - uses: ./.github/actions/setup-bazel-ci
+      - name: Run argument comment lint on codex-rs via Bazel
+        uses: ./.github/actions/run-argument-comment-lint
         with:
           target: ${{ runner.os }}
-      - name: Install Linux sandbox build dependencies
-        if: ${{ runner.os == 'Linux' }}
-        shell: bash
-        run: |
-          sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends pkg-config libcap-dev
-      - name: Run argument comment lint on codex-rs via Bazel
-        if: ${{ runner.os != 'Windows' }}
-        env:
-          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
-        shell: bash
-        run: |
-          bazel_targets="$(./tools/argument-comment-lint/list-bazel-targets.sh)"
-          ./.github/scripts/run-bazel-ci.sh \
-            -- \
-            build \
-            --config=argument-comment-lint \
-            --keep_going \
-            --build_metadata=COMMIT_SHA=${GITHUB_SHA} \
-            -- \
-            ${bazel_targets}
-      - name: Run argument comment lint on codex-rs via Bazel
-        if: ${{ runner.os == 'Windows' }}
-        env:
-          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
-        shell: bash
-        run: |
-          ./.github/scripts/run-argument-comment-lint-bazel.sh \
-            --config=argument-comment-lint \
-            --platforms=//:local_windows \
-            --keep_going \
-            --build_metadata=COMMIT_SHA=${GITHUB_SHA}
+          buildbuddy-api-key: ${{ secrets.BUILDBUDDY_API_KEY }}
 
   # --- CI to validate on different os/targets --------------------------------
   lint_build: