Merge c409464bfe into sapling-pr-archive-bolinfest

scripts/codex_package/README.md

@@ -13,6 +13,7 @@ The builder creates a canonical Codex package directory:
 │   └── <entrypoint>[.exe]
 ├── codex-resources
 │   ├── bwrap                             # Linux only
+│   ├── zsh/bin/zsh                       # supported Unix targets only
 │   ├── codex-command-runner.exe          # Windows only
 │   └── codex-windows-sandbox-setup.exe   # Windows only
 └── codex-path
@@ -67,3 +68,9 @@ DotSlash manifest at `scripts/codex_package/rg`. Downloaded archives are cached
 under `$TMPDIR/codex-package/<target>-rg` and are reused only after the recorded
 size and SHA-256 digest have been verified. Pass `--rg-bin` to use a local
 ripgrep executable instead.
+
+The patched zsh fork used by `shell_zsh_fork` is fetched from the DotSlash
+manifest at `scripts/codex_package/codex-zsh` when the selected target has a
+matching prebuilt artifact. Downloaded archives are cached under
+`$TMPDIR/codex-package/<target>-zsh` and installed at
+`codex-resources/zsh/bin/zsh`.

scripts/codex_package/cli.py

@@ -15,6 +15,7 @@ from .targets import TARGET_SPECS
 from .targets import PackageInputs
 from .targets import default_target
 from .targets import resolve_input_path
+from .zsh import resolve_zsh_bin
 from .version import read_workspace_version
 
 
@@ -161,13 +162,14 @@ def main() -> int:
     inputs = PackageInputs(
         entrypoint_bin=source_outputs.entrypoint_bin,
         rg_bin=resolve_rg_bin(spec, args.rg_bin),
+        zsh_bin=resolve_zsh_bin(spec),
         bwrap_bin=source_outputs.bwrap_bin,
         codex_command_runner_bin=source_outputs.codex_command_runner_bin,
         codex_windows_sandbox_setup_bin=source_outputs.codex_windows_sandbox_setup_bin,
     )
     prepare_package_dir(package_dir, force=args.force)
     build_package_dir(package_dir, version, variant, spec, inputs)
-    validate_package_dir(package_dir, variant, spec)
+    validate_package_dir(package_dir, variant, spec, include_zsh=inputs.zsh_bin is not None)
 
     for archive_output in args.archive_output:
         archive_path = archive_output.resolve()

scripts/codex_package/codex-zsh

@@ -0,0 +1,43 @@
+#!/usr/bin/env dotslash
+
+{
+  "name": "codex-zsh",
+  "platforms": {
+    "macos-aarch64": {
+      "size": 358776,
+      "hash": "sha256",
+      "digest": "c6dbb063a0135b947ab1cacc655b2b750874699472f412ec7daba97543a90c3c",
+      "format": "tar.gz",
+      "path": "codex-zsh/bin/zsh",
+      "providers": [
+        {
+          "url": "https://github.com/openai/codex/releases/download/rust-v0.132.0/codex-zsh-aarch64-apple-darwin.tar.gz"
+        }
+      ]
+    },
+    "linux-x86_64": {
+      "size": 433413,
+      "hash": "sha256",
+      "digest": "5f42d9fc8e9c8c399a727512002906006ae9de966ea7b3d87ca36b47efc59938",
+      "format": "tar.gz",
+      "path": "codex-zsh/bin/zsh",
+      "providers": [
+        {
+          "url": "https://github.com/openai/codex/releases/download/rust-v0.132.0/codex-zsh-x86_64-unknown-linux-musl.tar.gz"
+        }
+      ]
+    },
+    "linux-aarch64": {
+      "size": 411653,
+      "hash": "sha256",
+      "digest": "6c6e32c297425db02b4dbffb10925895875d14647fc3eb2f18767be97dc6a945",
+      "format": "tar.gz",
+      "path": "codex-zsh/bin/zsh",
+      "providers": [
+        {
+          "url": "https://github.com/openai/codex/releases/download/rust-v0.132.0/codex-zsh-aarch64-unknown-linux-musl.tar.gz"
+        }
+      ]
+    }
+  }
+}

scripts/codex_package/dotslash.py

@@ -0,0 +1,223 @@
+"""Fetch executable artifacts from checked-in DotSlash manifests."""
+
+import hashlib
+import json
+import shutil
+import stat
+import tarfile
+import tempfile
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlparse
+from urllib.request import urlopen
+
+from .targets import TargetSpec
+
+
+DOWNLOAD_TIMEOUT_SECS = 60
+
+
+@dataclass(frozen=True)
+class DotSlashArtifact:
+    size: int
+    digest: str
+    archive_format: str
+    archive_member: str
+    url: str
+
+
+def fetch_dotslash_executable(
+    spec: TargetSpec,
+    *,
+    manifest_path: Path,
+    artifact_label: str,
+    cache_key: str,
+    dest_name: str,
+    executable: bool,
+    missing_ok: bool = False,
+) -> Path | None:
+    artifact = artifact_for_target(
+        spec,
+        manifest_path,
+        artifact_label=artifact_label,
+        missing_ok=missing_ok,
+    )
+    if artifact is None:
+        return None
+
+    cache_dir = default_cache_root() / cache_key
+    archive_path = cache_dir / archive_filename(artifact.url)
+
+    if not archive_is_valid(archive_path, artifact, artifact_label):
+        download_archive(artifact.url, archive_path)
+        try:
+            verify_archive(archive_path, artifact, artifact_label)
+        except RuntimeError:
+            archive_path.unlink(missing_ok=True)
+            raise
+
+    dest = cache_dir / dest_name
+    extract_archive_member(archive_path, artifact, dest, artifact_label)
+    if executable:
+        mode = dest.stat().st_mode
+        dest.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+    return dest
+
+
+def artifact_for_target(
+    spec: TargetSpec,
+    manifest_path: Path,
+    *,
+    artifact_label: str,
+    missing_ok: bool = False,
+) -> DotSlashArtifact | None:
+    manifest = load_manifest(manifest_path)
+    platform_info = manifest.get("platforms", {}).get(spec.dotslash_platform)
+    if platform_info is None:
+        if missing_ok:
+            return None
+        raise RuntimeError(
+            f"{artifact_label} manifest {manifest_path} is missing platform "
+            f"{spec.dotslash_platform!r}"
+        )
+
+    providers = platform_info.get("providers")
+    if not providers:
+        raise RuntimeError(
+            f"{artifact_label} manifest {manifest_path} has no providers for "
+            f"{spec.dotslash_platform!r}"
+        )
+
+    hash_name = platform_info.get("hash")
+    if hash_name != "sha256":
+        raise RuntimeError(
+            f"Unsupported {artifact_label} hash {hash_name!r} for "
+            f"{spec.dotslash_platform!r}; expected sha256"
+        )
+
+    return DotSlashArtifact(
+        size=int(platform_info["size"]),
+        digest=str(platform_info["digest"]),
+        archive_format=str(platform_info["format"]),
+        archive_member=str(platform_info["path"]),
+        url=str(providers[0]["url"]),
+    )
+
+
+def load_manifest(manifest_path: Path) -> dict:
+    text = manifest_path.read_text(encoding="utf-8")
+    if text.startswith("#!"):
+        text = "\n".join(text.splitlines()[1:])
+    return json.loads(text)
+
+
+def default_cache_root() -> Path:
+    return Path(tempfile.gettempdir()) / "codex-package"
+
+
+def archive_filename(url: str) -> str:
+    filename = Path(urlparse(url).path).name
+    if not filename:
+        raise RuntimeError(f"Unable to determine archive filename from {url}")
+    return filename
+
+
+def archive_is_valid(
+    archive_path: Path,
+    artifact: DotSlashArtifact,
+    artifact_label: str,
+) -> bool:
+    if not archive_path.is_file():
+        return False
+    try:
+        verify_archive(archive_path, artifact, artifact_label)
+    except RuntimeError:
+        archive_path.unlink(missing_ok=True)
+        return False
+    return True
+
+
+def verify_archive(
+    archive_path: Path,
+    artifact: DotSlashArtifact,
+    artifact_label: str,
+) -> None:
+    actual_size = archive_path.stat().st_size
+    if actual_size != artifact.size:
+        raise RuntimeError(
+            f"{artifact_label} archive {archive_path} has size {actual_size}, "
+            f"expected {artifact.size}"
+        )
+
+    digest = hashlib.sha256()
+    with open(archive_path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            digest.update(chunk)
+
+    actual_digest = digest.hexdigest()
+    if actual_digest != artifact.digest:
+        raise RuntimeError(
+            f"{artifact_label} archive {archive_path} has sha256 {actual_digest}, "
+            f"expected {artifact.digest}"
+        )
+
+
+def download_archive(url: str, archive_path: Path) -> None:
+    archive_path.parent.mkdir(parents=True, exist_ok=True)
+    temp_path = archive_path.with_suffix(f"{archive_path.suffix}.tmp")
+    temp_path.unlink(missing_ok=True)
+    try:
+        with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response:
+            with open(temp_path, "wb") as out:
+                shutil.copyfileobj(response, out)
+        temp_path.replace(archive_path)
+    finally:
+        temp_path.unlink(missing_ok=True)
+
+
+def extract_archive_member(
+    archive_path: Path,
+    artifact: DotSlashArtifact,
+    dest: Path,
+    artifact_label: str,
+) -> None:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    dest.unlink(missing_ok=True)
+
+    if artifact.archive_format == "tar.gz":
+        with tarfile.open(archive_path, "r:gz") as archive:
+            try:
+                member = archive.getmember(artifact.archive_member)
+            except KeyError as exc:
+                raise RuntimeError(
+                    f"{artifact_label} archive {archive_path} is missing "
+                    f"{artifact.archive_member!r}"
+                ) from exc
+
+            extracted = archive.extractfile(member)
+            if extracted is None:
+                raise RuntimeError(
+                    f"{artifact_label} archive member {artifact.archive_member!r} is not a file"
+                )
+            with extracted, open(dest, "wb") as out:
+                shutil.copyfileobj(extracted, out)
+        return
+
+    if artifact.archive_format == "zip":
+        with zipfile.ZipFile(archive_path) as archive:
+            try:
+                with archive.open(artifact.archive_member) as extracted:
+                    with open(dest, "wb") as out:
+                        shutil.copyfileobj(extracted, out)
+            except KeyError as exc:
+                raise RuntimeError(
+                    f"{artifact_label} archive {archive_path} is missing "
+                    f"{artifact.archive_member!r}"
+                ) from exc
+        return
+
+    raise RuntimeError(
+        f"Unsupported {artifact_label} archive format {artifact.archive_format!r}; "
+        "expected tar.gz or zip"
+    )

scripts/codex_package/layout.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from .targets import PackageInputs
 from .targets import PackageVariant
 from .targets import TargetSpec
+from .zsh import ZSH_RESOURCE_PATH
 
 
 LAYOUT_VERSION = 1
@@ -50,6 +51,13 @@ def build_package_dir(
     )
     copy_executable(inputs.rg_bin, path_dir / spec.rg_name, is_windows=spec.is_windows)
 
+    if inputs.zsh_bin is not None:
+        copy_executable(
+            inputs.zsh_bin,
+            resources_dir / ZSH_RESOURCE_PATH,
+            is_windows=False,
+        )
+
     if inputs.bwrap_bin is not None:
         copy_executable(inputs.bwrap_bin, resources_dir / "bwrap", is_windows=False)
 
@@ -83,6 +91,8 @@ def validate_package_dir(
     package_dir: Path,
     variant: PackageVariant,
     spec: TargetSpec,
+    *,
+    include_zsh: bool,
 ) -> None:
     required_dirs = [
         Path("bin"),
@@ -122,6 +132,11 @@ def validate_package_dir(
     ]
     executable_files = list(required_files)
 
+    if include_zsh:
+        zsh_path = Path("codex-resources") / ZSH_RESOURCE_PATH
+        required_files.append(zsh_path)
+        executable_files.append(zsh_path)
+
     if spec.is_linux:
         required_files.append(Path("codex-resources") / "bwrap")
         executable_files.append(Path("codex-resources") / "bwrap")
@@ -148,7 +163,7 @@ def validate_package_dir(
 
 def copy_executable(src: Path, dest: Path, *, is_windows: bool) -> None:
     dest.parent.mkdir(parents=True, exist_ok=True)
-    shutil.copy2(src, dest)
+    shutil.copyfile(src, dest)
     if not is_windows:
         mode = dest.stat().st_mode
         dest.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)

scripts/codex_package/ripgrep.py

@@ -1,33 +1,14 @@
 """Fetch ripgrep from the DotSlash manifest used by the package builder."""
 
-import hashlib
-import json
-import shutil
-import stat
-import tarfile
-import tempfile
-import zipfile
-from dataclasses import dataclass
 from pathlib import Path
-from urllib.parse import urlparse
-from urllib.request import urlopen
 
+from .dotslash import fetch_dotslash_executable
 from .targets import REPO_ROOT
 from .targets import TargetSpec
 from .targets import resolve_input_path
 
 
 RG_MANIFEST = REPO_ROOT / "scripts" / "codex_package" / "rg"
-DOWNLOAD_TIMEOUT_SECS = 60
-
-
-@dataclass(frozen=True)
-class RgArtifact:
-    size: int
-    digest: str
-    archive_format: str
-    archive_member: str
-    url: str
 
 
 def resolve_rg_bin(spec: TargetSpec, rg_bin: Path | None) -> Path:
@@ -41,155 +22,15 @@ def fetch_rg(
     spec: TargetSpec,
     *,
     manifest_path: Path = RG_MANIFEST,
-    cache_root: Path | None = None,
 ) -> Path:
-    artifact = artifact_for_target(spec, manifest_path)
-    cache_dir = (cache_root or default_cache_root()) / f"{spec.target}-rg"
-    archive_path = cache_dir / archive_filename(artifact.url)
-
-    if not archive_is_valid(archive_path, artifact):
-        download_archive(artifact.url, archive_path)
-        try:
-            verify_archive(archive_path, artifact)
-        except RuntimeError:
-            archive_path.unlink(missing_ok=True)
-            raise
-
-    dest = cache_dir / spec.rg_name
-    extract_rg(archive_path, artifact, dest)
-    if not spec.is_windows:
-        mode = dest.stat().st_mode
-        dest.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
-    return dest
-
-
-def artifact_for_target(spec: TargetSpec, manifest_path: Path) -> RgArtifact:
-    manifest = load_manifest(manifest_path)
-    try:
-        platform_info = manifest["platforms"][spec.dotslash_platform]
-    except KeyError as exc:
-        raise RuntimeError(
-            f"ripgrep manifest {manifest_path} is missing platform {spec.dotslash_platform!r}"
-        ) from exc
-
-    providers = platform_info.get("providers")
-    if not providers:
-        raise RuntimeError(
-            f"ripgrep manifest {manifest_path} has no providers for {spec.dotslash_platform!r}"
-        )
-
-    hash_name = platform_info.get("hash")
-    if hash_name != "sha256":
-        raise RuntimeError(
-            f"Unsupported ripgrep hash {hash_name!r} for "
-            f"{spec.dotslash_platform!r}; expected sha256"
-        )
-
-    return RgArtifact(
-        size=int(platform_info["size"]),
-        digest=str(platform_info["digest"]),
-        archive_format=str(platform_info["format"]),
-        archive_member=str(platform_info["path"]),
-        url=str(providers[0]["url"]),
-    )
-
-
-def load_manifest(manifest_path: Path) -> dict:
-    text = manifest_path.read_text(encoding="utf-8")
-    if text.startswith("#!"):
-        text = "\n".join(text.splitlines()[1:])
-    return json.loads(text)
-
-
-def default_cache_root() -> Path:
-    return Path(tempfile.gettempdir()) / "codex-package"
-
-
-def archive_filename(url: str) -> str:
-    filename = Path(urlparse(url).path).name
-    if not filename:
-        raise RuntimeError(f"Unable to determine archive filename from {url}")
-    return filename
-
-
-def archive_is_valid(archive_path: Path, artifact: RgArtifact) -> bool:
-    if not archive_path.is_file():
-        return False
-    try:
-        verify_archive(archive_path, artifact)
-    except RuntimeError:
-        archive_path.unlink(missing_ok=True)
-        return False
-    return True
-
-
-def verify_archive(archive_path: Path, artifact: RgArtifact) -> None:
-    actual_size = archive_path.stat().st_size
-    if actual_size != artifact.size:
-        raise RuntimeError(
-            f"ripgrep archive {archive_path} has size {actual_size}, expected {artifact.size}"
-        )
-
-    digest = hashlib.sha256()
-    with open(archive_path, "rb") as fh:
-        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
-            digest.update(chunk)
-
-    actual_digest = digest.hexdigest()
-    if actual_digest != artifact.digest:
-        raise RuntimeError(
-            f"ripgrep archive {archive_path} has sha256 {actual_digest}, "
-            f"expected {artifact.digest}"
-        )
-
-
-def download_archive(url: str, archive_path: Path) -> None:
-    archive_path.parent.mkdir(parents=True, exist_ok=True)
-    temp_path = archive_path.with_suffix(f"{archive_path.suffix}.tmp")
-    temp_path.unlink(missing_ok=True)
-    try:
-        with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response:
-            with open(temp_path, "wb") as out:
-                shutil.copyfileobj(response, out)
-        temp_path.replace(archive_path)
-    finally:
-        temp_path.unlink(missing_ok=True)
-
-
-def extract_rg(archive_path: Path, artifact: RgArtifact, dest: Path) -> None:
-    dest.parent.mkdir(parents=True, exist_ok=True)
-    dest.unlink(missing_ok=True)
-
-    if artifact.archive_format == "tar.gz":
-        with tarfile.open(archive_path, "r:gz") as archive:
-            try:
-                member = archive.getmember(artifact.archive_member)
-            except KeyError as exc:
-                raise RuntimeError(
-                    f"ripgrep archive {archive_path} is missing {artifact.archive_member!r}"
-                ) from exc
-
-            extracted = archive.extractfile(member)
-            if extracted is None:
-                raise RuntimeError(
-                    f"ripgrep archive member {artifact.archive_member!r} is not a file"
-                )
-            with extracted, open(dest, "wb") as out:
-                shutil.copyfileobj(extracted, out)
-        return
-
-    if artifact.archive_format == "zip":
-        with zipfile.ZipFile(archive_path) as archive:
-            try:
-                with archive.open(artifact.archive_member) as extracted:
-                    with open(dest, "wb") as out:
-                        shutil.copyfileobj(extracted, out)
-            except KeyError as exc:
-                raise RuntimeError(
-                    f"ripgrep archive {archive_path} is missing {artifact.archive_member!r}"
-                ) from exc
-        return
-
-    raise RuntimeError(
-        f"Unsupported ripgrep archive format {artifact.archive_format!r}; expected tar.gz or zip"
+    rg_bin = fetch_dotslash_executable(
+        spec,
+        manifest_path=manifest_path,
+        artifact_label="ripgrep",
+        cache_key=f"{spec.target}-rg",
+        dest_name=spec.rg_name,
+        executable=not spec.is_windows,
     )
+    if rg_bin is None:
+        raise AssertionError("ripgrep is required for all package targets")
+    return rg_bin

scripts/codex_package/targets.py

@@ -40,6 +40,7 @@ class PackageVariant:
 class PackageInputs:
     entrypoint_bin: Path
     rg_bin: Path
+    zsh_bin: Path | None
     bwrap_bin: Path | None
     codex_command_runner_bin: Path | None
     codex_windows_sandbox_setup_bin: Path | None

scripts/codex_package/zsh.py

@@ -0,0 +1,23 @@
+"""Fetch the patched zsh fork used by shell_zsh_fork."""
+
+from pathlib import Path
+
+from .dotslash import fetch_dotslash_executable
+from .targets import REPO_ROOT
+from .targets import TargetSpec
+
+
+ZSH_MANIFEST = REPO_ROOT / "scripts" / "codex_package" / "codex-zsh"
+ZSH_RESOURCE_PATH = Path("zsh") / "bin" / "zsh"
+
+
+def resolve_zsh_bin(spec: TargetSpec) -> Path | None:
+    return fetch_dotslash_executable(
+        spec,
+        manifest_path=ZSH_MANIFEST,
+        artifact_label="codex-zsh",
+        cache_key=f"{spec.target}-zsh",
+        dest_name="zsh",
+        executable=True,
+        missing_ok=True,
+    )