From 9ccfe3cb295ffe45f35b89e68a224a7d3834520e Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 14:54:06 -0700 Subject: [PATCH] package: include zsh fork in Codex package Teach the Codex package builder to fetch the prebuilt zsh fork from a checked-in DotSlash manifest and install it under codex-resources/zsh/bin/zsh when an artifact is available for the package target. Generalize the DotSlash download/cache/verify helper previously embedded in ripgrep packaging so additional checked-in DotSlash manifests can use the same SHA-256 and size validation path. Add install-context support for locating the bundled zsh fork and thread that path through config loading as the lowest-precedence zsh_path default, preserving explicit CLI/profile/global config values. Also avoid preserving platform-specific file metadata when copying executables into the package directory so package smoke tests can use macOS system binaries as inputs. --- codex-cli/bin/codex-zsh | 43 ++++ codex-rs/Cargo.lock | 3 + codex-rs/app-server/Cargo.toml | 1 + codex-rs/app-server/src/config_manager.rs | 20 ++ codex-rs/cli/src/doctor.rs | 1 + codex-rs/cli/src/main.rs | 8 + codex-rs/core/src/config/config_tests.rs | 50 +++++ codex-rs/core/src/config/mod.rs | 6 +- codex-rs/exec/Cargo.toml | 1 + codex-rs/exec/src/lib.rs | 4 + codex-rs/install-context/src/lib.rs | 35 +++ codex-rs/mcp-server/Cargo.toml | 3 +- codex-rs/mcp-server/src/codex_tool_config.rs | 4 + codex-rs/tui/src/lib.rs | 7 + scripts/codex_package/README.md | 7 + scripts/codex_package/cli.py | 4 +- scripts/codex_package/dotslash.py | 223 +++++++++++++++++++ scripts/codex_package/layout.py | 17 +- scripts/codex_package/ripgrep.py | 183 +-------------- scripts/codex_package/targets.py | 1 + scripts/codex_package/zsh.py | 23 ++ 21 files changed, 468 insertions(+), 176 deletions(-) create mode 100755 codex-cli/bin/codex-zsh create mode 100644 scripts/codex_package/dotslash.py create mode 100644 scripts/codex_package/zsh.py diff --git a/codex-cli/bin/codex-zsh b/codex-cli/bin/codex-zsh new file mode 100755 index 0000000000..8876474d81 --- /dev/null +++ b/codex-cli/bin/codex-zsh @@ -0,0 +1,43 @@ +#!/usr/bin/env dotslash + +{ + "name": "codex-zsh", + "platforms": { + "macos-aarch64": { + "size": 358776, + "hash": "sha256", + "digest": "c6dbb063a0135b947ab1cacc655b2b750874699472f412ec7daba97543a90c3c", + "format": "tar.gz", + "path": "codex-zsh/bin/zsh", + "providers": [ + { + "url": "https://github.com/openai/codex/releases/download/rust-v0.132.0/codex-zsh-aarch64-apple-darwin.tar.gz" + } + ] + }, + "linux-x86_64": { + "size": 433413, + "hash": "sha256", + "digest": "5f42d9fc8e9c8c399a727512002906006ae9de966ea7b3d87ca36b47efc59938", + "format": "tar.gz", + "path": "codex-zsh/bin/zsh", + "providers": [ + { + "url": "https://github.com/openai/codex/releases/download/rust-v0.132.0/codex-zsh-x86_64-unknown-linux-musl.tar.gz" + } + ] + }, + "linux-aarch64": { + "size": 411653, + "hash": "sha256", + "digest": "6c6e32c297425db02b4dbffb10925895875d14647fc3eb2f18767be97dc6a945", + "format": "tar.gz", + "path": "codex-zsh/bin/zsh", + "providers": [ + { + "url": "https://github.com/openai/codex/releases/download/rust-v0.132.0/codex-zsh-aarch64-unknown-linux-musl.tar.gz" + } + ] + } + } +} diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 823eb6d3c1..e7ded3b3ac 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1905,6 +1905,7 @@ dependencies = [ "codex-git-utils", "codex-guardian", "codex-hooks", + "codex-install-context", "codex-login", "codex-mcp", "codex-memories-extension", @@ -2710,6 +2711,7 @@ dependencies = [ "codex-core", "codex-feedback", "codex-git-utils", + "codex-install-context", "codex-login", "codex-model-provider-info", "codex-otel", @@ -3141,6 +3143,7 @@ dependencies = [ "codex-core", "codex-exec-server", "codex-extension-api", + "codex-install-context", "codex-login", "codex-protocol", "codex-shell-command", diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 95baac4e9e..a2f5611201 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -45,6 +45,7 @@ codex-guardian = { workspace = true } codex-git-utils = { workspace = true } codex-file-watcher = { workspace = true } codex-hooks = { workspace = true } +codex-install-context = { workspace = true } codex-otel = { workspace = true } codex-plugin = { workspace = true } codex-shell-command = { workspace = true } diff --git a/codex-rs/app-server/src/config_manager.rs b/codex-rs/app-server/src/config_manager.rs index 25fdc5c0cb..5dd71499b0 100644 --- a/codex-rs/app-server/src/config_manager.rs +++ b/codex-rs/app-server/src/config_manager.rs @@ -9,6 +9,7 @@ use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_exec_server::LOCAL_FS; use codex_features::feature_for_key; +use codex_install_context::InstallContext; use codex_login::AuthManager; use codex_login::default_client::set_default_client_residency_requirement; use codex_utils_absolute_path::AbsolutePathBuf; @@ -159,6 +160,7 @@ impl ConfigManager { .await?; self.apply_runtime_feature_enablement(&mut config); self.apply_arg0_paths(&mut config); + apply_default_zsh_path(&mut config); Ok(config) } @@ -180,6 +182,7 @@ impl ConfigManager { } self.apply_runtime_feature_enablement(&mut config); self.apply_arg0_paths(&mut config); + apply_default_zsh_path(&mut config); Ok(config) } @@ -219,6 +222,11 @@ impl ConfigManager { typesafe_overrides: ConfigOverrides, fallback_cwd: Option, ) -> std::io::Result { + let mut typesafe_overrides = typesafe_overrides; + if typesafe_overrides.default_zsh_path.is_none() { + typesafe_overrides.default_zsh_path = default_zsh_path(); + } + let merged_cli_overrides = cli_overrides .iter() .cloned() @@ -319,6 +327,18 @@ impl ConfigManager { } } +fn apply_default_zsh_path(config: &mut Config) { + if config.zsh_path.is_none() { + config.zsh_path = default_zsh_path(); + } +} + +fn default_zsh_path() -> Option { + InstallContext::current() + .bundled_zsh_path() + .map(AbsolutePathBuf::into_path_buf) +} + pub(crate) fn protected_feature_keys(config_layer_stack: &ConfigLayerStack) -> BTreeSet { let mut protected_features = config_layer_stack .effective_config() diff --git a/codex-rs/cli/src/doctor.rs b/codex-rs/cli/src/doctor.rs index 3f8c3ff094..c8e6401a12 100644 --- a/codex-rs/cli/src/doctor.rs +++ b/codex-rs/cli/src/doctor.rs @@ -507,6 +507,7 @@ fn config_overrides_from_interactive( codex_self_exe: arg0_paths.codex_self_exe.clone(), codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe.clone(), main_execve_wrapper_exe: arg0_paths.main_execve_wrapper_exe.clone(), + default_zsh_path: crate::default_zsh_path(), show_raw_agent_reasoning: interactive.oss.then_some(true), additional_writable_roots: interactive.add_dir.clone(), ..Default::default() diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 95873191b9..6708ea6fcb 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -74,6 +74,7 @@ use codex_core::config::resolve_profile_v2_config_path; use codex_features::FEATURES; use codex_features::Stage; use codex_features::is_known_feature_key; +use codex_install_context::InstallContext; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_login::read_codex_access_token_from_env; @@ -1581,6 +1582,12 @@ async fn load_exec_server_config( .await?) } +pub(crate) fn default_zsh_path() -> Option { + InstallContext::current() + .bundled_zsh_path() + .map(AbsolutePathBuf::into_path_buf) +} + async fn load_exec_server_remote_auth( config: &codex_core::config::Config, missing_auth_error: &'static str, @@ -1716,6 +1723,7 @@ async fn run_debug_prompt_input_command( codex_self_exe: arg0_paths.codex_self_exe, codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe, main_execve_wrapper_exe: arg0_paths.main_execve_wrapper_exe, + default_zsh_path: default_zsh_path(), show_raw_agent_reasoning: shared.oss.then_some(true), ephemeral: Some(true), bypass_hook_trust: shared.bypass_hook_trust.then_some(true), diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 19fd762da9..c8a73fe547 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -4382,6 +4382,56 @@ async fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result< Ok(()) } +#[tokio::test] +async fn default_zsh_path_is_lowest_precedence() -> std::io::Result<()> { + let codex_home = TempDir::new()?; + let configured_zsh_path = codex_home.path().join("configured-zsh"); + let default_zsh_path = codex_home.path().join("packaged-zsh"); + let override_zsh_path = codex_home.path().join("override-zsh"); + + let config = Config::load_from_base_config_with_overrides( + ConfigToml::default(), + ConfigOverrides { + default_zsh_path: Some(default_zsh_path.clone()), + ..Default::default() + }, + codex_home.abs(), + ) + .await?; + assert_eq!(config.zsh_path, Some(default_zsh_path.clone())); + + let config = Config::load_from_base_config_with_overrides( + ConfigToml { + zsh_path: Some(configured_zsh_path.abs()), + ..Default::default() + }, + ConfigOverrides { + default_zsh_path: Some(default_zsh_path.clone()), + ..Default::default() + }, + codex_home.abs(), + ) + .await?; + assert_eq!(config.zsh_path, Some(configured_zsh_path.clone())); + + let config = Config::load_from_base_config_with_overrides( + ConfigToml { + zsh_path: Some(configured_zsh_path.abs()), + ..Default::default() + }, + ConfigOverrides { + zsh_path: Some(override_zsh_path.clone()), + default_zsh_path: Some(default_zsh_path), + ..Default::default() + }, + codex_home.abs(), + ) + .await?; + assert_eq!(config.zsh_path, Some(override_zsh_path)); + + Ok(()) +} + #[tokio::test] async fn sqlite_home_defaults_to_codex_home_for_workspace_write() -> std::io::Result<()> { let codex_home = TempDir::new()?; diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 434969e2c5..83d7ff7c92 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1366,6 +1366,7 @@ impl Config { cfg, ConfigOverrides { cwd: Some(self.cwd.to_path_buf()), + default_zsh_path: refreshed_config.zsh_path.clone(), ..Default::default() }, refreshed_config.codex_home.clone(), @@ -2072,6 +2073,7 @@ pub struct ConfigOverrides { pub codex_linux_sandbox_exe: Option, pub main_execve_wrapper_exe: Option, pub zsh_path: Option, + pub default_zsh_path: Option, pub base_instructions: Option, pub developer_instructions: Option, pub personality: Option, @@ -2469,6 +2471,7 @@ impl Config { codex_linux_sandbox_exe, main_execve_wrapper_exe, zsh_path: zsh_path_override, + default_zsh_path, base_instructions, developer_instructions, personality, @@ -3235,7 +3238,8 @@ impl Config { let compact_prompt = compact_prompt.or(file_compact_prompt); let zsh_path = zsh_path_override .or(config_profile.zsh_path.map(Into::into)) - .or(cfg.zsh_path.map(Into::into)); + .or(cfg.zsh_path.map(Into::into)) + .or(default_zsh_path); let review_model = override_review_model.or(cfg.review_model); diff --git a/codex-rs/exec/Cargo.toml b/codex-rs/exec/Cargo.toml index 37a577e2c8..68253754a3 100644 --- a/codex-rs/exec/Cargo.toml +++ b/codex-rs/exec/Cargo.toml @@ -32,6 +32,7 @@ codex-config = { workspace = true } codex-core = { workspace = true } codex-feedback = { workspace = true } codex-git-utils = { workspace = true } +codex-install-context = { workspace = true } codex-login = { workspace = true } codex-model-provider-info = { workspace = true } codex-otel = { workspace = true } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 3db7a51576..fc9ba00076 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -70,6 +70,7 @@ use codex_core::format_exec_policy_error_with_source; use codex_core::path_utils; use codex_feedback::CodexFeedback; use codex_git_utils::get_git_repo_root; +use codex_install_context::InstallContext; use codex_login::AuthConfig; use codex_login::default_client::set_default_client_residency_requirement; use codex_login::default_client::set_default_originator; @@ -424,6 +425,9 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe.clone(), main_execve_wrapper_exe: arg0_paths.main_execve_wrapper_exe.clone(), zsh_path: None, + default_zsh_path: InstallContext::current() + .bundled_zsh_path() + .map(AbsolutePathBuf::into_path_buf), base_instructions: None, developer_instructions: None, personality: None, diff --git a/codex-rs/install-context/src/lib.rs b/codex-rs/install-context/src/lib.rs index ec5c5217ca..d92a80cbef 100644 --- a/codex-rs/install-context/src/lib.rs +++ b/codex-rs/install-context/src/lib.rs @@ -11,6 +11,7 @@ const PATH_DIRNAME: &str = "codex-path"; const RELEASES_DIRNAME: &str = "releases"; const RESOURCES_DIRNAME: &str = "codex-resources"; const STANDALONE_PACKAGES_DIRNAME: &str = "standalone"; +const ZSH_DIRNAME: &str = "zsh"; static INSTALL_CONTEXT: OnceLock = OnceLock::new(); #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -166,6 +167,18 @@ impl InstallContext { None } + + pub fn bundled_zsh_path(&self) -> Option { + if cfg!(windows) { + None + } else { + self.bundled_resource(zsh_resource_path()) + } + } + + pub fn bundled_zsh_bin_dir(&self) -> Option { + self.bundled_zsh_path()?.parent() + } } impl CodexPackageLayout { @@ -260,6 +273,10 @@ fn default_rg_command() -> PathBuf { } } +fn zsh_resource_path() -> PathBuf { + PathBuf::from(ZSH_DIRNAME).join(BIN_DIRNAME).join("zsh") +} + #[cfg(test)] mod tests { use super::*; @@ -345,6 +362,11 @@ mod tests { fs::write(&exe_path, "")?; fs::write(resources_dir.join(TEST_RESOURCE_NAME), "")?; fs::write(path_dir.join(default_rg_command()), "")?; + if !cfg!(windows) { + let zsh_path = resources_dir.join(zsh_resource_path()); + fs::create_dir_all(zsh_path.parent().expect("zsh path should have parent"))?; + fs::write(&zsh_path, "")?; + } let canonical_package_dir = AbsolutePathBuf::from_absolute_path(package_dir.path().canonicalize()?)?; let canonical_bin_dir = AbsolutePathBuf::from_absolute_path(bin_dir.canonicalize()?)?; @@ -382,6 +404,19 @@ mod tests { context.bundled_resource(TEST_RESOURCE_NAME), Some(canonical_resources_dir.join(TEST_RESOURCE_NAME)) ); + if cfg!(windows) { + assert_eq!(context.bundled_zsh_path(), None); + assert_eq!(context.bundled_zsh_bin_dir(), None); + } else { + assert_eq!( + context.bundled_zsh_path(), + Some(canonical_resources_dir.join(zsh_resource_path())) + ); + assert_eq!( + context.bundled_zsh_bin_dir(), + Some(canonical_resources_dir.join(ZSH_DIRNAME).join(BIN_DIRNAME)) + ); + } Ok(()) } diff --git a/codex-rs/mcp-server/Cargo.toml b/codex-rs/mcp-server/Cargo.toml index 29b2d6c7d7..00a79ad020 100644 --- a/codex-rs/mcp-server/Cargo.toml +++ b/codex-rs/mcp-server/Cargo.toml @@ -23,8 +23,10 @@ codex-config = { workspace = true } codex-core = { workspace = true } codex-exec-server = { workspace = true } codex-extension-api = { workspace = true } +codex-install-context = { workspace = true } codex-login = { workspace = true } codex-protocol = { workspace = true } +codex-utils-absolute-path = { workspace = true } codex-utils-cli = { workspace = true } codex-utils-json-to-toml = { workspace = true } rmcp = { workspace = true } @@ -43,7 +45,6 @@ tracing = { workspace = true, features = ["log"] } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } [dev-dependencies] -codex-utils-absolute-path = { workspace = true } codex-shell-command = { workspace = true } core_test_support = { workspace = true } mcp_test_support = { workspace = true } diff --git a/codex-rs/mcp-server/src/codex_tool_config.rs b/codex-rs/mcp-server/src/codex_tool_config.rs index 2f9f354277..866202f7c6 100644 --- a/codex-rs/mcp-server/src/codex_tool_config.rs +++ b/codex-rs/mcp-server/src/codex_tool_config.rs @@ -4,6 +4,7 @@ use codex_arg0::Arg0DispatchPaths; use codex_core::config::Config; use codex_core::config::ConfigBuilder; use codex_core::config::ConfigOverrides; +use codex_install_context::InstallContext; use codex_protocol::ThreadId; use codex_protocol::config_types::SandboxMode; use codex_protocol::protocol::AskForApproval; @@ -180,6 +181,9 @@ impl CodexToolCallParam { codex_self_exe: arg0_paths.codex_self_exe.clone(), codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe.clone(), main_execve_wrapper_exe: arg0_paths.main_execve_wrapper_exe.clone(), + default_zsh_path: InstallContext::current() + .bundled_zsh_path() + .map(codex_utils_absolute_path::AbsolutePathBuf::into_path_buf), base_instructions, developer_instructions, compact_prompt, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 40cd121c71..7c46136857 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -1853,6 +1853,13 @@ async fn load_config_or_exit_with_fallback_cwd( strict_config: bool, fallback_cwd: Option, ) -> Config { + let mut overrides = overrides; + if overrides.default_zsh_path.is_none() { + overrides.default_zsh_path = codex_install_context::InstallContext::current() + .bundled_zsh_path() + .map(AbsolutePathBuf::into_path_buf); + } + #[allow(clippy::print_stderr)] match ConfigBuilder::default() .cli_overrides(cli_kv_overrides) diff --git a/scripts/codex_package/README.md b/scripts/codex_package/README.md index f53f1a41ad..f36a97eea3 100644 --- a/scripts/codex_package/README.md +++ b/scripts/codex_package/README.md @@ -13,6 +13,7 @@ The builder creates a canonical Codex package directory: │ └── [.exe] ├── codex-resources │ ├── bwrap # Linux only +│ ├── zsh/bin/zsh # supported Unix targets only │ ├── codex-command-runner.exe # Windows only │ └── codex-windows-sandbox-setup.exe # Windows only └── codex-path @@ -60,3 +61,9 @@ DotSlash manifest at `codex-cli/bin/rg`. Downloaded archives are cached under `$TMPDIR/codex-package/-rg` and are reused only after the recorded size and SHA-256 digest have been verified. Pass `--rg-bin` to use a local ripgrep executable instead. + +The patched zsh fork used by `shell_zsh_fork` is fetched from the DotSlash +manifest at `codex-cli/bin/codex-zsh` when the selected target has a matching +prebuilt artifact. Downloaded archives are cached under +`$TMPDIR/codex-package/-zsh` and installed at +`codex-resources/zsh/bin/zsh`. diff --git a/scripts/codex_package/cli.py b/scripts/codex_package/cli.py index 36ceda589e..9b7638e12a 100644 --- a/scripts/codex_package/cli.py +++ b/scripts/codex_package/cli.py @@ -15,6 +15,7 @@ from .targets import TARGET_SPECS from .targets import PackageInputs from .targets import default_target from .targets import resolve_input_path +from .zsh import resolve_zsh_bin from .version import read_workspace_version @@ -161,13 +162,14 @@ def main() -> int: inputs = PackageInputs( entrypoint_bin=source_outputs.entrypoint_bin, rg_bin=resolve_rg_bin(spec, args.rg_bin), + zsh_bin=resolve_zsh_bin(spec), bwrap_bin=source_outputs.bwrap_bin, codex_command_runner_bin=source_outputs.codex_command_runner_bin, codex_windows_sandbox_setup_bin=source_outputs.codex_windows_sandbox_setup_bin, ) prepare_package_dir(package_dir, force=args.force) build_package_dir(package_dir, version, variant, spec, inputs) - validate_package_dir(package_dir, variant, spec) + validate_package_dir(package_dir, variant, spec, include_zsh=inputs.zsh_bin is not None) for archive_output in args.archive_output: archive_path = archive_output.resolve() diff --git a/scripts/codex_package/dotslash.py b/scripts/codex_package/dotslash.py new file mode 100644 index 0000000000..e122de8509 --- /dev/null +++ b/scripts/codex_package/dotslash.py @@ -0,0 +1,223 @@ +"""Fetch executable artifacts from checked-in DotSlash manifests.""" + +import hashlib +import json +import shutil +import stat +import tarfile +import tempfile +import zipfile +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlparse +from urllib.request import urlopen + +from .targets import TargetSpec + + +DOWNLOAD_TIMEOUT_SECS = 60 + + +@dataclass(frozen=True) +class DotSlashArtifact: + size: int + digest: str + archive_format: str + archive_member: str + url: str + + +def fetch_dotslash_executable( + spec: TargetSpec, + *, + manifest_path: Path, + artifact_label: str, + cache_key: str, + dest_name: str, + executable: bool, + missing_ok: bool = False, +) -> Path | None: + artifact = artifact_for_target( + spec, + manifest_path, + artifact_label=artifact_label, + missing_ok=missing_ok, + ) + if artifact is None: + return None + + cache_dir = default_cache_root() / cache_key + archive_path = cache_dir / archive_filename(artifact.url) + + if not archive_is_valid(archive_path, artifact, artifact_label): + download_archive(artifact.url, archive_path) + try: + verify_archive(archive_path, artifact, artifact_label) + except RuntimeError: + archive_path.unlink(missing_ok=True) + raise + + dest = cache_dir / dest_name + extract_archive_member(archive_path, artifact, dest, artifact_label) + if executable: + mode = dest.stat().st_mode + dest.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + return dest + + +def artifact_for_target( + spec: TargetSpec, + manifest_path: Path, + *, + artifact_label: str, + missing_ok: bool = False, +) -> DotSlashArtifact | None: + manifest = load_manifest(manifest_path) + platform_info = manifest.get("platforms", {}).get(spec.dotslash_platform) + if platform_info is None: + if missing_ok: + return None + raise RuntimeError( + f"{artifact_label} manifest {manifest_path} is missing platform " + f"{spec.dotslash_platform!r}" + ) + + providers = platform_info.get("providers") + if not providers: + raise RuntimeError( + f"{artifact_label} manifest {manifest_path} has no providers for " + f"{spec.dotslash_platform!r}" + ) + + hash_name = platform_info.get("hash") + if hash_name != "sha256": + raise RuntimeError( + f"Unsupported {artifact_label} hash {hash_name!r} for " + f"{spec.dotslash_platform!r}; expected sha256" + ) + + return DotSlashArtifact( + size=int(platform_info["size"]), + digest=str(platform_info["digest"]), + archive_format=str(platform_info["format"]), + archive_member=str(platform_info["path"]), + url=str(providers[0]["url"]), + ) + + +def load_manifest(manifest_path: Path) -> dict: + text = manifest_path.read_text(encoding="utf-8") + if text.startswith("#!"): + text = "\n".join(text.splitlines()[1:]) + return json.loads(text) + + +def default_cache_root() -> Path: + return Path(tempfile.gettempdir()) / "codex-package" + + +def archive_filename(url: str) -> str: + filename = Path(urlparse(url).path).name + if not filename: + raise RuntimeError(f"Unable to determine archive filename from {url}") + return filename + + +def archive_is_valid( + archive_path: Path, + artifact: DotSlashArtifact, + artifact_label: str, +) -> bool: + if not archive_path.is_file(): + return False + try: + verify_archive(archive_path, artifact, artifact_label) + except RuntimeError: + archive_path.unlink(missing_ok=True) + return False + return True + + +def verify_archive( + archive_path: Path, + artifact: DotSlashArtifact, + artifact_label: str, +) -> None: + actual_size = archive_path.stat().st_size + if actual_size != artifact.size: + raise RuntimeError( + f"{artifact_label} archive {archive_path} has size {actual_size}, " + f"expected {artifact.size}" + ) + + digest = hashlib.sha256() + with open(archive_path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + + actual_digest = digest.hexdigest() + if actual_digest != artifact.digest: + raise RuntimeError( + f"{artifact_label} archive {archive_path} has sha256 {actual_digest}, " + f"expected {artifact.digest}" + ) + + +def download_archive(url: str, archive_path: Path) -> None: + archive_path.parent.mkdir(parents=True, exist_ok=True) + temp_path = archive_path.with_suffix(f"{archive_path.suffix}.tmp") + temp_path.unlink(missing_ok=True) + try: + with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response: + with open(temp_path, "wb") as out: + shutil.copyfileobj(response, out) + temp_path.replace(archive_path) + finally: + temp_path.unlink(missing_ok=True) + + +def extract_archive_member( + archive_path: Path, + artifact: DotSlashArtifact, + dest: Path, + artifact_label: str, +) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + dest.unlink(missing_ok=True) + + if artifact.archive_format == "tar.gz": + with tarfile.open(archive_path, "r:gz") as archive: + try: + member = archive.getmember(artifact.archive_member) + except KeyError as exc: + raise RuntimeError( + f"{artifact_label} archive {archive_path} is missing " + f"{artifact.archive_member!r}" + ) from exc + + extracted = archive.extractfile(member) + if extracted is None: + raise RuntimeError( + f"{artifact_label} archive member {artifact.archive_member!r} is not a file" + ) + with extracted, open(dest, "wb") as out: + shutil.copyfileobj(extracted, out) + return + + if artifact.archive_format == "zip": + with zipfile.ZipFile(archive_path) as archive: + try: + with archive.open(artifact.archive_member) as extracted: + with open(dest, "wb") as out: + shutil.copyfileobj(extracted, out) + except KeyError as exc: + raise RuntimeError( + f"{artifact_label} archive {archive_path} is missing " + f"{artifact.archive_member!r}" + ) from exc + return + + raise RuntimeError( + f"Unsupported {artifact_label} archive format {artifact.archive_format!r}; " + "expected tar.gz or zip" + ) diff --git a/scripts/codex_package/layout.py b/scripts/codex_package/layout.py index 6d8982a631..c763eb2604 100644 --- a/scripts/codex_package/layout.py +++ b/scripts/codex_package/layout.py @@ -8,6 +8,7 @@ from pathlib import Path from .targets import PackageInputs from .targets import PackageVariant from .targets import TargetSpec +from .zsh import ZSH_RESOURCE_PATH LAYOUT_VERSION = 1 @@ -50,6 +51,13 @@ def build_package_dir( ) copy_executable(inputs.rg_bin, path_dir / spec.rg_name, is_windows=spec.is_windows) + if inputs.zsh_bin is not None: + copy_executable( + inputs.zsh_bin, + resources_dir / ZSH_RESOURCE_PATH, + is_windows=False, + ) + if inputs.bwrap_bin is not None: copy_executable(inputs.bwrap_bin, resources_dir / "bwrap", is_windows=False) @@ -83,6 +91,8 @@ def validate_package_dir( package_dir: Path, variant: PackageVariant, spec: TargetSpec, + *, + include_zsh: bool, ) -> None: required_dirs = [ Path("bin"), @@ -122,6 +132,11 @@ def validate_package_dir( ] executable_files = list(required_files) + if include_zsh: + zsh_path = Path("codex-resources") / ZSH_RESOURCE_PATH + required_files.append(zsh_path) + executable_files.append(zsh_path) + if spec.is_linux: required_files.append(Path("codex-resources") / "bwrap") executable_files.append(Path("codex-resources") / "bwrap") @@ -148,7 +163,7 @@ def validate_package_dir( def copy_executable(src: Path, dest: Path, *, is_windows: bool) -> None: dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, dest) + shutil.copyfile(src, dest) if not is_windows: mode = dest.stat().st_mode dest.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) diff --git a/scripts/codex_package/ripgrep.py b/scripts/codex_package/ripgrep.py index 5411cb579a..70a5f120c0 100644 --- a/scripts/codex_package/ripgrep.py +++ b/scripts/codex_package/ripgrep.py @@ -1,33 +1,12 @@ -"""Fetch ripgrep from the DotSlash manifest used by the npm package.""" - -import hashlib -import json -import shutil -import stat -import tarfile -import tempfile -import zipfile -from dataclasses import dataclass from pathlib import Path -from urllib.parse import urlparse -from urllib.request import urlopen +from .dotslash import fetch_dotslash_executable from .targets import REPO_ROOT from .targets import TargetSpec from .targets import resolve_input_path RG_MANIFEST = REPO_ROOT / "codex-cli" / "bin" / "rg" -DOWNLOAD_TIMEOUT_SECS = 60 - - -@dataclass(frozen=True) -class RgArtifact: - size: int - digest: str - archive_format: str - archive_member: str - url: str def resolve_rg_bin(spec: TargetSpec, rg_bin: Path | None) -> Path: @@ -41,155 +20,15 @@ def fetch_rg( spec: TargetSpec, *, manifest_path: Path = RG_MANIFEST, - cache_root: Path | None = None, ) -> Path: - artifact = artifact_for_target(spec, manifest_path) - cache_dir = (cache_root or default_cache_root()) / f"{spec.target}-rg" - archive_path = cache_dir / archive_filename(artifact.url) - - if not archive_is_valid(archive_path, artifact): - download_archive(artifact.url, archive_path) - try: - verify_archive(archive_path, artifact) - except RuntimeError: - archive_path.unlink(missing_ok=True) - raise - - dest = cache_dir / spec.rg_name - extract_rg(archive_path, artifact, dest) - if not spec.is_windows: - mode = dest.stat().st_mode - dest.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - return dest - - -def artifact_for_target(spec: TargetSpec, manifest_path: Path) -> RgArtifact: - manifest = load_manifest(manifest_path) - try: - platform_info = manifest["platforms"][spec.dotslash_platform] - except KeyError as exc: - raise RuntimeError( - f"ripgrep manifest {manifest_path} is missing platform {spec.dotslash_platform!r}" - ) from exc - - providers = platform_info.get("providers") - if not providers: - raise RuntimeError( - f"ripgrep manifest {manifest_path} has no providers for {spec.dotslash_platform!r}" - ) - - hash_name = platform_info.get("hash") - if hash_name != "sha256": - raise RuntimeError( - f"Unsupported ripgrep hash {hash_name!r} for " - f"{spec.dotslash_platform!r}; expected sha256" - ) - - return RgArtifact( - size=int(platform_info["size"]), - digest=str(platform_info["digest"]), - archive_format=str(platform_info["format"]), - archive_member=str(platform_info["path"]), - url=str(providers[0]["url"]), - ) - - -def load_manifest(manifest_path: Path) -> dict: - text = manifest_path.read_text(encoding="utf-8") - if text.startswith("#!"): - text = "\n".join(text.splitlines()[1:]) - return json.loads(text) - - -def default_cache_root() -> Path: - return Path(tempfile.gettempdir()) / "codex-package" - - -def archive_filename(url: str) -> str: - filename = Path(urlparse(url).path).name - if not filename: - raise RuntimeError(f"Unable to determine archive filename from {url}") - return filename - - -def archive_is_valid(archive_path: Path, artifact: RgArtifact) -> bool: - if not archive_path.is_file(): - return False - try: - verify_archive(archive_path, artifact) - except RuntimeError: - archive_path.unlink(missing_ok=True) - return False - return True - - -def verify_archive(archive_path: Path, artifact: RgArtifact) -> None: - actual_size = archive_path.stat().st_size - if actual_size != artifact.size: - raise RuntimeError( - f"ripgrep archive {archive_path} has size {actual_size}, expected {artifact.size}" - ) - - digest = hashlib.sha256() - with open(archive_path, "rb") as fh: - for chunk in iter(lambda: fh.read(1024 * 1024), b""): - digest.update(chunk) - - actual_digest = digest.hexdigest() - if actual_digest != artifact.digest: - raise RuntimeError( - f"ripgrep archive {archive_path} has sha256 {actual_digest}, " - f"expected {artifact.digest}" - ) - - -def download_archive(url: str, archive_path: Path) -> None: - archive_path.parent.mkdir(parents=True, exist_ok=True) - temp_path = archive_path.with_suffix(f"{archive_path.suffix}.tmp") - temp_path.unlink(missing_ok=True) - try: - with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response: - with open(temp_path, "wb") as out: - shutil.copyfileobj(response, out) - temp_path.replace(archive_path) - finally: - temp_path.unlink(missing_ok=True) - - -def extract_rg(archive_path: Path, artifact: RgArtifact, dest: Path) -> None: - dest.parent.mkdir(parents=True, exist_ok=True) - dest.unlink(missing_ok=True) - - if artifact.archive_format == "tar.gz": - with tarfile.open(archive_path, "r:gz") as archive: - try: - member = archive.getmember(artifact.archive_member) - except KeyError as exc: - raise RuntimeError( - f"ripgrep archive {archive_path} is missing {artifact.archive_member!r}" - ) from exc - - extracted = archive.extractfile(member) - if extracted is None: - raise RuntimeError( - f"ripgrep archive member {artifact.archive_member!r} is not a file" - ) - with extracted, open(dest, "wb") as out: - shutil.copyfileobj(extracted, out) - return - - if artifact.archive_format == "zip": - with zipfile.ZipFile(archive_path) as archive: - try: - with archive.open(artifact.archive_member) as extracted: - with open(dest, "wb") as out: - shutil.copyfileobj(extracted, out) - except KeyError as exc: - raise RuntimeError( - f"ripgrep archive {archive_path} is missing {artifact.archive_member!r}" - ) from exc - return - - raise RuntimeError( - f"Unsupported ripgrep archive format {artifact.archive_format!r}; expected tar.gz or zip" + rg_bin = fetch_dotslash_executable( + spec, + manifest_path=manifest_path, + artifact_label="ripgrep", + cache_key=f"{spec.target}-rg", + dest_name=spec.rg_name, + executable=not spec.is_windows, ) + if rg_bin is None: + raise AssertionError("ripgrep is required for all package targets") + return rg_bin diff --git a/scripts/codex_package/targets.py b/scripts/codex_package/targets.py index 4af0d4a00d..8307a3e630 100644 --- a/scripts/codex_package/targets.py +++ b/scripts/codex_package/targets.py @@ -40,6 +40,7 @@ class PackageVariant: class PackageInputs: entrypoint_bin: Path rg_bin: Path + zsh_bin: Path | None bwrap_bin: Path | None codex_command_runner_bin: Path | None codex_windows_sandbox_setup_bin: Path | None diff --git a/scripts/codex_package/zsh.py b/scripts/codex_package/zsh.py new file mode 100644 index 0000000000..60669c881c --- /dev/null +++ b/scripts/codex_package/zsh.py @@ -0,0 +1,23 @@ +"""Fetch the patched zsh fork used by shell_zsh_fork.""" + +from pathlib import Path + +from .dotslash import fetch_dotslash_executable +from .targets import REPO_ROOT +from .targets import TargetSpec + + +ZSH_MANIFEST = REPO_ROOT / "codex-cli" / "bin" / "codex-zsh" +ZSH_RESOURCE_PATH = Path("zsh") / "bin" / "zsh" + + +def resolve_zsh_bin(spec: TargetSpec) -> Path | None: + return fetch_dotslash_executable( + spec, + manifest_path=ZSH_MANIFEST, + artifact_label="codex-zsh", + cache_key=f"{spec.target}-zsh", + dest_name="zsh", + executable=True, + missing_ok=True, + )