mirror of
https://github.com/openai/codex.git
synced 2026-10-21 17:48:44 +00:00
package: include zsh fork in Codex package
Teach the Codex package builder to fetch the prebuilt zsh fork from a checked-in DotSlash manifest and install it under codex-resources/zsh/bin/zsh when an artifact is available for the package target. Generalize the DotSlash download/cache/verify helper previously embedded in ripgrep packaging so additional checked-in DotSlash manifests can use the same SHA-256 and size validation path. Add install-context support for locating the bundled zsh fork and thread that path through config loading as the lowest-precedence zsh_path default, preserving explicit CLI/profile/global config values. Also avoid preserving platform-specific file metadata when copying executables into the package directory so package smoke tests can use macOS system binaries as inputs.
This commit is contained in:
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env dotslash
|
||||
|
||||
{
|
||||
"name": "codex-zsh",
|
||||
"platforms": {
|
||||
"macos-aarch64": {
|
||||
"size": 358776,
|
||||
"hash": "sha256",
|
||||
"digest": "c6dbb063a0135b947ab1cacc655b2b750874699472f412ec7daba97543a90c3c",
|
||||
"format": "tar.gz",
|
||||
"path": "codex-zsh/bin/zsh",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/openai/codex/releases/download/rust-v0.132.0/codex-zsh-aarch64-apple-darwin.tar.gz"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-x86_64": {
|
||||
"size": 433413,
|
||||
"hash": "sha256",
|
||||
"digest": "5f42d9fc8e9c8c399a727512002906006ae9de966ea7b3d87ca36b47efc59938",
|
||||
"format": "tar.gz",
|
||||
"path": "codex-zsh/bin/zsh",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/openai/codex/releases/download/rust-v0.132.0/codex-zsh-x86_64-unknown-linux-musl.tar.gz"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-aarch64": {
|
||||
"size": 411653,
|
||||
"hash": "sha256",
|
||||
"digest": "6c6e32c297425db02b4dbffb10925895875d14647fc3eb2f18767be97dc6a945",
|
||||
"format": "tar.gz",
|
||||
"path": "codex-zsh/bin/zsh",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/openai/codex/releases/download/rust-v0.132.0/codex-zsh-aarch64-unknown-linux-musl.tar.gz"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
+3
@@ -1905,6 +1905,7 @@ dependencies = [
|
||||
"codex-git-utils",
|
||||
"codex-guardian",
|
||||
"codex-hooks",
|
||||
"codex-install-context",
|
||||
"codex-login",
|
||||
"codex-mcp",
|
||||
"codex-memories-extension",
|
||||
@@ -2710,6 +2711,7 @@ dependencies = [
|
||||
"codex-core",
|
||||
"codex-feedback",
|
||||
"codex-git-utils",
|
||||
"codex-install-context",
|
||||
"codex-login",
|
||||
"codex-model-provider-info",
|
||||
"codex-otel",
|
||||
@@ -3141,6 +3143,7 @@ dependencies = [
|
||||
"codex-core",
|
||||
"codex-exec-server",
|
||||
"codex-extension-api",
|
||||
"codex-install-context",
|
||||
"codex-login",
|
||||
"codex-protocol",
|
||||
"codex-shell-command",
|
||||
|
||||
@@ -45,6 +45,7 @@ codex-guardian = { workspace = true }
|
||||
codex-git-utils = { workspace = true }
|
||||
codex-file-watcher = { workspace = true }
|
||||
codex-hooks = { workspace = true }
|
||||
codex-install-context = { workspace = true }
|
||||
codex-otel = { workspace = true }
|
||||
codex-plugin = { workspace = true }
|
||||
codex-shell-command = { workspace = true }
|
||||
|
||||
@@ -9,6 +9,7 @@ use codex_core::config::Config;
|
||||
use codex_core::config::ConfigOverrides;
|
||||
use codex_exec_server::LOCAL_FS;
|
||||
use codex_features::feature_for_key;
|
||||
use codex_install_context::InstallContext;
|
||||
use codex_login::AuthManager;
|
||||
use codex_login::default_client::set_default_client_residency_requirement;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
@@ -159,6 +160,7 @@ impl ConfigManager {
|
||||
.await?;
|
||||
self.apply_runtime_feature_enablement(&mut config);
|
||||
self.apply_arg0_paths(&mut config);
|
||||
apply_default_zsh_path(&mut config);
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
@@ -180,6 +182,7 @@ impl ConfigManager {
|
||||
}
|
||||
self.apply_runtime_feature_enablement(&mut config);
|
||||
self.apply_arg0_paths(&mut config);
|
||||
apply_default_zsh_path(&mut config);
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
@@ -219,6 +222,11 @@ impl ConfigManager {
|
||||
typesafe_overrides: ConfigOverrides,
|
||||
fallback_cwd: Option<PathBuf>,
|
||||
) -> std::io::Result<Config> {
|
||||
let mut typesafe_overrides = typesafe_overrides;
|
||||
if typesafe_overrides.default_zsh_path.is_none() {
|
||||
typesafe_overrides.default_zsh_path = default_zsh_path();
|
||||
}
|
||||
|
||||
let merged_cli_overrides = cli_overrides
|
||||
.iter()
|
||||
.cloned()
|
||||
@@ -319,6 +327,18 @@ impl ConfigManager {
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_default_zsh_path(config: &mut Config) {
|
||||
if config.zsh_path.is_none() {
|
||||
config.zsh_path = default_zsh_path();
|
||||
}
|
||||
}
|
||||
|
||||
fn default_zsh_path() -> Option<PathBuf> {
|
||||
InstallContext::current()
|
||||
.bundled_zsh_path()
|
||||
.map(AbsolutePathBuf::into_path_buf)
|
||||
}
|
||||
|
||||
pub(crate) fn protected_feature_keys(config_layer_stack: &ConfigLayerStack) -> BTreeSet<String> {
|
||||
let mut protected_features = config_layer_stack
|
||||
.effective_config()
|
||||
|
||||
@@ -507,6 +507,7 @@ fn config_overrides_from_interactive(
|
||||
codex_self_exe: arg0_paths.codex_self_exe.clone(),
|
||||
codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe.clone(),
|
||||
main_execve_wrapper_exe: arg0_paths.main_execve_wrapper_exe.clone(),
|
||||
default_zsh_path: crate::default_zsh_path(),
|
||||
show_raw_agent_reasoning: interactive.oss.then_some(true),
|
||||
additional_writable_roots: interactive.add_dir.clone(),
|
||||
..Default::default()
|
||||
|
||||
@@ -74,6 +74,7 @@ use codex_core::config::resolve_profile_v2_config_path;
|
||||
use codex_features::FEATURES;
|
||||
use codex_features::Stage;
|
||||
use codex_features::is_known_feature_key;
|
||||
use codex_install_context::InstallContext;
|
||||
use codex_login::AuthManager;
|
||||
use codex_login::CodexAuth;
|
||||
use codex_login::read_codex_access_token_from_env;
|
||||
@@ -1568,6 +1569,12 @@ async fn load_exec_server_remote_config(
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub(crate) fn default_zsh_path() -> Option<PathBuf> {
|
||||
InstallContext::current()
|
||||
.bundled_zsh_path()
|
||||
.map(AbsolutePathBuf::into_path_buf)
|
||||
}
|
||||
|
||||
async fn load_exec_server_remote_auth(
|
||||
config: &codex_core::config::Config,
|
||||
missing_auth_error: &'static str,
|
||||
@@ -1703,6 +1710,7 @@ async fn run_debug_prompt_input_command(
|
||||
codex_self_exe: arg0_paths.codex_self_exe,
|
||||
codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe,
|
||||
main_execve_wrapper_exe: arg0_paths.main_execve_wrapper_exe,
|
||||
default_zsh_path: default_zsh_path(),
|
||||
show_raw_agent_reasoning: shared.oss.then_some(true),
|
||||
ephemeral: Some(true),
|
||||
bypass_hook_trust: shared.bypass_hook_trust.then_some(true),
|
||||
|
||||
@@ -4142,6 +4142,56 @@ async fn add_dir_override_extends_workspace_writable_roots() -> std::io::Result<
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_zsh_path_is_lowest_precedence() -> std::io::Result<()> {
|
||||
let codex_home = TempDir::new()?;
|
||||
let configured_zsh_path = codex_home.path().join("configured-zsh");
|
||||
let default_zsh_path = codex_home.path().join("packaged-zsh");
|
||||
let override_zsh_path = codex_home.path().join("override-zsh");
|
||||
|
||||
let config = Config::load_from_base_config_with_overrides(
|
||||
ConfigToml::default(),
|
||||
ConfigOverrides {
|
||||
default_zsh_path: Some(default_zsh_path.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
codex_home.abs(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(config.zsh_path, Some(default_zsh_path.clone()));
|
||||
|
||||
let config = Config::load_from_base_config_with_overrides(
|
||||
ConfigToml {
|
||||
zsh_path: Some(configured_zsh_path.abs()),
|
||||
..Default::default()
|
||||
},
|
||||
ConfigOverrides {
|
||||
default_zsh_path: Some(default_zsh_path.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
codex_home.abs(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(config.zsh_path, Some(configured_zsh_path.clone()));
|
||||
|
||||
let config = Config::load_from_base_config_with_overrides(
|
||||
ConfigToml {
|
||||
zsh_path: Some(configured_zsh_path.abs()),
|
||||
..Default::default()
|
||||
},
|
||||
ConfigOverrides {
|
||||
zsh_path: Some(override_zsh_path.clone()),
|
||||
default_zsh_path: Some(default_zsh_path),
|
||||
..Default::default()
|
||||
},
|
||||
codex_home.abs(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(config.zsh_path, Some(override_zsh_path));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlite_home_defaults_to_codex_home_for_workspace_write() -> std::io::Result<()> {
|
||||
let codex_home = TempDir::new()?;
|
||||
|
||||
@@ -1364,6 +1364,7 @@ impl Config {
|
||||
cfg,
|
||||
ConfigOverrides {
|
||||
cwd: Some(self.cwd.to_path_buf()),
|
||||
default_zsh_path: refreshed_config.zsh_path.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
refreshed_config.codex_home.clone(),
|
||||
@@ -2070,6 +2071,7 @@ pub struct ConfigOverrides {
|
||||
pub codex_linux_sandbox_exe: Option<PathBuf>,
|
||||
pub main_execve_wrapper_exe: Option<PathBuf>,
|
||||
pub zsh_path: Option<PathBuf>,
|
||||
pub default_zsh_path: Option<PathBuf>,
|
||||
pub base_instructions: Option<String>,
|
||||
pub developer_instructions: Option<String>,
|
||||
pub personality: Option<Personality>,
|
||||
@@ -2467,6 +2469,7 @@ impl Config {
|
||||
codex_linux_sandbox_exe,
|
||||
main_execve_wrapper_exe,
|
||||
zsh_path: zsh_path_override,
|
||||
default_zsh_path,
|
||||
base_instructions,
|
||||
developer_instructions,
|
||||
personality,
|
||||
@@ -3225,7 +3228,8 @@ impl Config {
|
||||
let compact_prompt = compact_prompt.or(file_compact_prompt);
|
||||
let zsh_path = zsh_path_override
|
||||
.or(config_profile.zsh_path.map(Into::into))
|
||||
.or(cfg.zsh_path.map(Into::into));
|
||||
.or(cfg.zsh_path.map(Into::into))
|
||||
.or(default_zsh_path);
|
||||
|
||||
let review_model = override_review_model.or(cfg.review_model);
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ codex-config = { workspace = true }
|
||||
codex-core = { workspace = true }
|
||||
codex-feedback = { workspace = true }
|
||||
codex-git-utils = { workspace = true }
|
||||
codex-install-context = { workspace = true }
|
||||
codex-login = { workspace = true }
|
||||
codex-model-provider-info = { workspace = true }
|
||||
codex-otel = { workspace = true }
|
||||
|
||||
@@ -70,6 +70,7 @@ use codex_core::format_exec_policy_error_with_source;
|
||||
use codex_core::path_utils;
|
||||
use codex_feedback::CodexFeedback;
|
||||
use codex_git_utils::get_git_repo_root;
|
||||
use codex_install_context::InstallContext;
|
||||
use codex_login::AuthConfig;
|
||||
use codex_login::default_client::set_default_client_residency_requirement;
|
||||
use codex_login::default_client::set_default_originator;
|
||||
@@ -424,6 +425,9 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result
|
||||
codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe.clone(),
|
||||
main_execve_wrapper_exe: arg0_paths.main_execve_wrapper_exe.clone(),
|
||||
zsh_path: None,
|
||||
default_zsh_path: InstallContext::current()
|
||||
.bundled_zsh_path()
|
||||
.map(AbsolutePathBuf::into_path_buf),
|
||||
base_instructions: None,
|
||||
developer_instructions: None,
|
||||
personality: None,
|
||||
|
||||
@@ -11,6 +11,7 @@ const PATH_DIRNAME: &str = "codex-path";
|
||||
const RELEASES_DIRNAME: &str = "releases";
|
||||
const RESOURCES_DIRNAME: &str = "codex-resources";
|
||||
const STANDALONE_PACKAGES_DIRNAME: &str = "standalone";
|
||||
const ZSH_DIRNAME: &str = "zsh";
|
||||
static INSTALL_CONTEXT: OnceLock<InstallContext> = OnceLock::new();
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
@@ -166,6 +167,18 @@ impl InstallContext {
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub fn bundled_zsh_path(&self) -> Option<AbsolutePathBuf> {
|
||||
if cfg!(windows) {
|
||||
None
|
||||
} else {
|
||||
self.bundled_resource(zsh_resource_path())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bundled_zsh_bin_dir(&self) -> Option<AbsolutePathBuf> {
|
||||
self.bundled_zsh_path()?.parent()
|
||||
}
|
||||
}
|
||||
|
||||
impl CodexPackageLayout {
|
||||
@@ -260,6 +273,10 @@ fn default_rg_command() -> PathBuf {
|
||||
}
|
||||
}
|
||||
|
||||
fn zsh_resource_path() -> PathBuf {
|
||||
PathBuf::from(ZSH_DIRNAME).join(BIN_DIRNAME).join("zsh")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -345,6 +362,11 @@ mod tests {
|
||||
fs::write(&exe_path, "")?;
|
||||
fs::write(resources_dir.join(TEST_RESOURCE_NAME), "")?;
|
||||
fs::write(path_dir.join(default_rg_command()), "")?;
|
||||
if !cfg!(windows) {
|
||||
let zsh_path = resources_dir.join(zsh_resource_path());
|
||||
fs::create_dir_all(zsh_path.parent().expect("zsh path should have parent"))?;
|
||||
fs::write(&zsh_path, "")?;
|
||||
}
|
||||
let canonical_package_dir =
|
||||
AbsolutePathBuf::from_absolute_path(package_dir.path().canonicalize()?)?;
|
||||
let canonical_bin_dir = AbsolutePathBuf::from_absolute_path(bin_dir.canonicalize()?)?;
|
||||
@@ -382,6 +404,19 @@ mod tests {
|
||||
context.bundled_resource(TEST_RESOURCE_NAME),
|
||||
Some(canonical_resources_dir.join(TEST_RESOURCE_NAME))
|
||||
);
|
||||
if cfg!(windows) {
|
||||
assert_eq!(context.bundled_zsh_path(), None);
|
||||
assert_eq!(context.bundled_zsh_bin_dir(), None);
|
||||
} else {
|
||||
assert_eq!(
|
||||
context.bundled_zsh_path(),
|
||||
Some(canonical_resources_dir.join(zsh_resource_path()))
|
||||
);
|
||||
assert_eq!(
|
||||
context.bundled_zsh_bin_dir(),
|
||||
Some(canonical_resources_dir.join(ZSH_DIRNAME).join(BIN_DIRNAME))
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -23,8 +23,10 @@ codex-config = { workspace = true }
|
||||
codex-core = { workspace = true }
|
||||
codex-exec-server = { workspace = true }
|
||||
codex-extension-api = { workspace = true }
|
||||
codex-install-context = { workspace = true }
|
||||
codex-login = { workspace = true }
|
||||
codex-protocol = { workspace = true }
|
||||
codex-utils-absolute-path = { workspace = true }
|
||||
codex-utils-cli = { workspace = true }
|
||||
codex-utils-json-to-toml = { workspace = true }
|
||||
rmcp = { workspace = true }
|
||||
@@ -43,7 +45,6 @@ tracing = { workspace = true, features = ["log"] }
|
||||
tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] }
|
||||
|
||||
[dev-dependencies]
|
||||
codex-utils-absolute-path = { workspace = true }
|
||||
codex-shell-command = { workspace = true }
|
||||
core_test_support = { workspace = true }
|
||||
mcp_test_support = { workspace = true }
|
||||
|
||||
@@ -4,6 +4,7 @@ use codex_arg0::Arg0DispatchPaths;
|
||||
use codex_core::config::Config;
|
||||
use codex_core::config::ConfigBuilder;
|
||||
use codex_core::config::ConfigOverrides;
|
||||
use codex_install_context::InstallContext;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::config_types::SandboxMode;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
@@ -180,6 +181,9 @@ impl CodexToolCallParam {
|
||||
codex_self_exe: arg0_paths.codex_self_exe.clone(),
|
||||
codex_linux_sandbox_exe: arg0_paths.codex_linux_sandbox_exe.clone(),
|
||||
main_execve_wrapper_exe: arg0_paths.main_execve_wrapper_exe.clone(),
|
||||
default_zsh_path: InstallContext::current()
|
||||
.bundled_zsh_path()
|
||||
.map(codex_utils_absolute_path::AbsolutePathBuf::into_path_buf),
|
||||
base_instructions,
|
||||
developer_instructions,
|
||||
compact_prompt,
|
||||
|
||||
@@ -1853,6 +1853,13 @@ async fn load_config_or_exit_with_fallback_cwd(
|
||||
strict_config: bool,
|
||||
fallback_cwd: Option<PathBuf>,
|
||||
) -> Config {
|
||||
let mut overrides = overrides;
|
||||
if overrides.default_zsh_path.is_none() {
|
||||
overrides.default_zsh_path = codex_install_context::InstallContext::current()
|
||||
.bundled_zsh_path()
|
||||
.map(AbsolutePathBuf::into_path_buf);
|
||||
}
|
||||
|
||||
#[allow(clippy::print_stderr)]
|
||||
match ConfigBuilder::default()
|
||||
.cli_overrides(cli_kv_overrides)
|
||||
|
||||
@@ -13,6 +13,7 @@ The builder creates a canonical Codex package directory:
|
||||
│ └── <entrypoint>[.exe]
|
||||
├── codex-resources
|
||||
│ ├── bwrap # Linux only
|
||||
│ ├── zsh/bin/zsh # supported Unix targets only
|
||||
│ ├── codex-command-runner.exe # Windows only
|
||||
│ └── codex-windows-sandbox-setup.exe # Windows only
|
||||
└── codex-path
|
||||
@@ -52,3 +53,9 @@ DotSlash manifest at `codex-cli/bin/rg`. Downloaded archives are cached under
|
||||
`$TMPDIR/codex-package/<target>-rg` and are reused only after the recorded size
|
||||
and SHA-256 digest have been verified. Pass `--rg-bin` to use a local ripgrep
|
||||
executable instead.
|
||||
|
||||
The patched zsh fork used by `shell_zsh_fork` is fetched from the DotSlash
|
||||
manifest at `codex-cli/bin/codex-zsh` when the selected target has a matching
|
||||
prebuilt artifact. Downloaded archives are cached under
|
||||
`$TMPDIR/codex-package/<target>-zsh` and installed at
|
||||
`codex-resources/zsh/bin/zsh`.
|
||||
|
||||
@@ -15,6 +15,7 @@ from .targets import TARGET_SPECS
|
||||
from .targets import PackageInputs
|
||||
from .targets import default_target
|
||||
from .targets import resolve_input_path
|
||||
from .zsh import resolve_zsh_bin
|
||||
from .version import read_workspace_version
|
||||
|
||||
|
||||
@@ -124,13 +125,14 @@ def main() -> int:
|
||||
inputs = PackageInputs(
|
||||
entrypoint_bin=source_outputs.entrypoint_bin,
|
||||
rg_bin=resolve_rg_bin(spec, args.rg_bin),
|
||||
zsh_bin=resolve_zsh_bin(spec),
|
||||
bwrap_bin=source_outputs.bwrap_bin,
|
||||
codex_command_runner_bin=source_outputs.codex_command_runner_bin,
|
||||
codex_windows_sandbox_setup_bin=source_outputs.codex_windows_sandbox_setup_bin,
|
||||
)
|
||||
prepare_package_dir(package_dir, force=args.force)
|
||||
build_package_dir(package_dir, version, variant, spec, inputs)
|
||||
validate_package_dir(package_dir, variant, spec)
|
||||
validate_package_dir(package_dir, variant, spec, include_zsh=inputs.zsh_bin is not None)
|
||||
|
||||
for archive_output in args.archive_output:
|
||||
archive_path = archive_output.resolve()
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Fetch executable artifacts from checked-in DotSlash manifests."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import stat
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import urlopen
|
||||
|
||||
from .targets import TargetSpec
|
||||
|
||||
|
||||
DOWNLOAD_TIMEOUT_SECS = 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DotSlashArtifact:
|
||||
size: int
|
||||
digest: str
|
||||
archive_format: str
|
||||
archive_member: str
|
||||
url: str
|
||||
|
||||
|
||||
def fetch_dotslash_executable(
|
||||
spec: TargetSpec,
|
||||
*,
|
||||
manifest_path: Path,
|
||||
artifact_label: str,
|
||||
cache_key: str,
|
||||
dest_name: str,
|
||||
executable: bool,
|
||||
missing_ok: bool = False,
|
||||
) -> Path | None:
|
||||
artifact = artifact_for_target(
|
||||
spec,
|
||||
manifest_path,
|
||||
artifact_label=artifact_label,
|
||||
missing_ok=missing_ok,
|
||||
)
|
||||
if artifact is None:
|
||||
return None
|
||||
|
||||
cache_dir = default_cache_root() / cache_key
|
||||
archive_path = cache_dir / archive_filename(artifact.url)
|
||||
|
||||
if not archive_is_valid(archive_path, artifact, artifact_label):
|
||||
download_archive(artifact.url, archive_path)
|
||||
try:
|
||||
verify_archive(archive_path, artifact, artifact_label)
|
||||
except RuntimeError:
|
||||
archive_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
dest = cache_dir / dest_name
|
||||
extract_archive_member(archive_path, artifact, dest, artifact_label)
|
||||
if executable:
|
||||
mode = dest.stat().st_mode
|
||||
dest.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
return dest
|
||||
|
||||
|
||||
def artifact_for_target(
|
||||
spec: TargetSpec,
|
||||
manifest_path: Path,
|
||||
*,
|
||||
artifact_label: str,
|
||||
missing_ok: bool = False,
|
||||
) -> DotSlashArtifact | None:
|
||||
manifest = load_manifest(manifest_path)
|
||||
platform_info = manifest.get("platforms", {}).get(spec.dotslash_platform)
|
||||
if platform_info is None:
|
||||
if missing_ok:
|
||||
return None
|
||||
raise RuntimeError(
|
||||
f"{artifact_label} manifest {manifest_path} is missing platform "
|
||||
f"{spec.dotslash_platform!r}"
|
||||
)
|
||||
|
||||
providers = platform_info.get("providers")
|
||||
if not providers:
|
||||
raise RuntimeError(
|
||||
f"{artifact_label} manifest {manifest_path} has no providers for "
|
||||
f"{spec.dotslash_platform!r}"
|
||||
)
|
||||
|
||||
hash_name = platform_info.get("hash")
|
||||
if hash_name != "sha256":
|
||||
raise RuntimeError(
|
||||
f"Unsupported {artifact_label} hash {hash_name!r} for "
|
||||
f"{spec.dotslash_platform!r}; expected sha256"
|
||||
)
|
||||
|
||||
return DotSlashArtifact(
|
||||
size=int(platform_info["size"]),
|
||||
digest=str(platform_info["digest"]),
|
||||
archive_format=str(platform_info["format"]),
|
||||
archive_member=str(platform_info["path"]),
|
||||
url=str(providers[0]["url"]),
|
||||
)
|
||||
|
||||
|
||||
def load_manifest(manifest_path: Path) -> dict:
|
||||
text = manifest_path.read_text(encoding="utf-8")
|
||||
if text.startswith("#!"):
|
||||
text = "\n".join(text.splitlines()[1:])
|
||||
return json.loads(text)
|
||||
|
||||
|
||||
def default_cache_root() -> Path:
|
||||
return Path(tempfile.gettempdir()) / "codex-package"
|
||||
|
||||
|
||||
def archive_filename(url: str) -> str:
|
||||
filename = Path(urlparse(url).path).name
|
||||
if not filename:
|
||||
raise RuntimeError(f"Unable to determine archive filename from {url}")
|
||||
return filename
|
||||
|
||||
|
||||
def archive_is_valid(
|
||||
archive_path: Path,
|
||||
artifact: DotSlashArtifact,
|
||||
artifact_label: str,
|
||||
) -> bool:
|
||||
if not archive_path.is_file():
|
||||
return False
|
||||
try:
|
||||
verify_archive(archive_path, artifact, artifact_label)
|
||||
except RuntimeError:
|
||||
archive_path.unlink(missing_ok=True)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def verify_archive(
|
||||
archive_path: Path,
|
||||
artifact: DotSlashArtifact,
|
||||
artifact_label: str,
|
||||
) -> None:
|
||||
actual_size = archive_path.stat().st_size
|
||||
if actual_size != artifact.size:
|
||||
raise RuntimeError(
|
||||
f"{artifact_label} archive {archive_path} has size {actual_size}, "
|
||||
f"expected {artifact.size}"
|
||||
)
|
||||
|
||||
digest = hashlib.sha256()
|
||||
with open(archive_path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
|
||||
actual_digest = digest.hexdigest()
|
||||
if actual_digest != artifact.digest:
|
||||
raise RuntimeError(
|
||||
f"{artifact_label} archive {archive_path} has sha256 {actual_digest}, "
|
||||
f"expected {artifact.digest}"
|
||||
)
|
||||
|
||||
|
||||
def download_archive(url: str, archive_path: Path) -> None:
|
||||
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp_path = archive_path.with_suffix(f"{archive_path.suffix}.tmp")
|
||||
temp_path.unlink(missing_ok=True)
|
||||
try:
|
||||
with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response:
|
||||
with open(temp_path, "wb") as out:
|
||||
shutil.copyfileobj(response, out)
|
||||
temp_path.replace(archive_path)
|
||||
finally:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def extract_archive_member(
|
||||
archive_path: Path,
|
||||
artifact: DotSlashArtifact,
|
||||
dest: Path,
|
||||
artifact_label: str,
|
||||
) -> None:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.unlink(missing_ok=True)
|
||||
|
||||
if artifact.archive_format == "tar.gz":
|
||||
with tarfile.open(archive_path, "r:gz") as archive:
|
||||
try:
|
||||
member = archive.getmember(artifact.archive_member)
|
||||
except KeyError as exc:
|
||||
raise RuntimeError(
|
||||
f"{artifact_label} archive {archive_path} is missing "
|
||||
f"{artifact.archive_member!r}"
|
||||
) from exc
|
||||
|
||||
extracted = archive.extractfile(member)
|
||||
if extracted is None:
|
||||
raise RuntimeError(
|
||||
f"{artifact_label} archive member {artifact.archive_member!r} is not a file"
|
||||
)
|
||||
with extracted, open(dest, "wb") as out:
|
||||
shutil.copyfileobj(extracted, out)
|
||||
return
|
||||
|
||||
if artifact.archive_format == "zip":
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
try:
|
||||
with archive.open(artifact.archive_member) as extracted:
|
||||
with open(dest, "wb") as out:
|
||||
shutil.copyfileobj(extracted, out)
|
||||
except KeyError as exc:
|
||||
raise RuntimeError(
|
||||
f"{artifact_label} archive {archive_path} is missing "
|
||||
f"{artifact.archive_member!r}"
|
||||
) from exc
|
||||
return
|
||||
|
||||
raise RuntimeError(
|
||||
f"Unsupported {artifact_label} archive format {artifact.archive_format!r}; "
|
||||
"expected tar.gz or zip"
|
||||
)
|
||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
from .targets import PackageInputs
|
||||
from .targets import PackageVariant
|
||||
from .targets import TargetSpec
|
||||
from .zsh import ZSH_RESOURCE_PATH
|
||||
|
||||
|
||||
LAYOUT_VERSION = 1
|
||||
@@ -50,6 +51,13 @@ def build_package_dir(
|
||||
)
|
||||
copy_executable(inputs.rg_bin, path_dir / spec.rg_name, is_windows=spec.is_windows)
|
||||
|
||||
if inputs.zsh_bin is not None:
|
||||
copy_executable(
|
||||
inputs.zsh_bin,
|
||||
resources_dir / ZSH_RESOURCE_PATH,
|
||||
is_windows=False,
|
||||
)
|
||||
|
||||
if inputs.bwrap_bin is not None:
|
||||
copy_executable(inputs.bwrap_bin, resources_dir / "bwrap", is_windows=False)
|
||||
|
||||
@@ -83,6 +91,8 @@ def validate_package_dir(
|
||||
package_dir: Path,
|
||||
variant: PackageVariant,
|
||||
spec: TargetSpec,
|
||||
*,
|
||||
include_zsh: bool,
|
||||
) -> None:
|
||||
required_dirs = [
|
||||
Path("bin"),
|
||||
@@ -122,6 +132,11 @@ def validate_package_dir(
|
||||
]
|
||||
executable_files = list(required_files)
|
||||
|
||||
if include_zsh:
|
||||
zsh_path = Path("codex-resources") / ZSH_RESOURCE_PATH
|
||||
required_files.append(zsh_path)
|
||||
executable_files.append(zsh_path)
|
||||
|
||||
if spec.is_linux:
|
||||
required_files.append(Path("codex-resources") / "bwrap")
|
||||
executable_files.append(Path("codex-resources") / "bwrap")
|
||||
@@ -148,7 +163,7 @@ def validate_package_dir(
|
||||
|
||||
def copy_executable(src: Path, dest: Path, *, is_windows: bool) -> None:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dest)
|
||||
shutil.copyfile(src, dest)
|
||||
if not is_windows:
|
||||
mode = dest.stat().st_mode
|
||||
dest.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
@@ -1,33 +1,12 @@
|
||||
"""Fetch ripgrep from the DotSlash manifest used by the npm package."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import stat
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import urlopen
|
||||
|
||||
from .dotslash import fetch_dotslash_executable
|
||||
from .targets import REPO_ROOT
|
||||
from .targets import TargetSpec
|
||||
from .targets import resolve_input_path
|
||||
|
||||
|
||||
RG_MANIFEST = REPO_ROOT / "codex-cli" / "bin" / "rg"
|
||||
DOWNLOAD_TIMEOUT_SECS = 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RgArtifact:
|
||||
size: int
|
||||
digest: str
|
||||
archive_format: str
|
||||
archive_member: str
|
||||
url: str
|
||||
|
||||
|
||||
def resolve_rg_bin(spec: TargetSpec, rg_bin: Path | None) -> Path:
|
||||
@@ -41,155 +20,15 @@ def fetch_rg(
|
||||
spec: TargetSpec,
|
||||
*,
|
||||
manifest_path: Path = RG_MANIFEST,
|
||||
cache_root: Path | None = None,
|
||||
) -> Path:
|
||||
artifact = artifact_for_target(spec, manifest_path)
|
||||
cache_dir = (cache_root or default_cache_root()) / f"{spec.target}-rg"
|
||||
archive_path = cache_dir / archive_filename(artifact.url)
|
||||
|
||||
if not archive_is_valid(archive_path, artifact):
|
||||
download_archive(artifact.url, archive_path)
|
||||
try:
|
||||
verify_archive(archive_path, artifact)
|
||||
except RuntimeError:
|
||||
archive_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
dest = cache_dir / spec.rg_name
|
||||
extract_rg(archive_path, artifact, dest)
|
||||
if not spec.is_windows:
|
||||
mode = dest.stat().st_mode
|
||||
dest.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
return dest
|
||||
|
||||
|
||||
def artifact_for_target(spec: TargetSpec, manifest_path: Path) -> RgArtifact:
|
||||
manifest = load_manifest(manifest_path)
|
||||
try:
|
||||
platform_info = manifest["platforms"][spec.dotslash_platform]
|
||||
except KeyError as exc:
|
||||
raise RuntimeError(
|
||||
f"ripgrep manifest {manifest_path} is missing platform {spec.dotslash_platform!r}"
|
||||
) from exc
|
||||
|
||||
providers = platform_info.get("providers")
|
||||
if not providers:
|
||||
raise RuntimeError(
|
||||
f"ripgrep manifest {manifest_path} has no providers for {spec.dotslash_platform!r}"
|
||||
)
|
||||
|
||||
hash_name = platform_info.get("hash")
|
||||
if hash_name != "sha256":
|
||||
raise RuntimeError(
|
||||
f"Unsupported ripgrep hash {hash_name!r} for "
|
||||
f"{spec.dotslash_platform!r}; expected sha256"
|
||||
)
|
||||
|
||||
return RgArtifact(
|
||||
size=int(platform_info["size"]),
|
||||
digest=str(platform_info["digest"]),
|
||||
archive_format=str(platform_info["format"]),
|
||||
archive_member=str(platform_info["path"]),
|
||||
url=str(providers[0]["url"]),
|
||||
)
|
||||
|
||||
|
||||
def load_manifest(manifest_path: Path) -> dict:
|
||||
text = manifest_path.read_text(encoding="utf-8")
|
||||
if text.startswith("#!"):
|
||||
text = "\n".join(text.splitlines()[1:])
|
||||
return json.loads(text)
|
||||
|
||||
|
||||
def default_cache_root() -> Path:
|
||||
return Path(tempfile.gettempdir()) / "codex-package"
|
||||
|
||||
|
||||
def archive_filename(url: str) -> str:
|
||||
filename = Path(urlparse(url).path).name
|
||||
if not filename:
|
||||
raise RuntimeError(f"Unable to determine archive filename from {url}")
|
||||
return filename
|
||||
|
||||
|
||||
def archive_is_valid(archive_path: Path, artifact: RgArtifact) -> bool:
|
||||
if not archive_path.is_file():
|
||||
return False
|
||||
try:
|
||||
verify_archive(archive_path, artifact)
|
||||
except RuntimeError:
|
||||
archive_path.unlink(missing_ok=True)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def verify_archive(archive_path: Path, artifact: RgArtifact) -> None:
|
||||
actual_size = archive_path.stat().st_size
|
||||
if actual_size != artifact.size:
|
||||
raise RuntimeError(
|
||||
f"ripgrep archive {archive_path} has size {actual_size}, expected {artifact.size}"
|
||||
)
|
||||
|
||||
digest = hashlib.sha256()
|
||||
with open(archive_path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
|
||||
actual_digest = digest.hexdigest()
|
||||
if actual_digest != artifact.digest:
|
||||
raise RuntimeError(
|
||||
f"ripgrep archive {archive_path} has sha256 {actual_digest}, "
|
||||
f"expected {artifact.digest}"
|
||||
)
|
||||
|
||||
|
||||
def download_archive(url: str, archive_path: Path) -> None:
|
||||
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp_path = archive_path.with_suffix(f"{archive_path.suffix}.tmp")
|
||||
temp_path.unlink(missing_ok=True)
|
||||
try:
|
||||
with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response:
|
||||
with open(temp_path, "wb") as out:
|
||||
shutil.copyfileobj(response, out)
|
||||
temp_path.replace(archive_path)
|
||||
finally:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def extract_rg(archive_path: Path, artifact: RgArtifact, dest: Path) -> None:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.unlink(missing_ok=True)
|
||||
|
||||
if artifact.archive_format == "tar.gz":
|
||||
with tarfile.open(archive_path, "r:gz") as archive:
|
||||
try:
|
||||
member = archive.getmember(artifact.archive_member)
|
||||
except KeyError as exc:
|
||||
raise RuntimeError(
|
||||
f"ripgrep archive {archive_path} is missing {artifact.archive_member!r}"
|
||||
) from exc
|
||||
|
||||
extracted = archive.extractfile(member)
|
||||
if extracted is None:
|
||||
raise RuntimeError(
|
||||
f"ripgrep archive member {artifact.archive_member!r} is not a file"
|
||||
)
|
||||
with extracted, open(dest, "wb") as out:
|
||||
shutil.copyfileobj(extracted, out)
|
||||
return
|
||||
|
||||
if artifact.archive_format == "zip":
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
try:
|
||||
with archive.open(artifact.archive_member) as extracted:
|
||||
with open(dest, "wb") as out:
|
||||
shutil.copyfileobj(extracted, out)
|
||||
except KeyError as exc:
|
||||
raise RuntimeError(
|
||||
f"ripgrep archive {archive_path} is missing {artifact.archive_member!r}"
|
||||
) from exc
|
||||
return
|
||||
|
||||
raise RuntimeError(
|
||||
f"Unsupported ripgrep archive format {artifact.archive_format!r}; expected tar.gz or zip"
|
||||
rg_bin = fetch_dotslash_executable(
|
||||
spec,
|
||||
manifest_path=manifest_path,
|
||||
artifact_label="ripgrep",
|
||||
cache_key=f"{spec.target}-rg",
|
||||
dest_name=spec.rg_name,
|
||||
executable=not spec.is_windows,
|
||||
)
|
||||
if rg_bin is None:
|
||||
raise AssertionError("ripgrep is required for all package targets")
|
||||
return rg_bin
|
||||
|
||||
@@ -40,6 +40,7 @@ class PackageVariant:
|
||||
class PackageInputs:
|
||||
entrypoint_bin: Path
|
||||
rg_bin: Path
|
||||
zsh_bin: Path | None
|
||||
bwrap_bin: Path | None
|
||||
codex_command_runner_bin: Path | None
|
||||
codex_windows_sandbox_setup_bin: Path | None
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Fetch the patched zsh fork used by shell_zsh_fork."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .dotslash import fetch_dotslash_executable
|
||||
from .targets import REPO_ROOT
|
||||
from .targets import TargetSpec
|
||||
|
||||
|
||||
ZSH_MANIFEST = REPO_ROOT / "codex-cli" / "bin" / "codex-zsh"
|
||||
ZSH_RESOURCE_PATH = Path("zsh") / "bin" / "zsh"
|
||||
|
||||
|
||||
def resolve_zsh_bin(spec: TargetSpec) -> Path | None:
|
||||
return fetch_dotslash_executable(
|
||||
spec,
|
||||
manifest_path=ZSH_MANIFEST,
|
||||
artifact_label="codex-zsh",
|
||||
cache_key=f"{spec.target}-zsh",
|
||||
dest_name="zsh",
|
||||
executable=True,
|
||||
missing_ok=True,
|
||||
)
|
||||
Reference in New Issue
Block a user