Preserve Bash startup state in remote cache

This commit is contained in:
jif-oai
2026-07-08 12:12:12 +01:00
parent 84be35c7f6
commit 7261d53e8f
6 changed files with 596 additions and 240 deletions
+2 -1
View File
@@ -34,6 +34,8 @@ reqwest = { workspace = true, features = ["json", "rustls-tls", "stream"] }
prost = "0.14.3"
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sha2 = { workspace = true }
tempfile = { workspace = true }
thiserror = { workspace = true }
toml = { workspace = true }
tokio = { workspace = true, features = [
@@ -72,7 +74,6 @@ pretty_assertions = { workspace = true }
rcgen = { workspace = true }
rustls = { workspace = true }
serial_test = { workspace = true }
tempfile = { workspace = true }
test-case = "3.3.1"
tokio = { workspace = true, features = ["test-util"] }
tracing-opentelemetry = { workspace = true }
+9 -223
View File
@@ -1,242 +1,28 @@
#[cfg(not(unix))]
use std::collections::HashMap;
use crate::ExecServerRuntimePaths;
use crate::protocol::ExecParams;
#[cfg(unix)]
mod imp {
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use codex_file_system::FileSystemSandboxContext;
use codex_network_proxy::ManagedNetworkSandboxContext;
use codex_utils_path_uri::PathUri;
use tokio::sync::Mutex;
use tokio::sync::OnceCell;
use tokio::time::timeout;
use uuid::Uuid;
use super::*;
use crate::process_sandbox::prepare_exec_request;
const CAPTURE_TIMEOUT: Duration = Duration::from_secs(10);
const MAX_CAPTURE_BYTES: usize = 1024 * 1024;
#[derive(Default)]
pub(crate) struct BashEnvCache {
entry: Mutex<Option<CacheEntry>>,
}
struct CacheEntry {
key: CacheKey,
environment: Arc<OnceCell<Option<HashMap<String, String>>>>,
}
#[derive(PartialEq, Eq)]
struct CacheKey {
scope: PathUri,
shell: String,
cwd: PathUri,
environment: HashMap<String, String>,
sandbox: Option<FileSystemSandboxContext>,
enforce_managed_network: bool,
managed_network: Option<ManagedNetworkSandboxContext>,
}
impl BashEnvCache {
pub(crate) async fn environment_for_launch(
&self,
params: &ExecParams,
environment: HashMap<String, String>,
runtime_paths: Option<&ExecServerRuntimePaths>,
) -> HashMap<String, String> {
let Some(key) = CacheKey::new(params, &environment) else {
return environment;
};
let cached_environment = {
let mut entry = self.entry.lock().await;
if let Some(cached) = entry.as_ref()
&& cached.key == key
{
Arc::clone(&cached.environment)
} else {
let environment = Arc::new(OnceCell::new());
*entry = Some(CacheEntry {
key,
environment: Arc::clone(&environment),
});
environment
}
};
cached_environment
.get_or_init(|| capture_environment(params, &environment, runtime_paths))
.await
.clone()
.unwrap_or(environment)
}
}
impl CacheKey {
fn new(params: &ExecParams, environment: &HashMap<String, String>) -> Option<Self> {
let scope = params.env_policy.as_ref()?.bash_env_cache_scope.as_ref()?;
let [shell, option, _script] = params.argv.as_slice() else {
return None;
};
if params.tty
|| params.pipe_stdin
|| params.arg0.is_some()
|| option != "-c"
|| Path::new(shell).file_name()?.to_str()? != "bash"
|| !params.cwd.starts_with(scope)
|| environment.get("BASH_ENV").is_none_or(String::is_empty)
{
return None;
}
Some(Self {
scope: scope.clone(),
shell: shell.clone(),
cwd: params.cwd.clone(),
environment: environment.clone(),
sandbox: params.sandbox.clone(),
enforce_managed_network: params.enforce_managed_network,
managed_network: params.managed_network.clone(),
})
}
}
async fn capture_environment(
params: &ExecParams,
environment: &HashMap<String, String>,
runtime_paths: Option<&ExecServerRuntimePaths>,
) -> Option<HashMap<String, String>> {
let nonce = Uuid::new_v4().simple().to_string();
let start_marker = format!("__CODEX_BASH_ENV_START_{nonce}__");
let end_marker = format!("__CODEX_BASH_ENV_END_{nonce}__");
let mut capture = params.clone();
capture.argv = vec![
params.argv.first()?.clone(),
"-c".to_string(),
format!(
"builtin printf '%s' '{start_marker}'; if builtin command env -0; then builtin printf '%s' '{end_marker}'; else builtin exit 125; fi"
),
];
let prepared = prepare_exec_request(&capture, environment.clone(), runtime_paths).ok()?;
let (program, args) = prepared.command.split_first()?;
let spawned = codex_utils_pty::spawn_pipe_process_no_stdin(
program,
args,
prepared.cwd.as_path(),
&prepared.env,
&prepared.arg0,
)
.await
.ok()?;
let captured = timeout(CAPTURE_TIMEOUT, async move {
let _session = spawned.session;
let (stdout, stderr, exit_code) = tokio::join!(
collect_output(spawned.stdout_rx),
collect_output(spawned.stderr_rx),
spawned.exit_rx,
);
(exit_code.ok()? == 0 && stderr?.is_empty()).then_some(parse_capture(
stdout?,
&start_marker,
&end_marker,
)?)
})
.await
.ok()
.flatten()?;
Some(sanitize_environment(captured, environment))
}
async fn collect_output(mut receiver: tokio::sync::mpsc::Receiver<Vec<u8>>) -> Option<Vec<u8>> {
let mut output = Vec::new();
while let Some(chunk) = receiver.recv().await {
if output.len().checked_add(chunk.len())? > MAX_CAPTURE_BYTES {
return None;
}
output.extend_from_slice(&chunk);
}
Some(output)
}
fn parse_capture(
output: Vec<u8>,
start_marker: &str,
end_marker: &str,
) -> Option<HashMap<String, String>> {
let payload = output.strip_prefix(start_marker.as_bytes())?;
let payload = payload.strip_suffix(end_marker.as_bytes())?;
if !payload.is_empty() && !payload.ends_with(&[0]) {
return None;
}
payload
.split(|byte| *byte == 0)
.filter(|entry| !entry.is_empty())
.map(|entry| {
let separator = entry.iter().position(|byte| *byte == b'=')?;
let key = std::str::from_utf8(&entry[..separator]).ok()?.to_string();
let value = std::str::from_utf8(&entry[separator + 1..])
.ok()?
.to_string();
Some((key, value))
})
.collect()
}
fn sanitize_environment(
mut captured: HashMap<String, String>,
original: &HashMap<String, String>,
) -> HashMap<String, String> {
captured.remove("BASH_ENV");
captured.retain(|key, _| !shell_managed(key));
captured.extend(
original
.iter()
.filter(|(key, _)| shell_managed(key))
.map(|(key, value)| (key.clone(), value.clone())),
);
captured
}
fn shell_managed(key: &str) -> bool {
key.starts_with("BASH_FUNC_")
|| matches!(
key,
"BASHOPTS"
| "BASH_ARGV0"
| "BASH_EXECUTION_STRING"
| "OLDPWD"
| "PWD"
| "SHELLOPTS"
| "SHLVL"
| "_"
)
}
}
#[path = "bash_env_cache_unix.rs"]
mod imp;
#[cfg(unix)]
pub(crate) use imp::BashEnvCache;
#[cfg(not(unix))]
#[derive(Default)]
pub(crate) struct BashEnvCache;
pub(crate) struct BashEnvCache(());
#[cfg(not(unix))]
impl BashEnvCache {
pub(crate) async fn environment_for_launch(
pub(crate) async fn prepare_launch(
&self,
_params: &ExecParams,
environment: HashMap<String, String>,
params: &ExecParams,
environment: &HashMap<String, String>,
_runtime_paths: Option<&ExecServerRuntimePaths>,
) -> HashMap<String, String> {
environment
) -> (Vec<String>, HashMap<String, String>) {
(params.argv.clone(), environment.clone())
}
}
@@ -0,0 +1,493 @@
use std::collections::HashMap;
use std::fs::Metadata;
use std::io::Write;
use std::os::unix::fs::MetadataExt;
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;
use sha2::Digest;
use sha2::Sha256;
use tempfile::TempDir;
use tokio::sync::OnceCell;
use tokio::time::timeout;
use uuid::Uuid;
use super::ExecParams;
use super::ExecServerRuntimePaths;
use crate::process_sandbox::PreparedExecRequest;
use crate::process_sandbox::prepare_exec_request;
const CAPTURE_TIMEOUT: Duration = Duration::from_secs(10);
const MAX_CACHE_ENTRIES: usize = 16;
const MAX_ENV_BYTES: usize = 256 * 1024;
const MAX_SCOPE_BYTES: usize = 384 * 1024;
const MAX_PRESERVE_KEYS: usize = 128;
const MAX_SNAPSHOT_BYTES: usize = 1024 * 1024;
type SnapshotCell = OnceCell<Option<Arc<BashEnvSnapshot>>>;
pub(crate) struct BashEnvCache {
entries: Mutex<HashMap<[u8; 32], Arc<SnapshotCell>>>,
root: Option<TempDir>,
}
struct BashEnvSnapshotRequest {
key: [u8; 32],
environment: HashMap<String, String>,
preserve_env_keys: Vec<String>,
bash_env: String,
}
struct BashEnvSnapshot {
path: PathBuf,
bash_env: String,
checksum: String,
prefix: String,
}
impl Default for BashEnvCache {
fn default() -> Self {
Self {
entries: Mutex::new(HashMap::new()),
root: tempfile::Builder::new()
.prefix("codex-exec-server-bash-env-")
.permissions(std::fs::Permissions::from_mode(0o700))
.tempdir()
.ok(),
}
}
}
impl BashEnvCache {
pub(crate) async fn prepare_launch(
&self,
params: &ExecParams,
environment: &HashMap<String, String>,
runtime_paths: Option<&ExecServerRuntimePaths>,
) -> (Vec<String>, HashMap<String, String>) {
let Some(request) = BashEnvSnapshotRequest::new(params, environment) else {
return fallback(params, environment);
};
let preserve_env_keys = request.preserve_env_keys.clone();
let Some(snapshot) = self.snapshot(params, request, runtime_paths).await else {
return fallback(params, environment);
};
let Some(argv) = wrap_command(&params.argv, &snapshot, &preserve_env_keys) else {
return fallback(params, environment);
};
let mut environment = environment.clone();
environment.remove("BASH_ENV");
(argv, environment)
}
async fn snapshot(
&self,
params: &ExecParams,
request: BashEnvSnapshotRequest,
runtime_paths: Option<&ExecServerRuntimePaths>,
) -> Option<Arc<BashEnvSnapshot>> {
let key = request.key;
let cell = {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if entries.len() >= MAX_CACHE_ENTRIES && !entries.contains_key(&key) {
return None;
}
Arc::clone(
entries
.entry(key)
.or_insert_with(|| Arc::new(OnceCell::new())),
)
};
cell.get_or_init(|| async {
self.capture(params, request, runtime_paths)
.await
.map(Arc::new)
})
.await
.clone()
}
async fn capture(
&self,
params: &ExecParams,
request: BashEnvSnapshotRequest,
runtime_paths: Option<&ExecServerRuntimePaths>,
) -> Option<BashEnvSnapshot> {
let nonce = Uuid::new_v4().simple().to_string();
let prefix = format!("__CODEX_SNAPSHOT_{}", nonce.to_ascii_uppercase());
let marker = format!("# Codex remote Bash environment snapshot {nonce}");
let stderr_marker = format!("# Codex remote Bash stderr complete {nonce}");
let wrapper_end = format!("# Codex remote Bash capture complete {nonce}");
let root = self.root.as_ref()?;
let mut wrapper = tempfile::NamedTempFile::new_in(root.path()).ok()?;
let script = capture_script(
&prefix,
&marker,
&stderr_marker,
&wrapper_end,
&request.bash_env,
&request.preserve_env_keys,
);
wrapper.write_all(script.as_bytes()).ok()?;
wrapper.as_file().sync_all().ok()?;
let capture = params.clone();
let mut environment = request.environment;
environment.insert("BASH_ENV".into(), wrapper.path().to_string_lossy().into());
let prepared = prepare_exec_request(&capture, environment, runtime_paths).ok()?;
let (output, stderr) = capture_output(prepared).await?;
let marker_offset = output
.windows(marker.len())
.position(|part| part == marker.as_bytes())?;
let stdout = &output[..marker_offset];
let snapshot =
output[marker_offset..].strip_suffix(format!("{wrapper_end}\n").as_bytes())?;
let stderr = &stderr[..stderr
.windows(stderr_marker.len())
.position(|part| part == stderr_marker.as_bytes())?];
let mut contents = snapshot.to_vec();
contents.extend(replay_output(stdout, /*fd*/ 3));
contents.extend(replay_output(stderr, /*fd*/ 4));
contents.extend_from_slice(b"builtin true\n");
if !contents.ends_with(b"\n") {
contents.push(b'\n');
}
if contents.len() > MAX_SNAPSHOT_BYTES {
return None;
}
let mut temp = tempfile::NamedTempFile::new_in(root.path()).ok()?;
temp.write_all(&contents).ok()?;
temp.as_file().sync_all().ok()?;
let path = root.path().join(format!("{nonce}.sh"));
temp.persist(&path).ok()?;
Some(BashEnvSnapshot {
path,
bash_env: request.bash_env,
checksum: format!("{:x}", Sha256::digest(&contents)),
prefix,
})
}
}
impl BashEnvSnapshotRequest {
fn new(params: &ExecParams, environment: &HashMap<String, String>) -> Option<Self> {
let workspace_root = params.env_policy.as_ref()?.bash_env_cache_scope.as_ref()?;
if params.tty
|| params.pipe_stdin
|| params.arg0.is_some()
|| params.argv.len() < 3
|| params.argv.get(1).map(String::as_str) != Some("-c")
|| !params.cwd.starts_with(workspace_root)
|| environment.iter().any(|(key, value)| {
(key == "builtin" || key.starts_with("BASH_FUNC_builtin"))
&& value.trim_start().starts_with("()")
})
{
return None;
}
let shell = Path::new(params.argv.first()?);
let bash_env = Path::new(environment.get("BASH_ENV")?);
if !shell.is_absolute()
|| shell.file_name()?.to_str()? != "bash"
|| !shell.is_file()
|| !bash_env.is_absolute()
|| !bash_env.is_file()
{
return None;
}
let bash_env_source = std::fs::read(bash_env).ok()?;
if bash_env_source.len() > MAX_SNAPSHOT_BYTES || unsupported_startup(&bash_env_source) {
return None;
}
let mut preserve_env_keys = params
.env_policy
.as_ref()?
.r#set
.keys()
.cloned()
.collect::<Vec<_>>();
if preserve_env_keys.len() > MAX_PRESERVE_KEYS
|| preserve_env_keys
.iter()
.any(|key| key.len() > 128 || !valid_variable(key))
{
return None;
}
preserve_env_keys.retain(|key| key != "BASH_ENV");
preserve_env_keys.sort_unstable();
preserve_env_keys.dedup();
let mut environment_scope = environment
.iter()
.map(|(key, value)| (key.clone(), value.clone()))
.collect::<Vec<_>>();
environment_scope.sort_unstable();
let environment_bytes = environment_scope
.iter()
.try_fold(0usize, |size, (key, value)| {
size.checked_add(key.len())?.checked_add(value.len())
})?;
if environment_bytes > MAX_ENV_BYTES {
return None;
}
let mut scope = serde_json::to_value((
unsafe { libc::geteuid() },
workspace_root,
&params.cwd,
shell.to_string_lossy(),
file_identity(&shell.metadata().ok()?),
bash_env.to_string_lossy(),
file_identity(&bash_env.metadata().ok()?),
environment_scope,
&params.env_policy,
&params.sandbox,
params.enforce_managed_network,
&params.managed_network,
&preserve_env_keys,
))
.ok()?;
sort_json(&mut scope);
let scope = serde_json::to_vec(&scope).ok()?;
if scope.len() > MAX_SCOPE_BYTES {
return None;
}
let key = Sha256::digest(scope).into();
Some(Self {
key,
environment: environment.clone(),
preserve_env_keys,
bash_env: bash_env.to_string_lossy().into_owned(),
})
}
}
impl Drop for BashEnvSnapshot {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
async fn capture_output(prepared: PreparedExecRequest) -> Option<(Vec<u8>, Vec<u8>)> {
timeout(CAPTURE_TIMEOUT, async move {
let (program, args) = prepared.command.split_first()?;
let spawned = codex_utils_pty::spawn_pipe_process_no_stdin(
program,
args,
prepared.cwd.as_path(),
&prepared.env,
&prepared.arg0,
)
.await
.ok()?;
let codex_utils_pty::SpawnedProcess {
session: _session,
stdout_rx,
stderr_rx,
exit_rx,
} = spawned;
let (stdout, stderr, exit) = tokio::join!(
collect_output(stdout_rx),
collect_output(stderr_rx),
exit_rx
);
(exit.ok()? == 0).then_some((stdout?, stderr?))
})
.await
.ok()
.flatten()
}
async fn collect_output(mut receiver: tokio::sync::mpsc::Receiver<Vec<u8>>) -> Option<Vec<u8>> {
let mut output = Vec::new();
while let Some(chunk) = receiver.recv().await {
if output.len().checked_add(chunk.len())? > MAX_SNAPSHOT_BYTES {
return None;
}
output.extend_from_slice(&chunk);
}
Some(output)
}
fn capture_script(
prefix: &str,
marker: &str,
stderr_marker: &str,
wrapper_end: &str,
bash_env: &str,
preserve_env_keys: &[String],
) -> String {
let mut excluded = ":BASH:BASHOPTS:BASHPID:BASH_ALIASES:BASH_ARGC:BASH_ARGV:BASH_ARGV0:\
BASH_CMDS:BASH_COMMAND:BASH_ENV:BASH_EXECUTION_STRING:BASH_LINENO:BASH_REMATCH:BASH_SOURCE:BASH_SUBSHELL:\
BASH_VERSINFO:BASH_VERSION:COPROC:DIRSTACK:EPOCHREALTIME:EPOCHSECONDS:EUID:FUNCNAME:GROUPS:\
HISTCMD:HOSTTYPE:LINENO:MACHTYPE:OLDPWD:OSTYPE:PIPESTATUS:PPID:PWD:RANDOM:SECONDS:SHELLOPTS:SHLVL:SRANDOM:UID:_:"
.to_string();
for key in preserve_env_keys {
excluded.push_str(key);
excluded.push(':');
}
r#"if ! builtin type -P sha256sum >/dev/null 2>&1 && ! builtin type -P shasum >/dev/null 2>&1; then builtin exit 125; fi
builtin unset __CODEX_PREFIX___EXCLUDED __CODEX_PREFIX___P_NAMES __CODEX_PREFIX___NAME __CODEX_PREFIX___DECL __CODEX_PREFIX___ALIASES __CODEX_PREFIX___OPTS __CODEX_PREFIX___SHOPTS __CODEX_PREFIX___FUNC_ATTRS __CODEX_PREFIX___CWD __CODEX_PREFIX___UMASK
builtin export BASH_ENV='__CODEX_BASH_ENV__'
__CODEX_PREFIX___EXCLUDED='__CODEX_EXCLUDED__'; __CODEX_PREFIX___P_NAMES=; __CODEX_PREFIX___NAME=; __CODEX_PREFIX___DECL=
__CODEX_PREFIX___ALIASES=; __CODEX_PREFIX___OPTS=; __CODEX_PREFIX___SHOPTS=; __CODEX_PREFIX___FUNC_ATTRS=; __CODEX_PREFIX___CWD=; __CODEX_PREFIX___UMASK=
while IFS= builtin read -r __CODEX_PREFIX___NAME; do
[[ "$__CODEX_PREFIX___NAME" != __CODEX_PREFIX___* && "$__CODEX_PREFIX___EXCLUDED" != *":$__CODEX_PREFIX___NAME:"* && "$__CODEX_PREFIX___NAME" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
__CODEX_PREFIX___P_NAMES="${__CODEX_PREFIX___P_NAMES}${__CODEX_PREFIX___NAME}"$'\n'
done < <(builtin compgen -A variable)
builtin source "$BASH_ENV"
if [[ $- == *x* || $- == *v* ]] || [[ -n "$(builtin trap -p)" ]]; then builtin exit 125; fi
builtin printf '%s\n' '__CODEX_MARKER__'; builtin printf '%s\n' '__CODEX_STDERR_MARKER__' >&2
__CODEX_PREFIX___ALIASES=$(builtin alias -p 2>/dev/null || builtin true)
__CODEX_PREFIX___OPTS=$(builtin set +o); if [[ $- == *e* ]]; then __CODEX_PREFIX___OPTS=${__CODEX_PREFIX___OPTS/set +o errexit/set -o errexit}; fi
__CODEX_PREFIX___SHOPTS=$(builtin shopt -p 2>/dev/null || builtin true)
builtin shopt -u extdebug 2>/dev/null || builtin true; __CODEX_PREFIX___FUNC_ATTRS=$(builtin declare -F)
__CODEX_PREFIX___CWD=$(builtin pwd -L); __CODEX_PREFIX___UMASK=$(builtin umask)
builtin set +e; builtin set +u; builtin unalias -a 2>/dev/null || builtin true
__CODEX_PREFIX___ALIASES=${__CODEX_PREFIX___ALIASES//$'\n'/$'\nbuiltin '}; __CODEX_PREFIX___OPTS=${__CODEX_PREFIX___OPTS//$'\n'/$'\nbuiltin '}; __CODEX_PREFIX___SHOPTS=${__CODEX_PREFIX___SHOPTS//$'\n'/$'\nbuiltin '}
[[ -z "$__CODEX_PREFIX___ALIASES" ]] || __CODEX_PREFIX___ALIASES="builtin $__CODEX_PREFIX___ALIASES"
[[ -z "$__CODEX_PREFIX___OPTS" ]] || __CODEX_PREFIX___OPTS="builtin $__CODEX_PREFIX___OPTS"
[[ -z "$__CODEX_PREFIX___SHOPTS" ]] || __CODEX_PREFIX___SHOPTS="builtin $__CODEX_PREFIX___SHOPTS"
builtin printf 'builtin cd -L -- %q || builtin return 1\n' "$__CODEX_PREFIX___CWD"
builtin declare -f
while IFS= builtin read -r __CODEX_PREFIX___DECL; do
[[ "$__CODEX_PREFIX___DECL" == "declare -f "* ]] && continue
builtin printf 'builtin %s\n' "$__CODEX_PREFIX___DECL"
done <<< "$__CODEX_PREFIX___FUNC_ATTRS"
builtin printf '\n'
while IFS= builtin read -r __CODEX_PREFIX___NAME; do
[[ "$__CODEX_PREFIX___NAME" != __CODEX_PREFIX___* && "$__CODEX_PREFIX___EXCLUDED" != *":$__CODEX_PREFIX___NAME:"* && "$__CODEX_PREFIX___NAME" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
__CODEX_PREFIX___DECL=$(builtin declare -p "$__CODEX_PREFIX___NAME" 2>/dev/null) || continue; builtin printf 'builtin unset -v %s\n' "$__CODEX_PREFIX___NAME"; __CODEX_PREFIX___NAME=${__CODEX_PREFIX___DECL#declare }; __CODEX_PREFIX___NAME=${__CODEX_PREFIX___NAME%% *}
if [[ "$__CODEX_PREFIX___NAME" == -*[aA]* ]]; then __CODEX_PREFIX___DECL=${__CODEX_PREFIX___DECL#declare }; __CODEX_PREFIX___DECL=${__CODEX_PREFIX___DECL#* }; if [[ "$__CODEX_PREFIX___NAME" == *A* ]]; then __CODEX_PREFIX___CWD=A; else __CODEX_PREFIX___CWD=a; fi
if [[ "${__CODEX_PREFIX___DECL#*=}" == \(* ]]; then builtin printf 'builtin declare -%s %q\n' "$__CODEX_PREFIX___CWD" "$__CODEX_PREFIX___DECL"; else builtin printf 'builtin declare -%s %s\n' "$__CODEX_PREFIX___CWD" "$__CODEX_PREFIX___DECL"; fi; builtin printf 'builtin declare %s %s\n' "$__CODEX_PREFIX___NAME" "${__CODEX_PREFIX___DECL%%=*}"; else builtin printf 'builtin %s\n' "$__CODEX_PREFIX___DECL"; fi
done < <(builtin compgen -A variable)
while IFS= builtin read -r __CODEX_PREFIX___NAME; do
[[ -z "$__CODEX_PREFIX___NAME" ]] && continue
builtin declare -p "$__CODEX_PREFIX___NAME" >/dev/null 2>&1 || builtin printf 'builtin unset -v %s\n' "$__CODEX_PREFIX___NAME"
done < <(builtin printf '%s' "$__CODEX_PREFIX___P_NAMES")
builtin printf '\n'
if __CODEX_PREFIX___DECL=$(builtin declare -p BASH_ENV 2>/dev/null); then builtin printf 'builtin %s\n' "$__CODEX_PREFIX___DECL"; else builtin printf 'builtin unset -v BASH_ENV\n'; fi
builtin printf '\nbuiltin umask %s\n' "$__CODEX_PREFIX___UMASK"
builtin printf '%s\n' "$__CODEX_PREFIX___OPTS"
builtin printf '\n%s\n\n' "$__CODEX_PREFIX___SHOPTS"
[[ -z "$__CODEX_PREFIX___ALIASES" ]] || builtin printf '%s\n' "$__CODEX_PREFIX___ALIASES"
builtin printf '%s\n' '__CODEX_WRAPPER_END__'; builtin exit 0
"#
.replace("__CODEX_PREFIX__", prefix)
.replace("__CODEX_MARKER__", marker)
.replace("__CODEX_STDERR_MARKER__", stderr_marker)
.replace("__CODEX_WRAPPER_END__", wrapper_end)
.replace("__CODEX_BASH_ENV__", &quote(bash_env))
.replace("__CODEX_EXCLUDED__", &excluded)
}
fn replay_output(output: &[u8], fd: u8) -> Vec<u8> {
if output.is_empty() {
return Vec::new();
}
let escaped = output
.iter()
.map(|byte| format!("\\{byte:03o}"))
.collect::<String>();
format!("builtin printf '%b' '{escaped}' >&{fd}\n").into_bytes()
}
fn wrap_command(
argv: &[String],
snapshot: &BashEnvSnapshot,
preserve_env_keys: &[String],
) -> Option<Vec<String>> {
let mut script = String::new();
for (index, key) in preserve_env_keys.iter().enumerate() {
script.push_str(&format!(
"{prefix}_{index}_SET=\"${{{key}+x}}\"\n{prefix}_{index}_VALUE=\"${{{key}-}}\"\n",
prefix = snapshot.prefix,
));
}
script.push_str("builtin unset BASH_ENV\n");
script.push_str(&format!(
"{prefix}_HASH=$(if builtin command -v sha256sum >/dev/null 2>&1; then builtin command sha256sum '{path}'; elif builtin command -v shasum >/dev/null 2>&1; then builtin command shasum -a 256 '{path}'; fi)\n\
{prefix}_HASH=${{{prefix}_HASH%% *}}\n\
if [[ \"${{{prefix}_HASH}}\" = '{checksum}' ]] && builtin source '{path}' 3>&1 4>&2 >/dev/null 2>&1; then builtin true; else builtin export BASH_ENV='{bash_env}'; builtin source \"$BASH_ENV\" || builtin true; fi\n\
builtin unset {prefix}_HASH\n",
prefix = snapshot.prefix,
path = quote(&snapshot.path.to_string_lossy()),
checksum = snapshot.checksum,
bash_env = quote(&snapshot.bash_env),
));
for (index, key) in preserve_env_keys.iter().enumerate() {
script.push_str(&format!(
"if [[ \"${{{prefix}_{index}_SET}}\" = x ]]; then builtin export {key}=\"${{{prefix}_{index}_VALUE}}\"; else builtin unset {key}; fi\nbuiltin unset {prefix}_{index}_SET {prefix}_{index}_VALUE\n",
prefix = snapshot.prefix,
));
}
script.push_str(&format!(
"BASH_EXECUTION_STRING='{}'\n",
quote(argv.get(2)?)
));
script.push_str(argv.get(2)?);
let mut wrapped = vec![argv.first()?.clone(), "-c".to_string(), script];
wrapped.extend(argv.get(3..)?.iter().cloned());
Some(wrapped)
}
fn file_identity(metadata: &Metadata) -> (u64, u64, u64, i64, i64, i64, i64, u32) {
(
metadata.dev(),
metadata.ino(),
metadata.len(),
metadata.mtime(),
metadata.mtime_nsec(),
metadata.ctime(),
metadata.ctime_nsec(),
metadata.mode(),
)
}
fn sort_json(value: &mut serde_json::Value) {
match value {
serde_json::Value::Array(values) => values.iter_mut().for_each(sort_json),
serde_json::Value::Object(map) => {
map.values_mut().for_each(sort_json);
map.sort_keys();
}
_ => {}
}
}
fn valid_variable(name: &str) -> bool {
let mut chars = name.chars();
matches!(chars.next(), Some('_') | Some('a'..='z') | Some('A'..='Z'))
&& chars.all(|ch| ch == '_' || ch.is_ascii_alphanumeric())
}
fn unsupported_startup(source: &[u8]) -> bool {
let positional = |byte| matches!(byte, b'0'..=b'9' | b'@' | b'*' | b'#');
source
.windows(b"BASH_EXECUTION_STRING".len())
.any(|part| part == b"BASH_EXECUTION_STRING")
|| source
.windows(2)
.any(|part| part[0] == b'$' && positional(part[1]))
|| source
.windows(3)
.any(|part| part.starts_with(b"${") && positional(part[2]))
|| source
.split(|byte| !byte.is_ascii_alphanumeric() && *byte != b'_')
.any(|word| word == b"trap" || word == b"builtin")
}
fn quote(value: &str) -> String {
value.replace('\'', "'\"'\"'")
}
fn fallback(
params: &ExecParams,
environment: &HashMap<String, String>,
) -> (Vec<String>, HashMap<String, String>) {
(params.argv.clone(), environment.clone())
}
+5 -3
View File
@@ -234,14 +234,16 @@ impl LocalProcess {
async fn start_process(
&self,
params: ExecParams,
mut params: ExecParams,
) -> Result<(ExecResponse, watch::Sender<u64>, ExecProcessEventLog), JSONRPCErrorError> {
let process_id = params.process_id.clone();
let environment = self
let environment = child_env(&params);
let (argv, environment) = self
.inner
.bash_env_cache
.environment_for_launch(&params, child_env(&params), self.runtime_paths.as_ref())
.prepare_launch(&params, &environment, self.runtime_paths.as_ref())
.await;
params.argv = argv;
let prepared = prepare_exec_request(&params, environment, self.runtime_paths.as_ref())?;
let (program, args) = prepared
.command
+86 -13
View File
@@ -101,27 +101,98 @@ async fn assert_exec_process_starts_and_exits(use_remote: bool) -> Result<()> {
}
#[cfg(unix)]
async fn assert_bash_env_exports_are_cached(use_remote: bool) -> Result<()> {
async fn assert_bash_env_startup_state_is_cached(use_remote: bool) -> Result<()> {
let context = create_process_context(use_remote).await?;
let temp = tempfile::tempdir()?;
let bash_env = temp.path().join("bash_env.sh");
let counter = temp.path().join("counter");
let nested = temp.path().join("nested");
std::fs::create_dir(&nested)?;
std::fs::write(
&bash_env,
"printf x >> \"$COUNTER_FILE\"\nexport CACHED_VALUE=ready\nexport PATH=\"$PATH:/from-bash-env\"\n",
format!(
"printf '%s\\n' \"$PROFILE\" >> \"$COUNTER_FILE\"\n\
LOCAL_VALUE=ready\n\
snapshot_func() {{ printf function-ok; }}\n\
export PATH=\"$PATH:/from-bash-env\"\n\
umask 077\n\
cd -L -- '{}'\n\
set +B\n\
shopt -s expand_aliases\n\
alias snapshot_alias='printf alias-ok'\n",
nested.display()
),
)?;
let cwd = PathUri::from_host_native_path(temp.path())?;
for index in 0..2 {
for (index, profile) in ["A", "B", "A"].into_iter().enumerate() {
let script = format!(
"printf '%s|%s|%s|%s|%s|<%s>|%s|' \"$PROFILE\" \"$LOCAL_VALUE\" \
\"$(snapshot_func)\" \"$PWD\" \"$(umask)\" {{a,b}} \"$PATH\"; snapshot_alias; \
printf '|%s|{index}' \"$BASH_EXECUTION_STRING\""
);
let process = context
.backend
.start(ExecParams {
process_id: format!("bash-env-cache-{index}").into(),
argv: vec![
"/bin/bash".to_string(),
"-c".to_string(),
"printf '%s|%s' \"$CACHED_VALUE\" \"$PATH\"".to_string(),
],
argv: vec!["/bin/bash".to_string(), "-c".to_string(), script.clone()],
cwd: cwd.clone(),
env_policy: Some(ExecEnvPolicy {
inherit: ShellEnvironmentPolicyInherit::None,
ignore_default_excludes: true,
exclude: Vec::new(),
r#set: HashMap::new(),
include_only: Vec::new(),
bash_env_cache_scope: Some(cwd.clone()),
}),
env: HashMap::from([
(
"BASH_ENV".to_string(),
bash_env.to_string_lossy().into_owned(),
),
(
"COUNTER_FILE".to_string(),
counter.to_string_lossy().into_owned(),
),
("PROFILE".to_string(), profile.to_string()),
("PATH".to_string(), "/usr/bin:/bin".to_string()),
]),
tty: false,
pipe_stdin: false,
arg0: None,
sandbox: None,
enforce_managed_network: false,
managed_network: None,
})
.await?;
let wake = process.process.subscribe_wake();
let (output, exit_code, closed) =
collect_process_output_from_reads(process.process, wake).await?;
assert_eq!(
output,
format!(
"{profile}|ready|function-ok|{}|0077|<{{a,b}}>|\
/usr/bin:/bin:/from-bash-env|alias-ok|{script}|{index}",
nested.display()
)
);
assert_eq!(exit_code, Some(0));
assert!(closed);
}
assert_eq!(std::fs::read_to_string(&counter)?, "A\nB\n");
std::fs::write(
&bash_env,
"printf 'fallback\\n' >> \"$COUNTER_FILE\"\nSEEN=$BASH_EXECUTION_STRING\n",
)?;
for index in 0..2 {
let script = format!("printf '%s|{index}' \"$SEEN\"");
let process = context
.backend
.start(ExecParams {
process_id: format!("bash-env-fallback-{index}").into(),
argv: vec!["/bin/bash".to_string(), "-c".to_string(), script.clone()],
cwd: cwd.clone(),
env_policy: Some(ExecEnvPolicy {
inherit: ShellEnvironmentPolicyInherit::None,
@@ -153,12 +224,14 @@ async fn assert_bash_env_exports_are_cached(use_remote: bool) -> Result<()> {
let wake = process.process.subscribe_wake();
let (output, exit_code, closed) =
collect_process_output_from_reads(process.process, wake).await?;
assert_eq!(output, "ready|/usr/bin:/bin:/from-bash-env");
assert_eq!(output, format!("{script}|{index}"));
assert_eq!(exit_code, Some(0));
assert!(closed);
}
assert_eq!(std::fs::read_to_string(counter)?, "x");
assert_eq!(
std::fs::read_to_string(counter)?,
"A\nB\nfallback\nfallback\n"
);
Ok(())
}
@@ -1001,6 +1074,6 @@ async fn exec_process_preserves_queued_events_before_subscribe(use_remote: bool)
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
// Serialize tests that launch a real exec-server process through the full CLI.
#[serial_test::serial(remote_exec_server)]
async fn bash_env_exports_are_cached(use_remote: bool) -> Result<()> {
assert_bash_env_exports_are_cached(use_remote).await
async fn bash_env_startup_state_is_cached(use_remote: bool) -> Result<()> {
assert_bash_env_startup_state_is_cached(use_remote).await
}