openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-26 18:38:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Preserve complete shell snapshot exports through filtering and replay (#43907)

Browse Source 
## Why

Line-based export parsing can truncate multiline values or mistake their contents for declarations. Snapshot replay also needs to restore Bash options before parsing functions that use extended glob syntax.

## What changed

- Capture shell state, aliases, and complete export records with NUL-delimited boundaries, separating capture, credential preparation, and rendering.
- Apply credential and environment policy to whole exports, preserving multiline values and making credential aliases follow allowed overrides.
- Restore Bash `shopt` options before functions, including for Bash running as `sh`.
- Support non-evaluating POSIX `ENV` path expansion and preserve native environment metadata and unset exports.
- Move core and exec-server snapshot consumers to the shared capture parser. Allow capture overhead while enforcing the exec-server's 512 KiB state-and-environment limit before filtering.

## Testing

Add regression coverage for complete capture records, multiline export filtering and replay, credential alias overrides, `ENV` expansion, and `PATH` export state. Extend exec-server tests to cover Bash-backed `sh`, `extglob`, `nocasematch`, `set -u`, and large environments.

GitOrigin-RevId: 7589ab4d137529a395dcee10b5162718de24a621
This commit is contained in:
Winston Howes
2026-09-08 20:41:01 +00:00
committed by copyberry
parent f046cf35df
commit 1530f828cb
11 changed files with 1472 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/exec-server/src/shell_snapshot.rs
+39 -32
View File
@@ -10,7 +10,10 @@ use codex_network_proxy::strip_managed_proxy_env;
use codex_protocol::config_types::ShellEnvironmentPolicyInherit;
use codex_protocol::shell_environment;
use codex_shell_command::shell_detect::ShellType;
use codex_shell_command::shell_snapshot::snapshot_state_and_environment_script;
use codex_shell_command::shell_snapshot::CapturedSnapshot;
use codex_shell_command::shell_snapshot::SnapshotCaptureOptions;
use codex_shell_command::shell_snapshot::SnapshotStartup;
use codex_shell_command::shell_snapshot::snapshot_capture_script;
use codex_utils_path_uri::PathUri;
use tokio::io::AsyncReadExt;
use tokio::process::Command;
@@ -30,6 +33,8 @@ use crate::telemetry::ExecServerTelemetry;
const MAX_CACHED_SNAPSHOTS: usize = 16;
const MAX_SNAPSHOT_BYTES: usize = 512 * 1024;
// Capture also includes quoted export records and an optional pre-startup environment.
const MAX_SNAPSHOT_CAPTURE_BYTES: usize = 8 * MAX_SNAPSHOT_BYTES;
const MAX_SNAPSHOT_ENV_VALUE_BYTES: usize = 60 * 1024;
const MAX_SNAPSHOT_SCOPE_BYTES: usize = 256;
const SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(10);
@@ -255,7 +260,15 @@ async fn capture_snapshot(
prepared: &PreparedExecRequest,
shell_type: ShellType,
) -> CaptureResult {
let script = snapshot_state_and_environment_script(shell_type).ok_or_else(|| {
let script = snapshot_capture_script(
shell_type,
SnapshotCaptureOptions {
startup: SnapshotStartup::Interactive,
declarations: false,
environment: true,
},
)
.ok_or_else(|| {
(
"unsupported_shell",
invalid_params("unsupported shell snapshot script".to_string()),
@@ -299,7 +312,7 @@ async fn capture_snapshot(
let capture = async {
let mut output = Vec::new();
stdout
.take((MAX_SNAPSHOT_BYTES + 1) as u64)
.take((MAX_SNAPSHOT_CAPTURE_BYTES + 1) as u64)
.read_to_end(&mut output)
.await
.map_err(|err| {
@@ -308,10 +321,12 @@ async fn capture_snapshot(
internal_error(format!("cannot read shell snapshot: {err}")),
)
})?;
if output.len() > MAX_SNAPSHOT_BYTES {
if output.len() > MAX_SNAPSHOT_CAPTURE_BYTES {
return Err((
"too_large",
internal_error(format!("shell snapshot exceeds {MAX_SNAPSHOT_BYTES} bytes")),
internal_error(format!(
"shell snapshot capture exceeds {MAX_SNAPSHOT_CAPTURE_BYTES} bytes"
)),
));
}
let status = child.wait().await.map_err(|err| {
@@ -337,35 +352,30 @@ async fn capture_snapshot(
)
})??;
parse_snapshot(&output, params.env_policy.as_ref())
parse_snapshot(shell_type, &output, params.env_policy.as_ref())
}
fn parse_snapshot(output: &[u8], env_policy: Option<&ExecEnvPolicy>) -> CaptureResult {
let separator = output.iter().position(|byte| *byte == 0).ok_or_else(|| {
fn parse_snapshot(
shell_type: ShellType,
output: &[u8],
env_policy: Option<&ExecEnvPolicy>,
) -> CaptureResult {
let captured = CapturedSnapshot::parse(shell_type, output).ok_or_else(|| {
(
"missing_environment",
internal_error("shell snapshot is missing its environment".to_string()),
)
})?;
let state = &output[..separator];
let marker = b"# Snapshot file";
let start = state
.windows(marker.len())
.position(|window| window == marker)
.ok_or_else(|| {
(
"missing_state",
internal_error("shell snapshot is missing its state marker".to_string()),
)
})?;
let state = std::str::from_utf8(&state[start..]).map_err(|err| {
(
"invalid_utf8",
internal_error(format!("shell snapshot state is not UTF-8: {err}")),
"invalid_capture",
internal_error("invalid shell snapshot capture".to_string()),
)
})?;
let state = captured.render_state();
if state.len().saturating_add(captured.environment.len()) > MAX_SNAPSHOT_BYTES {
return Err((
"too_large",
internal_error(format!("shell snapshot exceeds {MAX_SNAPSHOT_BYTES} bytes")),
));
}
let mut environment = output[separator + 1..]
let mut environment = captured
.environment
.split(|byte| *byte == 0)
.filter(|entry| !entry.is_empty())
.filter_map(|entry| {
@@ -388,10 +398,7 @@ fn parse_snapshot(output: &[u8], env_policy: Option<&ExecEnvPolicy>) -> CaptureR
environment.remove("OLDPWD");
environment.retain(|name, _| !shell_environment::is_non_inheritable_env_var(name));
Ok(ShellSnapshot {
state: state.to_string(),
environment,
})
Ok(ShellSnapshot { state, environment })
}
#[cfg(test)]
codex-rs/exec-server/src/shell_snapshot_tests.rs
+24 -6
View File
@@ -12,6 +12,7 @@ use test_case::test_case;
use super::CapturePurpose;
use super::MAX_SNAPSHOT_ATTEMPTS;
use super::MAX_SNAPSHOT_BYTES;
use super::SNAPSHOT_RETRY_BACKOFF;
use super::ShellSnapshotCache;
use super::parse_snapshot;
@@ -228,6 +229,20 @@ async fn snapshot_failure_retries_are_bounded_and_single_flight(
Ok(())
}
#[test]
fn snapshot_size_limit_counts_state_and_environment_before_filtering() {
let half = "x".repeat(MAX_SNAPSHOT_BYTES / 2);
let oversized = format!("# Snapshot file\n# {half}\n\0\0\0FILTERED={half}\0");
let policy = ExecEnvPolicy {
inherit: ShellEnvironmentPolicyInherit::All,
ignore_default_excludes: false,
exclude: vec!["FILTERED".to_string()],
r#set: HashMap::new(),
include_only: Vec::new(),
};
assert!(parse_snapshot(ShellType::Bash, oversized.as_bytes(), Some(&policy)).is_err());
}
#[test]
fn snapshot_filters_profile_exports_after_capture() {
let policy = ExecEnvPolicy {
@@ -238,7 +253,8 @@ fn snapshot_filters_profile_exports_after_capture() {
include_only: vec!["PROFILE_*".to_string()],
};
let snapshot = parse_snapshot(
b"profile noise\n# Snapshot file\nfunction profile_helper() { :; }\n\0PROFILE_ALLOWED=profile\0PROFILE_DENIED=denied\0PROFILE_SECRET=secret\0PWD=/tmp\0",
ShellType::Bash,
b"profile \xff noise\n# Snapshot file\nfunction profile_helper() { :; }\n\0alias profile_alias='profile_helper'\n\0PROFILE_DENIED\0export PROFILE_DENIED=denied\n\0NON_UTF8\0export NON_UTF8='\xff'\n\0\0PROFILE_ALLOWED=profile\0PROFILE_DENIED=denied\0PROFILE_SECRET=secret\0PWD=/tmp\0NON_UTF8=\xff\0",
Some(&policy),
)
.expect("snapshot should parse");
@@ -249,7 +265,7 @@ fn snapshot_filters_profile_exports_after_capture() {
);
assert_eq!(
snapshot.state,
"# Snapshot file\nfunction profile_helper() { :; }\n"
"# Snapshot file\nfunction profile_helper() { :; }\nalias profile_alias='profile_helper'\n"
);
}
@@ -267,7 +283,8 @@ fn snapshot_preserves_profile_exports_with_restrictive_inheritance() {
include_only: Vec::new(),
};
let snapshot = parse_snapshot(
b"# Snapshot file\n\0PROFILE_ALLOWED=profile\0SDKROOT=/sdk\0PROFILE_SECRET=secret\0PROFILE_DENIED=denied\0",
ShellType::Bash,
b"# Snapshot file\n\0\0\0PROFILE_ALLOWED=profile\0SDKROOT=/sdk\0PROFILE_SECRET=secret\0PROFILE_DENIED=denied\0",
Some(&policy),
)
.expect("snapshot should parse");
@@ -300,10 +317,11 @@ fn snapshot_caches_only_unmanaged_proxy_state() {
]),
),
] {
let output = format!("# Snapshot file\n\0{exports}");
let snapshot =
parse_snapshot(output.as_bytes(), /*env_policy*/ None).expect("snapshot should parse");
let output = format!("# Snapshot file\n\0\0\0{exports}");
let snapshot = parse_snapshot(ShellType::Bash, output.as_bytes(), /*env_policy*/ None)
.expect("snapshot should parse");
assert_eq!(snapshot.environment, expected);
}
}
use codex_shell_command::shell_detect::ShellType;