openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-27 18:48:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Harden credential handling in shell snapshots and replay (#44040)

Browse Source 
## Why

Shell quoting can hide credentials from raw-text checks, and startup files can restore real credentials after the broker replaces them with dummy values.

## What changed

- Decode shell literals without evaluating them and reject snapshots containing credentials in executable source, including aliases, functions, and heredocs.
- Preserve credential policy overrides, explicit unsets, and aliases whose source variables were removed. Support credential aliases in Zsh tied arrays while rejecting credentials that span array elements.
- Guard snapshot replay against credential restoration through shell startup files and preserve unrelated `ENV` settings.
- Apply Windows environment-key casing rules to credential overrides, suppress unredacted sandbox diagnostics during snapshot capture, and clear inherited environment variables before launching escalated commands.

## Testing

Add regression coverage for shell quoting and escaped credentials, Zsh tied arrays, startup-file replay, readonly credentials, policy overrides, and sensitive capture timeout and cancellation handling.

GitOrigin-RevId: 58274c07c715423241d26ce6dd2c2b4230cf0f64
This commit is contained in:
Winston Howes
2026-09-09 04:00:10 +00:00
committed by copyberry
parent a548463b78
commit ec512d2347
21 changed files with 4215 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/shell-command/src/shell_snapshot.rs
+4
View File
@@ -6,6 +6,8 @@ mod capture;
mod credentials;
#[path = "shell_snapshot_exports.rs"]
mod exports;
#[path = "shell_snapshot_literals.rs"]
mod literals;
#[path = "shell_snapshot_render.rs"]
mod render;
@@ -17,6 +19,8 @@ pub use credentials::PreparedSnapshot;
pub use credentials::SnapshotCredentialEnvironment;
pub use credentials::prepare_snapshot_credentials;
use capture::BASH_SH_SNAPSHOT_HEADER;
#[cfg(all(test, unix))]
#[path = "shell_snapshot_tests.rs"]
mod tests;
codex-rs/shell-command/src/shell_snapshot_capture.rs
+91 -4
View File
@@ -1,8 +1,9 @@
//! Capture native shell state and decode complete export records without applying credential policy.
//! Retain native declaration flags and reject incomplete records.
//! Retain native declaration flags and tied-array boundaries; reject incomplete records.
use super::SnapshotStartup;
use super::exports;
use super::literals;
use super::posix_env_path_expansion_function;
use crate::shell_detect::ShellType;
use std::borrow::Cow;
@@ -156,6 +157,8 @@ SNAPSHOT_ENVIRONMENT
format!("{startup}{script}")
}
pub(super) const BASH_SH_SNAPSHOT_HEADER: &str = "# Snapshot file\n# Bash-backed sh\n";
fn sh_snapshot_script(shell_startup: SnapshotStartup) -> String {
let startup = match shell_startup {
SnapshotStartup::Interactive => [
@@ -229,6 +232,7 @@ SNAPSHOT_ENVIRONMENT
pub struct CapturedSnapshot<'a> {
/// Resolved POSIX startup file and its environment immediately before sourcing it.
pub startup_environment: Option<CapturedStartupEnvironment<'a>>,
pub(super) shell_type: ShellType,
pub(super) state: &'a str,
pub(super) aliases: &'a str,
pub(super) exports: Vec<CapturedExport<'a>>,
@@ -238,13 +242,26 @@ pub struct CapturedSnapshot<'a> {
/// Optional pre-startup state captured alongside the final snapshot records.
pub struct CapturedStartupEnvironment<'a> {
pub path: &'a str,
pub path: Cow<'a, str>,
pub environment: &'a [u8],
}
pub(super) struct CapturedExport<'a> {
pub source: Cow<'a, str>,
pub key: &'a str,
pub value: ExportValue,
}
pub(super) enum ExportValue {
Plain,
TiedArray(CapturedArray),
UnparsedArray,
}
pub(super) struct CapturedArray {
pub span: std::ops::Range<usize>,
pub values: Vec<Vec<u8>>,
pub separator: Vec<u8>,
}
impl<'a> CapturedSnapshot<'a> {
@@ -262,7 +279,7 @@ impl<'a> CapturedSnapshot<'a> {
.position(|part| part == startup_marker)
{
remaining = &remaining[start + startup_marker.len()..];
let path = std::str::from_utf8(take_record(&mut remaining)?).ok()?;
let path = String::from_utf8_lossy(take_record(&mut remaining)?);
let environment = remaining;
while !take_record(&mut remaining)?.is_empty() {}
let environment = &environment[..environment.len() - remaining.len() - 1];
@@ -310,11 +327,13 @@ impl<'a> CapturedSnapshot<'a> {
shlex::try_quote(&value).ok()?
)),
key,
value: ExportValue::Plain,
});
}
}
return Some(Self {
startup_environment,
shell_type,
state,
aliases,
exports: records,
@@ -322,7 +341,75 @@ impl<'a> CapturedSnapshot<'a> {
});
}
let source = String::from_utf8_lossy(take_record(&mut remaining)?);
records.push(CapturedExport { source, key });
// Only tied arrays need syntax decoding; scalar values remain native declarations.
let tied = shell_type == ShellType::Zsh
&& source
.split_whitespace()
.skip(/*n*/ 1)
.take_while(|word| word.starts_with('-'))
.any(|flags| flags.contains('T'));
let tree = tied
.then(|| crate::bash::try_parse_shell(&source))
.flatten();
let array = tree.as_ref().and_then(|tree| {
if tree.root_node().has_error() {
return None;
}
let mut cursor = tree.root_node().walk();
tree.root_node()
.named_child(/*i*/ 0)?
.named_children(&mut cursor)
.find_map(|child| {
child
.child_by_field_name("value")
.filter(|value| value.kind() == "array")
})
});
let value = if let Some(array) = array {
let mut cursor = array.walk();
let values = array
.named_children(&mut cursor)
.map(|node| {
literals::literal_word_bytes(
node,
&source,
shell_type,
literals::Quoting::Posix,
)
})
.collect::<Option<Vec<_>>>();
let separator = array.parent().and_then(|node| node.next_named_sibling());
let separator = if let Some(separator) = separator
&& separator.kind() == "variable_name"
{
separator
.utf8_text(source.as_bytes())
.ok()
.map(|value| value.as_bytes().to_vec())
} else if let Some(separator) = separator {
literals::literal_word_bytes(
separator,
&source,
shell_type,
literals::Quoting::Posix,
)
} else {
Some(vec![b':'])
};
match (values, separator) {
(Some(values), Some(separator)) => ExportValue::TiedArray(CapturedArray {
span: array.byte_range(),
values,
separator,
}),
_ => ExportValue::UnparsedArray,
}
} else if tied {
ExportValue::UnparsedArray
} else {
ExportValue::Plain
};
records.push(CapturedExport { source, key, value });
}
}
}
codex-rs/shell-command/src/shell_snapshot_credentials.rs
+126 -36
View File
@@ -1,7 +1,9 @@
//! Apply credential and environment policy to captured exports, producing typed render values.
//! Native declarations are rendered only after applying the supplied credential policy.
//! The shared literal decoder still validates alias and function bodies without evaluating them.
use super::capture::CapturedSnapshot;
use super::capture::ExportValue;
use super::literals;
use super::render;
use super::render::Export;
use super::render::Value;
@@ -32,7 +34,7 @@ pub struct PreparedSnapshot {
/// Prepare a replay script from captured state without changing the captured data.
///
/// Credential policy is applied before rendering. A rejected capture produces no script,
/// when the assembled source still contains a recognized credential.
/// including when executable source reconstructs a credential through shell quoting.
pub fn prepare_snapshot_credentials(
captured: &CapturedSnapshot<'_>,
environment: SnapshotCredentialEnvironment<'_>,
@@ -136,7 +138,7 @@ pub fn prepare_snapshot_credentials(
let mut alias_values = HashMap::new();
let mut rejected_alias_keys = Vec::new();
let mut invalid_export = false;
let mut exports = captured
let exports = captured
.exports
.iter()
.filter_map(|export| {
@@ -148,14 +150,46 @@ pub fn prepare_snapshot_credentials(
|| brokered_keys
.iter()
.any(|credential_key| credential_key == key)
|| brokered_alias_keys
.iter()
.any(|credential_key| credential_key == key)
|| is_disallowed_credential_alias(key, &mut virtualize_text)
{
return None;
}
if configured.contains_key(key)
&& let ExportValue::TiedArray(array) = &export.value
{
let value = allowed.get(key)?;
let mut virtualized = value.clone();
if !virtualize_text(&mut virtualized) || virtualized != *value {
invalid_export = true;
return Some(Export::Captured(line));
}
if credential_aliases.remove(key).is_some() {
alias_values.insert(key.to_string(), value.clone());
}
// An explicit scalar override replaces captured elements, not the array binding.
return Some(Export::ArrayBinding {
key,
declaration: line[..array.span.start].strip_suffix('=')?,
suffix: &line[array.span.end..],
});
}
if is_disallowed_credential_alias(key, &mut virtualize_text) {
return None;
}
if brokered_alias_keys.iter().any(|alias_key| alias_key == key) {
// Source-less aliases keep their live scalar value, including explicit unsets.
return match &export.value {
ExportValue::TiedArray(array) => Some(Export::ArrayBinding {
key,
declaration: line[..array.span.start].strip_suffix('=')?,
suffix: &line[array.span.end..],
}),
ExportValue::Plain | ExportValue::UnparsedArray => None,
};
}
if let Some(credential_keys) = credential_aliases.remove(key) {
if !credential_alias_is_allowed(&credential_keys) {
rejected_alias_keys.push(key.to_string());
@@ -167,12 +201,74 @@ pub fn prepare_snapshot_credentials(
let (assignment, value) =
credential_alias_assignment(allowed.get(key)?, &credential_keys)?;
alias_values.insert(key.to_string(), value);
if line
.split_whitespace()
.skip(/*n*/ 1)
.take_while(|word| word.starts_with('-'))
.any(|flags| flags.contains('T'))
{
if let ExportValue::TiedArray(array) = &export.value {
let scalar = array.values.join(array.separator.as_slice());
let crosses_element = credential_keys
.iter()
.flat_map(|key| {
[
real_credential_value(key),
original.get(*key),
discovered.get(*key),
]
})
.flatten()
.filter(|credential| !credential.is_empty())
.any(|credential| {
scalar
.windows(credential.len())
.enumerate()
.any(|(start, bytes)| {
if bytes != credential.as_bytes() {
return false;
}
let mut offset = 0;
!array.values.iter().any(|value| {
let contains = offset <= start
&& start + credential.len() <= offset + value.len();
offset += value.len() + array.separator.len();
contains
})
})
});
if crosses_element {
invalid_export = true;
return Some(Export::Captured(line));
}
let Some((elements, normalized)) = array
.values
.iter()
.map(|bytes| {
let mut text = String::from_utf8(bytes.clone()).ok()?;
if !virtualize_text(&mut text) {
return None;
}
credential_alias_assignment(&text, &credential_keys)
})
.collect::<Option<Vec<_>>>()
.map(|values| values.into_iter().unzip::<_, _, Vec<_>, Vec<_>>())
else {
invalid_export = true;
return Some(Export::Captured(line));
};
// Never replay a known credential that crosses array-element boundaries.
if normalized
.iter()
.map(String::as_bytes)
.collect::<Vec<_>>()
.join(array.separator.as_slice())
!= allowed.get(key)?.as_bytes()
{
invalid_export = true;
return Some(Export::Captured(line));
}
return Some(Export::Array {
prefix: &line[..array.span.start],
elements,
suffix: &line[array.span.end..],
});
}
if matches!(export.value, ExportValue::UnparsedArray) {
invalid_export = true;
return Some(Export::Captured(line));
}
@@ -185,6 +281,7 @@ pub fn prepare_snapshot_credentials(
Some(Export::Captured(line))
})
.collect::<Vec<_>>();
let snapshot = render::render(captured.state, captured.aliases, &exports)?;
if invalid_export {
return None;
}
@@ -192,38 +289,31 @@ pub fn prepare_snapshot_credentials(
for (key, credential_keys) in credential_aliases {
if !is_disallowed_credential_alias(key, &mut virtualize_text)
&& credential_alias_is_allowed(&credential_keys)
&& let Some((assignment, value)) = allowed
&& let Some((_, value)) = allowed
.get(key)
.and_then(|value| credential_alias_assignment(value, &credential_keys))
{
exports.push(Export::Alias {
key,
value: assignment,
});
// Native capture can deliberately omit readonly exports; retain only their metadata.
alias_values.insert(key.to_string(), value);
} else {
rejected_alias_keys.push(key.to_string());
}
}
let mut snapshot = render::render(captured.state, captured.aliases, &exports)?;
let mut credential_values = original
.values()
.chain(restored.values())
.chain(configured.values())
.collect::<Vec<_>>();
credential_values.sort_unstable_by_key(|value| (std::cmp::Reverse(value.len()), *value));
credential_values.dedup();
for value in credential_values {
let mut replacement = value.clone();
if virtualize_text(&mut replacement) && replacement != *value {
snapshot = snapshot.replace(value, &replacement);
}
let mut virtualized = snapshot.clone();
if !virtualize_text(&mut virtualized) || virtualized != snapshot {
return None;
}
Some(PreparedSnapshot {
script: snapshot,
aliases: alias_values,
rejected_alias_keys,
})
literals::literal_words(&snapshot, captured.shell_type)?
.into_iter()
.all(|word| {
let mut virtualized = word.clone();
virtualize_text(&mut virtualized) && virtualized == word
})
.then_some(PreparedSnapshot {
script: snapshot,
aliases: alias_values,
rejected_alias_keys,
})
}
codex-rs/shell-command/src/shell_snapshot_literals.rs
+715
View File
@@ -0,0 +1,715 @@
use crate::bash::try_parse_shell;
use crate::shell_detect::ShellType;
use std::collections::HashSet;
use tree_sitter::Node;
#[derive(Clone, Copy, Eq, Hash, PartialEq)]
pub(super) enum Quoting {
Posix,
LegacyBash,
ZshRcQuotes,
}
impl Quoting {
fn deferred_modes(shell_type: ShellType) -> &'static [Self] {
if matches!(shell_type, ShellType::Bash | ShellType::Sh) {
&[Self::Posix, Self::LegacyBash]
} else {
&[Self::Posix, Self::ZshRcQuotes]
}
}
}
#[derive(Clone, Copy)]
enum Parsing {
Snapshot,
Deferred(Quoting),
}
pub(super) fn literal_words(script: &str, shell_type: ShellType) -> Option<Vec<String>> {
// Bash invoked as sh still parses its array and alias syntax; other sh dialects do not.
let shell_type =
if shell_type == ShellType::Sh && script.starts_with(super::BASH_SH_SNAPSHOT_HEADER) {
ShellType::Bash
} else {
shell_type
};
let mut checked_aliases = HashSet::new();
let mut words =
literal_words_with_quoting(script, shell_type, Parsing::Snapshot, &mut checked_aliases)?;
if matches!(shell_type, ShellType::Bash | ShellType::Sh) {
words.extend(literal_words_with_quoting(
script,
shell_type,
Parsing::Deferred(Quoting::LegacyBash),
&mut checked_aliases,
)?);
}
Some(words)
}
fn literal_words_with_quoting(
script: &str,
shell_type: ShellType,
parsing: Parsing,
checked_aliases: &mut HashSet<(String, Quoting)>,
) -> Option<Vec<String>> {
let tree = try_parse_shell(script)?;
let mut cursor = tree.root_node().walk();
let mut quoting = match parsing {
Parsing::Snapshot => Quoting::Posix,
Parsing::Deferred(quoting) => quoting,
};
let mut pending = Vec::new();
// Zsh snapshots emit function definitions before restoring options and aliases.
for node in tree.root_node().named_children(&mut cursor) {
if matches!(parsing, Parsing::Snapshot) && node.kind() == "command" {
match node.utf8_text(script.as_bytes()).ok()? {
"setopt rcquotes" => quoting = Quoting::ZshRcQuotes,
"unsetopt rcquotes" => quoting = Quoting::Posix,
_ => {}
}
}
pending.push((node, quoting));
}
let mut words = Vec::new();
let mut checked_nodes = HashSet::new();
while let Some((node, quoting)) = pending.pop() {
if !checked_nodes.insert((node.id(), quoting)) {
continue;
}
if matches!(node.kind(), "command_substitution" | "process_substitution") {
if let Some(body) = node
.utf8_text(script.as_bytes())
.ok()?
.strip_prefix('`')
.and_then(|body| body.strip_suffix('`'))
{
// Backticks remove one escape layer before their command body is parsed.
let mut context = node.parent();
if shell_type == ShellType::Zsh {
while context.is_some_and(|parent| {
matches!(parent.kind(), "expansion" | "concatenation")
}) {
context = context.and_then(|parent| parent.parent());
}
}
let in_double_quotes = context.is_some_and(|parent| parent.kind() == "string");
let mut chars = body.chars().peekable();
let mut body = String::new();
while let Some(character) = chars.next() {
if character == '\\' {
match chars.peek() {
Some('\n') => {
chars.next();
continue;
}
Some('$' | '`' | '\\') => {
body.push(chars.next()?);
continue;
}
Some('"') if in_double_quotes => {
body.push(chars.next()?);
continue;
}
_ => {}
}
}
body.push(character);
}
for &quoting in Quoting::deferred_modes(shell_type) {
if checked_aliases.insert((body.clone(), quoting)) {
words.extend(literal_words_with_quoting(
&body,
shell_type,
Parsing::Deferred(quoting),
checked_aliases,
)?);
}
}
continue;
}
// Deferred bodies may be parsed after the snapshot restores RC_QUOTES.
let mut cursor = node.walk();
for child in node.named_children(&mut cursor) {
pending.extend(
Quoting::deferred_modes(shell_type)
.iter()
.map(|&quoting| (child, quoting)),
);
}
continue;
}
if node.kind() == "heredoc_body" {
let parent = node.parent()?;
let mut cursor = parent.walk();
let delimiter = parent
.named_children(&mut cursor)
.find(|child| child.kind() == "heredoc_start")?
.utf8_text(script.as_bytes())
.ok()?;
let quoted = delimiter.contains(['\'', '"', '\\']);
let strip_tabs = parent
.children(&mut cursor)
.any(|child| child.kind() == "<<-");
let mut start = node.start_byte();
let mut cursor = node.walk();
// Decode static heredoc text without joining across an expansion.
for end in node
.named_children(&mut cursor)
.filter(|child| !quoted && child.kind() != "heredoc_content")
.map(Some)
.chain(std::iter::once(None))
{
let text = &script[start..end.map_or(node.end_byte(), |child| child.start_byte())];
let text = if strip_tabs {
text.split_inclusive('\n')
.map(|line| line.trim_start_matches('\t'))
.collect::<String>()
} else {
text.to_string()
};
let mut chars = text.chars().peekable();
let mut decoded = String::new();
while let Some(character) = chars.next() {
if !quoted && character == '\\' {
match chars.peek() {
Some('\n') => {
chars.next();
continue;
}
Some('$' | '`' | '\\') => {
decoded.push(chars.next()?);
continue;
}
_ => {}
}
}
decoded.push(character);
}
words.push(decoded);
if let Some(child) = end {
start = child.end_byte();
pending.push((child, quoting));
}
}
continue;
}
let mut command_words: Vec<(std::ops::Range<usize>, Option<Vec<u8>>)> = Vec::new();
let declaration = if node.kind() == "command" {
let mut cursor = node.walk();
for argument in node
.child_by_field_name("name")
.into_iter()
.chain(node.children_by_field_name("argument", &mut cursor))
{
let word = literal_word_bytes(argument, script, shell_type, quoting);
if shell_type == ShellType::Zsh
&& let Some((previous_range, previous_word)) = command_words.last_mut()
&& has_continuation_gap(script, previous_range.end, argument.start_byte())
{
*previous_word = previous_word.take().zip(word).map(|(mut previous, word)| {
previous.extend(word);
previous
});
previous_range.end = argument.end_byte();
} else {
command_words.push((argument.byte_range(), word));
}
}
let mut arguments = command_words.iter().map(|(_, word)| {
word.as_ref()
.map(|word| String::from_utf8_lossy(word).into_owned())
});
let mut name = arguments.next().flatten();
loop {
match name.take().as_deref() {
Some(
name @ ("alias" | "declare" | "typeset" | "local" | "readonly" | "export"),
) => {
break Some(name.to_string());
}
Some("noglob" | "-") if shell_type == ShellType::Zsh => {
name = arguments.next().flatten();
}
Some("time") => {
name = arguments.next().flatten();
if shell_type == ShellType::Bash && name.as_deref() == Some("-p") {
name = arguments.next().flatten();
}
}
Some(prefix @ ("builtin" | "command")) => {
name = arguments.next().flatten();
if prefix == "command" {
while name.as_deref().is_some_and(|argument| {
argument.strip_prefix('-').is_some_and(|options| {
!options.is_empty() && options.bytes().all(|byte| byte == b'p')
})
}) {
name = arguments.next().flatten();
}
}
if (prefix == "command"
|| prefix == "builtin" && shell_type != ShellType::Zsh)
&& name.as_deref() == Some("--")
{
name = arguments.next().flatten();
}
}
_ => break None,
}
}
} else {
None
};
let is_alias = declaration.as_deref() == Some("alias");
let may_initialize_array = shell_type == ShellType::Bash
&& (node.kind() == "declaration_command" || declaration.is_some() && !is_alias);
if is_alias || may_initialize_array {
// Bash can inherit an array's type from an earlier declaration or calling scope.
// Aliases and quoted compound-array initializers are parsed again by the shell.
let mut cursor = node.walk();
for argument in node.named_children(&mut cursor) {
if command_words.iter().any(|(range, _)| {
range.start < argument.start_byte() && argument.end_byte() <= range.end
}) {
continue;
}
let continued = command_words.iter().find(|(range, _)| {
range.start == argument.start_byte() && range.end > argument.end_byte()
});
if let Some(assignment) = continued
.map_or_else(
|| literal_word_bytes(argument, script, shell_type, quoting),
|(_, word)| word.clone(),
)
.or_else(|| {
// A dynamic edge prevents full decoding, but not inspection of the
// static fragments in the rest of the continued argument.
let end = continued.map_or(argument.end_byte(), |(range, _)| range.end);
let mut pieces = std::iter::successors(Some(argument), |piece| {
piece.next_named_sibling()
})
.take_while(|piece| piece.end_byte() <= end)
.collect::<Vec<_>>();
pieces.reverse();
let mut word = Vec::new();
while let Some(piece) = pieces.pop() {
if let Some(literal) =
literal_word_bytes(piece, script, shell_type, quoting)
{
word.extend(literal);
} else {
match piece.kind() {
"variable_assignment" => {
word.extend_from_slice(b"_=");
pieces.push(piece.child_by_field_name("value")?);
}
"concatenation" | "string" => pieces.extend(
(0..piece.named_child_count())
.rev()
.filter_map(|index| piece.named_child(index)),
),
"string_content" => {
let text = piece.utf8_text(script.as_bytes()).ok()?;
let decoded = shlex::split(&format!("\"{text}\""))?;
let [literal] = decoded.as_slice() else {
return None;
};
word.extend_from_slice(literal.as_bytes());
}
"simple_expansion"
| "expansion"
| "command_substitution"
| "process_substitution"
| "arithmetic_expansion" => {
word.extend_from_slice(b"${__codex_snapshot_dynamic}");
}
_ => return None,
}
}
}
Some(word)
})
.map(|word| String::from_utf8_lossy(&word).into_owned())
&& let Some((_, body)) = assignment.split_once('=')
{
let body = if may_initialize_array {
let body = body
.trim_start_matches("${__codex_snapshot_dynamic}")
.trim_end_matches("${__codex_snapshot_dynamic}");
if !body.starts_with('(') || !body.ends_with(')') {
continue;
}
format!("_={body}")
} else {
body.to_string()
};
for &quoting in Quoting::deferred_modes(shell_type) {
if checked_aliases.insert((body.clone(), quoting)) {
words.extend(literal_words_with_quoting(
&body,
shell_type,
Parsing::Deferred(quoting),
checked_aliases,
)?);
}
}
}
}
}
let mut pieces = vec![node];
if shell_type == ShellType::Zsh
&& node.prev_named_sibling().is_none_or(|previous| {
!has_continuation_gap(script, previous.end_byte(), node.start_byte())
})
{
// Tree-sitter splits unquoted continuations into sibling words. Decode the
// pieces separately: removing the gap would change adjacent RC_QUOTES strings.
let mut previous = node;
while let Some(next) = previous.next_named_sibling() {
if !has_continuation_gap(script, previous.end_byte(), next.start_byte()) {
break;
}
pieces.push(next);
previous = next;
}
}
let continued = pieces.len() > 1;
if node.kind() == "ansi_c_string" {
words.push(decode_ansi_c(
node.utf8_text(script.as_bytes()).ok()?,
shell_type,
quoting,
)?);
if !continued {
continue;
}
} else if let Some(word) = literal_word(node, script, shell_type, quoting) {
words.push(word);
if !continued {
continue;
}
}
if continued || matches!(node.kind(), "string" | "concatenation") {
// Join adjacent static pieces, but never join across an expansion.
pieces.reverse();
let mut literal = Vec::new();
let mut raw_string_end = None;
while let Some(piece) = pieces.pop() {
if let Some(word) = literal_word_bytes(piece, script, shell_type, quoting) {
if matches!(quoting, Quoting::ZshRcQuotes)
&& piece.kind() == "raw_string"
&& raw_string_end == Some(piece.start_byte())
{
literal.push(b'\'');
}
literal.extend(word);
raw_string_end = (piece.kind() == "raw_string").then_some(piece.end_byte());
} else if piece.kind() == "variable_assignment" {
if !literal.is_empty() {
words.push(String::from_utf8_lossy(&literal).into_owned());
literal.clear();
}
// Only the value can continue into the next word, never the assignment name.
pieces.extend(piece.child_by_field_name("value"));
} else if piece.kind() == "string_content" {
let content = piece.utf8_text(script.as_bytes()).ok()?;
let decoded = shlex::split(&format!("\"{content}\""))?;
let [word] = decoded.as_slice() else {
return None;
};
literal.extend_from_slice(word.as_bytes());
} else if matches!(piece.kind(), "string" | "concatenation" | "command_name") {
if shell_type == ShellType::Zsh
&& piece.prev_sibling().is_some_and(|previous| {
previous.kind() == "$" && previous.end_byte() == piece.start_byte()
})
{
literal.push(b'$');
}
pieces.extend(
(0..piece.named_child_count())
.rev()
.filter_map(|index| piece.named_child(index)),
);
} else if !literal.is_empty() {
words.push(String::from_utf8_lossy(&literal).into_owned());
literal.clear();
}
}
if !literal.is_empty() {
words.push(String::from_utf8_lossy(&literal).into_owned());
}
}
let mut cursor = node.walk();
pending.extend(
node.named_children(&mut cursor)
.map(|child| (child, quoting)),
);
}
Some(words)
}
fn has_continuation_gap(script: &str, left_end: usize, right_start: usize) -> bool {
let gap = &script[left_end..right_start];
!gap.is_empty() && gap.as_bytes().chunks(2).all(|pair| pair == b"\\\n")
}
fn literal_word(
node: Node<'_>,
script: &str,
shell_type: ShellType,
quoting: Quoting,
) -> Option<String> {
let bytes = literal_word_bytes(node, script, shell_type, quoting)?;
Some(String::from_utf8_lossy(&bytes).into_owned())
}
pub(super) fn literal_word_bytes(
node: Node<'_>,
script: &str,
shell_type: ShellType,
quoting: Quoting,
) -> Option<Vec<u8>> {
let mut word = match node.kind() {
"command_name" => node
.named_child(0)
.and_then(|name| literal_word_bytes(name, script, shell_type, quoting)),
"concatenation" => {
let mut cursor = node.walk();
let mut word = Vec::new();
let mut raw_string_end = None;
for child in node.named_children(&mut cursor) {
// RC_QUOTES treats adjacent single-quoted strings as one quoted apostrophe.
if matches!(quoting, Quoting::ZshRcQuotes)
&& child.kind() == "raw_string"
&& raw_string_end == Some(child.start_byte())
{
word.push(b'\'');
}
word.extend(literal_word_bytes(child, script, shell_type, quoting)?);
raw_string_end = (child.kind() == "raw_string").then_some(child.end_byte());
}
Some(word)
}
"ansi_c_string" => {
decode_ansi_c_bytes(node.utf8_text(script.as_bytes()).ok()?, shell_type, quoting)
}
"word" | "number" | "string" | "raw_string" => {
let mut cursor = node.walk();
if node
.named_children(&mut cursor)
.any(|child| child.kind() != "string_content")
{
return None;
}
let text = node.utf8_text(script.as_bytes()).ok()?;
let mut context = node.parent();
while context
.is_some_and(|parent| matches!(parent.kind(), "expansion" | "concatenation"))
{
context = context.and_then(|parent| parent.parent());
}
let words = if shell_type == ShellType::Zsh
&& node.kind() == "word"
&& context.is_some_and(|parent| parent.kind() == "string")
{
shlex::split(&format!("\"{text}\""))?
} else {
shlex::split(text)?
};
let [word] = words.as_slice() else {
return None;
};
Some(word.as_bytes().to_vec())
}
_ => None,
}?;
if shell_type == ShellType::Zsh
&& matches!(node.kind(), "string" | "concatenation")
&& node.prev_sibling().is_some_and(|previous| {
previous.kind() == "$" && previous.end_byte() == node.start_byte()
})
{
word.insert(0, b'$');
}
Some(word)
}
// Decode the literal quoting emitted by Bash and Zsh, never shell expansions.
fn decode_ansi_c(raw: &str, shell_type: ShellType, quoting: Quoting) -> Option<String> {
let bytes = decode_ansi_c_bytes(raw, shell_type, quoting)?;
Some(String::from_utf8_lossy(&bytes).into_owned())
}
fn decode_ansi_c_bytes(raw: &str, shell_type: ShellType, quoting: Quoting) -> Option<Vec<u8>> {
let body = raw.strip_prefix("$'")?;
let body = body.strip_suffix('\'').or_else(|| {
// Array-shaped scalar templates may contain incomplete shell syntax.
(shell_type == ShellType::Bash).then_some(body)
})?;
let mut chars = body.chars().peekable();
let mut decoded = Vec::new();
let mut modifiers = ZshByteModifiers::default();
while let Some(character) = chars.next() {
if character != '\\' {
let mut bytes = [0; 4];
let bytes = character.encode_utf8(&mut bytes).as_bytes();
decoded.push(modifiers.apply(bytes[0]));
decoded.extend_from_slice(&bytes[1..]);
continue;
}
let Some(escape) = chars.next() else {
if shell_type == ShellType::Zsh {
return None;
}
decoded.push(b'\\');
break;
};
let byte = match escape {
'a' => 7,
'b' => 8,
'e' | 'E' => 27,
'f' => 12,
'n' => b'\n',
'r' => b'\r',
't' => b'\t',
'v' => 11,
'\\' | '\'' | '"' | '?' => escape as u8,
'c' if shell_type != ShellType::Zsh => {
let Some(character) = chars.next() else {
decoded.push(b'\\');
if quoting != Quoting::LegacyBash {
decoded.push(b'c');
}
break;
};
// Bash 3.2 leaves the second backslash; newer Bash consumes the pair.
if character == '\\'
&& quoting != Quoting::LegacyBash
&& chars.peek() == Some(&'\\')
{
chars.next();
}
let mut bytes = [0; 4];
let bytes = character.encode_utf8(&mut bytes).as_bytes();
decoded.push(if bytes[0] == b'?' { 127 } else { bytes[0] & 31 });
decoded.extend_from_slice(&bytes[1..]);
continue;
}
'C' | 'M' if shell_type == ShellType::Zsh => {
if chars.peek() == Some(&'-') {
chars.next();
}
if escape == 'C' {
modifiers.control = true;
} else {
modifiers.meta = Some(if modifiers.control {
MetaOrder::BeforeControl
} else {
MetaOrder::AfterControl
});
}
continue;
}
'0'..='7' | 'x' | 'u' | 'U' => {
if matches!(escape, 'u' | 'U') && quoting == Quoting::LegacyBash {
decoded.extend_from_slice(&[b'\\', escape as u8]);
continue;
}
let (radix, digits) = match escape {
'x' => (16, 2),
'u' => (16, 4),
'U' => (16, 8),
_ => (8, 2),
};
let mut value = escape.to_digit(8).unwrap_or(0);
let mut found = escape.is_ascii_digit();
for _ in 0..digits {
let Some(digit) = chars.peek().and_then(|next| next.to_digit(radix)) else {
break;
};
chars.next();
value = value.checked_mul(radix)?.checked_add(digit)?;
found = true;
}
if !found {
if shell_type == ShellType::Zsh {
return None;
}
decoded.extend_from_slice(&[b'\\', escape as u8]);
continue;
}
if matches!(escape, 'u' | 'U') {
let Some(character) = char::from_u32(value) else {
if shell_type == ShellType::Zsh {
return None;
}
// Bash emits extended UTF-8, including surrogate and 5/6-byte values.
let length = match value {
0..=0xffff => 3,
0x10000..=0x1fffff => 4,
0x200000..=0x3ffffff => 5,
0x4000000..=0x7fffffff => 6,
_ => continue,
};
let mut bytes = [0; 6];
for index in (1..length).rev() {
bytes[index] = 0x80 | (value & 0x3f) as u8;
value >>= 6;
}
bytes[0] = (0xff << (8 - length)) | value as u8;
decoded.extend_from_slice(&bytes[..length]);
continue;
};
// Zsh leaves byte modifiers pending across Unicode escapes.
let mut bytes = [0; 4];
decoded.extend_from_slice(character.encode_utf8(&mut bytes).as_bytes());
continue;
}
value as u8
}
_ => {
if shell_type != ShellType::Zsh {
decoded.push(b'\\');
}
let mut bytes = [0; 4];
let bytes = escape.encode_utf8(&mut bytes).as_bytes();
decoded.push(modifiers.apply(bytes[0]));
decoded.extend_from_slice(&bytes[1..]);
continue;
}
};
decoded.push(modifiers.apply(byte));
}
Some(decoded)
}
enum MetaOrder {
BeforeControl,
AfterControl,
}
#[derive(Default)]
struct ZshByteModifiers {
control: bool,
meta: Option<MetaOrder>,
}
impl ZshByteModifiers {
fn apply(&mut self, mut byte: u8) -> u8 {
// Escape order matters for control-? and bytes whose high bit is already set.
let meta = self.meta.take();
if matches!(meta, Some(MetaOrder::BeforeControl)) {
byte |= 0x80;
}
if std::mem::take(&mut self.control) {
byte = if byte == b'?' { 0x7f } else { byte & 0x9f };
}
if matches!(meta, Some(MetaOrder::AfterControl)) {
byte |= 0x80;
}
byte
}
}
codex-rs/shell-command/src/shell_snapshot_render.rs
+35 -5
View File
@@ -32,8 +32,20 @@ pub(super) enum ValuePart {
pub(super) enum Export<'a> {
Captured(&'a str),
Alias { key: &'a str, value: Value },
Assignment { declaration: &'a str, value: Value },
Assignment {
declaration: &'a str,
value: Value,
},
Array {
prefix: &'a str,
elements: Vec<Value>,
suffix: &'a str,
},
ArrayBinding {
key: &'a str,
declaration: &'a str,
suffix: &'a str,
},
}
impl Value {
@@ -64,12 +76,30 @@ pub(super) fn render(state: &str, aliases: &str, exports: &[Export<'_>]) -> Opti
for export in exports {
match export {
Export::Captured(source) => output.push_str(source),
Export::Alias { key, value } => {
output.push_str(&format!("export {key}={}\n", value.render()?));
}
Export::Assignment { declaration, value } => {
output.push_str(&format!("{declaration}={}\n", value.render()?));
}
Export::Array {
prefix,
elements,
suffix,
} => {
let elements = elements
.iter()
.map(Value::render)
.collect::<Option<Vec<_>>>()?;
output.push_str(&format!("{prefix}({}){suffix}", elements.join(" ")));
}
Export::ArrayBinding {
key,
declaration,
suffix,
} => {
output.push_str(&format!(
"if [ \"${{{key}+x}}\" = x ]; then\n{declaration}{}\nfi\n",
suffix.trim_end()
));
}
}
}
Some(output)
codex-rs/shell-command/src/shell_snapshot_tests.rs
+1349 -82
View File
File diff suppressed because it is too large Load Diff