Merge branch 'main' into codex/stateful-apply-patch-streaming

This commit is contained in:
Akshay Nathan
2026-05-04 15:18:11 -07:00
147 changed files with 3745 additions and 980 deletions
+21 -47
View File
@@ -20,7 +20,7 @@ use crate::MaybeApplyPatchVerified;
use crate::parser::Hunk;
use crate::parser::ParseError;
use crate::parser::ParsePatchMode;
use crate::parser::parse_patch_with_mode;
use crate::parser::parse_patch;
use crate::unified_diff_from_chunks;
use std::str::Utf8Error;
use tree_sitter::LanguageError;
@@ -103,16 +103,11 @@ fn extract_apply_patch_from_shell(
}
// TODO: make private once we remove tests in lib.rs
#[cfg_attr(not(test), allow(dead_code))]
pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch {
maybe_parse_apply_patch_with_mode(argv, ParsePatchMode::Legacy)
}
pub fn maybe_parse_apply_patch_with_mode(argv: &[String], mode: ParsePatchMode) -> MaybeApplyPatch {
pub fn maybe_parse_apply_patch(argv: &[String], mode: ParsePatchMode) -> MaybeApplyPatch {
match argv {
// Direct invocation: apply_patch <patch>
[cmd, body] if APPLY_PATCH_COMMANDS.contains(&cmd.as_str()) => {
match parse_patch_with_mode(body, mode) {
match parse_patch(body, mode) {
Ok(source) => MaybeApplyPatch::Body(source),
Err(e) => MaybeApplyPatch::PatchParseError(e),
}
@@ -120,7 +115,7 @@ pub fn maybe_parse_apply_patch_with_mode(argv: &[String], mode: ParsePatchMode)
// Shell heredoc form: (optional `cd <path> &&`) apply_patch <<'EOF' ...
_ => match parse_shell_script(argv) {
Some((shell, script)) => match extract_apply_patch_from_shell(shell, script) {
Ok((body, workdir)) => match parse_patch_with_mode(&body, mode) {
Ok((body, workdir)) => match parse_patch(&body, mode) {
Ok(mut source) => {
source.workdir = workdir;
MaybeApplyPatch::Body(source)
@@ -140,15 +135,6 @@ pub fn maybe_parse_apply_patch_with_mode(argv: &[String], mode: ParsePatchMode)
/// cwd must be an absolute path so that we can resolve relative paths in the
/// patch.
pub async fn maybe_parse_apply_patch_verified(
argv: &[String],
cwd: &AbsolutePathBuf,
fs: &dyn ExecutorFileSystem,
sandbox: Option<&codex_exec_server::FileSystemSandboxContext>,
) -> MaybeApplyPatchVerified {
maybe_parse_apply_patch_verified_with_mode(argv, ParsePatchMode::Legacy, cwd, fs, sandbox).await
}
pub async fn maybe_parse_apply_patch_verified_with_mode(
argv: &[String],
mode: ParsePatchMode,
cwd: &AbsolutePathBuf,
@@ -158,17 +144,17 @@ pub async fn maybe_parse_apply_patch_verified_with_mode(
// Detect a raw patch body passed directly as the command or as the body of a shell
// script. In these cases, report an explicit error rather than applying the patch.
if let [body] = argv
&& parse_patch_with_mode(body, mode).is_ok()
&& parse_patch(body, mode).is_ok()
{
return MaybeApplyPatchVerified::CorrectnessError(ApplyPatchError::ImplicitInvocation);
}
if let Some((_, script)) = parse_shell_script(argv)
&& parse_patch_with_mode(script, mode).is_ok()
&& parse_patch(script, mode).is_ok()
{
return MaybeApplyPatchVerified::CorrectnessError(ApplyPatchError::ImplicitInvocation);
}
match maybe_parse_apply_patch_with_mode(argv, mode) {
match maybe_parse_apply_patch(argv, mode) {
MaybeApplyPatch::Body(ApplyPatchArgs {
patch,
hunks,
@@ -179,15 +165,13 @@ pub async fn maybe_parse_apply_patch_verified_with_mode(
.map(|dir| cwd.join(Path::new(dir)))
.unwrap_or_else(|| cwd.clone());
let mut changes = HashMap::new();
for hunk in &hunks {
for hunk in hunks {
let path = hunk.resolve_path(&effective_cwd);
match hunk {
Hunk::AddFile { contents, .. } => {
changes.insert(
path.into_path_buf(),
ApplyPatchFileChange::Add {
content: contents.clone(),
},
ApplyPatchFileChange::Add { content: contents },
);
}
Hunk::DeleteFile { .. } => {
@@ -223,9 +207,7 @@ pub async fn maybe_parse_apply_patch_verified_with_mode(
path.into_path_buf(),
ApplyPatchFileChange::Update {
unified_diff,
move_path: move_path
.as_ref()
.map(|p| effective_cwd.join(p).into_path_buf()),
move_path: move_path.map(|p| effective_cwd.join(p).into_path_buf()),
new_content: contents,
},
);
@@ -234,7 +216,6 @@ pub async fn maybe_parse_apply_patch_verified_with_mode(
}
MaybeApplyPatchVerified::Body(ApplyPatchAction {
changes,
hunks,
patch,
cwd: effective_cwd,
})
@@ -400,7 +381,6 @@ fn extract_apply_patch_from_bash(
#[cfg(test)]
mod tests {
use super::*;
use crate::UpdateFileChunk;
use crate::parse_patch;
use crate::unified_diff_from_chunks;
use assert_matches::assert_matches;
@@ -462,7 +442,7 @@ mod tests {
}
fn assert_match_args(args: Vec<String>, expected_workdir: Option<&str>) {
match maybe_parse_apply_patch(&args) {
match maybe_parse_apply_patch(&args, ParsePatchMode::Legacy) {
MaybeApplyPatch::Body(ApplyPatchArgs { hunks, workdir, .. }) => {
assert_eq!(workdir.as_deref(), expected_workdir);
assert_eq!(hunks, expected_single_add());
@@ -479,7 +459,7 @@ mod tests {
fn assert_not_match(script: &str) {
let args = args_bash(script);
assert_matches!(
maybe_parse_apply_patch(&args),
maybe_parse_apply_patch(&args, ParsePatchMode::Legacy),
MaybeApplyPatch::NotApplyPatch
);
}
@@ -492,6 +472,7 @@ mod tests {
assert_matches!(
maybe_parse_apply_patch_verified(
&args,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
LOCAL_FS.as_ref(),
/*sandbox*/ None,
@@ -509,6 +490,7 @@ mod tests {
assert_matches!(
maybe_parse_apply_patch_verified(
&args,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
LOCAL_FS.as_ref(),
/*sandbox*/ None,
@@ -529,7 +511,7 @@ mod tests {
"#,
]);
match maybe_parse_apply_patch(&args) {
match maybe_parse_apply_patch(&args, ParsePatchMode::Legacy) {
MaybeApplyPatch::Body(ApplyPatchArgs { hunks, .. }) => {
assert_eq!(
hunks,
@@ -554,7 +536,7 @@ mod tests {
"#,
]);
match maybe_parse_apply_patch(&args) {
match maybe_parse_apply_patch(&args, ParsePatchMode::Legacy) {
MaybeApplyPatch::Body(ApplyPatchArgs { hunks, .. }) => {
assert_eq!(
hunks,
@@ -593,7 +575,7 @@ mod tests {
PATCH"#,
]);
match maybe_parse_apply_patch(&args) {
match maybe_parse_apply_patch(&args, ParsePatchMode::Legacy) {
MaybeApplyPatch::Body(ApplyPatchArgs { hunks, workdir, .. }) => {
assert_eq!(workdir, None);
assert_eq!(
@@ -714,7 +696,7 @@ PATCH"#,
path.display()
));
let patch = parse_patch(&patch).unwrap();
let patch = parse_patch(&patch, ParsePatchMode::Legacy).unwrap();
let chunks = match patch.hunks.as_slice() {
[Hunk::UpdateFile { chunks, .. }] => chunks,
_ => panic!("Expected a single UpdateFile hunk"),
@@ -753,7 +735,7 @@ PATCH"#,
path.display()
));
let patch = parse_patch(&patch).unwrap();
let patch = parse_patch(&patch, ParsePatchMode::Legacy).unwrap();
let chunks = match patch.hunks.as_slice() {
[Hunk::UpdateFile { chunks, .. }] => chunks,
_ => panic!("Expected a single UpdateFile hunk"),
@@ -798,6 +780,7 @@ PATCH"#,
let result = maybe_parse_apply_patch_verified(
&argv,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(session_dir.path()).unwrap(),
LOCAL_FS.as_ref(),
/*sandbox*/ None,
@@ -821,16 +804,6 @@ PATCH"#,
new_content: "updated session directory content\n".to_string(),
},
)]),
hunks: vec![Hunk::UpdateFile {
path: PathBuf::from(relative_path),
move_path: None,
chunks: vec![UpdateFileChunk {
change_context: None,
old_lines: vec!["session directory content".to_string()],
new_lines: vec!["updated session directory content".to_string()],
is_end_of_file: false,
}],
}],
patch: argv[1].clone(),
cwd: AbsolutePathBuf::from_absolute_path(session_dir.path()).unwrap(),
})
@@ -862,6 +835,7 @@ PATCH"#,
let result = maybe_parse_apply_patch_verified(
&argv,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(session_dir.path()).unwrap(),
LOCAL_FS.as_ref(),
/*sandbox*/ None,
+18 -17
View File
@@ -27,7 +27,6 @@ pub use streaming_parser::StreamingPatchParser;
use thiserror::Error;
pub use invocation::maybe_parse_apply_patch_verified;
pub use invocation::maybe_parse_apply_patch_verified_with_mode;
pub use standalone_executable::main;
use crate::invocation::ExtractHeredocError;
@@ -137,7 +136,6 @@ pub enum MaybeApplyPatchVerified {
#[derive(Debug, PartialEq)]
pub struct ApplyPatchAction {
changes: HashMap<PathBuf, ApplyPatchFileChange>,
hunks: Vec<Hunk>,
/// The raw patch argument that can be used to apply the patch. i.e., if the
/// original arg was parsed in "lenient" mode with a
@@ -158,10 +156,6 @@ impl ApplyPatchAction {
&self.changes
}
pub fn hunks(&self) -> &[Hunk] {
&self.hunks
}
/// Should be used exclusively for testing. (Not worth the overhead of
/// creating a feature flag for this.)
pub fn new_add_for_test(path: &AbsolutePathBuf, content: String) -> Self {
@@ -177,15 +171,10 @@ impl ApplyPatchAction {
+ {content}
*** End Patch"#,
);
let hunk = Hunk::AddFile {
path: path.to_path_buf(),
contents: content.clone(),
};
let changes = HashMap::from([(path.to_path_buf(), ApplyPatchFileChange::Add { content })]);
#[expect(clippy::expect_used)]
Self {
changes,
hunks: vec![hunk],
cwd: path.parent().expect("path should have parent"),
patch,
}
@@ -195,13 +184,14 @@ impl ApplyPatchAction {
/// Applies the patch and prints the result to stdout/stderr.
pub async fn apply_patch(
patch: &str,
mode: ParsePatchMode,
cwd: &AbsolutePathBuf,
stdout: &mut impl std::io::Write,
stderr: &mut impl std::io::Write,
fs: &dyn ExecutorFileSystem,
sandbox: Option<&FileSystemSandboxContext>,
) -> Result<(), ApplyPatchError> {
let hunks = match parse_patch(patch) {
let hunks = match parser::parse_patch(patch, mode) {
Ok(source) => source.hunks,
Err(e) => {
match &e {
@@ -652,6 +642,7 @@ mod tests {
let mut stderr = Vec::new();
apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
@@ -712,6 +703,7 @@ mod tests {
apply_patch(
&patch,
ParsePatchMode::Legacy,
&cwd,
&mut stdout,
&mut stderr,
@@ -755,6 +747,7 @@ mod tests {
let mut stderr = Vec::new();
apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
@@ -791,6 +784,7 @@ mod tests {
let mut stderr = Vec::new();
apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
@@ -831,6 +825,7 @@ mod tests {
let mut stderr = Vec::new();
apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
@@ -880,6 +875,7 @@ mod tests {
let mut stderr = Vec::new();
apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
@@ -938,6 +934,7 @@ mod tests {
let mut stderr = Vec::new();
apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
@@ -982,6 +979,7 @@ mod tests {
let mut stderr = Vec::new();
apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
@@ -1025,6 +1023,7 @@ mod tests {
let mut stderr = Vec::new();
apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
@@ -1069,7 +1068,7 @@ mod tests {
+QUX"#,
path.display()
));
let patch = parse_patch(&patch).unwrap();
let patch = parse_patch(&patch, ParsePatchMode::Legacy).unwrap();
let update_file_chunks = match patch.hunks.as_slice() {
[Hunk::UpdateFile { chunks, .. }] => chunks,
@@ -1116,7 +1115,7 @@ mod tests {
path.display()
));
let patch = parse_patch(&patch).unwrap();
let patch = parse_patch(&patch, ParsePatchMode::Legacy).unwrap();
let chunks = match patch.hunks.as_slice() {
[Hunk::UpdateFile { chunks, .. }] => chunks,
_ => panic!("Expected a single UpdateFile hunk"),
@@ -1157,7 +1156,7 @@ mod tests {
path.display()
));
let patch = parse_patch(&patch).unwrap();
let patch = parse_patch(&patch, ParsePatchMode::Legacy).unwrap();
let chunks = match patch.hunks.as_slice() {
[Hunk::UpdateFile { chunks, .. }] => chunks,
_ => panic!("Expected a single UpdateFile hunk"),
@@ -1196,7 +1195,7 @@ mod tests {
path.display()
));
let patch = parse_patch(&patch).unwrap();
let patch = parse_patch(&patch, ParsePatchMode::Legacy).unwrap();
let chunks = match patch.hunks.as_slice() {
[Hunk::UpdateFile { chunks, .. }] => chunks,
_ => panic!("Expected a single UpdateFile hunk"),
@@ -1246,7 +1245,7 @@ mod tests {
let patch = wrap_patch(&patch_body);
// Extract chunks then build the unified diff.
let parsed = parse_patch(&patch).unwrap();
let parsed = parse_patch(&patch, ParsePatchMode::Legacy).unwrap();
let chunks = match parsed.hunks.as_slice() {
[Hunk::UpdateFile { chunks, .. }] => chunks,
_ => panic!("Expected a single UpdateFile hunk"),
@@ -1281,6 +1280,7 @@ mod tests {
let mut stderr = Vec::new();
apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
@@ -1321,6 +1321,7 @@ g
let mut stderr = Vec::new();
let result = apply_patch(
&patch,
ParsePatchMode::Legacy,
&AbsolutePathBuf::from_absolute_path(dir.path()).unwrap(),
&mut stdout,
&mut stderr,
+16 -9
View File
@@ -129,16 +129,17 @@ pub enum ParsePatchMode {
Streaming,
}
pub fn parse_patch(patch: &str) -> Result<ApplyPatchArgs, ParseError> {
parse_patch_with_mode(patch, ParsePatchMode::Legacy)
}
pub fn parse_patch_with_mode(
patch: &str,
mode: ParsePatchMode,
) -> Result<ApplyPatchArgs, ParseError> {
pub fn parse_patch(patch: &str, mode: ParsePatchMode) -> Result<ApplyPatchArgs, ParseError> {
if mode == ParsePatchMode::Streaming {
return crate::streaming_parser::parse_patch(patch);
let patch = normalize_patch_text(patch)?;
let mut parser = crate::streaming_parser::StreamingPatchParser::default();
parser.push_delta(&patch)?;
let hunks = parser.finish()?;
return Ok(ApplyPatchArgs {
hunks,
patch,
workdir: None,
});
}
let mode = if PARSE_IN_STRICT_MODE {
@@ -149,6 +150,12 @@ pub fn parse_patch_with_mode(
parse_patch_text(patch, mode)
}
fn normalize_patch_text(patch: &str) -> Result<String, ParseError> {
let lines: Vec<&str> = patch.trim().lines().collect();
let (patch_lines, _) = check_patch_boundaries_lenient(&lines)?;
Ok(patch_lines.join("\n"))
}
enum ParseMode {
/// Parse the patch text argument as is.
Strict,
@@ -67,6 +67,7 @@ pub fn run_main() -> i32 {
};
match runtime.block_on(crate::apply_patch(
&patch_arg,
crate::ParsePatchMode::Legacy,
&cwd,
&mut stdout,
&mut stderr,
@@ -1,6 +1,5 @@
use std::path::PathBuf;
use crate::ApplyPatchArgs;
use crate::parser::ADD_FILE_MARKER;
use crate::parser::BEGIN_PATCH_MARKER;
use crate::parser::CHANGE_CONTEXT_MARKER;
@@ -17,67 +16,6 @@ use crate::parser::UpdateFileChunk;
use Hunk::*;
use ParseError::*;
pub fn parse_patch(patch: &str) -> Result<ApplyPatchArgs, ParseError> {
let patch = normalize_patch_text(patch)?;
let mut parser = StreamingPatchParser::default();
parser.push_delta(&patch)?;
let hunks = parser.finish()?;
Ok(ApplyPatchArgs {
patch,
hunks,
workdir: None,
})
}
fn normalize_patch_text(patch: &str) -> Result<String, ParseError> {
let lines: Vec<&str> = patch.trim().lines().collect();
let patch_lines = match check_patch_boundaries(&lines) {
Ok(lines) => lines,
Err(original_parse_error) => match lines.as_slice() {
[first, .., last]
if matches!(*first, "<<EOF" | "<<'EOF'" | "<<\"EOF\"")
&& last.ends_with("EOF")
&& lines.len() >= 4 =>
{
let inner_lines = &lines[1..lines.len() - 1];
check_patch_boundaries(inner_lines)?
}
_ => return Err(original_parse_error),
},
};
Ok(patch_lines.join("\n"))
}
fn check_patch_boundaries<'a>(lines: &'a [&'a str]) -> Result<&'a [&'a str], ParseError> {
let (first_line, last_line) = match lines {
[] => (None, None),
[first] => (Some(first), Some(first)),
[first, .., last] => (Some(first), Some(last)),
};
check_start_and_end_lines(first_line, last_line)?;
Ok(lines)
}
fn check_start_and_end_lines(
first_line: Option<&&str>,
last_line: Option<&&str>,
) -> Result<(), ParseError> {
let first_line = first_line.map(|line| line.trim());
let last_line = last_line.map(|line| line.trim());
match (first_line, last_line) {
(Some(first), Some(last)) if first == BEGIN_PATCH_MARKER && last == END_PATCH_MARKER => {
Ok(())
}
(Some(first), _) if first != BEGIN_PATCH_MARKER => Err(InvalidPatchError(String::from(
"The first line of the patch must be '*** Begin Patch'",
))),
_ => Err(InvalidPatchError(String::from(
"The last line of the patch must be '*** End Patch'",
))),
}
}
#[derive(Debug, Default, Clone)]
pub struct StreamingPatchParser {
line_buffer: String,
@@ -428,36 +366,6 @@ mod tests {
use std::path::PathBuf;
use super::*;
use crate::parser;
#[test]
fn test_parse_patch_matches_legacy_parser_for_complete_input() {
let patch = "\
*** Begin Patch
*** Add File: added.txt
+hello
*** Update File: changed.txt
@@
-old
+new
*** Delete File: removed.txt
*** End Patch";
assert_eq!(parse_patch(patch), parser::parse_patch(patch));
}
#[test]
fn test_parse_patch_supports_lenient_heredoc_wrapper() {
let patch = "\
<<'EOF'
*** Begin Patch
*** Add File: added.txt
+hello
*** End Patch
EOF";
assert_eq!(parse_patch(patch), parser::parse_patch(patch));
}
#[test]
fn test_streaming_patch_parser_streams_complete_lines_before_end_patch() {