From f1ee30daa385c5f7e0af8a5cc996f68918988959 Mon Sep 17 00:00:00 2001 From: Felipe Coury Date: Tue, 17 Feb 2026 16:54:22 -0300 Subject: [PATCH] tui: use source-mapped boundaries for stream resize remap --- codex-rs/tui/src/markdown.rs | 208 ++++++++++++++-- codex-rs/tui/src/markdown_render.rs | 66 ++++- codex-rs/tui/src/streaming/controller.rs | 301 ++++++++++++----------- 3 files changed, 400 insertions(+), 175 deletions(-) diff --git a/codex-rs/tui/src/markdown.rs b/codex-rs/tui/src/markdown.rs index f0c3ecba0f..3f23c59b56 100644 --- a/codex-rs/tui/src/markdown.rs +++ b/codex-rs/tui/src/markdown.rs @@ -20,10 +20,16 @@ //! info string is `md` or `markdown` AND whose body contains a //! header+delimiter pair, and degrades gracefully on unclosed fences. use ratatui::text::Line; +use std::ops::Range; use std::path::Path; use crate::table_detect; +pub(crate) struct AgentRenderedWithSourceMap { + pub(crate) lines: Vec>, + pub(crate) line_end_source_bytes: Vec, +} + /// Render markdown into `lines` while resolving local file-link display relative to `cwd`. /// /// Callers that already know the session working directory should pass it here so streamed and @@ -64,8 +70,39 @@ pub(crate) fn append_markdown_agent_with_cwd( cwd: Option<&Path>, lines: &mut Vec>, ) { - let normalized = unwrap_markdown_fences(markdown_source); - append_markdown(&normalized, width, cwd, lines); + let rendered = render_markdown_agent_with_source_map_with_cwd(markdown_source, width, cwd); + lines.extend(rendered.lines); +} + +#[cfg(test)] +pub(crate) fn render_markdown_agent_with_source_map( + markdown_source: &str, + width: Option, +) -> AgentRenderedWithSourceMap { + render_markdown_agent_with_source_map_with_cwd(markdown_source, width, /*cwd*/ None) +} + +pub(crate) fn render_markdown_agent_with_source_map_with_cwd( + markdown_source: &str, + width: Option, + cwd: Option<&Path>, +) -> AgentRenderedWithSourceMap { + let normalized = unwrap_markdown_fences_with_mapping(markdown_source); + let rendered = crate::markdown_render::render_markdown_text_with_width_and_cwd_and_source_map( + &normalized.text, + width, + cwd, + ); + let lines = rendered.text.lines; + let line_end_source_bytes = rendered + .line_end_input_bytes + .into_iter() + .map(|offset| normalized.raw_source_offset(offset)) + .collect(); + AgentRenderedWithSourceMap { + lines, + line_end_source_bytes, + } } /// Strip `` ```md ``/`` ```markdown `` fences that contain tables, emitting their content as bare @@ -79,7 +116,64 @@ pub(crate) fn append_markdown_agent_with_cwd( /// The fence unwrapping is intentionally conservative: it buffers the entire fence body before /// deciding, and an unclosed fence at end-of-input is re-emitted with its opening line so partial /// streams degrade to code display. +#[cfg(test)] fn unwrap_markdown_fences(markdown_source: &str) -> String { + unwrap_markdown_fences_with_mapping(markdown_source).text +} + +#[derive(Clone, Debug)] +struct NormalizedToRawSegment { + normalized_start: usize, + raw_start: usize, +} + +struct NormalizedMarkdown { + text: String, + raw_source_len: usize, + segments: Vec, +} + +impl NormalizedMarkdown { + fn new(raw_source_len: usize, capacity: usize) -> Self { + Self { + text: String::with_capacity(capacity), + raw_source_len, + segments: Vec::new(), + } + } + + fn push_source_range(&mut self, source: &str, range: Range) { + if range.is_empty() { + return; + } + let out_start = self.text.len(); + let out_text = &source[range.clone()]; + self.text.push_str(out_text); + self.segments.push(NormalizedToRawSegment { + normalized_start: out_start, + raw_start: range.start, + }); + } + + fn raw_source_offset(&self, normalized_offset: usize) -> usize { + if normalized_offset == 0 { + return 0; + } + if normalized_offset >= self.text.len() { + return self.raw_source_len; + } + let idx = self + .segments + .partition_point(|segment| segment.normalized_start < normalized_offset); + if idx == 0 { + return 0; + } + let segment = &self.segments[idx - 1]; + segment.raw_start + normalized_offset.saturating_sub(segment.normalized_start) + } +} + +fn unwrap_markdown_fences_with_mapping(markdown_source: &str) -> NormalizedMarkdown { #[derive(Clone, Copy)] struct Fence { marker: char, @@ -159,46 +253,66 @@ fn unwrap_markdown_fences(markdown_source: &str) -> String { false } + fn content_from_ranges(source: &str, ranges: &[Range]) -> String { + let mut content = String::new(); + for range in ranges { + content.push_str(&source[range.clone()]); + } + content + } + enum ActiveFence { Passthrough(Fence), MarkdownCandidate { fence: Fence, - opening_line: String, - content: String, + opening_range: Range, + content_ranges: Vec>, }, } - let mut out = String::with_capacity(markdown_source.len()); + let mut out = NormalizedMarkdown::new(markdown_source.len(), markdown_source.len()); let mut active_fence: Option = None; + let mut source_offset = 0usize; for line in markdown_source.split_inclusive('\n') { + let line_start = source_offset; + source_offset += line.len(); + let line_range = line_start..source_offset; + if let Some(active) = active_fence.take() { match active { ActiveFence::Passthrough(fence) => { - out.push_str(line); + out.push_source_range(markdown_source, line_range); if !is_close_fence(line, fence) { active_fence = Some(ActiveFence::Passthrough(fence)); } } ActiveFence::MarkdownCandidate { fence, - opening_line, - mut content, + opening_range, + mut content_ranges, } => { if is_close_fence(line, fence) { - if markdown_fence_contains_table(&content) { - out.push_str(&content); + if markdown_fence_contains_table(&content_from_ranges( + markdown_source, + &content_ranges, + )) { + for range in content_ranges { + out.push_source_range(markdown_source, range); + } } else { - out.push_str(&opening_line); - out.push_str(&content); - out.push_str(line); + out.push_source_range(markdown_source, opening_range); + for range in content_ranges { + out.push_source_range(markdown_source, range); + } + out.push_source_range(markdown_source, line_range); } } else { - content.push_str(line); + content_ranges.push(line_range); active_fence = Some(ActiveFence::MarkdownCandidate { fence, - opening_line, - content, + opening_range, + content_ranges, }); } } @@ -210,29 +324,31 @@ fn unwrap_markdown_fences(markdown_source: &str) -> String { if fence.is_markdown { active_fence = Some(ActiveFence::MarkdownCandidate { fence, - opening_line: line.to_string(), - content: String::new(), + opening_range: line_range, + content_ranges: Vec::new(), }); } else { - out.push_str(line); + out.push_source_range(markdown_source, line_range); active_fence = Some(ActiveFence::Passthrough(fence)); } continue; } - out.push_str(line); + out.push_source_range(markdown_source, line_range); } if let Some(active) = active_fence { match active { ActiveFence::Passthrough(_) => {} ActiveFence::MarkdownCandidate { - opening_line, - content, + opening_range, + content_ranges, .. } => { - out.push_str(&opening_line); - out.push_str(&content); + out.push_source_range(markdown_source, opening_range); + for range in content_ranges { + out.push_source_range(markdown_source, range); + } } } } @@ -421,4 +537,48 @@ mod tests { let normalized = unwrap_markdown_fences(src); assert_eq!(normalized, src); } + + #[test] + fn render_markdown_agent_with_source_map_offsets_align_with_line_count() { + let src = "alpha\n\n```md\n| A | B |\n|---|---|\n| 1 | 2 |\n```\nomega\n"; + let rendered = render_markdown_agent_with_source_map(src, Some(80)); + assert_eq!(rendered.lines.len(), rendered.line_end_source_bytes.len()); + + let mut prev = 0usize; + for offset in rendered.line_end_source_bytes { + assert!( + offset <= src.len(), + "line-end offset must stay within source bounds: {offset} > {}", + src.len() + ); + assert!( + offset >= prev, + "line-end offsets must be non-decreasing: {offset} < {prev}" + ); + prev = offset; + } + } + + #[test] + fn unwrap_markdown_fences_mapping_preserves_raw_offsets_across_removed_fence_lines() { + let src = "before\n```md\n| A | B |\n|---|---|\n| 1 | 2 |\n```\nafter\n"; + let normalized = unwrap_markdown_fences_with_mapping(src); + assert!( + !normalized.text.contains("```"), + "markdown table fence should be removed in normalized text" + ); + let after_in_normalized = normalized + .text + .find("after") + .expect("expected 'after' in normalized text"); + let after_in_raw = src.find("after").expect("expected 'after' in raw source"); + assert_eq!( + normalized.raw_source_offset(after_in_normalized + 1), + after_in_raw + 1 + ); + assert_eq!( + normalized.raw_source_offset(normalized.text.len()), + src.len() + ); + } } diff --git a/codex-rs/tui/src/markdown_render.rs b/codex-rs/tui/src/markdown_render.rs index ac4c6eb08f..e36f14aea0 100644 --- a/codex-rs/tui/src/markdown_render.rs +++ b/codex-rs/tui/src/markdown_render.rs @@ -59,6 +59,7 @@ use ratatui::text::Line; use ratatui::text::Span; use ratatui::text::Text; use regex_lite::Regex; +use std::ops::Range; use std::path::Path; use std::path::PathBuf; use std::sync::LazyLock; @@ -251,6 +252,11 @@ pub fn render_markdown_text(input: &str) -> Text<'static> { render_markdown_text_with_width(input, /*width*/ None) } +pub(crate) struct RenderedMarkdownWithSourceMap { + pub(crate) text: Text<'static>, + pub(crate) line_end_input_bytes: Vec, +} + /// Render markdown constrained to a known terminal width. /// /// The renderer preserves table structure when possible and falls back to @@ -273,13 +279,33 @@ pub(crate) fn render_markdown_text_with_width_and_cwd( width: Option, cwd: Option<&Path>, ) -> Text<'static> { + render_markdown_text_with_width_and_cwd_and_source_map(input, width, cwd).text +} + +#[cfg(test)] +pub(crate) fn render_markdown_text_with_width_and_source_map( + input: &str, + width: Option, +) -> RenderedMarkdownWithSourceMap { + let cwd = std::env::current_dir().ok(); + render_markdown_text_with_width_and_cwd_and_source_map(input, width, cwd.as_deref()) +} + +pub(crate) fn render_markdown_text_with_width_and_cwd_and_source_map( + input: &str, + width: Option, + cwd: Option<&Path>, +) -> RenderedMarkdownWithSourceMap { let mut options = Options::empty(); options.insert(Options::ENABLE_STRIKETHROUGH); options.insert(Options::ENABLE_TABLES); - let parser = Parser::new_ext(input, options); - let mut w = Writer::new(parser, width, cwd); + let parser = Parser::new_ext(input, options).into_offset_iter(); + let mut w = Writer::new(parser, width, input.len(), cwd); w.run(); - w.text + RenderedMarkdownWithSourceMap { + text: w.text, + line_end_input_bytes: w.line_end_input_bytes, + } } #[derive(Clone, Debug)] @@ -320,10 +346,13 @@ static HASH_LOCATION_SUFFIX_RE: LazyLock = /// allocation; when `None`, lines keep their intrinsic width. struct Writer<'a, I> where - I: Iterator>, + I: Iterator, Range)>, { iter: I, + source_len: usize, text: Text<'static>, + line_end_input_bytes: Vec, + current_event_end: usize, styles: MarkdownStyles, inline_styles: Vec