From 537e69ab663244e021e2c6400dc11a044ff319a2 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 18 Jul 2026 17:56:24 +0000 Subject: [PATCH] Render streamed Markdown incrementally (#34045) ## Why Streaming TUI responses repeatedly rendered the entire accumulated Markdown source for each committed delta, including blocks whose output could no longer change. ## What changed - Retain rendered output for completed top-level Markdown blocks and rerender only the final mutable block as new source arrives. - Track block boundaries during the existing Markdown parser pass and map them through table-fence normalization. - Preserve canonical full rendering when source-wide state may affect earlier output, including reference link definitions and inline visualizations, and when the width or render mode changes. - Append raw-mode lines directly while preserving blank lines. ## Testing Added coverage that compares incremental output with full rendering across paragraphs, headings, lists, block quotes, code fences, tables, HTML blocks, links, visualizations, render-mode changes, and normalized table fences. GitOrigin-RevId: 748171c4ed58923329b59c48ea3c2afa4819ec88 --- codex-rs/tui/src/inline_visualization.rs | 2 +- codex-rs/tui/src/markdown.rs | 28 ++ codex-rs/tui/src/markdown_render.rs | 4 + codex-rs/tui/src/markdown_render/streaming.rs | 92 +++++ codex-rs/tui/src/streaming/controller.rs | 123 ++++--- codex-rs/tui/src/streaming/mod.rs | 1 + codex-rs/tui/src/streaming/render.rs | 191 ++++++++++ codex-rs/tui/src/streaming/render_tests.rs | 333 ++++++++++++++++++ ...remental_render_representative_stream.snap | 127 +++++++ ...out_context_use_canonical_full_render.snap | 18 + 10 files changed, 874 insertions(+), 45 deletions(-) create mode 100644 codex-rs/tui/src/markdown_render/streaming.rs create mode 100644 codex-rs/tui/src/streaming/render.rs create mode 100644 codex-rs/tui/src/streaming/render_tests.rs create mode 100644 codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__incremental_render_representative_stream.snap create mode 100644 codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__inline_visualizations_without_context_use_canonical_full_render.snap diff --git a/codex-rs/tui/src/inline_visualization.rs b/codex-rs/tui/src/inline_visualization.rs index cbe07eebc7..7fd4cbc1a0 100644 --- a/codex-rs/tui/src/inline_visualization.rs +++ b/codex-rs/tui/src/inline_visualization.rs @@ -24,7 +24,7 @@ use uuid::Uuid; use self::viewer::materialize_document; -const DIRECTIVE_PREFIX: &str = "::codex-inline-vis{"; +pub(crate) const DIRECTIVE_PREFIX: &str = "::codex-inline-vis{"; const MAX_FRAGMENT_BYTES: u64 = 2 * 1024 * 1024; #[derive(Clone, Debug)] diff --git a/codex-rs/tui/src/markdown.rs b/codex-rs/tui/src/markdown.rs index b244af83f3..eb687457ca 100644 --- a/codex-rs/tui/src/markdown.rs +++ b/codex-rs/tui/src/markdown.rs @@ -106,6 +106,34 @@ pub(crate) fn render_markdown_agent_with_links_cwd_and_visualizations( lines } +/// Render an agent message and collect the block metadata needed for incremental rendering. +/// +/// Block offsets are mapped back to `markdown_source` after Markdown table fences are unwrapped. +/// If a normalized boundary cannot be expressed as a raw-source suffix, it is discarded so the +/// transformed block remains mutable. +pub(crate) fn render_streaming_markdown_agent_with_links_and_cwd( + markdown_source: &str, + width: Option, + cwd: Option<&Path>, +) -> crate::markdown_render::StreamingMarkdownRender { + let normalized = unwrap_markdown_fences(markdown_source); + let mut rendered = crate::markdown_render::render_streaming_markdown_lines_with_width_and_cwd( + &normalized, + width, + cwd, + ); + if normalized != markdown_source { + // Fence unwrapping removes opening/closing lines. A normalized tail that is still a raw + // suffix necessarily begins after those removed lines, so its boundary can safely be + // mapped back to the raw source; otherwise leave the transformed block mutable. + rendered.last_top_level_block_start = rendered + .last_top_level_block_start + .and_then(|boundary| markdown_source.strip_suffix(&normalized[boundary..])) + .map(str::len); + } + rendered +} + /// Strip `` ```md ``/`` ```markdown `` fences that contain tables, emitting their content as bare /// markdown so `pulldown-cmark` parses the tables natively. /// diff --git a/codex-rs/tui/src/markdown_render.rs b/codex-rs/tui/src/markdown_render.rs index 885e139811..a1a976f61c 100644 --- a/codex-rs/tui/src/markdown_render.rs +++ b/codex-rs/tui/src/markdown_render.rs @@ -77,8 +77,12 @@ use unicode_width::UnicodeWidthChar; use unicode_width::UnicodeWidthStr; use url::Url; +mod streaming; mod table_key_value; +pub(crate) use streaming::StreamingMarkdownRender; +pub(crate) use streaming::render_streaming_markdown_lines_with_width_and_cwd; + const TABLE_COLUMN_GAP: usize = 2; const TABLE_CELL_PADDING: usize = 1; const TABLE_HEADER_SEPARATOR_CHAR: char = '━'; diff --git a/codex-rs/tui/src/markdown_render/streaming.rs b/codex-rs/tui/src/markdown_render/streaming.rs new file mode 100644 index 0000000000..a4657c41ae --- /dev/null +++ b/codex-rs/tui/src/markdown_render/streaming.rs @@ -0,0 +1,92 @@ +//! Streaming markdown render metadata collected during the writer's single parse pass. +//! +//! Top-level block offsets always refer to the exact source passed to this renderer; callers that +//! normalize source before rendering must not apply those offsets to the original source. + +use super::DecodedTextMerge; +use super::Event; +use super::HyperlinkLine; +use super::Options; +use super::Parser; +use super::Tag; +use super::Writer; +use super::never_hide_link_destination; +use std::ops::Range; +use std::path::Path; + +/// Rendered lines and the block metadata needed to keep only the final block mutable. +pub(crate) struct StreamingMarkdownRender { + /// Styled output produced by the same parser pass that collected the metadata below. + pub(crate) lines: Vec, + /// Byte offset of the final top-level block when at least one earlier block exists. + pub(crate) last_top_level_block_start: Option, + /// Whether a reference definition can retroactively change another block's rendering. + pub(crate) has_reference_link_definition: bool, + /// Whether the first block is raw HTML, which joins a retained prefix without a separator. + pub(crate) first_top_level_block_is_html: bool, +} + +/// Render `input` while tracking the final mutable top-level block. +/// +/// Every reported byte offset indexes the exact `input` passed here. Callers that transform source +/// before rendering must map the offset back to their original source before retaining a prefix. +pub(crate) fn render_streaming_markdown_lines_with_width_and_cwd( + input: &str, + width: Option, + cwd: Option<&Path>, +) -> StreamingMarkdownRender { + let mut options = Options::empty(); + options.insert(Options::ENABLE_STRIKETHROUGH); + options.insert(Options::ENABLE_TABLES); + let parser = Parser::new_ext(input, options); + let has_reference_link_definition = parser.reference_definitions().iter().next().is_some(); + let parser = TopLevelBlockTracker { + iter: DecodedTextMerge::new(parser.into_offset_iter()), + depth: 0, + block_count: 0, + last_start: 0, + first_is_html: false, + }; + let mut writer = Writer::new(input, parser, width, cwd, &never_hide_link_destination); + writer.run(); + StreamingMarkdownRender { + lines: writer.text, + last_top_level_block_start: (writer.iter.block_count > 1).then_some(writer.iter.last_start), + has_reference_link_definition, + first_top_level_block_is_html: writer.iter.first_is_html, + } +} + +/// Records top-level block boundaries without adding a second parser traversal. +struct TopLevelBlockTracker { + iter: I, + depth: usize, + block_count: usize, + last_start: usize, + first_is_html: bool, +} + +impl<'a, I> Iterator for TopLevelBlockTracker +where + I: Iterator, Range)>, +{ + type Item = (Event<'a>, Range); + + fn next(&mut self) -> Option { + let (event, range) = self.iter.next()?; + if self.depth == 0 && matches!(&event, Event::Start(_) | Event::Rule | Event::Html(_)) { + self.block_count += 1; + self.last_start = range.start; + if self.block_count == 1 { + self.first_is_html = + matches!(&event, Event::Start(Tag::HtmlBlock) | Event::Html(_)); + } + } + match event { + Event::Start(_) => self.depth += 1, + Event::End(_) => self.depth = self.depth.saturating_sub(1), + _ => {} + } + Some((event, range)) + } +} diff --git a/codex-rs/tui/src/streaming/controller.rs b/codex-rs/tui/src/streaming/controller.rs index 8bff317be6..3ca3633d70 100644 --- a/codex-rs/tui/src/streaming/controller.rs +++ b/codex-rs/tui/src/streaming/controller.rs @@ -29,7 +29,7 @@ //! //! ## Invariants //! -//! - `emitted_stable_len <= enqueued_stable_len <= rendered_lines.len()`. +//! - `emitted_stable_len <= enqueued_stable_len <= render.lines.len()`. //! - `raw_source` is append-only until `reset()`; never modified mid-stream. //! - Tail starts exactly at `enqueued_stable_len`. //! - During confirmed table streaming, only lines from the table header onward @@ -37,13 +37,11 @@ use crate::history_cell::HistoryCell; use crate::history_cell::HistoryRenderMode; -use crate::history_cell::raw_lines_from_source; use crate::history_cell::{self}; use crate::inline_visualization::InlineVisualizationContext; use crate::markdown::render_markdown_agent_with_links_cwd_and_visualizations; use crate::style::proposed_plan_style; use crate::terminal_hyperlinks::HyperlinkLine; -use crate::terminal_hyperlinks::plain_hyperlink_lines; use crate::terminal_hyperlinks::prefix_hyperlink_lines; use ratatui::prelude::Stylize; use ratatui::text::Line; @@ -53,6 +51,8 @@ use std::time::Duration; use std::time::Instant; use super::StreamState; +use super::render::StreamingRender; +use super::render::render_source; use super::table_holdback::TableHoldbackScanner; use super::table_holdback::TableHoldbackState; #[cfg(test)] @@ -77,8 +77,8 @@ struct StreamCore { width: Option, /// Accumulated raw markdown source for the current stream. raw_source: String, - /// Full re-render of `raw_source` at `width`. Rebuilt on every committed delta. - rendered_lines: Vec, + /// Incremental render of `raw_source` at `width`. + render: StreamingRender, /// Lines enqueued into the commit-animation queue. enqueued_stable_len: usize, /// Lines actually emitted to scrollback. @@ -116,7 +116,7 @@ impl StreamCore { state: StreamState::new(width, cwd), width, raw_source: String::with_capacity(1024), - rendered_lines: Vec::with_capacity(64), + render: StreamingRender::new(), enqueued_stable_len: 0, emitted_stable_len: 0, cwd: cwd.to_path_buf(), @@ -146,7 +146,14 @@ impl StreamCore { { self.raw_source.push_str(&committed_source); self.holdback_scanner.push_source_chunk(&committed_source); - self.recompute_streaming_render(); + self.render.append( + &self.raw_source, + &committed_source, + self.width, + self.cwd.as_path(), + self.render_mode, + self.inline_visualization_context.as_ref(), + ); enqueued = self.sync_stable_queue(); } enqueued @@ -165,7 +172,13 @@ impl StreamCore { self.raw_source.push_str(&remainder_source); self.holdback_scanner.push_source_chunk(&remainder_source); } - let rendered = self.render_source(&self.raw_source); + let rendered = render_source( + &self.raw_source, + self.width, + self.cwd.as_path(), + self.render_mode, + self.inline_visualization_context.as_ref(), + ); if self.emitted_stable_len >= rendered.len() { Vec::new() } else { @@ -220,13 +233,13 @@ impl StreamCore { /// reappear in the active cell and duplicate content on screen. #[inline] fn current_tail_lines(&self) -> Vec { - let start = self.enqueued_stable_len.min(self.rendered_lines.len()); - self.rendered_lines[start..].to_vec() + let start = self.enqueued_stable_len.min(self.render.lines.len()); + self.render.lines[start..].to_vec() } #[inline] fn has_tail(&self) -> bool { - self.enqueued_stable_len < self.rendered_lines.len() + self.enqueued_stable_len < self.render.lines.len() } /// Update rendering width and rebuild queued stable lines for the new layout. @@ -250,10 +263,16 @@ impl StreamCore { return; } - self.recompute_streaming_render(); - self.emitted_stable_len = self.emitted_stable_len.min(self.rendered_lines.len()); + self.render.recompute( + &self.raw_source, + self.width, + self.cwd.as_path(), + self.render_mode, + self.inline_visualization_context.as_ref(), + ); + self.emitted_stable_len = self.emitted_stable_len.min(self.render.lines.len()); if had_pending_queue - && self.emitted_stable_len == self.rendered_lines.len() + && self.emitted_stable_len == self.render.lines.len() && self.emitted_stable_len > 0 { // If wrapped remainder compresses into fewer lines at the new width, @@ -266,7 +285,7 @@ impl StreamCore { // Avoid replaying already-emitted content after resize when no // stable lines were waiting in the queue and there was no mutable // tail to preserve. - self.enqueued_stable_len = self.rendered_lines.len(); + self.enqueued_stable_len = self.render.lines.len(); return; } self.rebuild_stable_queue_from_render(); @@ -276,29 +295,13 @@ impl StreamCore { fn reset(&mut self) { self.state.clear(); self.raw_source.clear(); - self.rendered_lines.clear(); + self.render.clear(); self.enqueued_stable_len = 0; self.emitted_stable_len = 0; self.stable_prefix_len_cache = None; self.holdback_scanner.reset(); } - fn render_source(&self, source: &str) -> Vec { - match self.render_mode { - HistoryRenderMode::Rich => render_markdown_agent_with_links_cwd_and_visualizations( - source, - self.width, - Some(self.cwd.as_path()), - self.inline_visualization_context.as_ref(), - ), - HistoryRenderMode::Raw => plain_hyperlink_lines(raw_lines_from_source(source)), - } - } - - fn recompute_streaming_render(&mut self) { - self.rendered_lines = self.render_source(&self.raw_source); - } - fn set_render_mode(&mut self, render_mode: HistoryRenderMode) { if self.render_mode == render_mode { return; @@ -311,17 +314,23 @@ impl StreamCore { return; } - self.recompute_streaming_render(); - self.emitted_stable_len = self.emitted_stable_len.min(self.rendered_lines.len()); + self.render.recompute( + &self.raw_source, + self.width, + self.cwd.as_path(), + self.render_mode, + self.inline_visualization_context.as_ref(), + ); + self.emitted_stable_len = self.emitted_stable_len.min(self.render.lines.len()); if had_pending_queue - && self.emitted_stable_len == self.rendered_lines.len() + && self.emitted_stable_len == self.render.lines.len() && self.emitted_stable_len > 0 { self.emitted_stable_len -= 1; } self.state.clear_queue(); if self.emitted_stable_len > 0 && !had_pending_queue && !had_live_tail { - self.enqueued_stable_len = self.rendered_lines.len(); + self.enqueued_stable_len = self.render.lines.len(); return; } self.rebuild_stable_queue_from_render(); @@ -330,7 +339,8 @@ impl StreamCore { /// Compute how many rendered lines should be in the stable region. fn compute_target_stable_len(&mut self) -> usize { let tail_budget = self.active_tail_budget_lines(); - self.rendered_lines + self.render + .lines .len() .saturating_sub(tail_budget) .max(self.emitted_stable_len) @@ -347,7 +357,7 @@ impl StreamCore { self.state.clear_queue(); if self.emitted_stable_len < target_stable_len { self.state.enqueue( - self.rendered_lines[self.emitted_stable_len..target_stable_len].to_vec(), + self.render.lines[self.emitted_stable_len..target_stable_len].to_vec(), ); } self.enqueued_stable_len = target_stable_len; @@ -359,7 +369,7 @@ impl StreamCore { } self.state - .enqueue(self.rendered_lines[self.enqueued_stable_len..target_stable_len].to_vec()); + .enqueue(self.render.lines[self.enqueued_stable_len..target_stable_len].to_vec()); self.enqueued_stable_len = target_stable_len; true } @@ -374,7 +384,7 @@ impl StreamCore { self.state.clear_queue(); if self.emitted_stable_len < target_stable_len { self.state - .enqueue(self.rendered_lines[self.emitted_stable_len..target_stable_len].to_vec()); + .enqueue(self.render.lines[self.emitted_stable_len..target_stable_len].to_vec()); } self.enqueued_stable_len = target_stable_len; } @@ -416,11 +426,11 @@ impl StreamCore { /// the only place where those coordinate systems are bridged. fn tail_budget_from_source_start(&mut self, source_start: usize) -> usize { if source_start == 0 { - return self.rendered_lines.len(); + return self.render.lines.len(); } let source_start = source_start.min(self.raw_source.len()); let stable_prefix_len = self.stable_prefix_len_for_source_start(source_start); - self.rendered_lines.len().saturating_sub(stable_prefix_len) + self.render.lines.len().saturating_sub(stable_prefix_len) } /// Render the stable prefix before `source_start` and return its line count. @@ -816,6 +826,31 @@ mod tests { .collect() } + #[test] + fn queued_heading_is_emitted_once_after_incremental_append() { + let mut ctrl = stream_controller(Some(80)); + assert!(ctrl.push("Paragraph.\n\n# Heading\n\n")); + ctrl.push("Next paragraph.\n"); + + let (cell, _) = ctrl.on_commit_tick_batch(usize::MAX); + let mut streamed = cell + .into_iter() + .flat_map(|cell| cell.transcript_lines(u16::MAX)) + .collect::>(); + if let (Some(cell), _source) = ctrl.finalize() { + streamed.extend(cell.transcript_lines(u16::MAX)); + } + let streamed = lines_to_plain_strings(&streamed); + assert_eq!( + streamed + .iter() + .filter(|line| line.contains("# Heading")) + .count(), + 1, + "expected the streamed heading to be emitted once: {streamed:?}", + ); + } + fn collect_plan_streamed_lines(deltas: &[&str], width: Option) -> Vec { let mut ctrl = plan_stream_controller(width); let mut lines = Vec::new(); @@ -896,7 +931,7 @@ mod tests { let mut ctrl = stream_controller(Some(80)); assert!(!ctrl.has_live_tail()); - ctrl.core.rendered_lines = vec![Line::from("tail line").into()]; + ctrl.core.render.lines = vec![Line::from("tail line").into()]; ctrl.core.enqueued_stable_len = 0; assert!(ctrl.has_live_tail()); @@ -909,7 +944,7 @@ mod tests { let mut ctrl = plan_stream_controller(Some(80)); assert!(!ctrl.has_live_tail()); - ctrl.core.rendered_lines = vec![Line::from("tail line").into()]; + ctrl.core.render.lines = vec![Line::from("tail line").into()]; ctrl.core.enqueued_stable_len = 0; assert!(ctrl.has_live_tail()); diff --git a/codex-rs/tui/src/streaming/mod.rs b/codex-rs/tui/src/streaming/mod.rs index 576277aa3e..804e9a3bdf 100644 --- a/codex-rs/tui/src/streaming/mod.rs +++ b/codex-rs/tui/src/streaming/mod.rs @@ -19,6 +19,7 @@ use crate::terminal_hyperlinks::HyperlinkLine; pub(crate) mod chunking; pub(crate) mod commit_tick; pub(crate) mod controller; +mod render; mod table_holdback; struct QueuedLine { diff --git a/codex-rs/tui/src/streaming/render.rs b/codex-rs/tui/src/streaming/render.rs new file mode 100644 index 0000000000..3e889c9478 --- /dev/null +++ b/codex-rs/tui/src/streaming/render.rs @@ -0,0 +1,191 @@ +//! Incremental markdown rendering for in-flight transcript streams. +//! +//! Completed top-level blocks are retained while the final block stays mutable, avoiding repeated +//! rendering of the stable prefix as newline-bearing deltas arrive. + +use crate::history_cell::HistoryRenderMode; +use crate::history_cell::raw_lines_from_source; +use crate::inline_visualization::DIRECTIVE_PREFIX; +use crate::inline_visualization::InlineVisualizationContext; +use crate::markdown::render_markdown_agent_with_links_cwd_and_visualizations; +use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd; +use crate::terminal_hyperlinks::HyperlinkLine; +use crate::terminal_hyperlinks::plain_hyperlink_lines; +use ratatui::text::Line; +use std::path::Path; + +/// Incremental render state split at source and rendered-line boundaries. +/// +/// The prefix before both boundaries is immutable; only the final top-level Markdown block is +/// re-rendered as committed source arrives. +pub(super) struct StreamingRender { + pub(super) lines: Vec, + /// Source prefix containing only completed top-level markdown blocks. + stable_source_len: usize, + /// Rendered-line boundary corresponding to `stable_source_len`. + stable_rendered_len: usize, + /// Reference-style link definitions can affect any earlier or later markdown block. + has_reference_link_definition: bool, +} + +impl StreamingRender { + pub(super) fn new() -> Self { + Self { + lines: Vec::with_capacity(64), + stable_source_len: 0, + stable_rendered_len: 0, + has_reference_link_definition: false, + } + } + + pub(super) fn clear(&mut self) { + self.lines.clear(); + self.stable_source_len = 0; + self.stable_rendered_len = 0; + self.has_reference_link_definition = false; + } + + /// Re-render the full source and reset both stable-prefix boundaries. + /// + /// This is used when width or render mode changes, and whenever source-wide rendering state + /// makes retaining previously rendered blocks unsafe. + pub(super) fn recompute( + &mut self, + source: &str, + width: Option, + cwd: &Path, + render_mode: HistoryRenderMode, + inline_visualization_context: Option<&InlineVisualizationContext>, + ) { + self.lines = match (render_mode, inline_visualization_context) { + (HistoryRenderMode::Rich, None) if !source.contains(DIRECTIVE_PREFIX) => { + let rendered = + render_streaming_markdown_agent_with_links_and_cwd(source, width, Some(cwd)); + self.has_reference_link_definition = rendered.has_reference_link_definition; + rendered.lines + } + _ => { + self.has_reference_link_definition = false; + render_source( + source, + width, + cwd, + render_mode, + inline_visualization_context, + ) + } + }; + self.stable_source_len = 0; + self.stable_rendered_len = 0; + } + + /// Append newly committed source while retaining only the final markdown block as mutable. + /// + /// The final top-level block can still change meaning when another line arrives (for example, + /// list tightness, a setext heading, a fenced block, or a table). Earlier top-level blocks are + /// rendered once and retained. Reference-style link definitions and inline visualization + /// rewriting fall back to a full render because they can affect source-wide rendering state. + pub(super) fn append( + &mut self, + raw_source: &str, + committed_source: &str, + width: Option, + cwd: &Path, + render_mode: HistoryRenderMode, + inline_visualization_context: Option<&InlineVisualizationContext>, + ) { + if render_mode == HistoryRenderMode::Raw { + self.lines + .extend(plain_hyperlink_lines(raw_lines_from_source( + committed_source, + ))); + return; + } + + if inline_visualization_context.is_some() || raw_source.contains(DIRECTIVE_PREFIX) { + self.recompute( + raw_source, + width, + cwd, + render_mode, + inline_visualization_context, + ); + return; + } + + if self.has_reference_link_definition { + self.recompute( + raw_source, + width, + cwd, + render_mode, + inline_visualization_context, + ); + return; + } + + let pending_source = &raw_source[self.stable_source_len..]; + let pending = + render_streaming_markdown_agent_with_links_and_cwd(pending_source, width, Some(cwd)); + if pending.has_reference_link_definition { + self.has_reference_link_definition = true; + self.recompute( + raw_source, + width, + cwd, + render_mode, + inline_visualization_context, + ); + return; + } + + let mut newly_stable_rendered_len = None; + if let Some(boundary) = pending.last_top_level_block_start { + let newly_stable_source = &pending_source[..boundary]; + let newly_stable = render_source( + newly_stable_source, + width, + cwd, + render_mode, + inline_visualization_context, + ); + self.stable_source_len += boundary; + newly_stable_rendered_len = Some(newly_stable.len()); + } + + self.lines.truncate(self.stable_rendered_len); + if !self.lines.is_empty() + && (!pending.lines.is_empty() || !pending_source.trim().is_empty()) + && !pending.first_top_level_block_is_html + { + self.lines.push(HyperlinkLine::new(Line::default())); + } + let pending_render_start = self.lines.len(); + self.lines.extend(pending.lines); + if let Some(newly_stable_rendered_len) = newly_stable_rendered_len { + self.stable_rendered_len = pending_render_start + newly_stable_rendered_len; + } + } +} + +pub(super) fn render_source( + source: &str, + width: Option, + cwd: &Path, + render_mode: HistoryRenderMode, + inline_visualization_context: Option<&InlineVisualizationContext>, +) -> Vec { + match render_mode { + HistoryRenderMode::Rich => render_markdown_agent_with_links_cwd_and_visualizations( + source, + width, + Some(cwd), + inline_visualization_context, + ), + HistoryRenderMode::Raw => plain_hyperlink_lines(raw_lines_from_source(source)), + } +} + +#[cfg(test)] +#[path = "render_tests.rs"] +mod tests; diff --git a/codex-rs/tui/src/streaming/render_tests.rs b/codex-rs/tui/src/streaming/render_tests.rs new file mode 100644 index 0000000000..3b699759ea --- /dev/null +++ b/codex-rs/tui/src/streaming/render_tests.rs @@ -0,0 +1,333 @@ +use super::StreamingRender; +use super::render_source; +use crate::history_cell::HistoryRenderMode; +use crate::inline_visualization::InlineVisualizationContext; +use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd; +use crate::terminal_hyperlinks::HyperlinkLine; +use crate::terminal_hyperlinks::visible_lines; +use codex_protocol::ThreadId; +use insta::assert_debug_snapshot; +use pretty_assertions::assert_eq; +use std::path::Path; +use std::path::PathBuf; + +fn test_cwd() -> PathBuf { + std::env::temp_dir() +} + +fn append( + render: &mut StreamingRender, + source: &mut String, + chunk: &str, + width: Option, + cwd: &Path, + render_mode: HistoryRenderMode, +) { + source.push_str(chunk); + render.append( + source, + chunk, + width, + cwd, + render_mode, + /*inline_visualization_context*/ None, + ); +} + +fn append_rich_and_assert_matches_full( + render: &mut StreamingRender, + source: &mut String, + chunk: &str, + width: Option, + cwd: &Path, +) { + append(render, source, chunk, width, cwd, HistoryRenderMode::Rich); + assert_eq!( + render.lines, + render_source( + source, + width, + cwd, + HistoryRenderMode::Rich, + /*inline_visualization_context*/ None, + ), + "incremental render diverged after chunk {chunk:?}", + ); +} + +fn lines_to_plain_strings(lines: &[HyperlinkLine]) -> Vec { + visible_lines(lines.to_vec()) + .iter() + .map(|line| { + line.spans + .iter() + .map(|span| span.content.as_ref()) + .collect::() + }) + .collect() +} + +fn assert_rich_stream_matches_full_render( + chunks: &[&str], + width: Option, +) -> (String, StreamingRender) { + let cwd = test_cwd(); + let mut source = String::new(); + let mut render = StreamingRender::new(); + + for chunk in chunks { + append_rich_and_assert_matches_full(&mut render, &mut source, chunk, width, &cwd); + } + + (source, render) +} + +#[test] +fn incremental_render_keeps_final_block_mutable_and_matches_full_render() { + let chunks = [ + "# Heading\n", + "\n", + "First paragraph with a [link](https://example.com).\n", + "continued on the next line.\n\n", + "1. First item\n", + "2. Second item\n\n", + "> Quoted paragraph\n\n", + "```rust\n", + "fn main() {}\n", + "```\n\n", + "| Key | Value |\n", + "| --- | --- |\n", + "| alpha | beta |\n", + ]; + let (source, render) = assert_rich_stream_matches_full_render(&chunks, Some(48)); + + assert!(render.stable_source_len > 0); + assert!(render.stable_source_len < source.len()); + assert_debug_snapshot!("incremental_render_representative_stream", render.lines); +} + +#[test] +fn growing_single_top_level_blocks_render_and_scan_in_one_pass() { + let streams: &[&[&str]] = &[ + &[ + "A paragraph that keeps growing\n", + "without a blank line between chunks.\n", + "It stays one top-level block.\n", + ], + &[ + "| Key | Value |\n", + "| --- | --- |\n", + "| alpha | beta |\n", + "| gamma | delta |\n", + ], + ]; + let cwd = test_cwd(); + let width = Some(80); + + for chunks in streams { + let mut source = String::new(); + let mut render = StreamingRender::new(); + for chunk in *chunks { + append_rich_and_assert_matches_full(&mut render, &mut source, chunk, width, &cwd); + let pending = render_streaming_markdown_agent_with_links_and_cwd( + &source, + width, + Some(cwd.as_path()), + ); + assert_eq!(pending.last_top_level_block_start, None); + assert_eq!(render.lines, pending.lines); + assert_eq!(render.stable_source_len, 0); + } + } +} + +#[test] +fn incremental_raw_render_preserves_blank_lines() { + let cwd = test_cwd(); + let width = Some(80); + let mut source = String::new(); + let mut render = StreamingRender::new(); + + for chunk in ["alpha\n", "\n", "beta\n", "\n"] { + append( + &mut render, + &mut source, + chunk, + width, + &cwd, + HistoryRenderMode::Raw, + ); + } + + assert_eq!( + lines_to_plain_strings(&render.lines), + vec!["alpha", "", "beta", ""], + ); +} + +#[test] +fn inline_visualizations_use_canonical_full_render() { + let cwd = test_cwd(); + let context = InlineVisualizationContext::new(&cwd, ThreadId::new()) + .expect("UUIDv7 thread id should provide a timestamp"); + let width = Some(80); + let mut source = String::new(); + let mut render = StreamingRender::new(); + + for chunk in ["Before.\n\n", "::codex-inline-vis{file=\"missing.html\"}\n"] { + source.push_str(chunk); + render.append( + &source, + chunk, + width, + &cwd, + HistoryRenderMode::Rich, + Some(&context), + ); + assert_eq!( + render.lines, + render_source( + &source, + width, + &cwd, + HistoryRenderMode::Rich, + Some(&context), + ), + ); + assert_eq!(render.stable_source_len, 0); + } +} + +#[test] +fn inline_visualizations_without_context_use_canonical_full_render() { + let (_, render) = assert_rich_stream_matches_full_render( + &["Before.\n\n", "::codex-inline-vis{file=\"missing.html\"}\n"], + Some(80), + ); + + assert_eq!(render.stable_source_len, 0); + assert_debug_snapshot!( + "inline_visualizations_without_context_use_canonical_full_render", + render.lines + ); +} + +#[test] +fn reference_link_definition_recomputes_earlier_and_later_blocks() { + let streams: &[&[&str]] = &[ + &[ + "Earlier [reference][id].\n\n", + "An unrelated paragraph.\n\n", + "[id]: https://example.com/reference\n", + "\n", + "Later [reference][id].\n", + ], + &[ + "Earlier [reference][id].\n\n", + "Another paragraph.\n\n", + "```markdown\n", + "| Key | Value |\n", + "| --- | --- |\n", + "| alpha | beta |\n", + "\n", + "[id]: https://example.com/reference\n", + "```\n", + ], + ]; + for chunks in streams { + let (_, render) = assert_rich_stream_matches_full_render(chunks, Some(80)); + assert!(render.has_reference_link_definition); + } +} + +#[test] +fn reference_link_definition_survives_raw_to_rich_render_mode_switch() { + let cwd = test_cwd(); + let width = Some(80); + let mut source = String::new(); + let mut render = StreamingRender::new(); + + append( + &mut render, + &mut source, + "[id]: https://example.com/reference\n\n", + width, + &cwd, + HistoryRenderMode::Raw, + ); + render.recompute( + &source, + width, + &cwd, + HistoryRenderMode::Rich, + /*inline_visualization_context*/ None, + ); + for chunk in ["First [reference][id].\n\n", "Later [reference][id].\n"] { + append_rich_and_assert_matches_full(&mut render, &mut source, chunk, width, &cwd); + } + + assert!(render.has_reference_link_definition); +} + +#[test] +fn incremental_render_does_not_add_blank_line_before_html_block() { + let streams: &[&[&str]] = &[ + &[ + "Paragraph.\n\n", + "
x
\n\n", + "Following paragraph.\n", + ], + &["Paragraph.\n\n
x
\n\n", "Following paragraph.\n"], + ]; + for chunks in streams { + let (_, render) = assert_rich_stream_matches_full_render(chunks, Some(80)); + assert_eq!( + lines_to_plain_strings(&render.lines), + vec!["Paragraph.", "
x
", "", "Following paragraph.",], + ); + } +} + +#[test] +fn incremental_render_preserves_heading_and_normalized_html_seams() { + let streams: &[&[&str]] = &[ + &["Paragraph.\n\n# Heading\n\n", "Next paragraph.\n"], + &[ + "Paragraph.\n\n", + "
x
\n\n", + "```markdown\n", + "| Key | Value |\n", + "| --- | --- |\n", + "| alpha | beta |\n", + "\n", + "Following paragraph.\n", + "```\n", + ], + ]; + for chunks in streams { + assert_rich_stream_matches_full_render(chunks, Some(80)); + } +} + +#[test] +fn paragraphs_after_unwrapped_table_fence_advance_stable_source() { + let cwd = test_cwd(); + let width = Some(80); + let (mut source, mut render) = assert_rich_stream_matches_full_render( + &[ + "```markdown\n", + "| Key | Value |\n", + "| --- | --- |\n", + "| alpha | beta |\n", + "```\n\n", + ], + width, + ); + + let mut previous_stable_source_len = render.stable_source_len; + for block in ["
Post-fence HTML.
\n\n", "First paragraph.\n\n"] { + append_rich_and_assert_matches_full(&mut render, &mut source, block, width, &cwd); + assert!(render.stable_source_len > previous_stable_source_len); + previous_stable_source_len = render.stable_source_len; + } +} diff --git a/codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__incremental_render_representative_stream.snap b/codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__incremental_render_representative_stream.snap new file mode 100644 index 0000000000..c93d483c22 --- /dev/null +++ b/codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__incremental_render_representative_stream.snap @@ -0,0 +1,127 @@ +--- +source: tui/src/streaming/render_tests.rs +expression: render.lines +--- +[ + HyperlinkLine { + line: Line::from_iter([ + Span::from("# ").bold().underlined(), + Span::from("Heading").bold().underlined(), + ]), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::default(), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::from_iter([ + Span::from("First paragraph with a "), + Span::from("link"), + ]), + hyperlinks: [ + TerminalHyperlink { + columns: 23..27, + destination: "https://example.com", + destination_kind: Web, + }, + ], + }, + HyperlinkLine { + line: Line::from_iter([ + Span::from("("), + Span::from("https://example.com").cyan().underlined(), + Span::from(")"), + Span::from("."), + ]), + hyperlinks: [ + TerminalHyperlink { + columns: 1..20, + destination: "https://example.com", + destination_kind: Web, + }, + ], + }, + HyperlinkLine { + line: Line::from("continued on the next line."), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::default(), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::from_iter([ + Span::from("1. ").light_blue(), + Span::from("First item"), + ]), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::from_iter([ + Span::from("2. ").light_blue(), + Span::from("Second item"), + ]), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::default(), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::from_iter([ + Span::from("> "), + Span::from("Quoted paragraph"), + ]).green(), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::default(), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::from_iter([ + Span::default(), + Span::from("fn").fg(Color::Rgb(203, 166, 247)), + Span::from(" ").fg(Color::Rgb(205, 214, 244)), + Span::from("main").fg(Color::Rgb(137, 180, 250)), + Span::from("(").fg(Color::Rgb(147, 153, 178)), + Span::from(")").fg(Color::Rgb(147, 153, 178)), + Span::from(" ").fg(Color::Rgb(205, 214, 244)), + Span::from("{").fg(Color::Rgb(147, 153, 178)), + Span::from("}").fg(Color::Rgb(147, 153, 178)), + ]), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::default(), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::from_iter([ + Span::from(" "), + Span::from("Key"), + Span::from(" "), + Span::from(" "), + Span::from(" "), + Span::from(" "), + Span::from("Value"), + ]).fg(Color::Rgb(249, 226, 175)).bold(), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::from(Span::from("━━━━━━━ ━━━━━━━").dim()), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::from_iter([ + Span::from(" "), + Span::from("alpha"), + Span::from(" "), + Span::from(" "), + Span::from(" "), + Span::from("beta"), + ]), + hyperlinks: [], + }, +] diff --git a/codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__inline_visualizations_without_context_use_canonical_full_render.snap b/codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__inline_visualizations_without_context_use_canonical_full_render.snap new file mode 100644 index 0000000000..687e703b40 --- /dev/null +++ b/codex-rs/tui/src/streaming/snapshots/codex_tui__streaming__render__tests__inline_visualizations_without_context_use_canonical_full_render.snap @@ -0,0 +1,18 @@ +--- +source: tui/src/streaming/render_tests.rs +expression: render.lines +--- +[ + HyperlinkLine { + line: Line::from("Before."), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::default(), + hyperlinks: [], + }, + HyperlinkLine { + line: Line::from(Span::from("Visualization unavailable on this device.").italic()), + hyperlinks: [], + }, +]