Render streamed Markdown incrementally (#34045)

## Why

Streaming TUI responses repeatedly rendered the entire accumulated Markdown source for each committed delta, including blocks whose output could no longer change.

## What changed

- Retain rendered output for completed top-level Markdown blocks and rerender only the final mutable block as new source arrives.
- Track block boundaries during the existing Markdown parser pass and map them through table-fence normalization.
- Preserve canonical full rendering when source-wide state may affect earlier output, including reference link definitions and inline visualizations, and when the width or render mode changes.
- Append raw-mode lines directly while preserving blank lines.

## Testing

Added coverage that compares incremental output with full rendering across paragraphs, headings, lists, block quotes, code fences, tables, HTML blocks, links, visualizations, render-mode changes, and normalized table fences.

GitOrigin-RevId: 748171c4ed58923329b59c48ea3c2afa4819ec88

codex-rs/tui/src/streaming/controller.rs

@@ -29,7 +29,7 @@
 //!
 //! ## Invariants
 //!
-//! - `emitted_stable_len <= enqueued_stable_len <= rendered_lines.len()`.
+//! - `emitted_stable_len <= enqueued_stable_len <= render.lines.len()`.
 //! - `raw_source` is append-only until `reset()`; never modified mid-stream.
 //! - Tail starts exactly at `enqueued_stable_len`.
 //! - During confirmed table streaming, only lines from the table header onward
@@ -37,13 +37,11 @@
 
 use crate::history_cell::HistoryCell;
 use crate::history_cell::HistoryRenderMode;
-use crate::history_cell::raw_lines_from_source;
 use crate::history_cell::{self};
 use crate::inline_visualization::InlineVisualizationContext;
 use crate::markdown::render_markdown_agent_with_links_cwd_and_visualizations;
 use crate::style::proposed_plan_style;
 use crate::terminal_hyperlinks::HyperlinkLine;
-use crate::terminal_hyperlinks::plain_hyperlink_lines;
 use crate::terminal_hyperlinks::prefix_hyperlink_lines;
 use ratatui::prelude::Stylize;
 use ratatui::text::Line;
@@ -53,6 +51,8 @@ use std::time::Duration;
 use std::time::Instant;
 
 use super::StreamState;
+use super::render::StreamingRender;
+use super::render::render_source;
 use super::table_holdback::TableHoldbackScanner;
 use super::table_holdback::TableHoldbackState;
 #[cfg(test)]
@@ -77,8 +77,8 @@ struct StreamCore {
     width: Option<usize>,
     /// Accumulated raw markdown source for the current stream.
     raw_source: String,
-    /// Full re-render of `raw_source` at `width`. Rebuilt on every committed delta.
-    rendered_lines: Vec<HyperlinkLine>,
+    /// Incremental render of `raw_source` at `width`.
+    render: StreamingRender,
     /// Lines enqueued into the commit-animation queue.
     enqueued_stable_len: usize,
     /// Lines actually emitted to scrollback.
@@ -116,7 +116,7 @@ impl StreamCore {
             state: StreamState::new(width, cwd),
             width,
             raw_source: String::with_capacity(1024),
-            rendered_lines: Vec::with_capacity(64),
+            render: StreamingRender::new(),
             enqueued_stable_len: 0,
             emitted_stable_len: 0,
             cwd: cwd.to_path_buf(),
@@ -146,7 +146,14 @@ impl StreamCore {
         {
             self.raw_source.push_str(&committed_source);
             self.holdback_scanner.push_source_chunk(&committed_source);
-            self.recompute_streaming_render();
+            self.render.append(
+                &self.raw_source,
+                &committed_source,
+                self.width,
+                self.cwd.as_path(),
+                self.render_mode,
+                self.inline_visualization_context.as_ref(),
+            );
             enqueued = self.sync_stable_queue();
         }
         enqueued
@@ -165,7 +172,13 @@ impl StreamCore {
             self.raw_source.push_str(&remainder_source);
             self.holdback_scanner.push_source_chunk(&remainder_source);
         }
-        let rendered = self.render_source(&self.raw_source);
+        let rendered = render_source(
+            &self.raw_source,
+            self.width,
+            self.cwd.as_path(),
+            self.render_mode,
+            self.inline_visualization_context.as_ref(),
+        );
         if self.emitted_stable_len >= rendered.len() {
             Vec::new()
         } else {
@@ -220,13 +233,13 @@ impl StreamCore {
     /// reappear in the active cell and duplicate content on screen.
     #[inline]
     fn current_tail_lines(&self) -> Vec<HyperlinkLine> {
-        let start = self.enqueued_stable_len.min(self.rendered_lines.len());
-        self.rendered_lines[start..].to_vec()
+        let start = self.enqueued_stable_len.min(self.render.lines.len());
+        self.render.lines[start..].to_vec()
     }
 
     #[inline]
     fn has_tail(&self) -> bool {
-        self.enqueued_stable_len < self.rendered_lines.len()
+        self.enqueued_stable_len < self.render.lines.len()
     }
 
     /// Update rendering width and rebuild queued stable lines for the new layout.
@@ -250,10 +263,16 @@ impl StreamCore {
             return;
         }
 
-        self.recompute_streaming_render();
-        self.emitted_stable_len = self.emitted_stable_len.min(self.rendered_lines.len());
+        self.render.recompute(
+            &self.raw_source,
+            self.width,
+            self.cwd.as_path(),
+            self.render_mode,
+            self.inline_visualization_context.as_ref(),
+        );
+        self.emitted_stable_len = self.emitted_stable_len.min(self.render.lines.len());
         if had_pending_queue
-            && self.emitted_stable_len == self.rendered_lines.len()
+            && self.emitted_stable_len == self.render.lines.len()
             && self.emitted_stable_len > 0
         {
             // If wrapped remainder compresses into fewer lines at the new width,
@@ -266,7 +285,7 @@ impl StreamCore {
             // Avoid replaying already-emitted content after resize when no
             // stable lines were waiting in the queue and there was no mutable
             // tail to preserve.
-            self.enqueued_stable_len = self.rendered_lines.len();
+            self.enqueued_stable_len = self.render.lines.len();
             return;
         }
         self.rebuild_stable_queue_from_render();
@@ -276,29 +295,13 @@ impl StreamCore {
     fn reset(&mut self) {
         self.state.clear();
         self.raw_source.clear();
-        self.rendered_lines.clear();
+        self.render.clear();
         self.enqueued_stable_len = 0;
         self.emitted_stable_len = 0;
         self.stable_prefix_len_cache = None;
         self.holdback_scanner.reset();
     }
 
-    fn render_source(&self, source: &str) -> Vec<HyperlinkLine> {
-        match self.render_mode {
-            HistoryRenderMode::Rich => render_markdown_agent_with_links_cwd_and_visualizations(
-                source,
-                self.width,
-                Some(self.cwd.as_path()),
-                self.inline_visualization_context.as_ref(),
-            ),
-            HistoryRenderMode::Raw => plain_hyperlink_lines(raw_lines_from_source(source)),
-        }
-    }
-
-    fn recompute_streaming_render(&mut self) {
-        self.rendered_lines = self.render_source(&self.raw_source);
-    }
-
     fn set_render_mode(&mut self, render_mode: HistoryRenderMode) {
         if self.render_mode == render_mode {
             return;
@@ -311,17 +314,23 @@ impl StreamCore {
             return;
         }
 
-        self.recompute_streaming_render();
-        self.emitted_stable_len = self.emitted_stable_len.min(self.rendered_lines.len());
+        self.render.recompute(
+            &self.raw_source,
+            self.width,
+            self.cwd.as_path(),
+            self.render_mode,
+            self.inline_visualization_context.as_ref(),
+        );
+        self.emitted_stable_len = self.emitted_stable_len.min(self.render.lines.len());
         if had_pending_queue
-            && self.emitted_stable_len == self.rendered_lines.len()
+            && self.emitted_stable_len == self.render.lines.len()
             && self.emitted_stable_len > 0
         {
             self.emitted_stable_len -= 1;
         }
         self.state.clear_queue();
         if self.emitted_stable_len > 0 && !had_pending_queue && !had_live_tail {
-            self.enqueued_stable_len = self.rendered_lines.len();
+            self.enqueued_stable_len = self.render.lines.len();
             return;
         }
         self.rebuild_stable_queue_from_render();
@@ -330,7 +339,8 @@ impl StreamCore {
     /// Compute how many rendered lines should be in the stable region.
     fn compute_target_stable_len(&mut self) -> usize {
         let tail_budget = self.active_tail_budget_lines();
-        self.rendered_lines
+        self.render
+            .lines
             .len()
             .saturating_sub(tail_budget)
             .max(self.emitted_stable_len)
@@ -347,7 +357,7 @@ impl StreamCore {
             self.state.clear_queue();
             if self.emitted_stable_len < target_stable_len {
                 self.state.enqueue(
-                    self.rendered_lines[self.emitted_stable_len..target_stable_len].to_vec(),
+                    self.render.lines[self.emitted_stable_len..target_stable_len].to_vec(),
                 );
             }
             self.enqueued_stable_len = target_stable_len;
@@ -359,7 +369,7 @@ impl StreamCore {
         }
 
         self.state
-            .enqueue(self.rendered_lines[self.enqueued_stable_len..target_stable_len].to_vec());
+            .enqueue(self.render.lines[self.enqueued_stable_len..target_stable_len].to_vec());
         self.enqueued_stable_len = target_stable_len;
         true
     }
@@ -374,7 +384,7 @@ impl StreamCore {
         self.state.clear_queue();
         if self.emitted_stable_len < target_stable_len {
             self.state
-                .enqueue(self.rendered_lines[self.emitted_stable_len..target_stable_len].to_vec());
+                .enqueue(self.render.lines[self.emitted_stable_len..target_stable_len].to_vec());
         }
         self.enqueued_stable_len = target_stable_len;
     }
@@ -416,11 +426,11 @@ impl StreamCore {
     /// the only place where those coordinate systems are bridged.
     fn tail_budget_from_source_start(&mut self, source_start: usize) -> usize {
         if source_start == 0 {
-            return self.rendered_lines.len();
+            return self.render.lines.len();
         }
         let source_start = source_start.min(self.raw_source.len());
         let stable_prefix_len = self.stable_prefix_len_for_source_start(source_start);
-        self.rendered_lines.len().saturating_sub(stable_prefix_len)
+        self.render.lines.len().saturating_sub(stable_prefix_len)
     }
 
     /// Render the stable prefix before `source_start` and return its line count.
@@ -816,6 +826,31 @@ mod tests {
             .collect()
     }
 
+    #[test]
+    fn queued_heading_is_emitted_once_after_incremental_append() {
+        let mut ctrl = stream_controller(Some(80));
+        assert!(ctrl.push("Paragraph.\n\n# Heading\n\n"));
+        ctrl.push("Next paragraph.\n");
+
+        let (cell, _) = ctrl.on_commit_tick_batch(usize::MAX);
+        let mut streamed = cell
+            .into_iter()
+            .flat_map(|cell| cell.transcript_lines(u16::MAX))
+            .collect::<Vec<_>>();
+        if let (Some(cell), _source) = ctrl.finalize() {
+            streamed.extend(cell.transcript_lines(u16::MAX));
+        }
+        let streamed = lines_to_plain_strings(&streamed);
+        assert_eq!(
+            streamed
+                .iter()
+                .filter(|line| line.contains("# Heading"))
+                .count(),
+            1,
+            "expected the streamed heading to be emitted once: {streamed:?}",
+        );
+    }
+
     fn collect_plan_streamed_lines(deltas: &[&str], width: Option<usize>) -> Vec<String> {
         let mut ctrl = plan_stream_controller(width);
         let mut lines = Vec::new();
@@ -896,7 +931,7 @@ mod tests {
         let mut ctrl = stream_controller(Some(80));
         assert!(!ctrl.has_live_tail());
 
-        ctrl.core.rendered_lines = vec![Line::from("tail line").into()];
+        ctrl.core.render.lines = vec![Line::from("tail line").into()];
         ctrl.core.enqueued_stable_len = 0;
         assert!(ctrl.has_live_tail());
 
@@ -909,7 +944,7 @@ mod tests {
         let mut ctrl = plan_stream_controller(Some(80));
         assert!(!ctrl.has_live_tail());
 
-        ctrl.core.rendered_lines = vec![Line::from("tail line").into()];
+        ctrl.core.render.lines = vec![Line::from("tail line").into()];
         ctrl.core.enqueued_stable_len = 0;
         assert!(ctrl.has_live_tail());