mirror of
https://github.com/openai/codex.git
synced 2026-11-12 21:20:34 +00:00
Avoid rerendering streamed code fences (#39061)
## Why The streaming TUI rerendered the entire mutable Markdown block whenever a newline-bearing delta arrived. For long fenced code blocks, this repeatedly parsed and syntax-highlighted the growing fence. ## What changed - Detect open, top-level, language-tagged code fences and append newly completed lines directly to the existing render. - Preserve Syntect parser state across appended lines so multiline syntax and the active theme remain consistent. - Fall back to the canonical Markdown renderer for ambiguous fence syntax, theme changes, incomplete lines, and highlighting limit transitions. ## Testing Added coverage that compares streamed output with full renders across syntax styles, Unicode, links, blank lines, fence variants, theme changes, and highlighting limits. A long-fence test also verifies that the rendered prefix is retained as new lines arrive. GitOrigin-RevId: 036db2b7bc93607083b8001f8b4fa3cbb58c713e
This commit is contained in:
Charlie Marsh
@@ -47,6 +47,11 @@ use syntect::parsing::SyntaxSet;
|
||||
use syntect::util::LinesWithEndings;
|
||||
use two_face::theme::EmbeddedThemeName;
|
||||
|
||||
#[path = "highlight_streaming.rs"]
|
||||
mod streaming;
|
||||
|
||||
pub(crate) use streaming::StreamingCodeHighlighter;
|
||||
|
||||
// -- Global singletons -------------------------------------------------------
|
||||
|
||||
static SYNTAX_SET: OnceLock<SyntaxSet> = OnceLock::new();
|
||||
@@ -630,25 +635,30 @@ fn highlight_to_line_spans_with_theme(
|
||||
|
||||
for line in LinesWithEndings::from(code) {
|
||||
let ranges = h.highlight_line(line, syntax_set()).ok()?;
|
||||
let mut spans: Vec<Span<'static>> = Vec::new();
|
||||
for (style, text) in ranges {
|
||||
// Strip trailing line endings (LF and CR) since we handle line
|
||||
// breaks ourselves. CRLF inputs would otherwise leave a stray \r.
|
||||
let text = text.trim_end_matches(['\n', '\r']);
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
spans.push(Span::styled(text.to_string(), convert_style(style)));
|
||||
}
|
||||
if spans.is_empty() {
|
||||
spans.push(Span::raw(String::new()));
|
||||
}
|
||||
lines.push(spans);
|
||||
lines.push(highlighted_line_spans(ranges));
|
||||
}
|
||||
|
||||
Some(lines)
|
||||
}
|
||||
|
||||
/// Convert one highlighted line into the spans shared by full and streaming renders.
|
||||
///
|
||||
/// Source line endings are omitted, while empty lines retain their canonical empty span.
|
||||
fn highlighted_line_spans(ranges: Vec<(SyntectStyle, &str)>) -> Vec<Span<'static>> {
|
||||
let mut spans = Vec::new();
|
||||
for (style, text) in ranges {
|
||||
// Line breaks are represented by the surrounding Line, not its spans.
|
||||
let text = text.trim_end_matches(['\n', '\r']);
|
||||
if !text.is_empty() {
|
||||
spans.push(Span::styled(text.to_string(), convert_style(style)));
|
||||
}
|
||||
}
|
||||
if spans.is_empty() {
|
||||
spans.push(Span::raw(String::new()));
|
||||
}
|
||||
spans
|
||||
}
|
||||
|
||||
/// Parse `code` using syntect for `lang` and return per-line styled spans.
|
||||
/// Each inner Vec represents one source line. Returns None when the language
|
||||
/// is not recognized or the input exceeds safety limits.
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
//! Append-only syntax highlighting for complete lines in an open code fence.
|
||||
|
||||
use super::MAX_HIGHLIGHT_LINE_BYTES;
|
||||
use super::exceeds_highlight_limits;
|
||||
use super::find_syntax;
|
||||
use super::highlighted_line_spans;
|
||||
use super::syntax_set;
|
||||
use super::syntax_theme_revision;
|
||||
use super::theme_lock;
|
||||
use ratatui::text::Line;
|
||||
use syntect::easy::HighlightLines;
|
||||
use syntect::highlighting::HighlightState;
|
||||
use syntect::parsing::ParseState;
|
||||
use syntect::util::LinesWithEndings;
|
||||
|
||||
/// Retains only Syntect's parser state, subject to the ordinary highlighting limits.
|
||||
///
|
||||
/// An absent state means the whole fence already uses the permanent plain-text fallback because
|
||||
/// its language is unknown or a size limit was exceeded. The caller owns the rendered prefix.
|
||||
pub(crate) struct StreamingCodeHighlighter {
|
||||
state: Option<HighlightedCode>,
|
||||
}
|
||||
|
||||
struct HighlightedCode {
|
||||
bytes: usize,
|
||||
lines: usize,
|
||||
theme_revision: u64,
|
||||
syntax: (HighlightState, ParseState),
|
||||
}
|
||||
|
||||
impl StreamingCodeHighlighter {
|
||||
/// Reconstruct highlighting state after the canonical renderer emitted `code`.
|
||||
///
|
||||
/// Unknown languages and oversized input retain a permanent plain-text fallback. A changed
|
||||
/// theme or failed replay returns `None`, requiring the caller to render the whole fence again.
|
||||
pub(crate) fn new(code: &str, lang: &str, theme_revision: u64) -> Option<Self> {
|
||||
let theme = match theme_lock().read() {
|
||||
Ok(theme) => theme,
|
||||
Err(poisoned) => poisoned.into_inner(),
|
||||
};
|
||||
if theme_revision != syntax_theme_revision() {
|
||||
return None;
|
||||
}
|
||||
let lines = code.lines().count();
|
||||
let syntax = find_syntax(lang).filter(|_| {
|
||||
!exceeds_highlight_limits(code.len(), lines)
|
||||
&& !code
|
||||
.lines()
|
||||
.any(|line| line.len() > MAX_HIGHLIGHT_LINE_BYTES)
|
||||
});
|
||||
let Some(syntax) = syntax else {
|
||||
return Some(Self { state: None });
|
||||
};
|
||||
let mut highlighter = HighlightLines::new(syntax, &theme);
|
||||
for line in LinesWithEndings::from(code) {
|
||||
highlighter.highlight_line(line, syntax_set()).ok()?;
|
||||
}
|
||||
Some(Self {
|
||||
state: Some(HighlightedCode {
|
||||
bytes: code.len(),
|
||||
lines,
|
||||
theme_revision,
|
||||
syntax: highlighter.state(),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Consume the previous state and highlight just the newly committed complete lines.
|
||||
///
|
||||
/// Every appended line must be newline-terminated to preserve Syntect's parser state.
|
||||
/// Returning `None` discards the state: the caller must render the whole fence again, which
|
||||
/// also removes old colors when the aggregate input crosses a highlighting limit.
|
||||
pub(crate) fn append(mut self, appended: &str) -> Option<(Self, Vec<Line<'static>>)> {
|
||||
if !appended.ends_with('\n') {
|
||||
return None;
|
||||
}
|
||||
let Some(mut state) = self.state.take() else {
|
||||
let lines = appended
|
||||
.lines()
|
||||
.map(|line| Line::from(line.to_string()))
|
||||
.collect();
|
||||
return Some((self, lines));
|
||||
};
|
||||
let bytes = state.bytes.checked_add(appended.len())?;
|
||||
let lines = state.lines.checked_add(appended.lines().count())?;
|
||||
if exceeds_highlight_limits(bytes, lines)
|
||||
|| appended
|
||||
.lines()
|
||||
.any(|line| line.len() > MAX_HIGHLIGHT_LINE_BYTES)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let theme = match theme_lock().read() {
|
||||
Ok(theme) => theme,
|
||||
Err(poisoned) => poisoned.into_inner(),
|
||||
};
|
||||
if state.theme_revision != syntax_theme_revision() {
|
||||
return None;
|
||||
}
|
||||
let (highlight_state, parse_state) = state.syntax;
|
||||
let mut highlighter = HighlightLines::from_state(&theme, highlight_state, parse_state);
|
||||
let mut rendered = Vec::with_capacity(lines - state.lines);
|
||||
for line in LinesWithEndings::from(appended) {
|
||||
let ranges = highlighter.highlight_line(line, syntax_set()).ok()?;
|
||||
rendered.push(Line::from(highlighted_line_spans(ranges)));
|
||||
}
|
||||
state.bytes = bytes;
|
||||
state.lines = lines;
|
||||
state.syntax = highlighter.state();
|
||||
self.state = Some(state);
|
||||
Some((self, rendered))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "highlight_streaming_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,75 @@
|
||||
use super::super::MAX_HIGHLIGHT_BYTES;
|
||||
use super::super::MAX_HIGHLIGHT_LINE_BYTES;
|
||||
use super::super::MAX_HIGHLIGHT_LINES;
|
||||
use super::super::highlight_code_to_lines;
|
||||
use super::super::syntax_theme_revision;
|
||||
use super::StreamingCodeHighlighter;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn appended_lines_preserve_multiline_syntax_state() {
|
||||
let first = "fn main() {\n /* comment\n";
|
||||
let appended = " continues */\n println!(\"界 e\u{301}\");\n}\n";
|
||||
let code = format!("{first}{appended}");
|
||||
let highlighter =
|
||||
StreamingCodeHighlighter::new(first, "rust", syntax_theme_revision()).unwrap();
|
||||
let (_, lines) = highlighter.append(appended).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
lines,
|
||||
highlight_code_to_lines(&code, "rust")
|
||||
.into_iter()
|
||||
.skip(first.lines().count())
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_and_unknown_language_fences_append() {
|
||||
for lang in ["rust", "unknown-language"] {
|
||||
let highlighter = StreamingCodeHighlighter::new("", lang, syntax_theme_revision()).unwrap();
|
||||
let (_, lines) = highlighter.append("fn main() {}\n\n").unwrap();
|
||||
assert_eq!(lines, highlight_code_to_lines("fn main() {}\n\n", lang));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_lines_and_theme_changes_discard_state() {
|
||||
let highlighter =
|
||||
StreamingCodeHighlighter::new("fn first() {}\n", "rust", syntax_theme_revision()).unwrap();
|
||||
assert!(highlighter.append("fn partial()").is_none());
|
||||
|
||||
let mut highlighter =
|
||||
StreamingCodeHighlighter::new("fn first() {}\n", "rust", syntax_theme_revision()).unwrap();
|
||||
let state = highlighter.state.as_mut().unwrap();
|
||||
state.theme_revision = state.theme_revision.wrapping_sub(1);
|
||||
assert!(highlighter.append("fn second() {}\n").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crossing_each_highlighting_limit_requires_a_canonical_render() {
|
||||
let cases = [
|
||||
(
|
||||
String::new(),
|
||||
format!("{}\n", "x".repeat(MAX_HIGHLIGHT_LINE_BYTES + 1)),
|
||||
),
|
||||
("x\n".repeat(MAX_HIGHLIGHT_LINES), "x\n".to_string()),
|
||||
(
|
||||
format!("{}\n", "x".repeat(1023)).repeat(MAX_HIGHLIGHT_BYTES / 1024),
|
||||
"x\n".to_string(),
|
||||
),
|
||||
];
|
||||
for (initial, appended) in cases {
|
||||
let highlighter =
|
||||
StreamingCodeHighlighter::new(&initial, "rust", syntax_theme_revision()).unwrap();
|
||||
assert!(highlighter.state.is_some());
|
||||
assert!(highlighter.append(&appended).is_none());
|
||||
|
||||
let complete = format!("{initial}{appended}");
|
||||
let highlighter =
|
||||
StreamingCodeHighlighter::new(&complete, "rust", syntax_theme_revision()).unwrap();
|
||||
assert!(highlighter.state.is_none());
|
||||
let (_, lines) = highlighter.append("next\n").unwrap();
|
||||
assert_eq!(lines, highlight_code_to_lines("next\n", "unknown-language"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
//! A conservative fast path for one open, top-level, language-tagged code fence.
|
||||
|
||||
use crate::render::highlight::MAX_HIGHLIGHT_LINE_BYTES;
|
||||
use crate::render::highlight::StreamingCodeHighlighter;
|
||||
use crate::render::highlight::syntax_theme_revision;
|
||||
use crate::terminal_hyperlinks::HyperlinkLine;
|
||||
use ratatui::text::Span;
|
||||
|
||||
/// An append-only fence whose original source is identical to pulldown-cmark's code text.
|
||||
pub(super) struct OpenCodeFence {
|
||||
marker: u8,
|
||||
marker_len: usize,
|
||||
language: String,
|
||||
content_start: usize,
|
||||
source_len: usize,
|
||||
theme_revision: u64,
|
||||
/// Initialized only if another chunk arrives, avoiding duplicate work on one-shot fences.
|
||||
highlighter: Option<StreamingCodeHighlighter>,
|
||||
}
|
||||
|
||||
impl OpenCodeFence {
|
||||
/// Recognize the final mutable top-level block after its canonical render.
|
||||
///
|
||||
/// `source` is the final block's suffix, while `source_len` includes the entire committed
|
||||
/// Markdown source and anchors the retained fence within it.
|
||||
///
|
||||
/// Indentation, CR/NUL normalization, and escaped info strings stay on the canonical path.
|
||||
/// Any line that could end the fence also stays there, including deliberately conservative
|
||||
/// false positives, so this fast path need not duplicate CommonMark's closing-fence grammar.
|
||||
pub(super) fn detect(source: &str, source_len: usize, theme_revision: u64) -> Option<Self> {
|
||||
let marker = *source.as_bytes().first()?;
|
||||
if marker != b'`' && marker != b'~' {
|
||||
return None;
|
||||
}
|
||||
let (opening, code) = source.split_once('\n')?;
|
||||
let marker_len = opening.bytes().take_while(|byte| *byte == marker).count();
|
||||
if marker_len < 3
|
||||
|| !source.ends_with('\n')
|
||||
|| theme_revision != syntax_theme_revision()
|
||||
|| source.contains(['\r', '\0'])
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let info = opening[marker_len..].trim_matches([' ', '\t', '\u{b}', '\u{c}']);
|
||||
if info.contains(['&', '\\']) || (marker == b'`' && info.contains('`')) {
|
||||
return None;
|
||||
}
|
||||
let language = info
|
||||
.split([',', ' ', '\t'])
|
||||
.next()
|
||||
.filter(|language| !language.is_empty())?;
|
||||
// Retain at most one bounded language token, never the streamed code itself.
|
||||
if language.len() > MAX_HIGHLIGHT_LINE_BYTES
|
||||
|| has_possible_closing_line(code, marker, marker_len)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
marker,
|
||||
marker_len,
|
||||
language: language.to_string(),
|
||||
content_start: source_len.checked_sub(code.len())?,
|
||||
source_len,
|
||||
theme_revision,
|
||||
highlighter: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Append code only while the caller's newline-committed source extends this same fence.
|
||||
///
|
||||
/// Returning `None` consumes the retained state and requires a canonical whole-fence render.
|
||||
/// Successful lines include the empty indent span expected by the canonical Markdown writer.
|
||||
pub(super) fn append(
|
||||
mut self,
|
||||
raw_source: &str,
|
||||
committed_source: &str,
|
||||
) -> Option<(Self, Vec<HyperlinkLine>)> {
|
||||
if self.source_len.checked_add(committed_source.len()) != Some(raw_source.len())
|
||||
|| !raw_source.ends_with(committed_source)
|
||||
|| committed_source.contains(['\r', '\0'])
|
||||
|| has_possible_closing_line(committed_source, self.marker, self.marker_len)
|
||||
|| self.theme_revision != syntax_theme_revision()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let highlighter = match self.highlighter.take() {
|
||||
Some(highlighter) => highlighter,
|
||||
None => StreamingCodeHighlighter::new(
|
||||
&raw_source[self.content_start..self.source_len],
|
||||
&self.language,
|
||||
self.theme_revision,
|
||||
)?,
|
||||
};
|
||||
let (highlighter, lines) = highlighter.append(committed_source)?;
|
||||
self.highlighter = Some(highlighter);
|
||||
self.source_len = raw_source.len();
|
||||
let lines = lines
|
||||
.into_iter()
|
||||
.map(|mut line| {
|
||||
// The canonical writer installs an empty indent span for top-level fences.
|
||||
line.spans.insert(/*index*/ 0, Span::default());
|
||||
HyperlinkLine::new(line)
|
||||
})
|
||||
.collect();
|
||||
Some((self, lines))
|
||||
}
|
||||
}
|
||||
|
||||
/// Conservatively detect any line that might close the fence.
|
||||
///
|
||||
/// False positives intentionally fall back to CommonMark instead of duplicating its closer rules.
|
||||
fn has_possible_closing_line(source: &str, marker: u8, marker_len: usize) -> bool {
|
||||
source.lines().any(|line| {
|
||||
line.trim_start_matches([' ', '\t'])
|
||||
.bytes()
|
||||
.take_while(|byte| *byte == marker)
|
||||
.count()
|
||||
>= marker_len
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "code_fence_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,147 @@
|
||||
use super::StreamingRender;
|
||||
use super::append;
|
||||
use super::append_rich_and_assert_matches_full;
|
||||
use super::assert_rich_stream_matches_full_render;
|
||||
use super::lines_to_plain_strings;
|
||||
use super::test_cwd;
|
||||
use crate::history_cell::HistoryRenderMode;
|
||||
use crate::render::highlight::MAX_HIGHLIGHT_LINE_BYTES;
|
||||
use insta::assert_debug_snapshot;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn growing_code_fences_preserve_styles_unicode_links_and_blank_lines() {
|
||||
let (_, render) = assert_rich_stream_matches_full_render(
|
||||
&[
|
||||
"Before [link](https://example.com).\n\n```rust,no_run\n",
|
||||
"fn main() {\n",
|
||||
" /* a multiline\n",
|
||||
" comment */\n\n",
|
||||
" println!(\"界 e\u{301} https://example.com\");\n",
|
||||
"}\n",
|
||||
"```\n\n~~~~python\n",
|
||||
"text = \"\"\"a multiline\n",
|
||||
"string\"\"\"\n",
|
||||
"~~~~\n",
|
||||
],
|
||||
Some(18),
|
||||
);
|
||||
assert!(render.open_code_fence.is_none());
|
||||
assert_debug_snapshot!(lines_to_plain_strings(&render.lines), @r#"
|
||||
[
|
||||
"Before link",
|
||||
"(https://example.com).",
|
||||
"",
|
||||
"fn main() {",
|
||||
" /* a multiline",
|
||||
" comment */",
|
||||
"",
|
||||
" println!(\"界 e\u{301} https://example.com\");",
|
||||
"}",
|
||||
"",
|
||||
"text = \"\"\"a multiline",
|
||||
"string\"\"\"",
|
||||
]
|
||||
"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_open_fence_retains_its_rendered_prefix() {
|
||||
let cwd = test_cwd();
|
||||
let mut source = String::new();
|
||||
let mut render = StreamingRender::new();
|
||||
append_rich_and_assert_matches_full(
|
||||
&mut render,
|
||||
&mut source,
|
||||
"```rust\nfn main() {\n",
|
||||
Some(80),
|
||||
&cwd,
|
||||
);
|
||||
let first_line = render.lines[0].line.spans[1].content.as_ptr();
|
||||
for index in 0..512 {
|
||||
append(
|
||||
&mut render,
|
||||
&mut source,
|
||||
&format!(" let value_{index} = {index};\n"),
|
||||
Some(80),
|
||||
&cwd,
|
||||
HistoryRenderMode::Rich,
|
||||
);
|
||||
assert!(render.open_code_fence.is_some());
|
||||
}
|
||||
assert_eq!(render.lines[0].line.spans[1].content.as_ptr(), first_line);
|
||||
append_rich_and_assert_matches_full(&mut render, &mut source, "}\n```\n", Some(80), &cwd);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguous_and_normalized_fences_use_canonical_semantics() {
|
||||
let streams: &[&[&str]] = &[
|
||||
&["```rust\n", "x\n", "``", "`\n"],
|
||||
&["```rust\n", "x\n", "```\t\n", "after\n"],
|
||||
&["```rust\n", "x\n", " ```` \n", "after\n"],
|
||||
&["~~~rust\n", "x\n", "~~~\u{a0}\n", "after\n"],
|
||||
&["```rust\r\n", "let value = 'é';\r\n", "```\r\n"],
|
||||
&["```rust\n", "let value = '\0';\n", "```\n"],
|
||||
&["```rust,no_run\n", "fn main() {}\n", "```\n"],
|
||||
&[" ```rust\n", " fn main() {}\n", " ```\n"],
|
||||
&["```unknown-language\n", "first\n", "second\n", "```\n"],
|
||||
&[
|
||||
"```markdown\n| A | B |\n",
|
||||
"| --- | --- |\n",
|
||||
"| 1 | 2 |\n```\n\n```rust\nfn first() {}\n",
|
||||
"fn second() {}\n",
|
||||
"```\n",
|
||||
],
|
||||
];
|
||||
for chunks in streams {
|
||||
assert_rich_stream_matches_full_render(chunks, Some(40));
|
||||
}
|
||||
for indent in 0..=3 {
|
||||
let closing = format!("{}~~~~~\t\n", " ".repeat(indent));
|
||||
assert_rich_stream_matches_full_render(
|
||||
&["~~~~rust\n", "x\n", &closing, "after\n"],
|
||||
Some(40),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crossing_a_highlight_limit_removes_previously_highlighted_colors() {
|
||||
let cwd = test_cwd();
|
||||
let mut source = String::new();
|
||||
let mut render = StreamingRender::new();
|
||||
for chunk in [
|
||||
"```rust\nlet first = 1;\n".to_string(),
|
||||
format!("{}\n", "x".repeat(MAX_HIGHLIGHT_LINE_BYTES + 1)),
|
||||
"let second = 2;\n".to_string(),
|
||||
"```\n".to_string(),
|
||||
] {
|
||||
append_rich_and_assert_matches_full(&mut render, &mut source, &chunk, Some(80), &cwd);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recomputing_after_resize_or_mode_change_drops_fence_state() {
|
||||
let (mut source, mut render) =
|
||||
assert_rich_stream_matches_full_render(&["```rust\nfn first() {}\n"], Some(80));
|
||||
assert!(render.open_code_fence.is_some());
|
||||
let cwd = test_cwd();
|
||||
render.recompute(
|
||||
&source,
|
||||
Some(20),
|
||||
&cwd,
|
||||
HistoryRenderMode::Rich,
|
||||
/*inline_visualization_context*/ None,
|
||||
);
|
||||
assert!(render.open_code_fence.is_none());
|
||||
append_rich_and_assert_matches_full(
|
||||
&mut render,
|
||||
&mut source,
|
||||
"fn second() {}\n",
|
||||
Some(20),
|
||||
&cwd,
|
||||
);
|
||||
assert!(render.open_code_fence.is_some());
|
||||
render.clear();
|
||||
assert!(render.open_code_fence.is_none());
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
use super::OpenCodeFence;
|
||||
use super::has_possible_closing_line;
|
||||
use crate::render::highlight::MAX_HIGHLIGHT_LINE_BYTES;
|
||||
use crate::render::highlight::syntax_theme_revision;
|
||||
|
||||
#[test]
|
||||
fn detects_only_source_preserving_open_fences() {
|
||||
for source in [
|
||||
"```rust,no_run\nfn main() {}\n",
|
||||
"~~~~python title=demo\nvalue = 1\n",
|
||||
"```\u{b}rust\u{c}\nfn main() {}\n",
|
||||
"```unknown-language\ntext\n",
|
||||
] {
|
||||
let fence =
|
||||
OpenCodeFence::detect(source, source.len(), syntax_theme_revision()).expect(source);
|
||||
assert!(fence.highlighter.is_none());
|
||||
}
|
||||
for source in [
|
||||
" ```rust\ncode\n",
|
||||
"```\ncode\n",
|
||||
"```rust\r\ncode\r\n",
|
||||
"```rust\n'\0'\n",
|
||||
"```rust\ncode\n",
|
||||
"```rust\\,no_run\ncode\n",
|
||||
"```rust\npartial",
|
||||
] {
|
||||
assert!(
|
||||
OpenCodeFence::detect(source, source.len(), syntax_theme_revision()).is_none(),
|
||||
"{source:?}"
|
||||
);
|
||||
}
|
||||
let source = format!("```{}\n", "x".repeat(MAX_HIGHLIGHT_LINE_BYTES + 1));
|
||||
assert!(OpenCodeFence::detect(&source, source.len(), syntax_theme_revision()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_possible_closer_returns_to_the_canonical_parser() {
|
||||
let opening = "~~~~rust\n";
|
||||
assert!(!has_possible_closing_line(
|
||||
"~~~\n```\n",
|
||||
/*marker*/ b'~',
|
||||
/*marker_len*/ 4
|
||||
));
|
||||
for line in [
|
||||
"~~~~\n",
|
||||
" ~~~~~~ \n",
|
||||
"~~~~\t\n",
|
||||
"~~~~\u{a0}\n",
|
||||
"\t~~~~\n",
|
||||
" ~~~~\n",
|
||||
"~~~~ trailing text\n",
|
||||
] {
|
||||
let source = format!("{opening}{line}");
|
||||
assert!(OpenCodeFence::detect(&source, source.len(), syntax_theme_revision()).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lazy_initialization_rejects_a_changed_theme() {
|
||||
let source = "```rust\nfn first() {}\n";
|
||||
let mut fence = OpenCodeFence::detect(source, source.len(), syntax_theme_revision()).unwrap();
|
||||
fence.theme_revision = fence.theme_revision.wrapping_sub(1);
|
||||
let appended = "fn second() {}\n";
|
||||
assert!(
|
||||
fence
|
||||
.append(&format!("{source}{appended}"), appended)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
@@ -17,6 +17,7 @@ use std::time::Instant;
|
||||
use crate::markdown_stream::MarkdownStreamCollector;
|
||||
use crate::terminal_hyperlinks::HyperlinkLine;
|
||||
pub(crate) mod chunking;
|
||||
mod code_fence;
|
||||
pub(crate) mod commit_tick;
|
||||
pub(crate) mod controller;
|
||||
mod render;
|
||||
|
||||
@@ -3,12 +3,14 @@
|
||||
//! Completed top-level blocks are retained while the final block stays mutable, avoiding repeated
|
||||
//! rendering of the stable prefix as newline-bearing deltas arrive.
|
||||
|
||||
use super::code_fence::OpenCodeFence;
|
||||
use crate::history_cell::HistoryRenderMode;
|
||||
use crate::history_cell::raw_lines_from_source;
|
||||
use crate::inline_visualization::InlineVisualizationContext;
|
||||
use crate::inline_visualization::contains_inline_visualization;
|
||||
use crate::markdown::render_markdown_agent_with_links_cwd_and_visualizations;
|
||||
use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd;
|
||||
use crate::render::highlight::syntax_theme_revision;
|
||||
use crate::terminal_hyperlinks::HyperlinkLine;
|
||||
use crate::terminal_hyperlinks::plain_hyperlink_lines;
|
||||
use ratatui::text::Line;
|
||||
@@ -28,6 +30,8 @@ pub(super) struct StreamingRender {
|
||||
has_reference_link_definition: bool,
|
||||
/// Inline visualization directives require source-wide rewriting once one is committed.
|
||||
has_inline_visualization_directive: bool,
|
||||
/// Parser state for a directly appendable, open top-level code fence.
|
||||
open_code_fence: Option<OpenCodeFence>,
|
||||
}
|
||||
|
||||
impl StreamingRender {
|
||||
@@ -38,6 +42,7 @@ impl StreamingRender {
|
||||
stable_rendered_len: 0,
|
||||
has_reference_link_definition: false,
|
||||
has_inline_visualization_directive: false,
|
||||
open_code_fence: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +52,7 @@ impl StreamingRender {
|
||||
self.stable_rendered_len = 0;
|
||||
self.has_reference_link_definition = false;
|
||||
self.has_inline_visualization_directive = false;
|
||||
self.open_code_fence = None;
|
||||
}
|
||||
|
||||
/// Re-render the full source and reset both stable-prefix boundaries.
|
||||
@@ -61,6 +67,7 @@ impl StreamingRender {
|
||||
render_mode: HistoryRenderMode,
|
||||
inline_visualization_context: Option<&InlineVisualizationContext>,
|
||||
) {
|
||||
self.open_code_fence = None;
|
||||
self.has_inline_visualization_directive = contains_inline_visualization(source);
|
||||
self.lines = match (render_mode, inline_visualization_context) {
|
||||
(HistoryRenderMode::Rich, None) if !self.has_inline_visualization_directive => {
|
||||
@@ -130,7 +137,16 @@ impl StreamingRender {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(fence) = self.open_code_fence.take()
|
||||
&& let Some((fence, lines)) = fence.append(raw_source, committed_source)
|
||||
{
|
||||
self.lines.extend(lines);
|
||||
self.open_code_fence = Some(fence);
|
||||
return;
|
||||
}
|
||||
|
||||
let pending_source = &raw_source[self.stable_source_len..];
|
||||
let theme_revision = syntax_theme_revision();
|
||||
let pending =
|
||||
render_streaming_markdown_agent_with_links_and_cwd(pending_source, width, Some(cwd));
|
||||
if pending.has_reference_link_definition {
|
||||
@@ -145,6 +161,13 @@ impl StreamingRender {
|
||||
return;
|
||||
}
|
||||
|
||||
let final_block_start = pending.last_top_level_block_start.unwrap_or(/*default*/ 0);
|
||||
self.open_code_fence = OpenCodeFence::detect(
|
||||
&pending_source[final_block_start..],
|
||||
raw_source.len(),
|
||||
theme_revision,
|
||||
);
|
||||
|
||||
let mut newly_stable_rendered_len = None;
|
||||
if let Some(boundary) = pending.last_top_level_block_start {
|
||||
let newly_stable_source = &pending_source[..boundary];
|
||||
|
||||
@@ -11,6 +11,9 @@ use pretty_assertions::assert_eq;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[path = "code_fence_render_tests.rs"]
|
||||
mod code_fence_tests;
|
||||
|
||||
fn test_cwd() -> PathBuf {
|
||||
std::env::temp_dir()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user