Show image previews in the fullscreen transcript (#54216)

## What changed

Render local viewed images and generated image artifacts as bounded thumbnails in the rich fullscreen transcript using Kitty or Sixel graphics. Viewed images appear when their activity is expanded or the detailed presentation is active; generated images appear alongside their saved-path entry.

Keep previews clipped to the viewport, retain graphics across unrelated redraws, and remove them when overlays open. Bound image decoding and cache memory, and preserve text-only entries for remote workspaces, unsupported terminals, missing files, and oversized images. Image rows do not enter searchable or selectable text.

## Testing

Add layout and snapshot coverage for disclosure, viewport clipping, rewrapping, raw-mode fallback, deleted files, and oversized rasters. Add terminal coverage for retained Kitty graphics, repainting after a clear, local-file transport, and overlay cleanup.

GitOrigin-RevId: fb2ec146331acfcfc82b2b283616671c9ee93757
This commit is contained in:
Eric Traut
2026-10-15 07:00:34 +00:00
committed by copyberry
parent bb64b894fd
commit dffa0deb69
24 changed files with 631 additions and 47 deletions
+30 -10
View File
@@ -11,7 +11,7 @@ use std::sync::Weak;
use super::*;
const MAX_CACHED_ENTRIES: usize = 64;
const MAX_CACHED_TEXT_BYTES: usize = 8 * 1024 * 1024;
const MAX_CACHED_LAYOUT_BYTES: usize = 8 * 1024 * 1024;
/// The selected activity presentation and any live content that follows it.
pub(crate) struct ActivityTranscriptLines {
@@ -36,7 +36,8 @@ pub(super) fn activity_layout(
return layout;
}
let label = match (expanded, lines.disclosure) {
(true, _) => "− Show less".to_owned(),
(true, Some(ActivityDisclosure::OutputLines(_))) => "− Show less".to_owned(),
(true, Some(ActivityDisclosure::Generic) | None) => "− Hide details".to_owned(),
(false, Some(ActivityDisclosure::OutputLines(count))) => {
let noun = if count == 1 { "line" } else { "lines" };
let mut label = format!("+ {count} {noun}");
@@ -70,6 +71,7 @@ struct RenderState {
foreground: Option<(u8, u8, u8)>,
background: Option<(u8, u8, u8)>,
color_level: crate::terminal_palette::StdoutColorLevel,
cell_size: Option<(u32, u32)>,
}
struct CachedLayout {
@@ -88,6 +90,7 @@ pub(super) struct CellPresentation {
expanded: bool,
disclosure: bool,
detailed: bool,
image_protocol: Option<crate::pets::ImageProtocol>,
}
impl TranscriptView {
@@ -168,6 +171,7 @@ impl TranscriptView {
let ids = cell.activity_ids();
let disclosure = !self.detailed && mode == HistoryRenderMode::Rich && !ids.is_empty();
let manually_expanded = self.disclosure.is_expanded(&ids);
let reveal_image = ids.is_empty() || manually_expanded || self.detailed;
let expanded = disclosure && (full_content || manually_expanded);
if manually_expanded {
self.disclosure.expanded.extend(ids);
@@ -179,13 +183,16 @@ impl TranscriptView {
expanded,
disclosure,
detailed: full_content,
image_protocol: self
.image_protocol
.filter(|_| mode == HistoryRenderMode::Rich && reveal_image),
};
let shortcut = self
.disclosure
.keymap
.primary_hint(KeymapContext::Global, "open_transcript");
Some(self.cache.get(cell, width, presentation, || {
if disclosure {
let mut layout = if disclosure {
activity_layout(
ActivityTranscriptLines {
activity: if expanded {
@@ -210,7 +217,15 @@ impl TranscriptView {
),
width,
)
};
if let Some(protocol) = presentation.image_protocol
&& let Some(path) = cell.image_path()
{
layout.image =
crate::transcript_image::TranscriptImage::load(path.as_path(), width, protocol)
.map(Arc::new);
}
layout
}))
}
}
@@ -223,6 +238,7 @@ impl LayoutCache {
foreground: crate::terminal_palette::default_fg(),
background: crate::terminal_palette::default_bg(),
color_level: crate::terminal_palette::stdout_color_level(),
cell_size: crate::transcript_image::terminal_cell_size(),
};
if self.render_state.as_ref() != Some(&state) {
self.entries.clear();
@@ -283,16 +299,20 @@ impl LayoutCache {
}
fn evict(&mut self) {
let mut bytes = self
.entries
.iter()
.map(|entry| entry.layout.text().len())
.sum::<usize>();
let retained_bytes = |entry: &CachedLayout| {
entry.layout.text().len()
+ entry
.layout
.image
.as_ref()
.map_or(0, |image| image.pixels.len())
};
let mut bytes = self.entries.iter().map(retained_bytes).sum::<usize>();
// Retain a single oversized entry rather than repeatedly laying it out while visible.
while self.entries.len() > 1
&& (self.entries.len() > MAX_CACHED_ENTRIES || bytes > MAX_CACHED_TEXT_BYTES)
&& (self.entries.len() > MAX_CACHED_ENTRIES || bytes > MAX_CACHED_LAYOUT_BYTES)
{
bytes -= self.entries.remove(/*index*/ 0).layout.text().len();
bytes -= retained_bytes(&self.entries.remove(/*index*/ 0));
}
}
}
@@ -51,6 +51,7 @@ fn layouts_refresh_for_width_animation_and_mutable_frames() {
expanded: false,
disclosure: false,
detailed: false,
image_protocol: None,
},
|| TextLayout::new(history.transcript_hyperlink_lines(width), width),
)
@@ -107,6 +108,7 @@ fn recent_entries_are_reused_and_old_entries_are_evicted() {
expanded: false,
disclosure: false,
detailed: false,
image_protocol: None,
},
|| {
TextLayout::new(
@@ -127,6 +129,7 @@ fn recent_entries_are_reused_and_old_entries_are_evicted() {
expanded: false,
disclosure: false,
detailed: false,
image_protocol: None,
},
|| {
TextLayout::new(
@@ -141,3 +144,133 @@ fn recent_entries_are_reused_and_old_entries_are_evicted() {
[1, 1, 2]
);
}
#[test]
fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() {
use crate::history_cell::new_image_generation_call;
use crate::history_cell::new_view_image_tool_call;
use crate::pets::ImageProtocol;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_path_uri::LegacyAppPathString;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("artifact.png");
image::RgbaImage::from_pixel(480, 160, image::Rgba([255, 0, 0, 255]))
.save(&path)
.unwrap();
let cells: Vec<Arc<dyn HistoryCell>> = vec![
Arc::new(new_view_image_tool_call(
"view-call".to_string(),
LegacyAppPathString::from_string(path.to_string_lossy().into_owned()),
)),
Arc::new(new_image_generation_call(
"image-call".to_string(),
"completed",
Some("A red rectangle".to_string()),
Some(AbsolutePathBuf::try_from(path.clone()).unwrap()),
)),
];
let area = Rect::new(
/*x*/ 0, /*y*/ 0, /*width*/ 140, /*height*/ 40,
);
let frame = |buf: &Buffer| {
let uri = url::Url::from_file_path(&path).unwrap().to_string();
crate::transcript_view::tests::text(buf)
.replace(&uri, "file:///workspace/artifact.png")
.replace(path.to_str().unwrap(), "/workspace/artifact.png")
.trim_end()
.to_string()
};
let mut view = TranscriptView::default();
let mut buf = Buffer::empty(area);
view.render(area, &mut buf, &cells);
let text_only = buf.clone();
view.image_protocol = Some(ImageProtocol::Kitty);
view.render(area, &mut buf, &cells);
let collapsed = frame(&buf);
let collapsed_areas: Vec<_> = view.image_placements().iter().map(|p| p.area).collect();
// Find may request full text, but only manual expansion may reveal inspected pixels.
assert!(
view.entry_layout(&cells, /*index*/ 0, /*full_content*/ true)
.unwrap()
.image
.is_none()
);
// An inspected image only appears after its existing activity control is expanded.
assert!(view.toggle_disclosure_at(&cells, /*column*/ 4, /*row*/ 1));
view.image_protocol = None;
view.render(area, &mut buf, &cells);
let text: Vec<_> = (0..cells.len())
.map(|index| view.layout(&cells, index).unwrap().text().to_string())
.collect();
view.image_protocol = Some(ImageProtocol::Kitty);
view.render(area, &mut buf, &cells);
let expanded_areas: Vec<_> = view.image_placements().iter().map(|p| p.area).collect();
insta::assert_snapshot!(
"fullscreen_image_artifact_rows",
format!(
"Collapsed inspection / generated output\n{collapsed}\nGraphics: {collapsed_areas:?}\n\nExpanded inspection / generated output\n{}\nGraphics: {expanded_areas:?}",
frame(&buf)
)
);
for (index, expected) in text.iter().enumerate() {
let layout = view.layout(&cells, index).unwrap();
assert_eq!(layout.text(), expected);
let pinned_narrow = layout.rewrap(/*width*/ 8);
assert!(pinned_narrow.visible_image().is_none());
let pinned_restored = pinned_narrow.rewrap(area.width);
assert_eq!(pinned_restored.row_count(), layout.row_count());
assert!(pinned_restored.visible_image().is_some());
}
// Short, inset viewports may begin in an image. Its placement stays inside the viewport.
let short = Rect::new(
/*x*/ 7, /*y*/ 5, /*width*/ 16, /*height*/ 3,
);
view.jump_to_latest();
view.render(short, &mut Buffer::empty(short), &cells);
let placements = view.image_placements();
assert!(!placements.is_empty());
assert!(
placements
.iter()
.all(|placement| short.contains(placement.area.as_position())
&& placement.area.right() <= short.right()
&& placement.area.bottom() <= short.bottom())
);
view.render(area, &mut buf, &cells);
let control_row = view
.visible
.iter()
.position(|visible| visible.layout.disclosure_row() == Some(visible.row))
.unwrap() as u16;
assert!(view.toggle_disclosure_at(&cells, /*column*/ 4, control_row));
view.render(area, &mut buf, &cells);
// Raw mode and unsupported terminals reserve no rows, even after rich previews were cached.
view.jump_to_latest();
view.set_presentation(/*detailed*/ false, HistoryRenderMode::Raw);
view.render(area, &mut buf, &cells);
assert!((0..cells.len()).all(|index| view.layout(&cells, index).unwrap().image.is_none()));
view.set_presentation(/*detailed*/ false, HistoryRenderMode::Rich);
view.image_protocol = None;
view.render(area, &mut buf, &cells);
assert_eq!(buf, text_only);
// Resumed sessions with a deleted file fall back to the original text entry.
std::fs::remove_file(&path).unwrap();
view.image_protocol = Some(ImageProtocol::Kitty);
view.render(area, &mut buf, &cells);
assert_eq!(buf, text_only);
assert!(view.image_placements().is_empty());
// A tiny compressed file with a large decoded raster must also fall back to text.
image::RgbImage::new(4000, 3000).save(&path).unwrap();
view.image_protocol = None;
view.render(area, &mut buf, &cells);
view.image_protocol = Some(ImageProtocol::Kitty);
view.render(area, &mut buf, &cells);
assert_eq!(buf, text_only);
assert!(view.image_placements().is_empty());
}
@@ -1,5 +1,6 @@
---
source: tui/src/transcript_view/footer_tests.rs
assertion_line: 515
expression: "hints.join(\"\\n\")"
---
collapsed
@@ -44,14 +45,14 @@ Buffer {
content: [
"Activity summary ",
"Hidden details ",
" − Show less ",
" − Hide details ",
],
styles: [
x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: UNDERLINED,
x: 0, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE,
x: 0, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 4, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD,
x: 15, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
x: 18, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: DIM,
]
}
"ctrl-x t": <composer hints>
@@ -0,0 +1,28 @@
---
source: tui/src/transcript_view/layout_tests.rs
assertion_line: 210
expression: "format!(\"Collapsed inspection / generated output\\n{collapsed}\\nGraphics: {collapsed_areas:?}\\n\\nExpanded inspection / generated output\\n{}\\nGraphics: {expanded_areas:?}\",\nframe(&buf))"
---
Collapsed inspection / generated output
• Viewed image artifact.png
+ Show details
• Generated Image:
└ A red rectangle
└ Saved to: file:///workspace/artifact.png
Graphics: [Rect { x: 52, y: 6, width: 36, height: 6 }]
Expanded inspection / generated output
• Viewed image /workspace/artifact.png
− Hide details
• Generated Image:
└ A red rectangle
└ Saved to: file:///workspace/artifact.png
Graphics: [Rect { x: 52, y: 2, width: 36, height: 6 }, Rect { x: 52, y: 12, width: 36, height: 6 }]
@@ -1,5 +1,6 @@
---
source: tui/src/transcript_view/search_tests.rs
assertion_line: 989
expression: "crate::transcript_view::tests::text(&buffer)"
---
• Running 'echo visible …
@@ -12,8 +13,8 @@ expression: "crate::transcript_view::tests::text(&buffer)"
$ 'echo visible
echo hidden newer needle'
− Show less
− Hide details
$ 'echo visible
echo hidden manual'
− Show less
− Hide details
+25
View File
@@ -42,6 +42,7 @@ use ratatui::text::Line;
/// One cell's logical text and its width-dependent display rows.
pub(super) struct TextLayout {
pub(super) image: Option<std::sync::Arc<crate::transcript_image::TranscriptImage>>,
logical: Vec<LogicalLine>,
text: String,
rows: Vec<TextRow>,
@@ -137,6 +138,7 @@ impl TextLayout {
layout =
layout.with_disclosure_control_at(control.label.clone(), control.source_offset);
}
layout.image = self.image.clone();
if self.separated {
layout.with_leading_separator()
} else {
@@ -271,9 +273,28 @@ impl TextLayout {
}
pub(super) fn row_count(&self) -> usize {
// Images occupy visual space, but have no searchable or selectable text rows.
self.text_row_count()
+ self
.visible_image()
.map_or(0, |image| usize::from(image.rows))
}
pub(super) fn text_row_count(&self) -> usize {
self.rows.len()
}
pub(super) fn visible_image(
&self,
) -> Option<&std::sync::Arc<crate::transcript_image::TranscriptImage>> {
self.image.as_ref().filter(|image| {
image.columns + 2 <= self.width
&& (image.protocol != crate::pets::ImageProtocol::Sixel
|| crate::transcript_image::terminal_cell_size()
== Some((image.cell_width, image.cell_height)))
})
}
pub(super) fn disclosure_row(&self) -> Option<usize> {
self.disclosure_control.as_ref().map(|control| control.row)
}
@@ -363,6 +384,9 @@ impl TextLayout {
/// Find the display row containing a source position, including omitted wrapping whitespace.
pub(super) fn row_for_offset(&self, offset: usize) -> usize {
if self.visible_image().is_some() && offset >= self.text.len() {
return self.row_count().saturating_sub(1);
}
let before_control = if let Some(control_row) = self.disclosure_row() {
let after_control = &self.rows[control_row + 1..];
let matching_rows = after_control.partition_point(|row| row.source.start <= offset);
@@ -500,6 +524,7 @@ impl TextLayout {
}
}
Self {
image: None,
logical,
text,
rows,