Add sidebar image previews and present generated images after answers (#61919)

## What changed

- Open PNG, JPEG, GIF, and WebP file links as sidebar image previews, fitted to the available space with format and dimension labels. Show a fallback when terminal graphics are unavailable, and preserve text previews for files such as Git LFS pointers.
- In the owned transcript, place completed, saved image results after the assistant's answer, with larger, left-aligned previews and a clickable `Generated image` label. Preserve this ordering during replay and history pagination, including interrupted turns.
- Reuse native Kitty image IDs and move existing placements without retransmitting their pixels. Probe Kitty graphics support in `herdr` before enabling previews.

## Testing

Add regression coverage for sidebar preview fallbacks, generated-image ordering, pagination and search, preview layout, Kitty placement reuse, and graphics probe replies interleaved with pasted input.

GitOrigin-RevId: 22d8aa2724f17bfa7f8280a2d11d4a7e2cb66728
This commit is contained in:
Felipe Coury
2026-11-17 00:13:04 +00:00
committed by copyberry
parent f8e58064fc
commit a669d8ffb2
45 changed files with 1242 additions and 186 deletions
+11 -3
View File
@@ -1438,8 +1438,16 @@ impl App {
Ok(())
}
fn prepare_frame_images(&mut self) -> Option<crate::pets::ImageProtocol> {
let protocol = self.chat_widget.pet_image_support().protocol().filter(|_| {
fn preview_image_protocol(&self, tui: &tui::Tui) -> Option<crate::pets::ImageProtocol> {
match tui.herdr_kitty_graphics {
Some(true) => Some(crate::pets::ImageProtocol::Kitty),
Some(false) => None,
None => self.chat_widget.pet_image_support().protocol(),
}
}
fn prepare_frame_images(&mut self, tui: &tui::Tui) -> Option<crate::pets::ImageProtocol> {
let protocol = self.preview_image_protocol(tui).filter(|_| {
!self.app_server_target.uses_remote_workspace()
&& crate::projectless::has_only_local_environments(&self.environment_manager)
});
@@ -1453,7 +1461,7 @@ impl App {
if tui.is_owned_screen() {
return self.render_owned_transcript(tui, screen_size);
}
self.prepare_frame_images();
self.prepare_frame_images(tui);
self.chat_widget
.empty_state_animation
.borrow_mut()
+100 -8
View File
@@ -10,6 +10,7 @@ use crate::history_cell::UserHistoryCell;
use crate::history_cell::is_session_history_prefix_cell;
use crate::pager_overlay::TranscriptHistoryState;
use crate::thread_transcript::RawReasoningVisibility;
use crate::thread_transcript::TranscriptCells;
use crate::thread_transcript::thread_items_to_transcript_cells;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ThreadItemsListResponse;
@@ -108,19 +109,46 @@ impl App {
};
let width = tui.terminal.last_known_screen_size.width;
self.remove_hidden_review_cells(tui, &turns, &hidden_item_ids, thread_id, &cwd, visibility);
let cells = self.project_older_history_cells(
let split_turn = items
.last()
.and_then(|(turn_id, last)| {
turns.iter().find(|turn| {
turn.id == *turn_id
&& matches!(turn.status, TurnStatus::Completed | TurnStatus::Interrupted)
&& turn.items.last().is_some_and(|item| item.id() != last.id())
})
})
.filter(|_| self.local_settings.transcript_mode.is_owned());
let existing = self
.side_companions
.get(&thread_id)
.filter(|_| parked)
.map_or(self.transcript_cells.as_slice(), |pane| {
pane.cells.as_slice()
});
let split_insertion = split_image_insertion(existing, split_turn);
let (cells, split_images) = self.project_older_history_cells(
items,
&turns,
&hidden_item_ids,
thread_id,
&cwd,
visibility,
split_insertion.is_some(),
);
if let Some(pane) = self.side_companions.get_mut(&thread_id).filter(|_| parked) {
let images = split_insertion.and_then(|index| {
if split_images.is_empty() {
return None;
}
let inserted = index..index + split_images.len();
pane.splice_older_cells(index..index, split_images);
Some(inserted)
});
let index = history_prefix_len(&pane.cells);
let inserted = index..index + cells.len();
pane.splice_older_cells(index..index, cells);
pane.view.history_loaded(&pane.cells, inserted);
pane.view.history_page_loaded(&pane.cells, inserted, images);
pane.view.history = if app_server.has_older_history(thread_id) {
TranscriptHistoryState::Partial
} else {
@@ -130,9 +158,20 @@ impl App {
tui.frame_requester().schedule_frame();
return Ok(());
}
let images = split_insertion.and_then(|index| {
if split_images.is_empty() {
return None;
}
let inserted = index..index + split_images.len();
if let Some(Overlay::Transcript(overlay)) = &mut self.overlay {
overlay.insert_history_page(index, split_images.clone());
}
self.transcript_cells.splice(index..index, split_images);
Some(inserted)
});
let inserted = self.prepend_older_transcript_cells(cells);
self.transcript_view
.history_loaded(&self.transcript_cells, inserted.clone());
.history_page_loaded(&self.transcript_cells, inserted.clone(), images);
if !inserted.is_empty() {
self.join_older_activity_group(inserted.end, &turns);
}
@@ -240,6 +279,10 @@ impl App {
}
/// Project each terminal row on the page containing the end of its turn.
#[allow(
clippy::too_many_arguments,
reason = "Projection needs the thread context and whether its split turn has a rendered end."
)]
fn project_older_history_cells(
&mut self,
items: Vec<(String, ThreadItemPayload)>,
@@ -248,9 +291,12 @@ impl App {
thread_id: ThreadId,
cwd: &AbsolutePathBuf,
visibility: RawReasoningVisibility,
) -> Vec<Arc<dyn HistoryCell>> {
relocate_split_images: bool,
) -> (TranscriptCells, TranscriptCells) {
let mut cells = Vec::new();
let mut split_images = Vec::new();
for (range, turn_end) in completion::group_turn_items(&items, turns) {
let start = cells.len();
// Internal prompts stay invisible, but still separate adjacent tool groups.
for visible in items[range]
.split(|(turn_id, item)| hidden_item_ids.contains(&(turn_id.as_str(), item.id())))
@@ -263,10 +309,24 @@ impl App {
Some(&self.config),
));
}
let Some(turn) = turn_end else { continue };
let Some(turn) = turn_end else {
if relocate_split_images {
split_images.extend(
cells.extract_if(start.., |cell| generated_image(cell.as_ref()).is_some()),
);
}
continue;
};
if self.local_settings.transcript_mode.is_owned()
&& turn.status != TurnStatus::InProgress
{
cells[start..].sort_by_key(|cell| generated_image(cell.as_ref()).is_some());
}
if turn.status == TurnStatus::Interrupted && !self.side_threads.contains_key(&thread_id)
{
cells.push(Arc::new(history_cell::new_interruption_notice()));
cells.push(Arc::new(history_cell::InterruptionNotice(Some(
turn.id.clone(),
))));
}
if turn.status == TurnStatus::Completed
&& let Some(completion) = if let Some(pane) =
@@ -287,10 +347,14 @@ impl App {
cells.push(Arc::new(completion));
}
}
for cell in cells.iter_mut().filter_map(Arc::get_mut) {
for cell in cells
.iter_mut()
.chain(&mut split_images)
.filter_map(Arc::get_mut)
{
self.link_viewed_image(cell);
}
cells
(cells, split_images)
}
/// Insert each page once, keeping session headers and any backtrack selection in place.
@@ -399,6 +463,34 @@ impl App {
}
}
fn generated_image(cell: &dyn HistoryCell) -> Option<&history_cell::ImageGenerationHistoryCell> {
cell.as_any()
.downcast_ref::<history_cell::ImageGenerationHistoryCell>()
.filter(|image| image.is_turn_output())
}
/// The first page can contain the answer while an older page contains its image.
fn split_image_insertion(
existing: &[Arc<dyn HistoryCell>],
split_turn: Option<&Turn>,
) -> Option<usize> {
let turn = split_turn?;
let end = existing.iter().position(|cell| {
let cell = cell.as_any();
cell.downcast_ref::<history_cell::FinalMessageSeparator>()
.is_some_and(|cell| cell.turn_id.as_deref() == Some(turn.id.as_str()))
|| cell
.downcast_ref::<history_cell::InterruptionNotice>()
.is_some_and(|cell| cell.0.as_deref() == Some(turn.id.as_str()))
})?;
Some(
existing[..end]
.iter()
.rposition(|cell| generated_image(cell.as_ref()).is_none())
.map_or(0, |index| index + 1),
)
}
/// Classify internal review prompts across page boundaries before rendering any older items.
fn hidden_review_item_ids(turns: &[Turn]) -> HashSet<(&str, &str)> {
let mut hidden = HashSet::new();
@@ -145,13 +145,14 @@ async fn resume_places_interruption_before_searches_split_across_pages() {
(audit.id.clone(), audit.items[0].payload.clone()),
];
let cwd = app.config.cwd.clone();
let older = app.project_older_history_cells(
let (older, _) = app.project_older_history_cells(
page,
&[canceled, audit],
&HashSet::new(),
ThreadId::new(),
&cwd,
RawReasoningVisibility::Hidden,
/*relocate_split_images*/ false,
);
cells.splice(0..0, older);
let rows = cells
@@ -170,6 +171,86 @@ async fn resume_places_interruption_before_searches_split_across_pages() {
);
}
#[tokio::test]
async fn generated_image_from_an_older_page_follows_its_answer() {
let (mut app, _, _) = crate::app::tests::make_test_app_with_channels().await;
app.local_settings.transcript_mode = crate::transcript_mode::TranscriptMode::Owned;
for status in [TurnStatus::Completed, TurnStatus::Interrupted] {
let completed = status == TurnStatus::Completed;
let mut turn = turn("image-turn", status, &[]);
let answer: ThreadItemPayload = serde_json::from_value(serde_json::json!({
"type": "agentMessage", "id": "answer", "text": "Here is the drawing.",
"phase": "final_answer",
}))
.unwrap();
turn.duration_ms = Some(2_000);
turn.items.push(ThreadItem::untimed(answer.clone()));
let cwd = app.config.cwd.clone();
let (mut existing, _) = app.project_older_history_cells(
vec![(turn.id.clone(), answer)],
std::slice::from_ref(&turn),
&HashSet::new(),
ThreadId::new(),
&cwd,
RawReasoningVisibility::Hidden,
/*relocate_split_images*/ false,
);
let path = app.config.cwd.as_path().join("drawing.png");
let image: ThreadItemPayload = serde_json::from_value(serde_json::json!({
"type": "imageGeneration", "id": "image", "status": "completed",
"revisedPrompt": "A drawing", "result": "", "savedPath": path,
}))
.unwrap();
turn.items
.insert(/*index*/ 0, ThreadItem::untimed(image.clone()));
let cwd = app.config.cwd.clone();
let mut previous = super::tests::turn("previous", TurnStatus::Completed, &[]);
previous.items.push(ThreadItem::untimed(image.clone()));
let items = vec![
(previous.id.clone(), image.clone()),
(turn.id.clone(), image),
];
let turns = [previous, turn];
let (older, images) = app.project_older_history_cells(
items,
&turns,
&HashSet::new(),
ThreadId::new(),
&cwd,
RawReasoningVisibility::Hidden,
/*relocate_split_images*/ true,
);
assert_eq!(
older
.iter()
.filter(|cell| generated_image(cell.as_ref()).is_some())
.count(),
1
);
assert_eq!(images.len(), 1);
let index = split_image_insertion(&existing, Some(&turns[1])).unwrap();
existing.splice(index..index, images);
let rows: Vec<_> = existing
.iter()
.flat_map(|cell| cell.compact_hyperlink_lines(/*width*/ 80))
.map(|line| line.line.to_string())
.collect();
assert_eq!(
rows,
[
"• Here is the drawing.",
" Generated image",
if completed {
" Worked for 2s"
} else {
"■ Conversation interrupted - use /feedback if something went wrong"
}
]
);
}
}
#[tokio::test]
async fn hidden_last_item_keeps_turn_groups_and_completion_boundaries() {
let computer = |id: &str| {
@@ -222,13 +303,14 @@ async fn hidden_last_item_keeps_turn_groups_and_completion_boundaries() {
})
.collect::<Vec<_>>();
let cells = app.project_older_history_cells(
let (cells, _) = app.project_older_history_cells(
items.clone(),
&turns,
&hidden,
thread_id,
&cwd,
RawReasoningVisibility::Hidden,
/*relocate_split_images*/ false,
);
let groups = cells
.iter()
@@ -252,13 +334,14 @@ async fn hidden_last_item_keeps_turn_groups_and_completion_boundaries() {
),
);
let repeated = app.project_older_history_cells(
let (repeated, _) = app.project_older_history_cells(
items,
&turns,
&hidden,
thread_id,
&cwd,
RawReasoningVisibility::Hidden,
/*relocate_split_images*/ false,
);
assert_eq!(
repeated
+14 -9
View File
@@ -8,6 +8,7 @@
//! Plain Enter returns an empty composer to latest after transcript interactions and prompt editing.
use super::sidebar::PaneFocus;
use super::sidebar_document::DocumentRenderOptions;
use super::turn_tips::TipSurface;
use super::*;
use crate::bottom_pane::CommandPopupPlacement;
@@ -244,7 +245,7 @@ impl App {
tui: &mut tui::Tui,
screen_size: Size,
) -> Result<Rect> {
self.transcript_view.image_protocol = self.prepare_frame_images();
self.transcript_view.image_protocol = self.prepare_frame_images(tui);
let frames = tui.frame_requester();
if self.conversation_frame.sidebar.is_some() || self.conversation_frame.chat_strip.is_some()
{
@@ -312,6 +313,7 @@ impl App {
})
});
let sidebar_parent = self.sidebar_parent();
let sidebar_image_protocol = self.preview_image_protocol(tui);
let sidebar_motion =
MotionMode::from_animations_enabled(self.local_settings.tui.animations);
// An input refresh opens a fresh terminal buffer. Carry the outer owner's
@@ -369,6 +371,9 @@ impl App {
}
Some(sidebar::SidebarLayout::Panel { sidebar, focus, .. }) => {
let focused = *focus == PaneFocus::Sidebar;
frame
.image_placements
.retain(|image| !sidebar.intersects(image.area));
if focused {
frame.cursor_position = None;
}
@@ -381,11 +386,14 @@ impl App {
let [_, body] = side_pane::tab_and_body(*sidebar);
if let Some(cursor) = doc.paint(
body,
frame.buffer,
sidebar_motion,
&frames,
&self.keymap,
resize_hint,
frame,
DocumentRenderOptions {
motion: sidebar_motion,
frames: &frames,
keymap: &self.keymap,
resize_hint,
image_protocol: sidebar_image_protocol,
},
) && focused
{
frame.cursor_position = Some(cursor);
@@ -508,9 +516,6 @@ impl App {
frame.buffer,
);
}
frame
.image_placements
.retain(|image| !sidebar.intersects(image.area));
}
None => {}
}
@@ -556,7 +556,13 @@ async fn owned_details_keep_the_composer_cursor_and_screen() -> Result<()> {
image::RgbaImage::from_pixel(8, 16, image::Rgba([80, 30, 20, 255])).save(&path)?;
let protocol = crate::pets::ImageProtocol::KittyLocalFile;
let image = Arc::new(
crate::transcript_image::TranscriptImage::load(&path, area.width, protocol).unwrap(),
crate::transcript_image::TranscriptImage::load(
&path,
area.width,
protocol,
crate::transcript_image::PreviewKind::Inspected,
)
.unwrap(),
);
let neighbor = Rect::new(/*x*/ 4, /*y*/ 9, image.columns, image.rows);
let presented = |tui: &tui::Tui| {
@@ -583,6 +589,7 @@ async fn owned_details_keep_the_composer_cursor_and_screen() -> Result<()> {
area: neighbor,
first_row: 0,
file: None,
terminal_id: None,
});
})?;
app.finish_conversation(&mut tui, painted)?;
+19
View File
@@ -54,6 +54,7 @@ impl SideCompanion {
Self {
display: CapturedSideDisplay {
rendered_completion_turn_ids: Default::default(),
generated_images: Default::default(),
active: None,
auxiliary: Vec::new(),
draft: Box::default(),
@@ -150,6 +151,13 @@ impl SideCompanion {
self.flush_answer();
self.answer = None;
}
self.cells.extend(
self.display
.generated_images
.take(&turn.id)
.into_iter()
.map(|image| Arc::new(image) as Arc<dyn HistoryCell>),
);
let work = self.display.work.take_if(|work| work.turn_id == turn.id);
if turn.status == TurnStatus::Completed
&& (work.is_some() || store.active_turn_id() == Some(turn.id.as_str()))
@@ -215,6 +223,17 @@ impl SideCompanion {
}
self.display.last_completed_agent_message = Some((turn.to_owned(), id.clone()));
}
ThreadItemPayload::ImageGeneration(item)
if item.status == "completed" && item.saved_path.is_some() =>
{
let image = crate::history_cell::new_image_generation_call(
item.id.clone(),
&item.status,
item.revised_prompt.clone(),
item.saved_path.clone(),
);
self.display.generated_images.record(turn.to_owned(), image);
}
ThreadItemPayload::CommandExecution {
id,
status,
+58 -13
View File
@@ -3,12 +3,16 @@
use super::sidebar::PaneFocus;
use super::sidebar::SidebarContent;
use super::*;
use crate::custom_terminal::Frame;
use crate::key_hint::KeyBinding;
use crate::keymap::KeymapContext;
use crate::keymap::RuntimeKeymap;
use crate::markdown_render::FileLink;
use crate::markdown_render::SourceLocation;
use crate::motion::MotionMode;
use crate::pets::ImageProtocol;
use crate::sidebar_image::DocumentContent;
use crate::sidebar_image::SidebarImage;
use crate::sidebar_tab_strip::SidebarTab;
use crate::terminal_hyperlinks::HyperlinkLine;
use crate::terminal_hyperlinks::visible_lines;
@@ -17,7 +21,6 @@ use crate::transcript_view::ViewAction;
use codex_app_server_client::AppServerPath;
use codex_utils_path_uri::LegacyAppPathString;
use codex_utils_path_uri::PathUri;
use ratatui::buffer::Buffer;
use ratatui::layout::Position;
use ratatui::widgets::Widget;
use unicode_segmentation::UnicodeSegmentation;
@@ -61,6 +64,14 @@ impl HistoryCell for FileCell {
}
}
pub(super) struct DocumentRenderOptions<'a> {
pub(super) motion: MotionMode,
pub(super) frames: &'a tui::FrameRequester,
pub(super) keymap: &'a RuntimeKeymap,
pub(super) resize_hint: Option<&'a str>,
pub(super) image_protocol: Option<ImageProtocol>,
}
pub(super) struct Document {
source: ThreadId,
pub(super) path: PathUri,
@@ -73,6 +84,7 @@ pub(super) struct Document {
enum State {
Loading(Instant),
Failed(String),
Image(Box<SidebarImage>),
Ready {
view: Box<TranscriptView>,
cells: [Arc<dyn HistoryCell>; 1],
@@ -129,9 +141,10 @@ impl Document {
}
}
pub(super) fn loaded(&mut self, result: Result<String, String>) {
pub(super) fn loaded(&mut self, result: Result<DocumentContent, String>) {
self.state = match result {
Ok(source) => {
Ok(DocumentContent::Image(image)) => State::Image(image),
Ok(DocumentContent::Text(source)) => {
let title = self.title();
let code =
self.location.is_some() || !crate::markdown_render::is_markdown_file(&title);
@@ -207,7 +220,7 @@ impl Document {
if restore_markdown || (location.is_some() && line_starts.is_empty()) {
// Coordinate links show source. Plain Markdown links restore rendering.
let source = source.to_owned();
self.loaded(Ok(source));
self.loaded(Ok(DocumentContent::Text(source)));
} else {
*target = location.and_then(|location| source_target(source, line_starts, location));
if let Some((_, offset)) = target {
@@ -221,15 +234,25 @@ impl Document {
pub(super) fn paint(
&mut self,
area: Rect,
buf: &mut Buffer,
motion: MotionMode,
frames: &tui::FrameRequester,
keymap: &RuntimeKeymap,
resize_hint: Option<&str>,
frame: &mut Frame<'_>,
options: DocumentRenderOptions<'_>,
) -> Option<Position> {
let DocumentRenderOptions {
motion,
frames,
keymap,
resize_hint,
image_protocol,
} = options;
let buf = &mut *frame.buffer;
ratatui::widgets::Clear.render(area, buf);
let body = area.inner(ratatui::layout::Margin::new(1, 0));
match &mut self.state {
State::Image(image) => {
frame
.image_placements
.extend(image.paint(body, buf, image_protocol, resize_hint))
}
State::Loading(started) => {
let glyph = crate::motion::loading_glyph(*started, motion, frames);
Line::from(vec![
@@ -510,10 +533,32 @@ impl App {
let reader = server.request_view();
let tx = self.app_event_tx.clone();
tokio::spawn(async move {
let result = reader
.read_document(source, &original_path, &path)
.await
.map_err(|error| format!("{error:#}"));
let result = async {
if (FileLink {
path: path.inferred_native_path_string(),
location,
})
.is_image()
{
let bytes = reader
.read_preview_file(source, &original_path, &path)
.await?;
tokio::task::spawn_blocking(move || match SidebarImage::decode(&bytes) {
Ok(image) => Ok(DocumentContent::Image(Box::new(image))),
Err(image_error) => AppServerSession::preview_text(bytes)
.map(DocumentContent::Text)
.map_err(|_| image_error),
})
.await?
} else {
reader
.read_document(source, &original_path, &path)
.await
.map(DocumentContent::Text)
}
}
.await
.map_err(|error| format!("{error:#}"));
tx.send(AppEvent::SidebarDocumentLoaded {
backend_id,
parent,
+66 -2
View File
@@ -493,6 +493,40 @@ async fn document_preview_scenario() -> Result<()> {
.await?;
assert_eq!(app.sidebar().unwrap().documents.len(), remaining);
}
app.chat_widget
.set_pet_image_support_for_tests(crate::pets::PetImageSupport::Unsupported(
crate::pets::PetImageUnsupportedReason::Terminal,
));
image::RgbImage::new(24, 16).save(dir.path().join("figure.png"))?;
app.open_document(
&mut tui,
&mut server,
parent,
FileLink {
path: "figure.png".into(),
location: None,
},
)
.await?;
Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(true).await)).await?;
Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(false).await)).await?;
let image = side_frame(&mut app, &mut tui, Size::new(160, 15))?;
let image_rows = image
.lines()
.map(|row| row.rsplit('│').next().unwrap_or(row).trim())
.filter(|row| {
row.contains("PNG ·")
|| row.contains("Image preview unavailable")
|| row.contains("←/→ tabs")
})
.collect::<Vec<_>>()
.join("\n");
insta::assert_snapshot!(image_rows, @"
PNG · 24 × 16
Image preview unavailable in this terminal
←/→ tabs · Esc close
");
assert_eq!(app.sidebar().unwrap().documents.len(), 2);
app.open_document(
&mut tui,
&mut server,
@@ -507,11 +541,41 @@ async fn document_preview_scenario() -> Result<()> {
crate::pager_overlay::TranscriptOverlay::new(Vec::new(), app.keymap.pager.clone()),
));
Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(true).await)).await?;
assert_eq!(app.sidebar().unwrap().documents.len(), 2);
assert_eq!(app.sidebar().unwrap().documents.len(), 3);
assert!(app.overlay.take().is_some());
Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(false).await)).await?;
let after_overlay = side_frame(&mut app, &mut tui, size)?;
assert!(after_overlay.contains("Last section."), "{after_overlay}");
assert!(after_overlay.contains("PNG · 24 × 16"), "{after_overlay}");
std::fs::write(
dir.path().join("pointer.png"),
"version https://git-lfs.github.com/spec/v1
size 123
",
)?;
app.open_document(
&mut tui,
&mut server,
parent,
FileLink {
path: "pointer.png".into(),
location: None,
},
)
.await?;
Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(true).await)).await?;
Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(false).await)).await?;
let pointer = side_frame(&mut app, &mut tui, Size::new(200, 15))?;
insta::assert_snapshot!(pointer.lines().filter_map(|row| row.split_once('│')).map(|(_, row)| row.trim()).filter_map(|row| row.split_once(' ').map(|(_, line)| line.trim())).filter(|row| row.starts_with("version ") || row.starts_with("size ")).collect::<Vec<_>>().join("
"), @"
version https://git-lfs.github.com/spec/v1
size 123
");
let Some(SidebarContent::Document(pointer_id)) = app.sidebar().unwrap().visible_content()
else {
panic!("the pointer preview should be visible");
};
app.sidebar_mut().remove_document(pointer_id);
let child = ThreadId::new();
app.side_threads.insert(child, SideThreadState::new(parent));
let (stream, display) = app.chat_widget.take_side_display(/*width*/ 120);
+2 -1
View File
@@ -315,6 +315,7 @@ pub(crate) enum CapturedSideStream {
#[derive(Debug)]
pub(crate) struct CapturedSideDisplay {
pub(crate) rendered_completion_turn_ids: std::collections::HashMap<String, bool>,
pub(crate) generated_images: crate::history_cell::PendingGeneratedImages,
pub(crate) active: Option<Box<dyn HistoryCell>>,
pub(crate) auxiliary: Vec<crate::terminal_hyperlinks::HyperlinkLine>,
pub(crate) draft: Box<crate::bottom_pane::TextArea>,
@@ -540,7 +541,7 @@ pub(crate) enum AppEvent {
backend_id: crate::backend_id::BackendId,
parent: ThreadId,
request: uuid::Uuid,
result: Result<String, String>,
result: Result<crate::sidebar_image::DocumentContent, String>,
},
SavedSideChatsLoaded {
backend_id: crate::backend_id::BackendId,
@@ -81,6 +81,10 @@ impl AppServerSession {
let bytes = self
.read_preview_file(source, original_path, resolved_path)
.await?;
Self::preview_text(bytes)
}
pub(crate) fn preview_text(bytes: Vec<u8>) -> Result<String> {
let text = String::from_utf8(bytes).wrap_err("Not a UTF-8 text file")?;
color_eyre::eyre::ensure!(
!text
@@ -341,8 +341,9 @@ impl AppServerSession {
if let Some(turn) = turns.iter_mut().find(|turn| turn.id == entry.turn_id)
&& !turn.items.iter().any(|item| item.id() == entry.item.id())
{
let turn_id = entry.turn_id.clone();
let item = ThreadItem::try_from(entry)?;
items.push((turn.id.clone(), item.payload.clone()));
items.push((turn_id, item.payload.clone()));
turn.items.insert(/*index*/ 0, item);
turn.items_view = TurnItemsView::Summary;
}
+15 -6
View File
@@ -419,7 +419,7 @@ impl ChatWidget {
}
}
}
self.on_interrupted_turn(reason);
self.on_interrupted_turn_with_id(reason, Some(turn_id));
self.input_queue.pending_steers.extend(later);
for (message, record) in unconfirmed {
self.input_queue.queued_user_messages.push_back(message);
@@ -430,7 +430,12 @@ impl ChatWidget {
self.refresh_pending_input_preview();
}
#[cfg(test)]
pub(super) fn on_interrupted_turn(&mut self, reason: TurnAbortReason) {
self.on_interrupted_turn_with_id(reason, /*turn_id*/ None);
}
fn on_interrupted_turn_with_id(&mut self, reason: TurnAbortReason, turn_id: Option<&str>) {
self.requeue_image_submission();
// Finalize, log a gentle prompt, and clear running state.
self.finalize_turn();
@@ -454,12 +459,16 @@ impl ChatWidget {
/*hint*/ None,
));
} else {
self.add_to_history(match reason {
TurnAbortReason::BudgetLimited => history_cell::new_error_event(
"Goal budget reached - the turn was stopped.".to_string(),
match reason {
TurnAbortReason::BudgetLimited => {
self.add_to_history(history_cell::new_error_event(
"Goal budget reached - the turn was stopped.".to_string(),
))
}
TurnAbortReason::Interrupted => self.add_to_history(
history_cell::InterruptionNotice(turn_id.map(str::to_owned)),
),
TurnAbortReason::Interrupted => history_cell::new_interruption_notice(),
});
}
}
}
+3
View File
@@ -501,6 +501,7 @@ impl ChatWidget {
question_drafts = self.take_question_drafts();
}
self.last_non_retry_error = None;
self.flush_generated_images(&notification.turn.id);
let completion = self.completion_cell(&notification.turn, replay_kind);
self.on_task_complete(
last_agent_message.map(|(_, text)| text.to_owned()),
@@ -513,6 +514,7 @@ impl ChatWidget {
question_drafts = self.take_question_drafts();
}
self.last_non_retry_error = None;
self.flush_generated_images(&notification.turn.id);
let reason = if self
.turn_lifecycle
.take_budget_limited(notification.turn.id.as_str())
@@ -527,6 +529,7 @@ impl ChatWidget {
if replay_kind.is_none() {
question_drafts = self.take_question_drafts();
}
self.flush_generated_images(&notification.turn.id);
let error = notification
.turn
.error
+1 -6
View File
@@ -413,12 +413,7 @@ impl ChatWidget {
self.on_view_image_tool_call(id, path);
}
ThreadItemPayload::ImageGeneration(item) => {
self.on_image_generation_end(
item.id,
item.status,
item.revised_prompt,
item.saved_path,
);
self.on_image_generation_end(&turn_id, item);
}
ThreadItemPayload::EnteredReviewMode { review, .. } => {
if from_replay {
@@ -1,7 +1,12 @@
---
source: tui/src/chatwidget/tests.rs
source: tui/src/chatwidget/tests/exec_flow.rs
assertion_line: 1189
expression: combined
---
• Here is the square.
• Generated Image:
└ A tiny blue square
└ Saved to: file:///tmp/ig-1.png
Worked for 2s
+1
View File
@@ -258,6 +258,7 @@ impl ChatWidget {
});
let display = CapturedSideDisplay {
rendered_completion_turn_ids: self.turn_lifecycle.rendered_completion_turn_ids.clone(),
generated_images: std::mem::take(&mut self.turn_lifecycle.generated_images),
active,
last_completed_agent_message: self.transcript.last_completed_agent_message.clone(),
auxiliary: self
+21 -4
View File
@@ -1153,6 +1153,8 @@ async fn image_generation_begin_restores_working_status_after_single_line_preamb
#[tokio::test]
async fn image_generation_call_adds_history_cell() {
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
chat.local_settings.transcript_mode = crate::transcript_mode::TranscriptMode::Owned;
handle_turn_started(&mut chat, "turn-1");
handle_image_generation_end(
&mut chat,
@@ -1162,13 +1164,28 @@ async fn image_generation_call_adds_history_cell() {
Some(test_path_buf("/tmp/ig-1.png").abs()),
);
let cells = drain_insert_history(&mut rx);
assert_eq!(cells.len(), 1, "expected a single history cell");
assert!(drain_insert_history(&mut rx).is_empty());
complete_assistant_message(
&mut chat,
"image-answer",
"Here is the square.",
Some(MessagePhase::FinalAnswer),
);
handle_turn_completed(&mut chat, "turn-1", Some(2_000));
let cells = drain_insert_history_with(&mut rx, |cell| {
if cell.as_any().is::<history_cell::FinalMessageSeparator>() {
return vec!["Worked for 2s".into()];
}
cell.display_lines(/*width*/ 80)
});
let platform_file_url = url::Url::from_file_path(test_path_buf("/tmp/ig-1.png"))
.expect("test path should convert to file URL")
.to_string();
let combined =
lines_to_single_string(&cells[0]).replace(&platform_file_url, "file:///tmp/ig-1.png");
let combined = cells
.iter()
.map(|lines| lines_to_single_string(lines))
.collect::<String>()
.replace(&platform_file_url, "file:///tmp/ig-1.png");
assert_chatwidget_snapshot!("image_generation_call_history_snapshot", combined);
handle_image_generation_end(
+28 -10
View File
@@ -27,21 +27,39 @@ impl ChatWidget {
pub(super) fn on_image_generation_end(
&mut self,
call_id: String,
status: String,
revised_prompt: Option<String>,
saved_path: Option<AbsolutePathBuf>,
turn_id: &str,
item: codex_app_server_protocol::ImageGenerationItem,
) {
self.flush_answer_stream_with_separator();
self.add_to_history(history_cell::new_image_generation_call(
call_id,
&status,
revised_prompt,
saved_path,
));
let image = history_cell::new_image_generation_call(
item.id,
&item.status,
item.revised_prompt,
item.saved_path,
);
if self.local_settings.transcript_mode.is_owned()
&& !turn_id.is_empty()
&& image.is_turn_output()
{
self.turn_lifecycle
.generated_images
.record(turn_id.to_owned(), image);
} else {
self.add_to_history(image);
}
self.request_redraw();
}
pub(super) fn flush_generated_images(&mut self, turn_id: &str) {
let images = self.turn_lifecycle.generated_images.take(turn_id);
if !images.is_empty() {
self.flush_answer_and_plan_streams();
for image in images {
self.add_to_history(image);
}
}
}
pub(super) fn on_file_change_completed(&mut self, item: ThreadItemPayload) {
self.defer_or_handle(
item,
@@ -19,6 +19,7 @@ pub(super) struct TurnLifecycleState {
pub(super) budget_limited_turn_ids: HashSet<String>,
/// False records an untimed turn boundary that may still receive a completion label.
pub(super) rendered_completion_turn_ids: HashMap<String, bool>,
pub(super) generated_images: crate::history_cell::PendingGeneratedImages,
pub(super) goal_status_active_turn_started_at: Option<Instant>,
}
@@ -32,6 +33,7 @@ impl TurnLifecycleState {
admitted_turn_id: None,
budget_limited_turn_ids: HashSet::new(),
rendered_completion_turn_ids: HashMap::new(),
generated_images: Default::default(),
goal_status_active_turn_started_at: None,
}
}
@@ -66,6 +68,7 @@ impl TurnLifecycleState {
self.admitted_turn_id = None;
self.budget_limited_turn_ids.clear();
self.rendered_completion_turn_ids.clear();
self.generated_images = Default::default();
}
pub(super) fn set_prevent_idle_sleep(&mut self, enabled: bool) {
+178 -47
View File
@@ -25,6 +25,8 @@ use std::io;
use std::io::Write;
use crossterm::cursor::MoveTo;
use crossterm::cursor::RestorePosition;
use crossterm::cursor::SavePosition;
use crossterm::cursor::SetCursorStyle;
use crossterm::queue;
use crossterm::style::Colored;
@@ -406,64 +408,118 @@ where
} else {
crate::transcript_image::FIRST_IMAGE_ID
};
let changed = *retained_images != images;
if changed {
if self.tmux_kitty_image_base.is_some() && !retained_images.is_empty() {
// tmux retains placeholder text after Ghostty deletes the associated pixels.
self.invalidate_viewport();
let tmux = self.tmux_kitty_image_base.is_some();
// tmux retains placeholder cells, so keep their existing positional IDs and redraw
// behavior. Native Kitty can keep the same ID and pixels as other images change.
if !tmux || *retained_images == images {
for image in &mut images {
if let Some(index) = retained_images.iter().position(|old| old == image) {
*image = retained_images.swap_remove(index);
}
}
// Remove old graphics before text painting, including when switching to overlays.
for (index, placement) in retained_images.iter().enumerate() {
match placement.image.protocol {
crate::pets::ImageProtocol::Kitty
| crate::pets::ImageProtocol::KittyLocalFile => {
let id = first_id + index as u32;
self.backend
.write_all(crate::pets::kitty_delete_image(id).as_bytes())?;
}
crate::pets::ImageProtocol::Sixel => self.invalidate_viewport(),
} else if !retained_images.is_empty() {
self.invalidate_viewport();
}
let mut moved = Vec::new();
// iTerm2 can discard the pixel data when a placement is removed. Redraw it
// from the source in that terminal, and keep the fast path for native Kitty.
if !tmux
&& codex_terminal_detection::terminal_info().name
!= codex_terminal_detection::TerminalName::Iterm2
{
for (index, image) in images.iter_mut().enumerate() {
if image.terminal_id.is_none()
&& image.image.protocol != crate::pets::ImageProtocol::Sixel
&& let Some(previous) = retained_images.iter().position(|old| {
std::sync::Arc::ptr_eq(&old.image, &image.image)
&& old.first_row == image.first_row
&& old.area.as_size() == image.area.as_size()
})
{
let mut old = retained_images.swap_remove(previous);
old.area = image.area;
*image = old;
moved.push(index);
}
}
}
for old in retained_images.drain(..) {
match old.image.protocol {
crate::pets::ImageProtocol::Kitty | crate::pets::ImageProtocol::KittyLocalFile => {
if let Some(id) = old.terminal_id {
self.backend
.write_all(crate::pets::kitty_delete_image(id).as_bytes())?;
}
}
crate::pets::ImageProtocol::Sixel => self.invalidate_viewport(),
}
}
// Not every terminal or multiplexer hides intermediate cursor moves inside a
// synchronized update, especially when the frame spans multiple writes.
// A text clear, focus highlight, or full repaint can erase an otherwise stable image.
let updates = diff_buffers(self.previous_buffer(), self.current_buffer());
// A text clear, focus highlight, or full repaint may erase an otherwise stable image.
let repaint_images = changed
|| updates.iter().any(|update| {
images.iter().any(|placement| match update {
DrawCommand::Put { x, y, .. } => placement.area.contains((*x, *y).into()),
let mut redraw = Vec::new();
let mut next_id = first_id;
for index in 0..images.len() {
let image = &images[index];
if image.terminal_id.is_none() {
while images
.iter()
.any(|other| other.terminal_id == Some(next_id))
{
next_id += 1;
}
images[index].terminal_id = Some(next_id);
redraw.push(index);
next_id += 1;
} else if !moved.contains(&index)
&& updates.iter().any(|update| match update {
DrawCommand::Put { x, y, .. } => image.area.contains((*x, *y).into()),
DrawCommand::ClearToEnd { x, y, .. } => {
*y >= placement.area.y
&& *y < placement.area.bottom()
&& *x < placement.area.right()
(image.area.y..image.area.bottom()).contains(y) && *x < image.area.right()
}
})
});
{
redraw.push(index);
}
}
if !updates.is_empty() && !self.hidden_cursor {
self.hide_cursor()?;
}
self.flush_updates(updates)?;
if repaint_images {
*retained_images = images;
for (index, placement) in retained_images.iter_mut().enumerate() {
let id = first_id + index as u32;
let kitty_placement = if self.tmux_kitty_image_base.is_some() {
crate::pets::KittyPlacement::Virtual(id)
} else {
crate::pets::KittyPlacement::Cursor(id)
};
placement.file = placement
.image
.draw(
&mut self.backend,
placement.area,
placement.first_row,
kitty_placement,
)
.map_err(io::Error::other)?;
}
*retained_images = images;
for index in moved {
let image = &retained_images[index];
let id = image
.terminal_id
.ok_or_else(|| io::Error::other("missing terminal ID for image"))?;
queue!(
self.backend,
SavePosition,
MoveTo(image.area.x, image.area.y)
)?;
self.backend
.write_all(crate::pets::kitty_move_image(id, image.area).as_bytes())?;
queue!(self.backend, RestorePosition)?;
}
for index in redraw {
let placement = &mut retained_images[index];
let id = placement
.terminal_id
.ok_or_else(|| io::Error::other("missing terminal ID for placement"))?;
let kitty_placement = if tmux {
crate::pets::KittyPlacement::Virtual(id)
} else {
crate::pets::KittyPlacement::Cursor(id)
};
placement.file = placement
.image
.draw(
&mut self.backend,
placement.area,
placement.first_row,
kitty_placement,
)
.map_err(io::Error::other)?;
}
match cursor_position {
@@ -958,8 +1014,13 @@ mod tests {
(crate::pets::ImageProtocol::Kitty, Some(0x80_0100)),
] {
let image = std::sync::Arc::new(
crate::transcript_image::TranscriptImage::load(&path, /*width*/ 40, protocol)
.unwrap(),
crate::transcript_image::TranscriptImage::load(
&path,
/*width*/ 40,
protocol,
crate::transcript_image::PreviewKind::Inspected,
)
.unwrap(),
);
let mut terminal =
Terminal::with_options(CaptureBackend::new(/*width*/ 40, /*height*/ 12)).unwrap();
@@ -997,6 +1058,7 @@ mod tests {
),
first_row: 0,
file: None,
terminal_id: None,
});
})
.unwrap();
@@ -1027,6 +1089,75 @@ mod tests {
}
}
#[test]
fn native_kitty_moves_an_image_without_resending_other_visible_images() {
use crate::pets::ImageProtocol;
use crate::transcript_image::ImagePlacement;
use crate::transcript_image::TranscriptImage;
use std::sync::Arc;
let mut terminal =
Terminal::with_options(CaptureBackend::new(/*width*/ 40, /*height*/ 12)).unwrap();
let size = terminal.last_known_screen_size;
terminal.set_viewport_area(size.into());
let raster = Arc::new(TranscriptImage {
protocol: ImageProtocol::Kitty,
columns: 2,
rows: 2,
pixels: image::RgbaImage::from_pixel(16, 32, image::Rgba([10, 110, 200, 255])),
cell_width: 8,
cell_height: 16,
});
let fixed = ImagePlacement {
image: Arc::clone(&raster),
area: Rect::new(
/*x*/ 2, /*y*/ 2, /*width*/ 2, /*height*/ 2,
),
first_row: 0,
file: None,
terminal_id: None,
};
let mut moving = fixed.registration();
moving.area.x = 16;
let mut retained = Vec::new();
terminal
.draw_with_size(size, &mut retained, |frame| {
frame
.image_placements
.extend([fixed.registration(), moving.registration()]);
})
.unwrap();
let original_ids: Vec<_> = retained.iter().map(|image| image.terminal_id).collect();
assert_eq!(terminal.backend().output().matches("_Ga=T").count(), 2);
moving.area.x += 2;
terminal
.draw_with_size(size, &mut retained, |frame| {
frame
.image_placements
.extend([fixed.registration(), moving.registration()]);
})
.unwrap();
let output = terminal.backend().output();
assert_eq!(output.matches("_Ga=T").count(), 2);
assert_eq!(output.matches("_Ga=p").count(), 1);
assert_eq!(
retained
.iter()
.map(|image| image.terminal_id)
.collect::<Vec<_>>(),
original_ids
);
terminal
.draw_with_size(size, &mut retained, |frame| {
frame.image_placements.push(moving.registration());
})
.unwrap();
assert_eq!(terminal.backend().output().matches("_Ga=T").count(), 2);
assert_eq!(retained[0].terminal_id, original_ids[1]);
}
impl Backend for CaptureBackend {
type Error = io::Error;
+14 -5
View File
@@ -344,11 +344,20 @@ pub(crate) fn new_info_event(message: String, hint: Option<String>) -> PlainHist
PlainHistoryCell { lines }
}
pub(crate) fn new_interruption_notice() -> PlainHistoryCell {
PlainHistoryCell::new(vec![
Line::from("■ Conversation interrupted - use /feedback if something went wrong")
.style(crate::style::secondary_text_style()),
])
#[derive(Debug)]
pub(crate) struct InterruptionNotice(pub(crate) Option<String>);
impl HistoryCell for InterruptionNotice {
fn display_lines(&self, _width: u16) -> Vec<Line<'static>> {
vec![
Line::from("■ Conversation interrupted - use /feedback if something went wrong")
.style(crate::style::secondary_text_style()),
]
}
fn raw_lines(&self) -> Vec<Line<'static>> {
plain_lines(self.display_lines(/*width*/ u16::MAX))
}
}
pub(crate) fn new_error_event(message: String) -> PlainHistoryCell {
+73 -2
View File
@@ -280,7 +280,7 @@ pub(crate) fn new_image_generation_call(
revised_prompt: Option<String>,
saved_path: Option<AbsolutePathBuf>,
) -> ImageGenerationHistoryCell {
let detail = revised_prompt.unwrap_or(call_id);
let detail = revised_prompt.unwrap_or_else(|| call_id.clone());
let failed = status == "failed";
let heading = if failed {
vec!["✗ ".red().bold(), "Image generation failed".bold()].into()
@@ -296,21 +296,52 @@ pub(crate) fn new_image_generation_call(
}
ImageGenerationHistoryCell {
id: call_id,
lines,
failed,
saved_path: saved_path.filter(|_| !failed),
saved_path: saved_path.filter(|_| status == "completed"),
}
}
#[derive(Debug)]
pub(crate) struct ImageGenerationHistoryCell {
pub(crate) id: String,
failed: bool,
lines: Vec<Line<'static>>,
saved_path: Option<AbsolutePathBuf>,
}
impl ImageGenerationHistoryCell {
/// A completed generation with a saved artifact belongs to the answer.
pub(crate) fn is_turn_output(&self) -> bool {
self.saved_path.is_some()
}
}
/// Hold saved results until the turn can place them after the answer.
#[derive(Debug, Default)]
pub(crate) struct PendingGeneratedImages(HashMap<String, Vec<ImageGenerationHistoryCell>>);
impl PendingGeneratedImages {
pub(crate) fn record(&mut self, turn_id: String, image: ImageGenerationHistoryCell) {
let images = self.0.entry(turn_id).or_default();
if let Some(existing) = images.iter_mut().find(|existing| existing.id == image.id) {
*existing = image;
} else {
images.push(image);
}
}
pub(crate) fn take(&mut self, turn_id: &str) -> Vec<ImageGenerationHistoryCell> {
self.0.remove(turn_id).unwrap_or_default()
}
}
impl HistoryCell for ImageGenerationHistoryCell {
fn tool_call_labels(&self) -> Vec<String> {
if self.is_turn_output() {
return Vec::new();
}
let label = if self.failed {
"Image generation failed"
} else {
@@ -323,6 +354,46 @@ impl HistoryCell for ImageGenerationHistoryCell {
self.lines.clone()
}
fn display_hyperlink_lines(&self, width: u16) -> Vec<HyperlinkLine> {
let mut lines =
crate::terminal_hyperlinks::plain_hyperlink_lines(self.display_lines(width));
if let Some(path) = &self.saved_path
&& let Some(line) = lines.last_mut()
&& let Some(span) = line.line.spans.last()
{
let width = crate::line_truncation::span_width(span);
let start = line.line.width().saturating_sub(width);
line.hyperlinks
.push(crate::terminal_hyperlinks::TerminalHyperlink::document(
start..start + width,
crate::markdown_render::FileLink {
path: path.to_string_lossy().into_owned(),
location: None,
},
));
}
lines
}
fn compact_hyperlink_lines(&self, width: u16) -> Vec<HyperlinkLine> {
let Some(path) = &self.saved_path else {
return self.display_hyperlink_lines(width);
};
let mut line = HyperlinkLine::from(Line::from(" Generated image".dim()));
line.hyperlinks.push(TerminalHyperlink::document(
2..line.line.width(),
crate::markdown_render::FileLink {
path: path.to_string_lossy().into_owned(),
location: None,
},
));
vec![line]
}
fn transcript_hyperlink_lines(&self, width: u16) -> Vec<HyperlinkLine> {
self.display_hyperlink_lines(width)
}
fn raw_lines(&self) -> Vec<Line<'static>> {
plain_lines(self.lines.clone())
}
+6 -1
View File
@@ -19,6 +19,7 @@ use std::collections::HashMap;
/// restoring the conversation constructs new cells and refreshes whether the timestamp needs a date.
#[derive(Debug)]
pub struct FinalMessageSeparator {
pub(crate) turn_id: Option<String>,
elapsed_seconds: Option<u64>,
runtime_metrics: Option<RuntimeMetricsSummary>,
completed_at: Option<DateTime<Local>>,
@@ -51,7 +52,10 @@ impl FinalMessageSeparator {
return None;
}
rendered_completion_turn_ids.insert(turn.id.clone(), has_label);
let cell = Self::new(elapsed_seconds, /*runtime_metrics*/ None);
let cell = Self {
turn_id: Some(turn.id.clone()),
..Self::new(elapsed_seconds, /*runtime_metrics*/ None)
};
Some(match completed_at {
Some(completed_at) => cell.with_completed_at(completed_at, ClockFormat::system()),
None => cell,
@@ -64,6 +68,7 @@ impl FinalMessageSeparator {
runtime_metrics: Option<RuntimeMetricsSummary>,
) -> Self {
Self {
turn_id: None,
elapsed_seconds,
runtime_metrics,
completed_at: None,
+13 -1
View File
@@ -609,7 +609,7 @@ fn image_generation_call_renders_saved_path() {
"call-image-generation".to_string(),
"completed",
Some("A tiny blue square".to_string()),
Some(saved_path),
Some(saved_path.clone()),
);
assert_eq!(
@@ -620,6 +620,18 @@ fn image_generation_call_renders_saved_path() {
expected_saved_path,
],
);
let lines = cell.transcript_hyperlink_lines(/*width*/ 80);
let link = &lines.last().unwrap().hyperlinks[0];
assert_eq!(
link.reader_destination(/*documents*/ true),
Some(crate::terminal_hyperlinks::ReaderLink::Document(
crate::markdown_render::FileLink {
path: saved_path.to_string_lossy().into_owned(),
location: None,
},
)),
);
assert_eq!(link.reader_destination(/*documents*/ false), None);
}
fn session_configured_event(model: &str) -> ThreadSessionState {
+1
View File
@@ -219,6 +219,7 @@ mod session_start;
mod session_state;
mod shortcut_help;
mod sidebar_chooser;
mod sidebar_image;
mod sidebar_tab_strip;
mod skills_helpers;
mod slash_command;
@@ -39,6 +39,12 @@ pub(crate) struct FileLink {
pub(crate) location: Option<SourceLocation>,
}
impl FileLink {
pub(crate) fn is_image(&self) -> bool {
self.location.is_none() && crate::sidebar_image::supports_path(&self.path)
}
}
pub(crate) fn is_markdown_file(path: &str) -> bool {
path.rsplit_once('.').is_some_and(|(_, ext)| {
ext.eq_ignore_ascii_case("md") || ext.eq_ignore_ascii_case("markdown")
+5 -1
View File
@@ -340,6 +340,11 @@ impl TranscriptOverlay {
.iter()
.rposition(|cell| is_session_history_prefix_cell(cell.as_ref()))
.map_or(/*default*/ 0, |index| index + 1);
self.insert_history_page(index, cells);
index
}
pub(crate) fn insert_history_page(&mut self, index: usize, cells: Vec<Arc<dyn HistoryCell>>) {
let added = cells.len();
self.cells.splice(index..index, cells);
for highlight in [&mut self.highlight_cell, &mut self.pending_highlight] {
@@ -349,7 +354,6 @@ impl TranscriptOverlay {
}
self.view.set_highlight(self.highlight_cell);
self.view.history_loaded(&self.cells, index..index + added);
index
}
pub(crate) fn replace_cells(&mut self, cells: Vec<Arc<dyn HistoryCell>>) {
+8
View File
@@ -259,6 +259,14 @@ impl KittyPlacement {
}
}
pub(crate) fn kitty_move_image(id: u32, area: ratatui::layout::Rect) -> String {
let (columns, rows) = (area.width, area.height);
// Lowercase i removes placements but keeps the uploaded pixel data.
wrap_for_tmux_if_needed(&format!(
"{ESC}_Ga=d,d=i,i={id},q=2;{ST}{ESC}_Ga=p,i={id},c={columns},r={rows},C=1,q=2;{ST}"
))
}
pub fn kitty_transmit_png_with_id(
path: &Path,
columns: u16,
+1
View File
@@ -48,6 +48,7 @@ pub(crate) use image_protocol::PetImageUnsupportedReason;
#[cfg(not(test))]
pub(crate) use image_protocol::detect_pet_image_support;
pub(crate) use image_protocol::kitty_delete_image;
pub(crate) use image_protocol::kitty_move_image;
pub(crate) use image_protocol::kitty_transmit_png_file_with_id;
pub(crate) use image_protocol::kitty_transmit_png_with_id;
pub(crate) use picker::PET_PICKER_VIEW_ID;
+174
View File
@@ -0,0 +1,174 @@
//! Decode a workspace image once and reuse the transcript's terminal image lifecycle.
use std::io::Cursor;
use std::sync::Arc;
use color_eyre::eyre::OptionExt;
use color_eyre::eyre::Result;
use image::DynamicImage;
use image::ImageFormat;
use ratatui::buffer::Buffer;
use ratatui::layout::Rect;
use ratatui::style::Stylize;
use ratatui::text::Line;
use ratatui::widgets::Widget;
use crate::pets::ImageProtocol;
use crate::transcript_image::ImagePlacement;
use crate::transcript_image::TranscriptImage;
use crate::transcript_image::terminal_cell_size;
#[derive(Debug)]
pub(crate) enum DocumentContent {
Text(String),
Image(Box<SidebarImage>),
}
pub(crate) struct SidebarImage {
label: String,
source_size: (u32, u32),
raster: DynamicImage,
fitted: Option<Arc<TranscriptImage>>,
}
impl std::fmt::Debug for SidebarImage {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SidebarImage")
.field("label", &self.label)
.finish_non_exhaustive()
}
}
pub(crate) fn supports_path(path: &str) -> bool {
path.rsplit_once('.')
.and_then(|(_, extension)| ImageFormat::from_extension(extension))
.is_some_and(supported_format)
}
fn supported_format(format: ImageFormat) -> bool {
matches!(
format,
ImageFormat::Png | ImageFormat::Jpeg | ImageFormat::Gif | ImageFormat::WebP
)
}
impl SidebarImage {
pub(crate) fn decode(bytes: &[u8]) -> Result<Self> {
let reader = image::ImageReader::new(Cursor::new(bytes)).with_guessed_format()?;
let format = reader
.format()
.filter(|format| supported_format(*format))
.ok_or_eyre("Unsupported image format")?;
let mut limits = image::Limits::default();
limits.max_alloc = Some(32 * 1024 * 1024);
let image = crate::image_decode::decode_oriented(reader, limits)?;
let (width, height) = (image.width(), image.height());
Ok(Self {
label: format!(
"{} · {width} × {height}",
format.extensions_str()[0].to_ascii_uppercase()
),
source_size: (width, height),
raster: if width > 768 || height > 768 {
image.thumbnail(768, 768)
} else {
image
},
fitted: None,
})
}
pub(crate) fn paint(
&mut self,
area: Rect,
buf: &mut Buffer,
protocol: Option<ImageProtocol>,
resize_hint: Option<&str>,
) -> Option<ImagePlacement> {
Line::from(self.label.as_str()).dim().render(area, buf);
let available_rows = area.height.saturating_sub(2 + u16::from(area.height > 3));
if area.height > 1 {
let footer = Rect::new(area.x, area.bottom() - 1, area.width, 1);
if let Some(hint) = resize_hint {
crate::sidebar_chooser::resize_hint(hint, footer.width).render(footer, buf);
} else {
crate::footer_hint::HintLine::new(
[
crate::footer_hint::shortcut("←/→", "tabs"),
crate::footer_hint::shortcut("Esc", "close"),
],
footer.width,
)
.line
.render(footer, buf);
}
}
// tmux uses Kitty's virtual placements. The shared placeholder renderer
// currently emits row marks for up to six rows.
let rows = if crate::tui::tmux_fullscreen_image_base().is_some() {
available_rows.min(6)
} else {
available_rows
};
let body = Rect::new(area.x, area.y.saturating_add(1), area.width, rows);
if body.is_empty() {
return None;
}
let Some((protocol, (cw, ch))) = protocol.and_then(|protocol| {
let size = terminal_cell_size()
.or_else(|| (protocol != ImageProtocol::Sixel).then_some((8, 16)))?;
Some((protocol, size))
}) else {
Line::from("Image preview unavailable in this terminal").render(body, buf);
return None;
};
let (width, height) = self.source_size;
let (max_width, max_height) = match protocol {
ImageProtocol::Sixel => (160, 80),
ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (width, height),
};
let scale = (f64::from((u32::from(body.width) * cw).min(max_width)) / f64::from(width))
.min(f64::from((u32::from(body.height) * ch).min(max_height)) / f64::from(height))
.min(1.0);
let pixel_width = (f64::from(width) * scale).round().max(1.0) as u32;
let pixel_height = (f64::from(height) * scale).round().max(1.0) as u32;
let columns = pixel_width.div_ceil(cw) as u16;
let rows = pixel_height.div_ceil(ch) as u16;
if !self.fitted.as_ref().is_some_and(|image| {
image.protocol == protocol
&& image.cell_width == cw
&& image.cell_height == ch
&& image.columns == columns
&& image.rows == rows
}) {
let pixels = self
.raster
.thumbnail(
pixel_width.min(self.raster.width()),
pixel_height.min(self.raster.height()),
)
.to_rgba8();
self.fitted = Some(Arc::new(TranscriptImage {
protocol,
columns,
rows,
pixels,
cell_width: cw,
cell_height: ch,
}));
}
let image = self.fitted.as_ref()?;
Some(ImagePlacement {
image: Arc::clone(image),
area: Rect::new(
body.x + (body.width - image.columns) / 2,
body.y + (body.height - image.rows) / 2,
image.columns,
image.rows,
),
first_row: 0,
file: None,
terminal_id: None,
})
}
}
+1
View File
@@ -138,6 +138,7 @@ impl StartupDraft {
);
terminal_restore_guard.events = Some(tui.event_broker.clone());
tui.terminal_app_over_ssh = initialized_terminal.terminal_app_over_ssh;
tui.herdr_kitty_graphics = initialized_terminal.herdr_kitty_graphics;
tui.set_alt_screen_enabled(screen.use_alt_screen);
let mut pump = StartupDraftPump::new(&tui, initial_screen, session_action);
pump.motion = screen.welcome_motion;
+30 -10
View File
@@ -49,7 +49,7 @@ fn alternate_scroll_report(input: &[u8]) -> Option<bool> {
mod imp {
use super::DefaultColors;
use super::parse_default_colors;
use super::startup_replay::startup_replay_input;
use super::startup_replay::parse_startup_input;
use super::terminal_identity;
use std::fs::File;
use std::fs::OpenOptions;
@@ -73,13 +73,14 @@ mod imp {
static DA1_SUPPORTED: AtomicBool = AtomicBool::new(false);
/// Results from the TUI's one-shot startup terminal probe.
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)]
pub(crate) struct StartupProbe {
pub(crate) alternate_scroll_enabled: Option<bool>,
pub(crate) cursor_position: Option<Position>,
pub(crate) default_colors: Option<DefaultColors>,
pub(crate) keyboard_enhancement_supported: Option<bool>,
pub(crate) terminal_app_over_ssh: Option<bool>,
pub(crate) kitty_graphics: bool,
}
/// Whether the startup probe should query keyboard enhancement support.
@@ -305,6 +306,7 @@ mod imp {
pub(crate) fn startup(
timeout: Duration,
keyboard_probe: StartupKeyboardEnhancementProbe,
probe_herdr: bool,
) -> io::Result<StartupProbe> {
DA1_SUPPORTED.store(false, Ordering::Relaxed);
let mut tty = Tty::open()?;
@@ -319,6 +321,10 @@ mod imp {
}
// Query before DA1 so its reply acts as a barrier even if mode 1007 is unsupported.
query.extend_from_slice(b"\x1B[?1007$p");
if probe_herdr {
// Query a single inline RGB pixel without storing or displaying an image.
query.extend_from_slice(b"\x1b_Gi=31,s=1,v=1,a=q,t=d,f=24;AAAA\x1b\\");
}
// DA1 finishes keyboard detection and establishes whether we can use it at exit.
query.extend_from_slice(b"\x1B[c\x1B[6n\x1B]10;?\x1B\\\x1B]11;?\x1B\\");
if query_identity {
@@ -332,12 +338,17 @@ mod imp {
timeout,
keyboard_probe,
query_identity,
probe_herdr,
&mut buffer,
);
crossterm::event::buffer_input(&startup_replay_input(&buffer))?;
let (replay, kitty_graphics) = parse_startup_input(&buffer, probe_herdr);
crossterm::event::buffer_input(&replay)?;
let saw_da1 = find_primary_device_attributes(&buffer).is_some();
DA1_SUPPORTED.store(saw_da1, Ordering::Relaxed);
result
result.map(|probe| StartupProbe {
kitty_graphics,
..probe
})
}
/// Reads available terminal bytes until `parse` recognizes a probe response or time expires.
@@ -373,15 +384,13 @@ mod imp {
timeout: Duration,
keyboard_probe: StartupKeyboardEnhancementProbe,
query_identity: bool,
probe_herdr: bool,
buffer: &mut Vec<u8>,
) -> io::Result<StartupProbe> {
let deadline = Instant::now() + timeout;
let mut probe = StartupProbe {
alternate_scroll_enabled: None,
cursor_position: None,
default_colors: None,
keyboard_enhancement_supported: None,
terminal_app_over_ssh: (!query_identity).then_some(/*t*/ false),
..Default::default()
};
let mut saw_supported_keyboard = false;
loop {
@@ -392,7 +401,9 @@ mod imp {
buffer,
keyboard_probe,
);
if startup_probe_complete(&probe, keyboard_probe) {
if startup_probe_complete(&probe, keyboard_probe)
&& (!probe_herdr || find_primary_device_attributes(buffer).is_some())
{
return Ok(probe);
}
let now = Instant::now();
@@ -698,6 +709,7 @@ mod imp {
default_colors: None,
keyboard_enhancement_supported: None,
terminal_app_over_ssh: Some(false),
kitty_graphics: false,
};
let mut saw_supported_keyboard = false;
update_startup_probe(
@@ -718,6 +730,7 @@ mod imp {
}),
keyboard_enhancement_supported: Some(true),
terminal_app_over_ssh: Some(false),
kitty_graphics: false,
}
);
assert!(startup_probe_complete(
@@ -734,6 +747,7 @@ mod imp {
default_colors: None,
keyboard_enhancement_supported: None,
terminal_app_over_ssh: None,
kitty_graphics: false,
};
let mut supported_keyboard = false;
let input =
@@ -772,6 +786,7 @@ mod imp {
Duration::from_millis(/*millis*/ 5),
StartupKeyboardEnhancementProbe::Skip,
/*query_identity*/ true,
/*probe_herdr*/ false,
&mut buffer,
)
.expect("finish within probe deadline");
@@ -792,6 +807,7 @@ mod imp {
Duration::from_secs(/*secs*/ 1),
StartupKeyboardEnhancementProbe::Query,
/*query_identity*/ false,
/*probe_herdr*/ false,
&mut buffer,
)
.expect("read terminal responses after buffered typeahead");
@@ -807,11 +823,15 @@ mod imp {
}),
keyboard_enhancement_supported: Some(true),
terminal_app_over_ssh: Some(false),
kitty_graphics: false,
}
);
let mut expected_replay = vec![b'x'; 2_048];
expected_replay.extend_from_slice(b"\x1B[20;10R\x1B[?64;1;2c\x1B[?7u");
assert_eq!(startup_replay_input(&buffer), expected_replay);
assert_eq!(
parse_startup_input(&buffer, /*probe_herdr*/ false),
(expected_replay, false)
);
}
#[test]
@@ -1,15 +1,17 @@
//! Preserve startup terminal input while filtering completed color and scroll-mode replies.
//! Preserve startup terminal input while filtering replies to terminal probes.
const MAX_REPLAYED_OSC_BYTES: usize = 1024;
/// Preserve input and bracketed paste while omitting completed terminal-color replies.
/// Return the input to replay and whether Herdr acknowledged the Kitty graphics query.
///
/// Keep bounded incomplete OSC prefixes so unread continuations stay parser-framed.
pub(super) fn startup_replay_input(input: &[u8]) -> Vec<u8> {
pub(super) fn parse_startup_input(input: &[u8], probe_herdr: bool) -> (Vec<u8>, bool) {
const PASTE_START: &[u8] = b"\x1b[200~";
const PASTE_END: &[u8] = b"\x1b[201~";
const KITTY_REPLY: &[u8] = b"\x1b_Gi=31;";
let mut replay = Vec::with_capacity(input.len());
let mut kitty_graphics = false;
let mut cursor = 0;
let mut inside_paste = false;
while cursor < input.len() {
@@ -52,6 +54,17 @@ pub(super) fn startup_replay_input(input: &[u8]) -> Vec<u8> {
continue;
}
// Only consume replies to our query. Bracketed paste above remains user input.
if probe_herdr
&& let Some(message) = input[cursor..].strip_prefix(KITTY_REPLY)
&& let Some(end) = message.iter().position(|b| !(0x20..=0x7e).contains(b))
&& message[end..].starts_with(b"\x1b\\")
{
kitty_graphics = &message[..end] == b"OK";
cursor += KITTY_REPLY.len() + end + 2;
continue;
}
if input[cursor..].starts_with(b"\x1b]") {
if let Some((end, terminator_len)) = super::osc_payload_end(&input[cursor + 2..]) {
cursor += 2 + end + terminator_len;
@@ -65,7 +78,7 @@ pub(super) fn startup_replay_input(input: &[u8]) -> Vec<u8> {
replay.push(input[cursor]);
cursor += 1;
}
replay
(replay, kitty_graphics)
}
#[cfg(test)]
@@ -1,7 +1,13 @@
#![cfg(test)]
use super::MAX_REPLAYED_OSC_BYTES;
use super::startup_replay_input;
use super::parse_startup_input;
use pretty_assertions::assert_eq;
fn startup_replay_input(input: &[u8]) -> Vec<u8> {
parse_startup_input(input, /*probe_herdr*/ false).0
}
#[test]
fn startup_replay_bounds_oversized_osc_without_exposing_its_payload() {
let mut input = b"typed\x1b]10;".to_vec();
@@ -56,3 +62,19 @@ fn startup_replay_omits_terminal_color_replies_interleaved_with_bracketed_paste(
response_after_paste
);
}
#[test]
fn herdr_graphics_reply_selects_previews_without_replaying_the_response_as_keys() {
let paste = b"\x1b[200~\x1b_Gi=31;OK\x1b\\\x1b[201~";
for (reply, expected) in [
(b"\x1b_Gi=31;OK\x1b\\".as_slice(), true),
(b"\x1b_Gi=31;EINVAL: unsupported\x1b\\", false),
(b"", false),
] {
let input = [paste.as_slice(), reply, b"typed"].concat();
assert_eq!(
parse_startup_input(&input, /*probe_herdr*/ true),
([paste.as_slice(), b"typed"].concat(), expected)
);
}
}
+62 -20
View File
@@ -30,6 +30,7 @@ pub(crate) struct ImagePlacement {
pub(crate) area: Rect,
pub(crate) first_row: usize,
pub(crate) file: Option<tempfile::NamedTempFile>,
pub(crate) terminal_id: Option<u32>,
}
impl ImagePlacement {
@@ -40,6 +41,7 @@ impl ImagePlacement {
area: self.area,
first_row: self.first_row,
file: None,
terminal_id: None,
}
}
}
@@ -73,10 +75,29 @@ pub(crate) fn terminal_cell_size() -> Option<(u32, u32)> {
})
}
/// Generated output is larger than an image shown as a tool detail.
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum PreviewKind {
Inspected,
Generated,
}
impl TranscriptImage {
fn load_pixels(path: &Path, max_width: u32, max_height: u32) -> Option<RgbaImage> {
if max_width == 0 || max_height == 0 {
return None;
}
let image = Self::decode(path)?;
Some(
image
.thumbnail(max_width.min(image.width()), max_height.min(image.height()))
.to_rgba8(),
)
}
fn decode(path: &Path) -> Option<DynamicImage> {
let metadata = path.metadata().ok()?;
if max_width == 0 || !metadata.is_file() || metadata.len() > 32 * 1024 * 1024 {
if !metadata.is_file() || metadata.len() > 32 * 1024 * 1024 {
return None;
}
let reader = image::ImageReader::open(path)
@@ -87,32 +108,53 @@ impl TranscriptImage {
// screenshots usable, but never allocate hundreds of MB for an optional thumbnail.
let mut limits = image::Limits::default();
limits.max_alloc = Some(32 * 1024 * 1024);
let image = crate::image_decode::decode_oriented(reader, limits).ok()?;
Some(
image
.thumbnail(max_width.min(image.width()), max_height.min(image.height()))
.to_rgba8(),
)
crate::image_decode::decode_oriented(reader, limits).ok()
}
pub(crate) fn load(path: &Path, width: u16, protocol: ImageProtocol) -> Option<Self> {
let columns = width.saturating_sub(2).min(48);
pub(crate) fn load(
path: &Path,
width: u16,
protocol: ImageProtocol,
kind: PreviewKind,
) -> Option<Self> {
let columns = width.saturating_sub(2);
if columns == 0 {
return None;
}
// Sixel paints physical pixels. Guessing could paint outside the reserved rows.
let (cell_width, cell_height) = match protocol {
ImageProtocol::Sixel => terminal_cell_size()?,
ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (8, 16),
ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => match kind {
PreviewKind::Inspected => (8, 16),
PreviewKind::Generated => terminal_cell_size().unwrap_or((8, 16)),
},
};
// The sprite encoder scans each active color per Sixel band. Bound both its CPU
// work and worst-case output even with large terminal cells or high-entropy images.
let (max_width, max_height) = match protocol {
ImageProtocol::Sixel => (160, 80),
ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (384, 160),
let image = Self::decode(path)?;
// Keep the current Sixel encoding and tmux placeholder budgets.
let (columns, max_width, max_height) = match (protocol, kind) {
(ImageProtocol::Sixel, _) => (columns.min(48), 160, 80.min(6 * cell_height)),
(_, PreviewKind::Inspected) => (columns.min(48), 384, 6 * cell_height),
(_, PreviewKind::Generated) => {
let ratio = f64::from(image.width()) / f64::from(image.height());
let max_width = if ratio >= 2.0 {
768
} else if ratio >= 1.01 {
480
} else {
400
};
let max_height = if crate::tui::tmux_fullscreen_image_base().is_some() {
6 * cell_height
} else {
400
};
(columns, max_width, max_height)
}
};
let pixels = Self::load_pixels(
path,
(u32::from(columns) * cell_width).min(max_width),
(6 * cell_height).min(max_height),
)?;
let max_width = (u32::from(columns) * cell_width).min(max_width);
let pixels = image
.thumbnail(max_width.min(image.width()), max_height.min(image.height()))
.to_rgba8();
Some(Self {
protocol,
columns: pixels.width().div_ceil(cell_width) as u16,
@@ -125,6 +125,7 @@ impl ComposerImages {
area: tile,
first_row: 0,
file: None,
terminal_id: None,
});
} else {
Paragraph::new(attachment.placeholder.as_str())
+8 -1
View File
@@ -439,8 +439,14 @@ impl TranscriptView {
if skip >= rows.len() {
continue;
}
let image_indent = if visible.layout.image_left_aligned {
// Match the two-column prefix the assistant's Markdown uses.
2
} else {
self.area.width.saturating_sub(image.columns) / 2
};
let area = Rect::new(
self.area.x + self.area.width.saturating_sub(image.columns) / 2,
self.area.x + image_indent,
entry_y + skip as u16,
image.columns,
(rows.len() - skip) as u16,
@@ -450,6 +456,7 @@ impl TranscriptView {
area,
first_row: visible.row.saturating_sub(image_start),
file: None,
terminal_id: None,
});
}
placements
+11 -1
View File
@@ -89,9 +89,19 @@ fn cell_content_layout(
if let Some(protocol) = presentation.image_protocol
&& let Some(path) = cell.image_path()
{
use crate::transcript_image::PreviewKind;
let kind = if cell
.as_any()
.is::<crate::history_cell::ImageGenerationHistoryCell>()
{
PreviewKind::Generated
} else {
PreviewKind::Inspected
};
layout.image =
crate::transcript_image::TranscriptImage::load(path.as_path(), width, protocol)
crate::transcript_image::TranscriptImage::load(path.as_path(), width, protocol, kind)
.map(Arc::new);
layout.image_left_aligned = kind == PreviewKind::Generated;
}
layout
}
@@ -164,16 +164,29 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() {
image::RgbaImage::from_pixel(480, 160, image::Rgba([255, 0, 0, 255]))
.save(&path)
.unwrap();
let square_path = dir.path().join("generated.png");
image::RgbImage::new(768, 768).save(&square_path).unwrap();
let cells: Vec<Arc<dyn HistoryCell>> = vec![
Arc::new(new_view_image_tool_call(
"view-call".to_string(),
LegacyAppPathString::from_string(path.to_string_lossy().into_owned()),
)),
Arc::new(
crate::history_cell::AgentMarkdownCell::new(
"Here is the image.".to_string(),
dir.path(),
)
.with_phase(Some(codex_protocol::models::MessagePhase::FinalAnswer)),
),
Arc::new(new_image_generation_call(
"image-call".to_string(),
"completed",
Some("A red rectangle".to_string()),
Some(AbsolutePathBuf::try_from(path.clone()).unwrap()),
Some("A square".to_string()),
Some(AbsolutePathBuf::try_from(square_path.clone()).unwrap()),
)),
Arc::new(crate::history_cell::FinalMessageSeparator::new(
Some(2),
/*runtime_metrics*/ None,
)),
];
let area = Rect::new(
@@ -193,8 +206,18 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() {
let text_only = buf.clone();
view.image_protocol = Some(ImageProtocol::Kitty);
view.render(area, &mut buf, &cells);
view.sync_turn_details(&cells, /*enabled*/ true, /*active*/ None);
view.render(area, &mut buf, &cells);
let collapsed = frame(&buf);
let collapsed_areas: Vec<_> = view.image_placements().iter().map(|p| p.area).collect();
assert_eq!(collapsed_areas.len(), 1);
assert!(
collapsed.find("Here is the image.").unwrap() < collapsed.find("Generated image").unwrap()
);
assert_eq!(collapsed_areas[0].x, area.x + 2);
assert!(collapsed_areas[0].height > 6);
view.sync_turn_details(&cells, /*enabled*/ false, /*active*/ None);
view.render(area, &mut buf, &cells);
// Find may request full text, but only manual expansion may reveal inspected pixels.
assert!(
view.entry_layout(&cells, /*index*/ 0, /*full_content*/ true)
@@ -220,7 +243,8 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() {
frame(&buf)
)
);
for (index, expected) in text.iter().enumerate() {
for index in [0, 2] {
let expected = &text[index];
let layout = view.layout(&cells, index).unwrap();
assert_eq!(layout.text(), expected);
let pinned_narrow = layout.rewrap(/*width*/ 8);
@@ -231,7 +255,7 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() {
}
// Short, inset viewports may begin in an image. Its placement stays inside the viewport.
let short = Rect::new(
/*x*/ 7, /*y*/ 5, /*width*/ 16, /*height*/ 3,
/*x*/ 7, /*y*/ 5, /*width*/ 16, /*height*/ 5,
);
view.jump_to_latest();
view.render(short, &mut Buffer::empty(short), &cells);
@@ -266,6 +290,7 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() {
// Resumed sessions with a deleted file fall back to the original text entry.
std::fs::remove_file(&path).unwrap();
std::fs::remove_file(&square_path).unwrap();
view.image_protocol = Some(ImageProtocol::Kitty);
view.render(area, &mut buf, &cells);
assert_eq!(buf, text_only);
@@ -595,6 +595,15 @@ impl TranscriptView {
&mut self,
cells: &[Arc<dyn HistoryCell>],
inserted: Range<usize>,
) {
self.history_page_loaded(cells, inserted, /*inserted_images*/ None);
}
pub(crate) fn history_page_loaded(
&mut self,
cells: &[Arc<dyn HistoryCell>],
inserted: Range<usize>,
inserted_images: Option<Range<usize>>,
) {
self.invalidate_turn_details_from(/*index*/ 0);
// With only a header and live output, the first older page extends the canonical tail.
@@ -604,7 +613,17 @@ impl TranscriptView {
{
self.last_tail = cells.last().map(EntryKey::cell);
}
let image_span = inserted_images.map(|images| {
// The older-page prepend shifts the separately inserted images.
let len = inserted.len();
self.prepend_snapshot_history(cells, images.start + len..images.end + len)
});
let inserted = self.prepend_snapshot_history(cells, inserted);
let inserted = if inserted.is_empty() {
image_span.unwrap_or(inserted)
} else {
inserted
};
if let Position::Reading(anchor) = self.position {
let index = self.resolve(cells, anchor);
self.position = Position::Reading(Anchor {
@@ -918,6 +918,33 @@ fn search_scans_a_page_inserted_after_the_retained_session_header() {
assert!(Arc::ptr_eq(&cells[0], &header));
}
#[test]
fn search_resumes_from_the_text_in_a_page_with_a_relocated_image() {
let mut cells = vec![cell("answer"), cell("completion")];
let mut view = TranscriptView {
area: Rect::new(
/*x*/ 0, /*y*/ 0, /*width*/ 80, /*height*/ 6,
),
history: TranscriptHistoryState::Partial,
..TranscriptView::default()
};
view.begin_search();
view.paste_search("needle");
finish_scan(&mut view, &cells);
assert!(view.search.needs_history(view.history));
let match_cell = cell("older needle");
cells.insert(/*index*/ 1, cell("Generated image"));
cells.insert(/*index*/ 0, Arc::clone(&match_cell));
view.history = TranscriptHistoryState::Complete;
view.history_page_loaded(&cells, 0..1, Some(1..2));
finish_scan(&mut view, &cells);
assert_eq!(
view.search.current.as_ref().unwrap().anchor.key,
EntryKey::cell(&match_cell)
);
}
#[test]
fn search_finds_the_first_older_entry_after_replaying_the_session_header() {
let header = || -> Arc<dyn HistoryCell> {
@@ -1,16 +1,42 @@
---
source: tui/src/transcript_view/layout_tests.rs
assertion_line: 210
assertion_line: 239
expression: "format!(\"Collapsed inspection / generated output\\n{collapsed}\\nGraphics: {collapsed_areas:?}\\n\\nExpanded inspection / generated output\\n{}\\nGraphics: {expanded_areas:?}\",\nframe(&buf))"
---
Collapsed inspection / generated output
• Viewed image artifact.png
+ Show details
▸ Viewed image /workspace/artifact.png
• Generated Image:
└ A red rectangle
└ Saved to: file:///workspace/artifact.png
Graphics: [Rect { x: 52, y: 6, width: 36, height: 6 }]
• Here is the image.
Generated image
Worked for 2s
Graphics: [Rect { x: 2, y: 5, width: 50, height: 25 }]
Expanded inspection / generated output
• Viewed image /workspace/artifact.png
@@ -22,7 +48,34 @@ Expanded inspection / generated output
• Generated Image:
└ A red rectangle
└ Saved to: file:///workspace/artifact.png
Graphics: [Rect { x: 52, y: 2, width: 36, height: 6 }, Rect { x: 52, y: 12, width: 36, height: 6 }]
• Here is the image.
Generated image
Worked for 2s
Graphics: [Rect { x: 52, y: 2, width: 36, height: 6 }, Rect { x: 2, y: 12, width: 50, height: 25 }]
+3
View File
@@ -47,6 +47,7 @@ use ratatui::text::Line;
/// One cell's logical text and its width-dependent display rows.
pub(crate) struct TextLayout {
pub(super) image: Option<std::sync::Arc<crate::transcript_image::TranscriptImage>>,
pub(super) image_left_aligned: bool,
logical: Vec<LogicalLine>,
text: String,
rows: Vec<TextRow>,
@@ -212,6 +213,7 @@ impl TextLayout {
layout = layout.with_control(control.label.clone(), control.anchor);
}
layout.image = self.image.clone();
layout.image_left_aligned = self.image_left_aligned;
if self.leading_blank_hidden {
layout = layout.without_leading_blank_row();
}
@@ -941,6 +943,7 @@ impl TextLayout {
}
Self {
image: None,
image_left_aligned: false,
logical,
text,
rows,
+19 -9
View File
@@ -112,6 +112,7 @@ pub(crate) struct InitializedTerminal {
pub(crate) terminal: Terminal,
pub(crate) enhanced_keys_supported: bool,
pub(crate) terminal_app_over_ssh: bool,
pub(crate) herdr_kitty_graphics: Option<bool>,
pub(crate) stderr_guard: terminal_stderr::TerminalStderrGuard,
}
@@ -516,6 +517,12 @@ pub(crate) fn init() -> Result<InitializedTerminal> {
#[cfg(unix)]
let backend = CrosstermBackend::new(stdout());
#[cfg(unix)]
let probe_herdr = {
let info = codex_terminal_detection::terminal_info();
info.multiplexer.is_none() && info.term_program.as_deref() == Some("herdr")
};
#[cfg(unix)]
let startup_probe = {
use crate::terminal_probe::StartupKeyboardEnhancementProbe;
@@ -526,8 +533,11 @@ pub(crate) fn init() -> Result<InitializedTerminal> {
} else {
StartupKeyboardEnhancementProbe::Query
};
match crate::terminal_probe::startup(crate::terminal_probe::DEFAULT_TIMEOUT, keyboard_probe)
{
match crate::terminal_probe::startup(
crate::terminal_probe::DEFAULT_TIMEOUT,
keyboard_probe,
probe_herdr,
) {
Ok(probe) => {
tracing::info!(
duration_ms = %started_at.elapsed().as_millis(),
@@ -543,13 +553,7 @@ pub(crate) fn init() -> Result<InitializedTerminal> {
duration_ms = %started_at.elapsed().as_millis(),
"terminal startup probes failed: {err}"
);
crate::terminal_probe::StartupProbe {
alternate_scroll_enabled: None,
cursor_position: None,
default_colors: None,
keyboard_enhancement_supported: None,
terminal_app_over_ssh: None,
}
crate::terminal_probe::StartupProbe::default()
}
}
};
@@ -613,6 +617,10 @@ pub(crate) fn init() -> Result<InitializedTerminal> {
.unwrap_or(/*default*/ false),
#[cfg(not(unix))]
terminal_app_over_ssh: false,
#[cfg(unix)]
herdr_kitty_graphics: probe_herdr.then_some(startup_probe.kitty_graphics),
#[cfg(not(unix))]
herdr_kitty_graphics: None,
stderr_guard,
};
restore_guard.active = false;
@@ -744,6 +752,7 @@ pub struct Tui {
enhanced_keys_supported: bool,
// Cache the startup result so later configuration loads use the same terminal policy.
pub(crate) terminal_app_over_ssh: bool,
pub(crate) herdr_kitty_graphics: Option<bool>,
notification_backend: Option<DesktopNotificationBackend>,
notification_condition: NotificationCondition,
scrollback: ScrollbackStrategy,
@@ -822,6 +831,7 @@ impl Tui {
terminal_focused: Arc::new(AtomicBool::new(true)),
enhanced_keys_supported,
terminal_app_over_ssh: false,
herdr_kitty_graphics: None,
notification_backend: Some(detect_backend(NotificationMethod::default())),
notification_condition: NotificationCondition::default(),
scrollback,
@@ -49,6 +49,7 @@ fn preview_at(protocol: ImageProtocol, y: u16) -> ImagePlacement {
area: Rect::new(/*x*/ 0, y, /*width*/ 2, /*height*/ 1),
first_row: 0,
file: None,
terminal_id: None,
}
}