From dffa0deb69268ed4ac31578794c9ea897b336ae3 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 15 Oct 2026 06:53:44 +0000 Subject: [PATCH] Show image previews in the fullscreen transcript (#54216) ## What changed Render local viewed images and generated image artifacts as bounded thumbnails in the rich fullscreen transcript using Kitty or Sixel graphics. Viewed images appear when their activity is expanded or the detailed presentation is active; generated images appear alongside their saved-path entry. Keep previews clipped to the viewport, retain graphics across unrelated redraws, and remove them when overlays open. Bound image decoding and cache memory, and preserve text-only entries for remote workspaces, unsupported terminals, missing files, and oversized images. Image rows do not enter searchable or selectable text. ## Testing Add layout and snapshot coverage for disclosure, viewport clipping, rewrapping, raw-mode fallback, deleted files, and oversized rasters. Add terminal coverage for retained Kitty graphics, repainting after a clear, local-file transport, and overlay cleanup. GitOrigin-RevId: fb2ec146331acfcfc82b2b283616671c9ee93757 --- codex-rs/tui/src/app/owned_transcript.rs | 7 + .../tui/src/app/owned_transcript_tests.rs | 1 + codex-rs/tui/src/chatwidget/pets.rs | 2 +- .../tui/src/chatwidget/rendering_tests.rs | 4 +- codex-rs/tui/src/chatwidget/replay.rs | 4 +- codex-rs/tui/src/chatwidget/tool_lifecycle.rs | 4 +- codex-rs/tui/src/custom_terminal.rs | 128 ++++++++++++- codex-rs/tui/src/history_cell/mod.rs | 5 + codex-rs/tui/src/history_cell/patches.rs | 44 ++++- .../tui/src/history_cell/patches_tests.rs | 10 +- codex-rs/tui/src/lib.rs | 1 + codex-rs/tui/src/pets/image_protocol.rs | 9 +- codex-rs/tui/src/pets/mod.rs | 7 +- .../tui/src/thread_transcript/other_items.rs | 4 +- codex-rs/tui/src/transcript_image.rs | 169 ++++++++++++++++++ codex-rs/tui/src/transcript_view.rs | 33 ++++ codex-rs/tui/src/transcript_view/layout.rs | 40 +++-- .../tui/src/transcript_view/layout_tests.rs | 133 ++++++++++++++ ..._keeps_controls_without_passive_hints.snap | 5 +- ...tests__fullscreen_image_artifact_rows.snap | 28 +++ ...arch__tests__find_selective_expansion.snap | 5 +- codex-rs/tui/src/transcript_view/text.rs | 25 +++ codex-rs/tui/src/tui.rs | 8 +- codex-rs/tui/src/tui/size_monitor_tests.rs | 2 +- 24 files changed, 631 insertions(+), 47 deletions(-) create mode 100644 codex-rs/tui/src/transcript_image.rs create mode 100644 codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__layout__tests__fullscreen_image_artifact_rows.snap diff --git a/codex-rs/tui/src/app/owned_transcript.rs b/codex-rs/tui/src/app/owned_transcript.rs index 23993b29cd..7c43e1799b 100644 --- a/codex-rs/tui/src/app/owned_transcript.rs +++ b/codex-rs/tui/src/app/owned_transcript.rs @@ -80,6 +80,10 @@ impl App { let chat_widget = &self.chat_widget; let transcript_width = chat_widget.history_wrap_width(width); let view = &mut self.transcript_view; + view.image_protocol = chat_widget.pet_image_support().protocol().filter(|_| { + !self.app_server_target.uses_remote_workspace() + && crate::projectless::has_only_local_environments(&self.environment_manager) + }); view.mouse_scroll_speed = self.local_settings.tui.mouse_scroll_speed.unwrap_or(1.0); view.primary_selection = self.right_click_paste_environment.primary; view.copy_on_select = self @@ -358,6 +362,9 @@ impl App { frame.set_cursor_style(foreground.cursor_style(foreground_area)); frame.set_cursor_position(position); } + if chat_widget.no_modal_or_popup_active() { + frame.transcript_images = view.image_placements(); + } })?; if let Some((surface, tip)) = &turn_tip && tip.rendered.get().is_some() diff --git a/codex-rs/tui/src/app/owned_transcript_tests.rs b/codex-rs/tui/src/app/owned_transcript_tests.rs index ff6e0201bb..1ca9ddc35a 100644 --- a/codex-rs/tui/src/app/owned_transcript_tests.rs +++ b/codex-rs/tui/src/app/owned_transcript_tests.rs @@ -353,6 +353,7 @@ async fn owned_transcript_reserves_a_row_above_the_composer() -> Result<()> { app.chat_widget.apply_external_edit(draft.to_string()); if label == "Detailed" { app.transcript_cells = vec![Arc::new(crate::history_cell::new_view_image_tool_call( + "detail-image".to_string(), codex_utils_path_uri::LegacyAppPathString::from_string("assets/detail-image.png"), ))]; app.open_transcript_overlay(&mut tui); diff --git a/codex-rs/tui/src/chatwidget/pets.rs b/codex-rs/tui/src/chatwidget/pets.rs index 90be6317da..aec83e79e9 100644 --- a/codex-rs/tui/src/chatwidget/pets.rs +++ b/codex-rs/tui/src/chatwidget/pets.rs @@ -210,7 +210,7 @@ impl ChatWidget { true } - fn pet_image_support(&self) -> crate::pets::PetImageSupport { + pub(crate) fn pet_image_support(&self) -> crate::pets::PetImageSupport { #[cfg(test)] if let Some(support) = self.pet_image_support_override { return support; diff --git a/codex-rs/tui/src/chatwidget/rendering_tests.rs b/codex-rs/tui/src/chatwidget/rendering_tests.rs index 6a95ec466b..4129eca7be 100644 --- a/codex-rs/tui/src/chatwidget/rendering_tests.rs +++ b/codex-rs/tui/src/chatwidget/rendering_tests.rs @@ -367,7 +367,9 @@ fn active_transcript_preserves_clipped_markdown_hyperlinks() { size.height, )); terminal - .draw_with_size(size, |frame| renderable.render(area, frame.buffer_mut())) + .draw_with_size(size, &mut Vec::new(), |frame| { + renderable.render(area, frame.buffer_mut()) + }) .expect("render terminal frame"); let output = String::from_utf8(terminal.backend().writer().clone()).expect("UTF-8 output"); assert!(output.contains("\x1b]8;;https://example.com/\x07OSC8 label\x1b]8;;\x07")); diff --git a/codex-rs/tui/src/chatwidget/replay.rs b/codex-rs/tui/src/chatwidget/replay.rs index 8f573351ca..6c8a7fb7fb 100644 --- a/codex-rs/tui/src/chatwidget/replay.rs +++ b/codex-rs/tui/src/chatwidget/replay.rs @@ -413,8 +413,8 @@ impl ChatWidget { .unwrap_or(codex_app_server_protocol::WebSearchAction::Other), ); } - ThreadItem::ImageView { id: _, path } => { - self.on_view_image_tool_call(path); + ThreadItem::ImageView { id, path } => { + self.on_view_image_tool_call(id, path); } ThreadItem::ImageGeneration(item) => { self.on_image_generation_end( diff --git a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs index 135aabca87..60617254f6 100644 --- a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs +++ b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs @@ -12,9 +12,9 @@ impl ChatWidget { self.add_to_history(history_cell::new_patch_event(changes, &self.config.cwd)); } - pub(super) fn on_view_image_tool_call(&mut self, path: LegacyAppPathString) { + pub(super) fn on_view_image_tool_call(&mut self, id: String, path: LegacyAppPathString) { self.flush_answer_stream_with_separator(); - self.add_to_history(history_cell::new_view_image_tool_call(path)); + self.add_to_history(history_cell::new_view_image_tool_call(id, path)); self.request_redraw(); } diff --git a/codex-rs/tui/src/custom_terminal.rs b/codex-rs/tui/src/custom_terminal.rs index c2a6f05895..0607a8ca6a 100644 --- a/codex-rs/tui/src/custom_terminal.rs +++ b/codex-rs/tui/src/custom_terminal.rs @@ -81,6 +81,9 @@ pub struct Frame<'a> { /// The buffer that is used to draw the current frame pub(crate) buffer: &'a mut Buffer, + + /// Fullscreen graphics to send after text, before the synchronized frame is committed. + pub(crate) transcript_images: Vec, } impl Frame<'_> { @@ -254,6 +257,7 @@ where /// Get a Frame object which provides a consistent view into the terminal state for rendering. pub fn get_frame(&mut self) -> Frame<'_> { Frame { + transcript_images: Vec::new(), cursor_position: None, cursor_style: SetCursorStyle::DefaultUserShape, viewport_area: self.viewport_area, @@ -346,13 +350,14 @@ where F: FnOnce(&mut Frame), { let screen_size = self.size()?; - self.draw_with_size(screen_size, render_callback) + self.draw_with_size(screen_size, &mut Vec::new(), render_callback) } /// Draws a single frame using a screen size already obtained by the caller. pub(crate) fn draw_with_size( &mut self, screen_size: Size, + retained_images: &mut Vec, render_callback: F, ) -> io::Result<()> where @@ -370,14 +375,52 @@ where // Buffer. Thus, we're taking the important data out of the Frame and dropping it. let cursor_position = frame.cursor_position; let cursor_style = frame.cursor_style; + let images = std::mem::take(&mut frame.transcript_images); + let changed = *retained_images != images; + if changed { + // Remove old graphics before text painting, including when switching to overlays. + for (index, placement) in retained_images.iter().enumerate() { + match placement.image.protocol { + crate::pets::ImageProtocol::Kitty + | crate::pets::ImageProtocol::KittyLocalFile => { + let id = crate::transcript_image::FIRST_IMAGE_ID + index as u32; + self.backend + .write_all(crate::pets::kitty_delete_image(id).as_bytes())?; + } + crate::pets::ImageProtocol::Sixel => self.invalidate_viewport(), + } + } + } // Not every terminal or multiplexer hides intermediate cursor moves inside a // synchronized update, especially when the frame spans multiple writes. let updates = diff_buffers(self.previous_buffer(), self.current_buffer()); + // A text clear, focus highlight, or full repaint may erase an otherwise stable image. + let repaint_images = changed + || updates.iter().any(|update| { + images.iter().any(|placement| match update { + DrawCommand::Put { x, y, .. } => placement.area.contains((*x, *y).into()), + DrawCommand::ClearToEnd { x, y, .. } => { + *y >= placement.area.y + && *y < placement.area.bottom() + && *x < placement.area.right() + } + }) + }); if !updates.is_empty() && !self.hidden_cursor { self.hide_cursor()?; } self.flush_updates(updates)?; + if repaint_images { + *retained_images = images; + for (index, placement) in retained_images.iter_mut().enumerate() { + let id = crate::transcript_image::FIRST_IMAGE_ID + index as u32; + placement.file = placement + .image + .draw(&mut self.backend, placement.area, placement.first_row, id) + .map_err(io::Error::other)?; + } + } match cursor_position { None if !self.hidden_cursor => self.hide_cursor()?, @@ -451,7 +494,7 @@ where self.backend.set_cursor_position(position)?; self.backend.clear_region(ClearType::AfterCursor)?; // Reset the back buffer to make sure the next update will redraw everything. - self.previous_buffer_mut().reset(); + self.invalidate_viewport(); Ok(()) } @@ -478,7 +521,7 @@ where self.set_cursor_position(home)?; std::io::Write::flush(&mut self.backend)?; self.visible_history_rows = 0; - self.previous_buffer_mut().reset(); + self.invalidate_viewport(); Ok(()) } @@ -497,7 +540,7 @@ where std::io::Write::flush(&mut self.backend)?; self.last_known_cursor_pos = Position { x: 0, y: 0 }; self.visible_history_rows = 0; - self.previous_buffer_mut().reset(); + self.invalidate_viewport(); Ok(()) } @@ -849,6 +892,77 @@ mod tests { } } + #[test] + fn transcript_images_survive_unrelated_redraws_and_are_removed_for_overlays() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("viewed.png"); + image::RgbaImage::from_pixel(32, 32, image::Rgba([10, 110, 200, 255])) + .save(&path) + .unwrap(); + for protocol in [ + crate::pets::ImageProtocol::Kitty, + crate::pets::ImageProtocol::KittyLocalFile, + ] { + let image = std::sync::Arc::new( + crate::transcript_image::TranscriptImage::load(&path, /*width*/ 40, protocol) + .unwrap(), + ); + let mut terminal = + Terminal::with_options(CaptureBackend::new(/*width*/ 40, /*height*/ 12)).unwrap(); + let size = terminal.last_known_screen_size; + terminal.set_viewport_area(Rect::new( + /*x*/ 0, + /*y*/ 0, + size.width, + size.height, + )); + let mut retained = Vec::new(); + for (index, text) in ["", "a", "ab", "ab"].into_iter().enumerate() { + if index == 3 { + assert_eq!(terminal.backend().output().matches("_Ga=T").count(), 1); + terminal.clear().unwrap(); + } + terminal + .draw_with_size(size, &mut retained, |frame| { + Line::from(text).render( + Rect::new( + /*x*/ 0, /*y*/ 10, /*width*/ 40, /*height*/ 1, + ), + frame.buffer_mut(), + ); + frame + .transcript_images + .push(crate::transcript_image::ImagePlacement { + image: std::sync::Arc::clone(&image), + area: Rect::new( + /*x*/ 2, + /*y*/ 2, + image.columns, + image.rows, + ), + first_row: 0, + file: None, + }); + }) + .unwrap(); + } + let output = terminal.backend().output(); + assert_eq!(output.matches("_Ga=T").count(), 2); + assert_eq!(output.matches("_Ga=d").count(), 0); + if protocol == crate::pets::ImageProtocol::KittyLocalFile { + assert!(output.contains("t=f") && output.contains("C=1")); + assert!(retained[0].file.as_ref().unwrap().path().exists()); + } else { + assert!(retained[0].file.is_none()); + } + terminal + .draw_with_size(size, &mut retained, |_| {}) + .unwrap(); + assert_eq!(terminal.backend().output().matches("_Ga=d").count(), 1); + assert!(retained.is_empty()); + } + } + impl Backend for CaptureBackend { type Error = io::Error; @@ -1047,7 +1161,9 @@ mod tests { let screen_size = terminal.last_known_screen_size; for _ in 0..3 { - terminal.draw_with_size(screen_size, |_| {}).expect("draw"); + terminal + .draw_with_size(screen_size, &mut Vec::new(), |_| {}) + .expect("draw"); } terminal.set_viewport_area(Rect::new( @@ -1071,7 +1187,7 @@ mod tests { let area = Rect::new(/*x*/ 0, /*y*/ 0, size.width, size.height); terminal.set_viewport_area(area); terminal - .draw_with_size(size, |frame| { + .draw_with_size(size, &mut Vec::new(), |frame| { Paragraph::new("alpha beta") .wrap(Wrap { trim: false }) .render(area, frame.buffer_mut()); diff --git a/codex-rs/tui/src/history_cell/mod.rs b/codex-rs/tui/src/history_cell/mod.rs index ac39aef420..5b14559f63 100644 --- a/codex-rs/tui/src/history_cell/mod.rs +++ b/codex-rs/tui/src/history_cell/mod.rs @@ -224,6 +224,11 @@ pub(crate) trait HistoryCell: std::fmt::Debug + Send + Sync + Any { /// Returns copy-friendly plain logical lines for raw scrollback mode. fn raw_lines(&self) -> Vec>; + /// A local artifact the owned fullscreen viewport may preview. Text surfaces ignore it. + fn image_path(&self) -> Option { + None + } + /// Raw live presentation may omit transcript-only diagnostics without losing their source. fn live_raw_lines(&self) -> Vec> { self.raw_lines() diff --git a/codex-rs/tui/src/history_cell/patches.rs b/codex-rs/tui/src/history_cell/patches.rs index 3569c7c9ab..49191948f0 100644 --- a/codex-rs/tui/src/history_cell/patches.rs +++ b/codex-rs/tui/src/history_cell/patches.rs @@ -159,11 +159,20 @@ impl HistoryCell for PatchFailureCell { #[derive(Debug)] pub(crate) struct ViewImageHistoryCell { + activity_id: String, filename: String, path_label: String, } impl HistoryCell for ViewImageHistoryCell { + fn activity_ids(&self) -> Vec { + vec![self.activity_id.clone()] + } + + fn image_path(&self) -> Option { + LegacyAppPathString::from_string(self.path_label.clone()).to_inferred_abs_path() + } + fn display_lines(&self, width: u16) -> Vec> { let line = vec![ "• ".dim(), @@ -191,12 +200,16 @@ impl HistoryCell for ViewImageHistoryCell { } } -pub(crate) fn new_view_image_tool_call(path: LegacyAppPathString) -> ViewImageHistoryCell { +pub(crate) fn new_view_image_tool_call( + id: String, + path: LegacyAppPathString, +) -> ViewImageHistoryCell { let filename = path .to_inferred_path_uri() .and_then(|path| path.basename()) .unwrap_or_else(|| path.render_for_ui()); ViewImageHistoryCell { + activity_id: format!("image-view:{id}"), filename, path_label: path.into_string(), } @@ -207,7 +220,7 @@ pub(crate) fn new_image_generation_call( status: &str, revised_prompt: Option, saved_path: Option, -) -> PlainHistoryCell { +) -> ImageGenerationHistoryCell { let detail = revised_prompt.unwrap_or(call_id); let heading = if status == "failed" { vec!["✗ ".red().bold(), "Image generation failed".bold()].into() @@ -215,12 +228,35 @@ pub(crate) fn new_image_generation_call( vec!["• ".dim(), "Generated Image:".bold()].into() }; let mut lines: Vec> = vec![heading, vec![" └ ".dim(), detail.dim()].into()]; - if let Some(saved_path) = saved_path { + if let Some(saved_path) = &saved_path { let saved_path = Url::from_file_path(saved_path.as_path()) .map(|url| url.to_string()) .unwrap_or_else(|_| saved_path.display().to_string()); lines.push(vec![" └ ".dim(), "Saved to: ".dim(), saved_path.into()].into()); } - PlainHistoryCell { lines } + ImageGenerationHistoryCell { + lines, + saved_path: saved_path.filter(|_| status != "failed"), + } +} + +#[derive(Debug)] +pub(crate) struct ImageGenerationHistoryCell { + lines: Vec>, + saved_path: Option, +} + +impl HistoryCell for ImageGenerationHistoryCell { + fn display_lines(&self, _width: u16) -> Vec> { + self.lines.clone() + } + + fn raw_lines(&self) -> Vec> { + plain_lines(self.lines.clone()) + } + + fn image_path(&self) -> Option { + self.saved_path.clone() + } } diff --git a/codex-rs/tui/src/history_cell/patches_tests.rs b/codex-rs/tui/src/history_cell/patches_tests.rs index 9844d9b4ec..9950b4350d 100644 --- a/codex-rs/tui/src/history_cell/patches_tests.rs +++ b/codex-rs/tui/src/history_cell/patches_tests.rs @@ -46,7 +46,8 @@ fn viewed_image_retains_original_path_in_details() { "/workspace/assets/example.png", r"C:\workspace\assets\example.png", ] { - let cell = new_view_image_tool_call(LegacyAppPathString::from_string(path)); + let cell = + new_view_image_tool_call("view".to_string(), LegacyAppPathString::from_string(path)); assert_eq!( cell.display_lines(/*width*/ 80) .iter() @@ -70,9 +71,10 @@ fn viewed_image_retains_original_path_in_details() { #[test] fn viewed_image_narrow_summary() { - let cell = new_view_image_tool_call(LegacyAppPathString::from_string( - "/workspace/very-long-screenshot-name.png", - )); + let cell = new_view_image_tool_call( + "view".to_string(), + LegacyAppPathString::from_string("/workspace/very-long-screenshot-name.png"), + ); insta::assert_snapshot!(cell.display_lines(/*width*/ 32)[0].to_string()); } diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index e4ab86c347..e586a61fa4 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -241,6 +241,7 @@ mod thread_transcript; mod token_usage; mod tool_output; mod tooltips; +mod transcript_image; mod transcript_mode; mod transcript_reflow; mod transcript_view; diff --git a/codex-rs/tui/src/pets/image_protocol.rs b/codex-rs/tui/src/pets/image_protocol.rs index 06b6fc77bc..947cedc180 100644 --- a/codex-rs/tui/src/pets/image_protocol.rs +++ b/codex-rs/tui/src/pets/image_protocol.rs @@ -241,7 +241,7 @@ pub fn kitty_transmit_png_with_id( if index == 0 { let image_id = kitty_image_id_arg(image_id); command.push_str(&format!( - "{ESC}_Ga=T,t=d,f=100,c={columns},r={rows},q=2{image_id},m={more_flag};{chunk}{ST}", + "{ESC}_Ga=T,t=d,f=100,c={columns},r={rows},q=2{image_id},m={more_flag},C=1;{chunk}{ST}", )); } else { command.push_str(&format!("{ESC}_Gm={more_flag};{chunk}{ST}")); @@ -262,7 +262,8 @@ pub fn kitty_transmit_png_file_with_id( .with_context(|| format!("canonicalize {}", path.display()))?; let payload = general_purpose::STANDARD.encode(path.to_string_lossy().as_bytes()); let image_id = kitty_image_id_arg(image_id); - let command = format!("{ESC}_Ga=T,t=f,f=100,c={columns},r={rows},q=2{image_id};{payload}{ST}"); + let command = + format!("{ESC}_Ga=T,t=f,f=100,c={columns},r={rows},q=2{image_id},C=1;{payload}{ST}"); Ok(wrap_for_tmux_if_needed(&command)) } @@ -353,7 +354,7 @@ mod tests { ) .unwrap(); - assert!(command.starts_with("\x1b_Ga=T,t=d,f=100,c=4,r=3,q=2,m=0;")); + assert!(command.starts_with("\x1b_Ga=T,t=d,f=100,c=4,r=3,q=2,m=0,C=1;")); assert!(command.contains("cG5n")); assert!(command.ends_with("\x1b\\")); } @@ -691,7 +692,7 @@ mod tests { assert_eq!( command, - format!("\x1b_Ga=T,t=f,f=100,c=4,r=3,q=2,i=7;{payload}\x1b\\") + format!("\x1b_Ga=T,t=f,f=100,c=4,r=3,q=2,i=7,C=1;{payload}\x1b\\") ); } } diff --git a/codex-rs/tui/src/pets/mod.rs b/codex-rs/tui/src/pets/mod.rs index f12313ff4c..1026f62ef4 100644 --- a/codex-rs/tui/src/pets/mod.rs +++ b/codex-rs/tui/src/pets/mod.rs @@ -40,16 +40,19 @@ pub(crate) use ambient::test_ambient_pet; pub(crate) use asset_pack::builtin_spritesheet_path; #[cfg(test)] pub(crate) use asset_pack::write_test_pack; -#[cfg(test)] pub(crate) use image_protocol::ImageProtocol; pub(crate) use image_protocol::PetImageSupport; #[cfg(test)] pub(crate) use image_protocol::PetImageUnsupportedReason; #[cfg(not(test))] pub(crate) use image_protocol::detect_pet_image_support; +pub(crate) use image_protocol::kitty_delete_image; +pub(crate) use image_protocol::kitty_transmit_png_file_with_id; +pub(crate) use image_protocol::kitty_transmit_png_with_id; pub(crate) use picker::PET_PICKER_VIEW_ID; pub(crate) use picker::build_pet_picker_params; pub(crate) use preview::PetPickerPreviewState; +pub(crate) use sixel::encode_rgba; pub(crate) const DEFAULT_PET_ID: &str = "codex"; pub(crate) const DISABLED_PET_ID: &str = "disabled"; @@ -368,7 +371,7 @@ mod tests { let output = String::from_utf8(output).unwrap(); assert!(output.contains("a=d,d=I,i=49374,q=2;")); assert!(output.contains("\x1b[4;3H")); - assert!(output.contains("a=T,t=f,f=100,c=4,r=2,q=2,i=49374;")); + assert!(output.contains("a=T,t=f,f=100,c=4,r=2,q=2,i=49374,C=1;")); assert!(!output.contains("cG5n")); assert!(output.contains("\x1b8")); } diff --git a/codex-rs/tui/src/thread_transcript/other_items.rs b/codex-rs/tui/src/thread_transcript/other_items.rs index 6811613390..3d81922dfa 100644 --- a/codex-rs/tui/src/thread_transcript/other_items.rs +++ b/codex-rs/tui/src/thread_transcript/other_items.rs @@ -55,8 +55,8 @@ pub(super) fn cells(item: ThreadItem, cwd: &AbsolutePathBuf) -> TranscriptCells item.query, item.action.unwrap_or(WebSearchAction::Other), ))), - ThreadItem::ImageView { path, .. } => { - cells.push(Arc::new(history_cell::new_view_image_tool_call(path))) + ThreadItem::ImageView { id, path } => { + cells.push(Arc::new(history_cell::new_view_image_tool_call(id, path))) } ThreadItem::ImageGeneration(item) if !matches!(item.status.as_str(), "completed" | "failed") => diff --git a/codex-rs/tui/src/transcript_image.rs b/codex-rs/tui/src/transcript_image.rs new file mode 100644 index 0000000000..9aa9159603 --- /dev/null +++ b/codex-rs/tui/src/transcript_image.rs @@ -0,0 +1,169 @@ +//! Bounded image previews for the owned fullscreen transcript. +//! +//! Only decoded local image pixels reach the terminal encoders. Previews are never source text; +//! their reserved rows belong to layout. The terminal retains visible placements and any local +//! transport files until those placements change. + +use std::io::Write; +use std::path::Path; +use std::sync::Arc; + +use anyhow::Result; +use crossterm::cursor::MoveTo; +use crossterm::cursor::RestorePosition; +use crossterm::cursor::SavePosition; +use crossterm::queue; +use image::DynamicImage; +use image::ImageFormat; +use image::RgbaImage; +use ratatui::layout::Rect; + +use crate::pets::ImageProtocol; + +// Separate from the two image ids reserved for pets (0xC0DE and 0xC0DF). +pub(crate) const FIRST_IMAGE_ID: u32 = 0xC100; + +pub(crate) struct ImagePlacement { + pub(crate) image: Arc, + pub(crate) area: Rect, + pub(crate) first_row: usize, + pub(crate) file: Option, +} + +impl PartialEq for ImagePlacement { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.image, &other.image) + && self.area == other.area + && self.first_row == other.first_row + } +} + +pub(crate) struct TranscriptImage { + pub(crate) protocol: ImageProtocol, + pub(crate) columns: u16, + pub(crate) rows: u16, + pub(crate) pixels: RgbaImage, + pub(crate) cell_width: u32, + pub(crate) cell_height: u32, +} + +pub(crate) fn terminal_cell_size() -> Option<(u32, u32)> { + crossterm::terminal::window_size() + .ok() + .filter(|size| size.columns > 0 && size.rows > 0 && size.width > 0 && size.height > 0) + .map(|size| { + ( + u32::from(size.width / size.columns).max(1), + u32::from(size.height / size.rows).max(1), + ) + }) +} + +impl TranscriptImage { + pub(crate) fn load(path: &Path, width: u16, protocol: ImageProtocol) -> Option { + let columns = width.saturating_sub(2).min(48); + let metadata = path.metadata().ok()?; + // Bound encoded input as well as the raster: metadata alone can dwarf a tiny image. + if columns == 0 || !metadata.is_file() || metadata.len() > 32 * 1024 * 1024 { + return None; + } + // Sixel paints physical pixels rather than sizing to a Kitty cell rectangle. Guessing + // the cell size could paint beyond the reserved rows and cover the composer. + let (cell_width, cell_height) = match protocol { + ImageProtocol::Sixel => terminal_cell_size()?, + // Kitty scales to cell coordinates itself. Use a stable, typical cell aspect ratio. + ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (8, 16), + }; + let mut reader = image::ImageReader::open(path) + .ok()? + .with_guessed_format() + .ok()?; + // The reader checks the decoded size before allocating the raster. Keep normal + // screenshots usable, but never allocate hundreds of MB for an optional thumbnail. + let mut limits = image::Limits::default(); + limits.max_alloc = Some(32 * 1024 * 1024); + reader.limits(limits); + let image = reader.decode().ok()?; + // The sprite encoder scans each active color per Sixel band. Bound both its CPU + // work and worst-case output even with large terminal cells or high-entropy images. + let (max_width, max_height) = match protocol { + ImageProtocol::Sixel => (160, 80), + ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (384, 160), + }; + let pixels = image + .thumbnail( + (u32::from(columns) * cell_width) + .min(image.width()) + .min(max_width), + (6 * cell_height).min(image.height()).min(max_height), + ) + .to_rgba8(); + Some(Self { + protocol, + columns: pixels.width().div_ceil(cell_width) as u16, + rows: pixels.height().div_ceil(cell_height) as u16, + pixels, + cell_width, + cell_height, + }) + } + + /// Draw only the visible vertical slice, never covering the prompt header or composer. + pub(crate) fn draw( + &self, + writer: &mut impl Write, + area: Rect, + first_row: usize, + id: u32, + ) -> Result> { + let (top, bottom) = match self.protocol { + ImageProtocol::Sixel => ( + first_row as u32 * self.cell_height, + (first_row as u32 + u32::from(area.height)) * self.cell_height, + ), + // Kitty scales the entire raster to r=rows; clip at those scaled boundaries. + ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => ( + first_row as u32 * self.pixels.height() / u32::from(self.rows), + (first_row as u32 + u32::from(area.height)) * self.pixels.height() + / u32::from(self.rows), + ), + }; + let top = top.min(self.pixels.height()); + let height = bottom.min(self.pixels.height()) - top; + if height == 0 { + return Ok(None); + } + let cropped = + image::imageops::crop_imm(&self.pixels, 0, top, self.pixels.width(), height).to_image(); + let mut file = None; + // Encoding and temp-file failures must not abort the TUI. Only writes to the actual + // terminal below are fatal; the existing text entry remains usable without its preview. + let payload = (|| -> Result<_> { + Ok(match self.protocol { + ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => { + let mut png = tempfile::Builder::new().suffix(".png").tempfile()?; + DynamicImage::ImageRgba8(cropped).write_to(&mut png, ImageFormat::Png)?; + png.flush()?; + let transmit = if self.protocol == ImageProtocol::KittyLocalFile { + crate::pets::kitty_transmit_png_file_with_id + } else { + crate::pets::kitty_transmit_png_with_id + }; + let command = transmit(png.path(), self.columns, area.height, Some(id))?; + file = (self.protocol == ImageProtocol::KittyLocalFile).then_some(png); + command.into_bytes() + } + ImageProtocol::Sixel => { + crate::pets::encode_rgba(cropped.as_raw(), cropped.width(), cropped.height())? + } + }) + })(); + let Ok(payload) = payload else { + return Ok(None); + }; + queue!(writer, SavePosition, MoveTo(area.x, area.y))?; + writer.write_all(&payload)?; + queue!(writer, RestorePosition)?; + Ok(file) + } +} diff --git a/codex-rs/tui/src/transcript_view.rs b/codex-rs/tui/src/transcript_view.rs index 80b8d67873..0e4fc5ec61 100644 --- a/codex-rs/tui/src/transcript_view.rs +++ b/codex-rs/tui/src/transcript_view.rs @@ -89,6 +89,7 @@ struct VisibleRow { /// Shared scrolling and interaction state for compact and detailed transcript presentations. pub(crate) struct TranscriptView { + pub(crate) image_protocol: Option, pub(crate) mouse_scroll_speed: f64, pending_mouse_scroll: f64, pub(crate) copy_on_select: bool, @@ -128,6 +129,7 @@ pub(crate) struct TranscriptView { impl Default for TranscriptView { fn default() -> Self { Self { + image_protocol: None, mouse_scroll_speed: 1.0, pending_mouse_scroll: 0.0, copy_on_select: false, @@ -305,6 +307,37 @@ impl TranscriptView { self.render_search(buf); } + pub(crate) fn image_placements(&self) -> Vec { + let mut placements = Vec::new(); + let mut y = self.area.y; + for rows in self.visible.chunk_by(|a, b| a.index == b.index) { + let entry_y = y; + y += rows.len() as u16; + let visible = &rows[0]; + let Some(image) = visible.layout.visible_image() else { + continue; + }; + let image_start = visible.layout.text_row_count(); + let skip = image_start.saturating_sub(visible.row); + if skip >= rows.len() { + continue; + } + let area = Rect::new( + self.area.x + self.area.width.saturating_sub(image.columns) / 2, + entry_y + skip as u16, + image.columns, + (rows.len() - skip) as u16, + ); + placements.push(crate::transcript_image::ImagePlacement { + image: Arc::clone(image), + area, + first_row: visible.row.saturating_sub(image_start), + file: None, + }); + } + placements + } + pub(crate) fn sync_live_tail( &mut self, width: u16, diff --git a/codex-rs/tui/src/transcript_view/layout.rs b/codex-rs/tui/src/transcript_view/layout.rs index 5e912a0627..f91e3d0225 100644 --- a/codex-rs/tui/src/transcript_view/layout.rs +++ b/codex-rs/tui/src/transcript_view/layout.rs @@ -11,7 +11,7 @@ use std::sync::Weak; use super::*; const MAX_CACHED_ENTRIES: usize = 64; -const MAX_CACHED_TEXT_BYTES: usize = 8 * 1024 * 1024; +const MAX_CACHED_LAYOUT_BYTES: usize = 8 * 1024 * 1024; /// The selected activity presentation and any live content that follows it. pub(crate) struct ActivityTranscriptLines { @@ -36,7 +36,8 @@ pub(super) fn activity_layout( return layout; } let label = match (expanded, lines.disclosure) { - (true, _) => "− Show less".to_owned(), + (true, Some(ActivityDisclosure::OutputLines(_))) => "− Show less".to_owned(), + (true, Some(ActivityDisclosure::Generic) | None) => "− Hide details".to_owned(), (false, Some(ActivityDisclosure::OutputLines(count))) => { let noun = if count == 1 { "line" } else { "lines" }; let mut label = format!("+ {count} {noun}"); @@ -70,6 +71,7 @@ struct RenderState { foreground: Option<(u8, u8, u8)>, background: Option<(u8, u8, u8)>, color_level: crate::terminal_palette::StdoutColorLevel, + cell_size: Option<(u32, u32)>, } struct CachedLayout { @@ -88,6 +90,7 @@ pub(super) struct CellPresentation { expanded: bool, disclosure: bool, detailed: bool, + image_protocol: Option, } impl TranscriptView { @@ -168,6 +171,7 @@ impl TranscriptView { let ids = cell.activity_ids(); let disclosure = !self.detailed && mode == HistoryRenderMode::Rich && !ids.is_empty(); let manually_expanded = self.disclosure.is_expanded(&ids); + let reveal_image = ids.is_empty() || manually_expanded || self.detailed; let expanded = disclosure && (full_content || manually_expanded); if manually_expanded { self.disclosure.expanded.extend(ids); @@ -179,13 +183,16 @@ impl TranscriptView { expanded, disclosure, detailed: full_content, + image_protocol: self + .image_protocol + .filter(|_| mode == HistoryRenderMode::Rich && reveal_image), }; let shortcut = self .disclosure .keymap .primary_hint(KeymapContext::Global, "open_transcript"); Some(self.cache.get(cell, width, presentation, || { - if disclosure { + let mut layout = if disclosure { activity_layout( ActivityTranscriptLines { activity: if expanded { @@ -210,7 +217,15 @@ impl TranscriptView { ), width, ) + }; + if let Some(protocol) = presentation.image_protocol + && let Some(path) = cell.image_path() + { + layout.image = + crate::transcript_image::TranscriptImage::load(path.as_path(), width, protocol) + .map(Arc::new); } + layout })) } } @@ -223,6 +238,7 @@ impl LayoutCache { foreground: crate::terminal_palette::default_fg(), background: crate::terminal_palette::default_bg(), color_level: crate::terminal_palette::stdout_color_level(), + cell_size: crate::transcript_image::terminal_cell_size(), }; if self.render_state.as_ref() != Some(&state) { self.entries.clear(); @@ -283,16 +299,20 @@ impl LayoutCache { } fn evict(&mut self) { - let mut bytes = self - .entries - .iter() - .map(|entry| entry.layout.text().len()) - .sum::(); + let retained_bytes = |entry: &CachedLayout| { + entry.layout.text().len() + + entry + .layout + .image + .as_ref() + .map_or(0, |image| image.pixels.len()) + }; + let mut bytes = self.entries.iter().map(retained_bytes).sum::(); // Retain a single oversized entry rather than repeatedly laying it out while visible. while self.entries.len() > 1 - && (self.entries.len() > MAX_CACHED_ENTRIES || bytes > MAX_CACHED_TEXT_BYTES) + && (self.entries.len() > MAX_CACHED_ENTRIES || bytes > MAX_CACHED_LAYOUT_BYTES) { - bytes -= self.entries.remove(/*index*/ 0).layout.text().len(); + bytes -= retained_bytes(&self.entries.remove(/*index*/ 0)); } } } diff --git a/codex-rs/tui/src/transcript_view/layout_tests.rs b/codex-rs/tui/src/transcript_view/layout_tests.rs index 486adc4308..366a978f07 100644 --- a/codex-rs/tui/src/transcript_view/layout_tests.rs +++ b/codex-rs/tui/src/transcript_view/layout_tests.rs @@ -51,6 +51,7 @@ fn layouts_refresh_for_width_animation_and_mutable_frames() { expanded: false, disclosure: false, detailed: false, + image_protocol: None, }, || TextLayout::new(history.transcript_hyperlink_lines(width), width), ) @@ -107,6 +108,7 @@ fn recent_entries_are_reused_and_old_entries_are_evicted() { expanded: false, disclosure: false, detailed: false, + image_protocol: None, }, || { TextLayout::new( @@ -127,6 +129,7 @@ fn recent_entries_are_reused_and_old_entries_are_evicted() { expanded: false, disclosure: false, detailed: false, + image_protocol: None, }, || { TextLayout::new( @@ -141,3 +144,133 @@ fn recent_entries_are_reused_and_old_entries_are_evicted() { [1, 1, 2] ); } + +#[test] +fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() { + use crate::history_cell::new_image_generation_call; + use crate::history_cell::new_view_image_tool_call; + use crate::pets::ImageProtocol; + use codex_utils_absolute_path::AbsolutePathBuf; + use codex_utils_path_uri::LegacyAppPathString; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("artifact.png"); + image::RgbaImage::from_pixel(480, 160, image::Rgba([255, 0, 0, 255])) + .save(&path) + .unwrap(); + let cells: Vec> = vec![ + Arc::new(new_view_image_tool_call( + "view-call".to_string(), + LegacyAppPathString::from_string(path.to_string_lossy().into_owned()), + )), + Arc::new(new_image_generation_call( + "image-call".to_string(), + "completed", + Some("A red rectangle".to_string()), + Some(AbsolutePathBuf::try_from(path.clone()).unwrap()), + )), + ]; + let area = Rect::new( + /*x*/ 0, /*y*/ 0, /*width*/ 140, /*height*/ 40, + ); + let frame = |buf: &Buffer| { + let uri = url::Url::from_file_path(&path).unwrap().to_string(); + crate::transcript_view::tests::text(buf) + .replace(&uri, "file:///workspace/artifact.png") + .replace(path.to_str().unwrap(), "/workspace/artifact.png") + .trim_end() + .to_string() + }; + let mut view = TranscriptView::default(); + let mut buf = Buffer::empty(area); + view.render(area, &mut buf, &cells); + let text_only = buf.clone(); + view.image_protocol = Some(ImageProtocol::Kitty); + view.render(area, &mut buf, &cells); + let collapsed = frame(&buf); + let collapsed_areas: Vec<_> = view.image_placements().iter().map(|p| p.area).collect(); + // Find may request full text, but only manual expansion may reveal inspected pixels. + assert!( + view.entry_layout(&cells, /*index*/ 0, /*full_content*/ true) + .unwrap() + .image + .is_none() + ); + + // An inspected image only appears after its existing activity control is expanded. + assert!(view.toggle_disclosure_at(&cells, /*column*/ 4, /*row*/ 1)); + view.image_protocol = None; + view.render(area, &mut buf, &cells); + let text: Vec<_> = (0..cells.len()) + .map(|index| view.layout(&cells, index).unwrap().text().to_string()) + .collect(); + view.image_protocol = Some(ImageProtocol::Kitty); + view.render(area, &mut buf, &cells); + let expanded_areas: Vec<_> = view.image_placements().iter().map(|p| p.area).collect(); + insta::assert_snapshot!( + "fullscreen_image_artifact_rows", + format!( + "Collapsed inspection / generated output\n{collapsed}\nGraphics: {collapsed_areas:?}\n\nExpanded inspection / generated output\n{}\nGraphics: {expanded_areas:?}", + frame(&buf) + ) + ); + for (index, expected) in text.iter().enumerate() { + let layout = view.layout(&cells, index).unwrap(); + assert_eq!(layout.text(), expected); + let pinned_narrow = layout.rewrap(/*width*/ 8); + assert!(pinned_narrow.visible_image().is_none()); + let pinned_restored = pinned_narrow.rewrap(area.width); + assert_eq!(pinned_restored.row_count(), layout.row_count()); + assert!(pinned_restored.visible_image().is_some()); + } + // Short, inset viewports may begin in an image. Its placement stays inside the viewport. + let short = Rect::new( + /*x*/ 7, /*y*/ 5, /*width*/ 16, /*height*/ 3, + ); + view.jump_to_latest(); + view.render(short, &mut Buffer::empty(short), &cells); + let placements = view.image_placements(); + assert!(!placements.is_empty()); + assert!( + placements + .iter() + .all(|placement| short.contains(placement.area.as_position()) + && placement.area.right() <= short.right() + && placement.area.bottom() <= short.bottom()) + ); + + view.render(area, &mut buf, &cells); + let control_row = view + .visible + .iter() + .position(|visible| visible.layout.disclosure_row() == Some(visible.row)) + .unwrap() as u16; + assert!(view.toggle_disclosure_at(&cells, /*column*/ 4, control_row)); + view.render(area, &mut buf, &cells); + + // Raw mode and unsupported terminals reserve no rows, even after rich previews were cached. + view.jump_to_latest(); + view.set_presentation(/*detailed*/ false, HistoryRenderMode::Raw); + view.render(area, &mut buf, &cells); + assert!((0..cells.len()).all(|index| view.layout(&cells, index).unwrap().image.is_none())); + view.set_presentation(/*detailed*/ false, HistoryRenderMode::Rich); + view.image_protocol = None; + view.render(area, &mut buf, &cells); + assert_eq!(buf, text_only); + + // Resumed sessions with a deleted file fall back to the original text entry. + std::fs::remove_file(&path).unwrap(); + view.image_protocol = Some(ImageProtocol::Kitty); + view.render(area, &mut buf, &cells); + assert_eq!(buf, text_only); + assert!(view.image_placements().is_empty()); + + // A tiny compressed file with a large decoded raster must also fall back to text. + image::RgbImage::new(4000, 3000).save(&path).unwrap(); + view.image_protocol = None; + view.render(area, &mut buf, &cells); + view.image_protocol = Some(ImageProtocol::Kitty); + view.render(area, &mut buf, &cells); + assert_eq!(buf, text_only); + assert!(view.image_placements().is_empty()); +} diff --git a/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__footer__tests__activity_focus_keeps_controls_without_passive_hints.snap b/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__footer__tests__activity_focus_keeps_controls_without_passive_hints.snap index dabbfed8a0..5a9b56193b 100644 --- a/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__footer__tests__activity_focus_keeps_controls_without_passive_hints.snap +++ b/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__footer__tests__activity_focus_keeps_controls_without_passive_hints.snap @@ -1,5 +1,6 @@ --- source: tui/src/transcript_view/footer_tests.rs +assertion_line: 515 expression: "hints.join(\"\\n\")" --- collapsed @@ -44,14 +45,14 @@ Buffer { content: [ "Activity summary ", "Hidden details ", - " − Show less ", + " − Hide details ", ], styles: [ x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: UNDERLINED, x: 0, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, x: 0, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, x: 4, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD, - x: 15, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 18, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, ] } "ctrl-x t": diff --git a/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__layout__tests__fullscreen_image_artifact_rows.snap b/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__layout__tests__fullscreen_image_artifact_rows.snap new file mode 100644 index 0000000000..7827815393 --- /dev/null +++ b/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__layout__tests__fullscreen_image_artifact_rows.snap @@ -0,0 +1,28 @@ +--- +source: tui/src/transcript_view/layout_tests.rs +assertion_line: 210 +expression: "format!(\"Collapsed inspection / generated output\\n{collapsed}\\nGraphics: {collapsed_areas:?}\\n\\nExpanded inspection / generated output\\n{}\\nGraphics: {expanded_areas:?}\",\nframe(&buf))" +--- +Collapsed inspection / generated output +• Viewed image artifact.png + + Show details + +• Generated Image: + └ A red rectangle + └ Saved to: file:///workspace/artifact.png +Graphics: [Rect { x: 52, y: 6, width: 36, height: 6 }] + +Expanded inspection / generated output +• Viewed image /workspace/artifact.png + − Hide details + + + + + + + +• Generated Image: + └ A red rectangle + └ Saved to: file:///workspace/artifact.png +Graphics: [Rect { x: 52, y: 2, width: 36, height: 6 }, Rect { x: 52, y: 12, width: 36, height: 6 }] diff --git a/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__search__tests__find_selective_expansion.snap b/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__search__tests__find_selective_expansion.snap index 0d2d28c15b..fb8773ec73 100644 --- a/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__search__tests__find_selective_expansion.snap +++ b/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__search__tests__find_selective_expansion.snap @@ -1,5 +1,6 @@ --- source: tui/src/transcript_view/search_tests.rs +assertion_line: 989 expression: "crate::transcript_view::tests::text(&buffer)" --- • Running 'echo visible … @@ -12,8 +13,8 @@ expression: "crate::transcript_view::tests::text(&buffer)" $ 'echo visible echo hidden newer needle' - − Show less + − Hide details $ 'echo visible echo hidden manual' - − Show less + − Hide details diff --git a/codex-rs/tui/src/transcript_view/text.rs b/codex-rs/tui/src/transcript_view/text.rs index 4aac01ddda..e2bc38ddd9 100644 --- a/codex-rs/tui/src/transcript_view/text.rs +++ b/codex-rs/tui/src/transcript_view/text.rs @@ -42,6 +42,7 @@ use ratatui::text::Line; /// One cell's logical text and its width-dependent display rows. pub(super) struct TextLayout { + pub(super) image: Option>, logical: Vec, text: String, rows: Vec, @@ -137,6 +138,7 @@ impl TextLayout { layout = layout.with_disclosure_control_at(control.label.clone(), control.source_offset); } + layout.image = self.image.clone(); if self.separated { layout.with_leading_separator() } else { @@ -271,9 +273,28 @@ impl TextLayout { } pub(super) fn row_count(&self) -> usize { + // Images occupy visual space, but have no searchable or selectable text rows. + self.text_row_count() + + self + .visible_image() + .map_or(0, |image| usize::from(image.rows)) + } + + pub(super) fn text_row_count(&self) -> usize { self.rows.len() } + pub(super) fn visible_image( + &self, + ) -> Option<&std::sync::Arc> { + self.image.as_ref().filter(|image| { + image.columns + 2 <= self.width + && (image.protocol != crate::pets::ImageProtocol::Sixel + || crate::transcript_image::terminal_cell_size() + == Some((image.cell_width, image.cell_height))) + }) + } + pub(super) fn disclosure_row(&self) -> Option { self.disclosure_control.as_ref().map(|control| control.row) } @@ -363,6 +384,9 @@ impl TextLayout { /// Find the display row containing a source position, including omitted wrapping whitespace. pub(super) fn row_for_offset(&self, offset: usize) -> usize { + if self.visible_image().is_some() && offset >= self.text.len() { + return self.row_count().saturating_sub(1); + } let before_control = if let Some(control_row) = self.disclosure_row() { let after_control = &self.rows[control_row + 1..]; let matching_rows = after_control.partition_point(|row| row.source.start <= offset); @@ -500,6 +524,7 @@ impl TextLayout { } } Self { + image: None, logical, text, rows, diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 01201ec1a0..2a573f1e97 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -671,6 +671,7 @@ pub struct Tui { screen_size: ScreenSizePolicy, ambient_pet_image_state: crate::pets::PetImageRenderState, pet_picker_preview_image_state: crate::pets::PetImageRenderState, + pub(crate) transcript_images: Vec, alt_saved_viewport: Option, #[cfg(unix)] suspend_context: SuspendContext, @@ -749,6 +750,7 @@ impl Tui { screen_size: ScreenSizePolicy::default(), ambient_pet_image_state: crate::pets::PetImageRenderState::default(), pet_picker_preview_image_state: crate::pets::PetImageRenderState::default(), + transcript_images: Vec::new(), alt_saved_viewport: None, #[cfg(unix)] suspend_context: SuspendContext::new(), @@ -1367,9 +1369,7 @@ impl Tui { self.suspend_context.set_cursor_y(inline_area_bottom); } - terminal.draw_with_size(screen_size, |frame| { - draw_fn(frame); - }) + terminal.draw_with_size(screen_size, &mut self.transcript_images, draw_fn) })??; Ok(()) } @@ -1505,7 +1505,7 @@ impl Tui { self.suspend_context.set_cursor_y(inline_area_bottom); } - terminal.draw_with_size(screen_size, |frame| { + terminal.draw_with_size(screen_size, &mut self.transcript_images, |frame| { draw_fn(frame); }) })??; diff --git a/codex-rs/tui/src/tui/size_monitor_tests.rs b/codex-rs/tui/src/tui/size_monitor_tests.rs index f15fe04730..1986104da4 100644 --- a/codex-rs/tui/src/tui/size_monitor_tests.rs +++ b/codex-rs/tui/src/tui/size_monitor_tests.rs @@ -141,7 +141,7 @@ async fn missed_resize_recovers_and_slow_query_does_not_block_input() { let area = Rect::new(/*x*/ 0, /*y*/ 0, size.width, size.height); terminal.set_viewport_area(area); terminal - .draw_with_size(size, |frame| { + .draw_with_size(size, &mut Vec::new(), |frame| { Paragraph::new("alpha beta") .wrap(Wrap { trim: false }) .render(area, frame.buffer_mut());