From a669d8ffb230fb117e8763245dc3a93ef81f923e Mon Sep 17 00:00:00 2001 From: Felipe Coury Date: Mon, 16 Nov 2026 23:48:27 +0000 Subject: [PATCH] Add sidebar image previews and present generated images after answers (#61919) ## What changed - Open PNG, JPEG, GIF, and WebP file links as sidebar image previews, fitted to the available space with format and dimension labels. Show a fallback when terminal graphics are unavailable, and preserve text previews for files such as Git LFS pointers. - In the owned transcript, place completed, saved image results after the assistant's answer, with larger, left-aligned previews and a clickable `Generated image` label. Preserve this ordering during replay and history pagination, including interrupted turns. - Reuse native Kitty image IDs and move existing placements without retransmitting their pixels. Probe Kitty graphics support in `herdr` before enabling previews. ## Testing Add regression coverage for sidebar preview fallbacks, generated-image ordering, pagination and search, preview layout, Kitty placement reuse, and graphics probe replies interleaved with pasted input. GitOrigin-RevId: 22d8aa2724f17bfa7f8280a2d11d4a7e2cb66728 --- codex-rs/tui/src/app.rs | 14 +- codex-rs/tui/src/app/history_pagination.rs | 108 ++++++++- .../tui/src/app/history_pagination_tests.rs | 89 ++++++- codex-rs/tui/src/app/owned_transcript.rs | 23 +- .../tui/src/app/owned_transcript_tests.rs | 9 +- codex-rs/tui/src/app/side_companion.rs | 19 ++ codex-rs/tui/src/app/sidebar_document.rs | 71 +++++- .../tui/src/app/sidebar_document_tests.rs | 68 +++++- codex-rs/tui/src/app_event.rs | 3 +- codex-rs/tui/src/app_server_session/fs.rs | 4 + .../tui/src/app_server_session/history.rs | 3 +- codex-rs/tui/src/chatwidget/input_restore.rs | 21 +- codex-rs/tui/src/chatwidget/protocol.rs | 3 + codex-rs/tui/src/chatwidget/replay.rs | 7 +- ...mage_generation_call_history_snapshot.snap | 7 +- codex-rs/tui/src/chatwidget/streaming.rs | 1 + .../tui/src/chatwidget/tests/exec_flow.rs | 25 +- codex-rs/tui/src/chatwidget/tool_lifecycle.rs | 38 ++- codex-rs/tui/src/chatwidget/turn_lifecycle.rs | 3 + codex-rs/tui/src/custom_terminal.rs | 225 ++++++++++++++---- codex-rs/tui/src/history_cell/notices.rs | 19 +- codex-rs/tui/src/history_cell/patches.rs | 75 +++++- codex-rs/tui/src/history_cell/separators.rs | 7 +- codex-rs/tui/src/history_cell/tests.rs | 14 +- codex-rs/tui/src/lib.rs | 1 + .../tui/src/markdown_render/local_links.rs | 6 + codex-rs/tui/src/pager_overlay/transcript.rs | 6 +- codex-rs/tui/src/pets/image_protocol.rs | 8 + codex-rs/tui/src/pets/mod.rs | 1 + codex-rs/tui/src/sidebar_image.rs | 174 ++++++++++++++ codex-rs/tui/src/startup_draft.rs | 1 + codex-rs/tui/src/terminal_probe.rs | 40 +++- .../tui/src/terminal_probe/startup_replay.rs | 21 +- .../terminal_probe/startup_replay_tests.rs | 24 +- codex-rs/tui/src/transcript_image.rs | 82 +++++-- codex-rs/tui/src/transcript_image/composer.rs | 1 + codex-rs/tui/src/transcript_view.rs | 9 +- codex-rs/tui/src/transcript_view/layout.rs | 12 +- .../tui/src/transcript_view/layout_tests.rs | 33 ++- codex-rs/tui/src/transcript_view/search.rs | 19 ++ .../tui/src/transcript_view/search_tests.rs | 27 +++ ...tests__fullscreen_image_artifact_rows.snap | 75 +++++- codex-rs/tui/src/transcript_view/text.rs | 3 + codex-rs/tui/src/tui.rs | 28 ++- .../tui/src/tui/inline_image_history_tests.rs | 1 + 45 files changed, 1242 insertions(+), 186 deletions(-) create mode 100644 codex-rs/tui/src/sidebar_image.rs diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 7f89755fcc..e1804f7993 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -1438,8 +1438,16 @@ impl App { Ok(()) } - fn prepare_frame_images(&mut self) -> Option { - let protocol = self.chat_widget.pet_image_support().protocol().filter(|_| { + fn preview_image_protocol(&self, tui: &tui::Tui) -> Option { + match tui.herdr_kitty_graphics { + Some(true) => Some(crate::pets::ImageProtocol::Kitty), + Some(false) => None, + None => self.chat_widget.pet_image_support().protocol(), + } + } + + fn prepare_frame_images(&mut self, tui: &tui::Tui) -> Option { + let protocol = self.preview_image_protocol(tui).filter(|_| { !self.app_server_target.uses_remote_workspace() && crate::projectless::has_only_local_environments(&self.environment_manager) }); @@ -1453,7 +1461,7 @@ impl App { if tui.is_owned_screen() { return self.render_owned_transcript(tui, screen_size); } - self.prepare_frame_images(); + self.prepare_frame_images(tui); self.chat_widget .empty_state_animation .borrow_mut() diff --git a/codex-rs/tui/src/app/history_pagination.rs b/codex-rs/tui/src/app/history_pagination.rs index a660d62192..8a8d11cb12 100644 --- a/codex-rs/tui/src/app/history_pagination.rs +++ b/codex-rs/tui/src/app/history_pagination.rs @@ -10,6 +10,7 @@ use crate::history_cell::UserHistoryCell; use crate::history_cell::is_session_history_prefix_cell; use crate::pager_overlay::TranscriptHistoryState; use crate::thread_transcript::RawReasoningVisibility; +use crate::thread_transcript::TranscriptCells; use crate::thread_transcript::thread_items_to_transcript_cells; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ThreadItemsListResponse; @@ -108,19 +109,46 @@ impl App { }; let width = tui.terminal.last_known_screen_size.width; self.remove_hidden_review_cells(tui, &turns, &hidden_item_ids, thread_id, &cwd, visibility); - let cells = self.project_older_history_cells( + let split_turn = items + .last() + .and_then(|(turn_id, last)| { + turns.iter().find(|turn| { + turn.id == *turn_id + && matches!(turn.status, TurnStatus::Completed | TurnStatus::Interrupted) + && turn.items.last().is_some_and(|item| item.id() != last.id()) + }) + }) + .filter(|_| self.local_settings.transcript_mode.is_owned()); + let existing = self + .side_companions + .get(&thread_id) + .filter(|_| parked) + .map_or(self.transcript_cells.as_slice(), |pane| { + pane.cells.as_slice() + }); + let split_insertion = split_image_insertion(existing, split_turn); + let (cells, split_images) = self.project_older_history_cells( items, &turns, &hidden_item_ids, thread_id, &cwd, visibility, + split_insertion.is_some(), ); if let Some(pane) = self.side_companions.get_mut(&thread_id).filter(|_| parked) { + let images = split_insertion.and_then(|index| { + if split_images.is_empty() { + return None; + } + let inserted = index..index + split_images.len(); + pane.splice_older_cells(index..index, split_images); + Some(inserted) + }); let index = history_prefix_len(&pane.cells); let inserted = index..index + cells.len(); pane.splice_older_cells(index..index, cells); - pane.view.history_loaded(&pane.cells, inserted); + pane.view.history_page_loaded(&pane.cells, inserted, images); pane.view.history = if app_server.has_older_history(thread_id) { TranscriptHistoryState::Partial } else { @@ -130,9 +158,20 @@ impl App { tui.frame_requester().schedule_frame(); return Ok(()); } + let images = split_insertion.and_then(|index| { + if split_images.is_empty() { + return None; + } + let inserted = index..index + split_images.len(); + if let Some(Overlay::Transcript(overlay)) = &mut self.overlay { + overlay.insert_history_page(index, split_images.clone()); + } + self.transcript_cells.splice(index..index, split_images); + Some(inserted) + }); let inserted = self.prepend_older_transcript_cells(cells); self.transcript_view - .history_loaded(&self.transcript_cells, inserted.clone()); + .history_page_loaded(&self.transcript_cells, inserted.clone(), images); if !inserted.is_empty() { self.join_older_activity_group(inserted.end, &turns); } @@ -240,6 +279,10 @@ impl App { } /// Project each terminal row on the page containing the end of its turn. + #[allow( + clippy::too_many_arguments, + reason = "Projection needs the thread context and whether its split turn has a rendered end." + )] fn project_older_history_cells( &mut self, items: Vec<(String, ThreadItemPayload)>, @@ -248,9 +291,12 @@ impl App { thread_id: ThreadId, cwd: &AbsolutePathBuf, visibility: RawReasoningVisibility, - ) -> Vec> { + relocate_split_images: bool, + ) -> (TranscriptCells, TranscriptCells) { let mut cells = Vec::new(); + let mut split_images = Vec::new(); for (range, turn_end) in completion::group_turn_items(&items, turns) { + let start = cells.len(); // Internal prompts stay invisible, but still separate adjacent tool groups. for visible in items[range] .split(|(turn_id, item)| hidden_item_ids.contains(&(turn_id.as_str(), item.id()))) @@ -263,10 +309,24 @@ impl App { Some(&self.config), )); } - let Some(turn) = turn_end else { continue }; + let Some(turn) = turn_end else { + if relocate_split_images { + split_images.extend( + cells.extract_if(start.., |cell| generated_image(cell.as_ref()).is_some()), + ); + } + continue; + }; + if self.local_settings.transcript_mode.is_owned() + && turn.status != TurnStatus::InProgress + { + cells[start..].sort_by_key(|cell| generated_image(cell.as_ref()).is_some()); + } if turn.status == TurnStatus::Interrupted && !self.side_threads.contains_key(&thread_id) { - cells.push(Arc::new(history_cell::new_interruption_notice())); + cells.push(Arc::new(history_cell::InterruptionNotice(Some( + turn.id.clone(), + )))); } if turn.status == TurnStatus::Completed && let Some(completion) = if let Some(pane) = @@ -287,10 +347,14 @@ impl App { cells.push(Arc::new(completion)); } } - for cell in cells.iter_mut().filter_map(Arc::get_mut) { + for cell in cells + .iter_mut() + .chain(&mut split_images) + .filter_map(Arc::get_mut) + { self.link_viewed_image(cell); } - cells + (cells, split_images) } /// Insert each page once, keeping session headers and any backtrack selection in place. @@ -399,6 +463,34 @@ impl App { } } +fn generated_image(cell: &dyn HistoryCell) -> Option<&history_cell::ImageGenerationHistoryCell> { + cell.as_any() + .downcast_ref::() + .filter(|image| image.is_turn_output()) +} + +/// The first page can contain the answer while an older page contains its image. +fn split_image_insertion( + existing: &[Arc], + split_turn: Option<&Turn>, +) -> Option { + let turn = split_turn?; + let end = existing.iter().position(|cell| { + let cell = cell.as_any(); + cell.downcast_ref::() + .is_some_and(|cell| cell.turn_id.as_deref() == Some(turn.id.as_str())) + || cell + .downcast_ref::() + .is_some_and(|cell| cell.0.as_deref() == Some(turn.id.as_str())) + })?; + Some( + existing[..end] + .iter() + .rposition(|cell| generated_image(cell.as_ref()).is_none()) + .map_or(0, |index| index + 1), + ) +} + /// Classify internal review prompts across page boundaries before rendering any older items. fn hidden_review_item_ids(turns: &[Turn]) -> HashSet<(&str, &str)> { let mut hidden = HashSet::new(); diff --git a/codex-rs/tui/src/app/history_pagination_tests.rs b/codex-rs/tui/src/app/history_pagination_tests.rs index 1f64f80395..885be16568 100644 --- a/codex-rs/tui/src/app/history_pagination_tests.rs +++ b/codex-rs/tui/src/app/history_pagination_tests.rs @@ -145,13 +145,14 @@ async fn resume_places_interruption_before_searches_split_across_pages() { (audit.id.clone(), audit.items[0].payload.clone()), ]; let cwd = app.config.cwd.clone(); - let older = app.project_older_history_cells( + let (older, _) = app.project_older_history_cells( page, &[canceled, audit], &HashSet::new(), ThreadId::new(), &cwd, RawReasoningVisibility::Hidden, + /*relocate_split_images*/ false, ); cells.splice(0..0, older); let rows = cells @@ -170,6 +171,86 @@ async fn resume_places_interruption_before_searches_split_across_pages() { ); } +#[tokio::test] +async fn generated_image_from_an_older_page_follows_its_answer() { + let (mut app, _, _) = crate::app::tests::make_test_app_with_channels().await; + app.local_settings.transcript_mode = crate::transcript_mode::TranscriptMode::Owned; + for status in [TurnStatus::Completed, TurnStatus::Interrupted] { + let completed = status == TurnStatus::Completed; + let mut turn = turn("image-turn", status, &[]); + let answer: ThreadItemPayload = serde_json::from_value(serde_json::json!({ + "type": "agentMessage", "id": "answer", "text": "Here is the drawing.", + "phase": "final_answer", + })) + .unwrap(); + turn.duration_ms = Some(2_000); + turn.items.push(ThreadItem::untimed(answer.clone())); + let cwd = app.config.cwd.clone(); + let (mut existing, _) = app.project_older_history_cells( + vec![(turn.id.clone(), answer)], + std::slice::from_ref(&turn), + &HashSet::new(), + ThreadId::new(), + &cwd, + RawReasoningVisibility::Hidden, + /*relocate_split_images*/ false, + ); + + let path = app.config.cwd.as_path().join("drawing.png"); + let image: ThreadItemPayload = serde_json::from_value(serde_json::json!({ + "type": "imageGeneration", "id": "image", "status": "completed", + "revisedPrompt": "A drawing", "result": "", "savedPath": path, + })) + .unwrap(); + turn.items + .insert(/*index*/ 0, ThreadItem::untimed(image.clone())); + let cwd = app.config.cwd.clone(); + let mut previous = super::tests::turn("previous", TurnStatus::Completed, &[]); + previous.items.push(ThreadItem::untimed(image.clone())); + let items = vec![ + (previous.id.clone(), image.clone()), + (turn.id.clone(), image), + ]; + let turns = [previous, turn]; + let (older, images) = app.project_older_history_cells( + items, + &turns, + &HashSet::new(), + ThreadId::new(), + &cwd, + RawReasoningVisibility::Hidden, + /*relocate_split_images*/ true, + ); + assert_eq!( + older + .iter() + .filter(|cell| generated_image(cell.as_ref()).is_some()) + .count(), + 1 + ); + assert_eq!(images.len(), 1); + let index = split_image_insertion(&existing, Some(&turns[1])).unwrap(); + existing.splice(index..index, images); + let rows: Vec<_> = existing + .iter() + .flat_map(|cell| cell.compact_hyperlink_lines(/*width*/ 80)) + .map(|line| line.line.to_string()) + .collect(); + assert_eq!( + rows, + [ + "• Here is the drawing.", + " Generated image", + if completed { + " Worked for 2s" + } else { + "■ Conversation interrupted - use /feedback if something went wrong" + } + ] + ); + } +} + #[tokio::test] async fn hidden_last_item_keeps_turn_groups_and_completion_boundaries() { let computer = |id: &str| { @@ -222,13 +303,14 @@ async fn hidden_last_item_keeps_turn_groups_and_completion_boundaries() { }) .collect::>(); - let cells = app.project_older_history_cells( + let (cells, _) = app.project_older_history_cells( items.clone(), &turns, &hidden, thread_id, &cwd, RawReasoningVisibility::Hidden, + /*relocate_split_images*/ false, ); let groups = cells .iter() @@ -252,13 +334,14 @@ async fn hidden_last_item_keeps_turn_groups_and_completion_boundaries() { ), ); - let repeated = app.project_older_history_cells( + let (repeated, _) = app.project_older_history_cells( items, &turns, &hidden, thread_id, &cwd, RawReasoningVisibility::Hidden, + /*relocate_split_images*/ false, ); assert_eq!( repeated diff --git a/codex-rs/tui/src/app/owned_transcript.rs b/codex-rs/tui/src/app/owned_transcript.rs index 2026c7c448..2701fc7e33 100644 --- a/codex-rs/tui/src/app/owned_transcript.rs +++ b/codex-rs/tui/src/app/owned_transcript.rs @@ -8,6 +8,7 @@ //! Plain Enter returns an empty composer to latest after transcript interactions and prompt editing. use super::sidebar::PaneFocus; +use super::sidebar_document::DocumentRenderOptions; use super::turn_tips::TipSurface; use super::*; use crate::bottom_pane::CommandPopupPlacement; @@ -244,7 +245,7 @@ impl App { tui: &mut tui::Tui, screen_size: Size, ) -> Result { - self.transcript_view.image_protocol = self.prepare_frame_images(); + self.transcript_view.image_protocol = self.prepare_frame_images(tui); let frames = tui.frame_requester(); if self.conversation_frame.sidebar.is_some() || self.conversation_frame.chat_strip.is_some() { @@ -312,6 +313,7 @@ impl App { }) }); let sidebar_parent = self.sidebar_parent(); + let sidebar_image_protocol = self.preview_image_protocol(tui); let sidebar_motion = MotionMode::from_animations_enabled(self.local_settings.tui.animations); // An input refresh opens a fresh terminal buffer. Carry the outer owner's @@ -369,6 +371,9 @@ impl App { } Some(sidebar::SidebarLayout::Panel { sidebar, focus, .. }) => { let focused = *focus == PaneFocus::Sidebar; + frame + .image_placements + .retain(|image| !sidebar.intersects(image.area)); if focused { frame.cursor_position = None; } @@ -381,11 +386,14 @@ impl App { let [_, body] = side_pane::tab_and_body(*sidebar); if let Some(cursor) = doc.paint( body, - frame.buffer, - sidebar_motion, - &frames, - &self.keymap, - resize_hint, + frame, + DocumentRenderOptions { + motion: sidebar_motion, + frames: &frames, + keymap: &self.keymap, + resize_hint, + image_protocol: sidebar_image_protocol, + }, ) && focused { frame.cursor_position = Some(cursor); @@ -508,9 +516,6 @@ impl App { frame.buffer, ); } - frame - .image_placements - .retain(|image| !sidebar.intersects(image.area)); } None => {} } diff --git a/codex-rs/tui/src/app/owned_transcript_tests.rs b/codex-rs/tui/src/app/owned_transcript_tests.rs index cd4c1fb4ba..e3f3a99ed2 100644 --- a/codex-rs/tui/src/app/owned_transcript_tests.rs +++ b/codex-rs/tui/src/app/owned_transcript_tests.rs @@ -556,7 +556,13 @@ async fn owned_details_keep_the_composer_cursor_and_screen() -> Result<()> { image::RgbaImage::from_pixel(8, 16, image::Rgba([80, 30, 20, 255])).save(&path)?; let protocol = crate::pets::ImageProtocol::KittyLocalFile; let image = Arc::new( - crate::transcript_image::TranscriptImage::load(&path, area.width, protocol).unwrap(), + crate::transcript_image::TranscriptImage::load( + &path, + area.width, + protocol, + crate::transcript_image::PreviewKind::Inspected, + ) + .unwrap(), ); let neighbor = Rect::new(/*x*/ 4, /*y*/ 9, image.columns, image.rows); let presented = |tui: &tui::Tui| { @@ -583,6 +589,7 @@ async fn owned_details_keep_the_composer_cursor_and_screen() -> Result<()> { area: neighbor, first_row: 0, file: None, + terminal_id: None, }); })?; app.finish_conversation(&mut tui, painted)?; diff --git a/codex-rs/tui/src/app/side_companion.rs b/codex-rs/tui/src/app/side_companion.rs index 4cd96f1a43..a2710befb9 100644 --- a/codex-rs/tui/src/app/side_companion.rs +++ b/codex-rs/tui/src/app/side_companion.rs @@ -54,6 +54,7 @@ impl SideCompanion { Self { display: CapturedSideDisplay { rendered_completion_turn_ids: Default::default(), + generated_images: Default::default(), active: None, auxiliary: Vec::new(), draft: Box::default(), @@ -150,6 +151,13 @@ impl SideCompanion { self.flush_answer(); self.answer = None; } + self.cells.extend( + self.display + .generated_images + .take(&turn.id) + .into_iter() + .map(|image| Arc::new(image) as Arc), + ); let work = self.display.work.take_if(|work| work.turn_id == turn.id); if turn.status == TurnStatus::Completed && (work.is_some() || store.active_turn_id() == Some(turn.id.as_str())) @@ -215,6 +223,17 @@ impl SideCompanion { } self.display.last_completed_agent_message = Some((turn.to_owned(), id.clone())); } + ThreadItemPayload::ImageGeneration(item) + if item.status == "completed" && item.saved_path.is_some() => + { + let image = crate::history_cell::new_image_generation_call( + item.id.clone(), + &item.status, + item.revised_prompt.clone(), + item.saved_path.clone(), + ); + self.display.generated_images.record(turn.to_owned(), image); + } ThreadItemPayload::CommandExecution { id, status, diff --git a/codex-rs/tui/src/app/sidebar_document.rs b/codex-rs/tui/src/app/sidebar_document.rs index 809fc56ce7..786eaf8bd0 100644 --- a/codex-rs/tui/src/app/sidebar_document.rs +++ b/codex-rs/tui/src/app/sidebar_document.rs @@ -3,12 +3,16 @@ use super::sidebar::PaneFocus; use super::sidebar::SidebarContent; use super::*; +use crate::custom_terminal::Frame; use crate::key_hint::KeyBinding; use crate::keymap::KeymapContext; use crate::keymap::RuntimeKeymap; use crate::markdown_render::FileLink; use crate::markdown_render::SourceLocation; use crate::motion::MotionMode; +use crate::pets::ImageProtocol; +use crate::sidebar_image::DocumentContent; +use crate::sidebar_image::SidebarImage; use crate::sidebar_tab_strip::SidebarTab; use crate::terminal_hyperlinks::HyperlinkLine; use crate::terminal_hyperlinks::visible_lines; @@ -17,7 +21,6 @@ use crate::transcript_view::ViewAction; use codex_app_server_client::AppServerPath; use codex_utils_path_uri::LegacyAppPathString; use codex_utils_path_uri::PathUri; -use ratatui::buffer::Buffer; use ratatui::layout::Position; use ratatui::widgets::Widget; use unicode_segmentation::UnicodeSegmentation; @@ -61,6 +64,14 @@ impl HistoryCell for FileCell { } } +pub(super) struct DocumentRenderOptions<'a> { + pub(super) motion: MotionMode, + pub(super) frames: &'a tui::FrameRequester, + pub(super) keymap: &'a RuntimeKeymap, + pub(super) resize_hint: Option<&'a str>, + pub(super) image_protocol: Option, +} + pub(super) struct Document { source: ThreadId, pub(super) path: PathUri, @@ -73,6 +84,7 @@ pub(super) struct Document { enum State { Loading(Instant), Failed(String), + Image(Box), Ready { view: Box, cells: [Arc; 1], @@ -129,9 +141,10 @@ impl Document { } } - pub(super) fn loaded(&mut self, result: Result) { + pub(super) fn loaded(&mut self, result: Result) { self.state = match result { - Ok(source) => { + Ok(DocumentContent::Image(image)) => State::Image(image), + Ok(DocumentContent::Text(source)) => { let title = self.title(); let code = self.location.is_some() || !crate::markdown_render::is_markdown_file(&title); @@ -207,7 +220,7 @@ impl Document { if restore_markdown || (location.is_some() && line_starts.is_empty()) { // Coordinate links show source. Plain Markdown links restore rendering. let source = source.to_owned(); - self.loaded(Ok(source)); + self.loaded(Ok(DocumentContent::Text(source))); } else { *target = location.and_then(|location| source_target(source, line_starts, location)); if let Some((_, offset)) = target { @@ -221,15 +234,25 @@ impl Document { pub(super) fn paint( &mut self, area: Rect, - buf: &mut Buffer, - motion: MotionMode, - frames: &tui::FrameRequester, - keymap: &RuntimeKeymap, - resize_hint: Option<&str>, + frame: &mut Frame<'_>, + options: DocumentRenderOptions<'_>, ) -> Option { + let DocumentRenderOptions { + motion, + frames, + keymap, + resize_hint, + image_protocol, + } = options; + let buf = &mut *frame.buffer; ratatui::widgets::Clear.render(area, buf); let body = area.inner(ratatui::layout::Margin::new(1, 0)); match &mut self.state { + State::Image(image) => { + frame + .image_placements + .extend(image.paint(body, buf, image_protocol, resize_hint)) + } State::Loading(started) => { let glyph = crate::motion::loading_glyph(*started, motion, frames); Line::from(vec![ @@ -510,10 +533,32 @@ impl App { let reader = server.request_view(); let tx = self.app_event_tx.clone(); tokio::spawn(async move { - let result = reader - .read_document(source, &original_path, &path) - .await - .map_err(|error| format!("{error:#}")); + let result = async { + if (FileLink { + path: path.inferred_native_path_string(), + location, + }) + .is_image() + { + let bytes = reader + .read_preview_file(source, &original_path, &path) + .await?; + tokio::task::spawn_blocking(move || match SidebarImage::decode(&bytes) { + Ok(image) => Ok(DocumentContent::Image(Box::new(image))), + Err(image_error) => AppServerSession::preview_text(bytes) + .map(DocumentContent::Text) + .map_err(|_| image_error), + }) + .await? + } else { + reader + .read_document(source, &original_path, &path) + .await + .map(DocumentContent::Text) + } + } + .await + .map_err(|error| format!("{error:#}")); tx.send(AppEvent::SidebarDocumentLoaded { backend_id, parent, diff --git a/codex-rs/tui/src/app/sidebar_document_tests.rs b/codex-rs/tui/src/app/sidebar_document_tests.rs index bb4ba26f52..0c345e5387 100644 --- a/codex-rs/tui/src/app/sidebar_document_tests.rs +++ b/codex-rs/tui/src/app/sidebar_document_tests.rs @@ -493,6 +493,40 @@ async fn document_preview_scenario() -> Result<()> { .await?; assert_eq!(app.sidebar().unwrap().documents.len(), remaining); } + app.chat_widget + .set_pet_image_support_for_tests(crate::pets::PetImageSupport::Unsupported( + crate::pets::PetImageUnsupportedReason::Terminal, + )); + image::RgbImage::new(24, 16).save(dir.path().join("figure.png"))?; + app.open_document( + &mut tui, + &mut server, + parent, + FileLink { + path: "figure.png".into(), + location: None, + }, + ) + .await?; + Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(true).await)).await?; + Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(false).await)).await?; + let image = side_frame(&mut app, &mut tui, Size::new(160, 15))?; + let image_rows = image + .lines() + .map(|row| row.rsplit('│').next().unwrap_or(row).trim()) + .filter(|row| { + row.contains("PNG ·") + || row.contains("Image preview unavailable") + || row.contains("←/→ tabs") + }) + .collect::>() + .join("\n"); + insta::assert_snapshot!(image_rows, @" + PNG · 24 × 16 + Image preview unavailable in this terminal + ←/→ tabs · Esc close + "); + assert_eq!(app.sidebar().unwrap().documents.len(), 2); app.open_document( &mut tui, &mut server, @@ -507,11 +541,41 @@ async fn document_preview_scenario() -> Result<()> { crate::pager_overlay::TranscriptOverlay::new(Vec::new(), app.keymap.pager.clone()), )); Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(true).await)).await?; - assert_eq!(app.sidebar().unwrap().documents.len(), 2); + assert_eq!(app.sidebar().unwrap().documents.len(), 3); assert!(app.overlay.take().is_some()); Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(false).await)).await?; let after_overlay = side_frame(&mut app, &mut tui, size)?; - assert!(after_overlay.contains("Last section."), "{after_overlay}"); + assert!(after_overlay.contains("PNG · 24 × 16"), "{after_overlay}"); + std::fs::write( + dir.path().join("pointer.png"), + "version https://git-lfs.github.com/spec/v1 +size 123 +", + )?; + app.open_document( + &mut tui, + &mut server, + parent, + FileLink { + path: "pointer.png".into(), + location: None, + }, + ) + .await?; + Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(true).await)).await?; + Box::pin(app.handle_event(&mut tui, &mut server, next_document_event(false).await)).await?; + let pointer = side_frame(&mut app, &mut tui, Size::new(200, 15))?; + insta::assert_snapshot!(pointer.lines().filter_map(|row| row.split_once('│')).map(|(_, row)| row.trim()).filter_map(|row| row.split_once(' ').map(|(_, line)| line.trim())).filter(|row| row.starts_with("version ") || row.starts_with("size ")).collect::>().join(" +"), @" + version https://git-lfs.github.com/spec/v1 + size 123 + "); + let Some(SidebarContent::Document(pointer_id)) = app.sidebar().unwrap().visible_content() + else { + panic!("the pointer preview should be visible"); + }; + app.sidebar_mut().remove_document(pointer_id); + let child = ThreadId::new(); app.side_threads.insert(child, SideThreadState::new(parent)); let (stream, display) = app.chat_widget.take_side_display(/*width*/ 120); diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index 5d8b330dad..4cca3dffa8 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -315,6 +315,7 @@ pub(crate) enum CapturedSideStream { #[derive(Debug)] pub(crate) struct CapturedSideDisplay { pub(crate) rendered_completion_turn_ids: std::collections::HashMap, + pub(crate) generated_images: crate::history_cell::PendingGeneratedImages, pub(crate) active: Option>, pub(crate) auxiliary: Vec, pub(crate) draft: Box, @@ -540,7 +541,7 @@ pub(crate) enum AppEvent { backend_id: crate::backend_id::BackendId, parent: ThreadId, request: uuid::Uuid, - result: Result, + result: Result, }, SavedSideChatsLoaded { backend_id: crate::backend_id::BackendId, diff --git a/codex-rs/tui/src/app_server_session/fs.rs b/codex-rs/tui/src/app_server_session/fs.rs index ce521048b4..d8b7e8a71d 100644 --- a/codex-rs/tui/src/app_server_session/fs.rs +++ b/codex-rs/tui/src/app_server_session/fs.rs @@ -81,6 +81,10 @@ impl AppServerSession { let bytes = self .read_preview_file(source, original_path, resolved_path) .await?; + Self::preview_text(bytes) + } + + pub(crate) fn preview_text(bytes: Vec) -> Result { let text = String::from_utf8(bytes).wrap_err("Not a UTF-8 text file")?; color_eyre::eyre::ensure!( !text diff --git a/codex-rs/tui/src/app_server_session/history.rs b/codex-rs/tui/src/app_server_session/history.rs index 2b10177c96..15eb721475 100644 --- a/codex-rs/tui/src/app_server_session/history.rs +++ b/codex-rs/tui/src/app_server_session/history.rs @@ -341,8 +341,9 @@ impl AppServerSession { if let Some(turn) = turns.iter_mut().find(|turn| turn.id == entry.turn_id) && !turn.items.iter().any(|item| item.id() == entry.item.id()) { + let turn_id = entry.turn_id.clone(); let item = ThreadItem::try_from(entry)?; - items.push((turn.id.clone(), item.payload.clone())); + items.push((turn_id, item.payload.clone())); turn.items.insert(/*index*/ 0, item); turn.items_view = TurnItemsView::Summary; } diff --git a/codex-rs/tui/src/chatwidget/input_restore.rs b/codex-rs/tui/src/chatwidget/input_restore.rs index 888521b82a..79bfba5a8f 100644 --- a/codex-rs/tui/src/chatwidget/input_restore.rs +++ b/codex-rs/tui/src/chatwidget/input_restore.rs @@ -419,7 +419,7 @@ impl ChatWidget { } } } - self.on_interrupted_turn(reason); + self.on_interrupted_turn_with_id(reason, Some(turn_id)); self.input_queue.pending_steers.extend(later); for (message, record) in unconfirmed { self.input_queue.queued_user_messages.push_back(message); @@ -430,7 +430,12 @@ impl ChatWidget { self.refresh_pending_input_preview(); } + #[cfg(test)] pub(super) fn on_interrupted_turn(&mut self, reason: TurnAbortReason) { + self.on_interrupted_turn_with_id(reason, /*turn_id*/ None); + } + + fn on_interrupted_turn_with_id(&mut self, reason: TurnAbortReason, turn_id: Option<&str>) { self.requeue_image_submission(); // Finalize, log a gentle prompt, and clear running state. self.finalize_turn(); @@ -454,12 +459,16 @@ impl ChatWidget { /*hint*/ None, )); } else { - self.add_to_history(match reason { - TurnAbortReason::BudgetLimited => history_cell::new_error_event( - "Goal budget reached - the turn was stopped.".to_string(), + match reason { + TurnAbortReason::BudgetLimited => { + self.add_to_history(history_cell::new_error_event( + "Goal budget reached - the turn was stopped.".to_string(), + )) + } + TurnAbortReason::Interrupted => self.add_to_history( + history_cell::InterruptionNotice(turn_id.map(str::to_owned)), ), - TurnAbortReason::Interrupted => history_cell::new_interruption_notice(), - }); + } } } diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index 3f1ea5a7c3..79759069d2 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -501,6 +501,7 @@ impl ChatWidget { question_drafts = self.take_question_drafts(); } self.last_non_retry_error = None; + self.flush_generated_images(¬ification.turn.id); let completion = self.completion_cell(¬ification.turn, replay_kind); self.on_task_complete( last_agent_message.map(|(_, text)| text.to_owned()), @@ -513,6 +514,7 @@ impl ChatWidget { question_drafts = self.take_question_drafts(); } self.last_non_retry_error = None; + self.flush_generated_images(¬ification.turn.id); let reason = if self .turn_lifecycle .take_budget_limited(notification.turn.id.as_str()) @@ -527,6 +529,7 @@ impl ChatWidget { if replay_kind.is_none() { question_drafts = self.take_question_drafts(); } + self.flush_generated_images(¬ification.turn.id); let error = notification .turn .error diff --git a/codex-rs/tui/src/chatwidget/replay.rs b/codex-rs/tui/src/chatwidget/replay.rs index a33c326030..d575997242 100644 --- a/codex-rs/tui/src/chatwidget/replay.rs +++ b/codex-rs/tui/src/chatwidget/replay.rs @@ -413,12 +413,7 @@ impl ChatWidget { self.on_view_image_tool_call(id, path); } ThreadItemPayload::ImageGeneration(item) => { - self.on_image_generation_end( - item.id, - item.status, - item.revised_prompt, - item.saved_path, - ); + self.on_image_generation_end(&turn_id, item); } ThreadItemPayload::EnteredReviewMode { review, .. } => { if from_replay { diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__image_generation_call_history_snapshot.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__image_generation_call_history_snapshot.snap index 452834c96b..bd8cb6f7a7 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__image_generation_call_history_snapshot.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__image_generation_call_history_snapshot.snap @@ -1,7 +1,12 @@ --- -source: tui/src/chatwidget/tests.rs +source: tui/src/chatwidget/tests/exec_flow.rs +assertion_line: 1189 expression: combined --- +• Here is the square. + • Generated Image: └ A tiny blue square └ Saved to: file:///tmp/ig-1.png + +Worked for 2s diff --git a/codex-rs/tui/src/chatwidget/streaming.rs b/codex-rs/tui/src/chatwidget/streaming.rs index 1934c3d750..de3f7b05bd 100644 --- a/codex-rs/tui/src/chatwidget/streaming.rs +++ b/codex-rs/tui/src/chatwidget/streaming.rs @@ -258,6 +258,7 @@ impl ChatWidget { }); let display = CapturedSideDisplay { rendered_completion_turn_ids: self.turn_lifecycle.rendered_completion_turn_ids.clone(), + generated_images: std::mem::take(&mut self.turn_lifecycle.generated_images), active, last_completed_agent_message: self.transcript.last_completed_agent_message.clone(), auxiliary: self diff --git a/codex-rs/tui/src/chatwidget/tests/exec_flow.rs b/codex-rs/tui/src/chatwidget/tests/exec_flow.rs index 9eb88d5034..838363ffc2 100644 --- a/codex-rs/tui/src/chatwidget/tests/exec_flow.rs +++ b/codex-rs/tui/src/chatwidget/tests/exec_flow.rs @@ -1153,6 +1153,8 @@ async fn image_generation_begin_restores_working_status_after_single_line_preamb #[tokio::test] async fn image_generation_call_adds_history_cell() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.local_settings.transcript_mode = crate::transcript_mode::TranscriptMode::Owned; + handle_turn_started(&mut chat, "turn-1"); handle_image_generation_end( &mut chat, @@ -1162,13 +1164,28 @@ async fn image_generation_call_adds_history_cell() { Some(test_path_buf("/tmp/ig-1.png").abs()), ); - let cells = drain_insert_history(&mut rx); - assert_eq!(cells.len(), 1, "expected a single history cell"); + assert!(drain_insert_history(&mut rx).is_empty()); + complete_assistant_message( + &mut chat, + "image-answer", + "Here is the square.", + Some(MessagePhase::FinalAnswer), + ); + handle_turn_completed(&mut chat, "turn-1", Some(2_000)); + let cells = drain_insert_history_with(&mut rx, |cell| { + if cell.as_any().is::() { + return vec!["Worked for 2s".into()]; + } + cell.display_lines(/*width*/ 80) + }); let platform_file_url = url::Url::from_file_path(test_path_buf("/tmp/ig-1.png")) .expect("test path should convert to file URL") .to_string(); - let combined = - lines_to_single_string(&cells[0]).replace(&platform_file_url, "file:///tmp/ig-1.png"); + let combined = cells + .iter() + .map(|lines| lines_to_single_string(lines)) + .collect::() + .replace(&platform_file_url, "file:///tmp/ig-1.png"); assert_chatwidget_snapshot!("image_generation_call_history_snapshot", combined); handle_image_generation_end( diff --git a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs index 830b81c83d..443b3a50cb 100644 --- a/codex-rs/tui/src/chatwidget/tool_lifecycle.rs +++ b/codex-rs/tui/src/chatwidget/tool_lifecycle.rs @@ -27,21 +27,39 @@ impl ChatWidget { pub(super) fn on_image_generation_end( &mut self, - call_id: String, - status: String, - revised_prompt: Option, - saved_path: Option, + turn_id: &str, + item: codex_app_server_protocol::ImageGenerationItem, ) { self.flush_answer_stream_with_separator(); - self.add_to_history(history_cell::new_image_generation_call( - call_id, - &status, - revised_prompt, - saved_path, - )); + let image = history_cell::new_image_generation_call( + item.id, + &item.status, + item.revised_prompt, + item.saved_path, + ); + if self.local_settings.transcript_mode.is_owned() + && !turn_id.is_empty() + && image.is_turn_output() + { + self.turn_lifecycle + .generated_images + .record(turn_id.to_owned(), image); + } else { + self.add_to_history(image); + } self.request_redraw(); } + pub(super) fn flush_generated_images(&mut self, turn_id: &str) { + let images = self.turn_lifecycle.generated_images.take(turn_id); + if !images.is_empty() { + self.flush_answer_and_plan_streams(); + for image in images { + self.add_to_history(image); + } + } + } + pub(super) fn on_file_change_completed(&mut self, item: ThreadItemPayload) { self.defer_or_handle( item, diff --git a/codex-rs/tui/src/chatwidget/turn_lifecycle.rs b/codex-rs/tui/src/chatwidget/turn_lifecycle.rs index a6ed3332b2..6855c99e51 100644 --- a/codex-rs/tui/src/chatwidget/turn_lifecycle.rs +++ b/codex-rs/tui/src/chatwidget/turn_lifecycle.rs @@ -19,6 +19,7 @@ pub(super) struct TurnLifecycleState { pub(super) budget_limited_turn_ids: HashSet, /// False records an untimed turn boundary that may still receive a completion label. pub(super) rendered_completion_turn_ids: HashMap, + pub(super) generated_images: crate::history_cell::PendingGeneratedImages, pub(super) goal_status_active_turn_started_at: Option, } @@ -32,6 +33,7 @@ impl TurnLifecycleState { admitted_turn_id: None, budget_limited_turn_ids: HashSet::new(), rendered_completion_turn_ids: HashMap::new(), + generated_images: Default::default(), goal_status_active_turn_started_at: None, } } @@ -66,6 +68,7 @@ impl TurnLifecycleState { self.admitted_turn_id = None; self.budget_limited_turn_ids.clear(); self.rendered_completion_turn_ids.clear(); + self.generated_images = Default::default(); } pub(super) fn set_prevent_idle_sleep(&mut self, enabled: bool) { diff --git a/codex-rs/tui/src/custom_terminal.rs b/codex-rs/tui/src/custom_terminal.rs index d0dde7f37c..992bd26bd9 100644 --- a/codex-rs/tui/src/custom_terminal.rs +++ b/codex-rs/tui/src/custom_terminal.rs @@ -25,6 +25,8 @@ use std::io; use std::io::Write; use crossterm::cursor::MoveTo; +use crossterm::cursor::RestorePosition; +use crossterm::cursor::SavePosition; use crossterm::cursor::SetCursorStyle; use crossterm::queue; use crossterm::style::Colored; @@ -406,64 +408,118 @@ where } else { crate::transcript_image::FIRST_IMAGE_ID }; - let changed = *retained_images != images; - if changed { - if self.tmux_kitty_image_base.is_some() && !retained_images.is_empty() { - // tmux retains placeholder text after Ghostty deletes the associated pixels. - self.invalidate_viewport(); + let tmux = self.tmux_kitty_image_base.is_some(); + // tmux retains placeholder cells, so keep their existing positional IDs and redraw + // behavior. Native Kitty can keep the same ID and pixels as other images change. + if !tmux || *retained_images == images { + for image in &mut images { + if let Some(index) = retained_images.iter().position(|old| old == image) { + *image = retained_images.swap_remove(index); + } } - // Remove old graphics before text painting, including when switching to overlays. - for (index, placement) in retained_images.iter().enumerate() { - match placement.image.protocol { - crate::pets::ImageProtocol::Kitty - | crate::pets::ImageProtocol::KittyLocalFile => { - let id = first_id + index as u32; - self.backend - .write_all(crate::pets::kitty_delete_image(id).as_bytes())?; - } - crate::pets::ImageProtocol::Sixel => self.invalidate_viewport(), + } else if !retained_images.is_empty() { + self.invalidate_viewport(); + } + let mut moved = Vec::new(); + // iTerm2 can discard the pixel data when a placement is removed. Redraw it + // from the source in that terminal, and keep the fast path for native Kitty. + if !tmux + && codex_terminal_detection::terminal_info().name + != codex_terminal_detection::TerminalName::Iterm2 + { + for (index, image) in images.iter_mut().enumerate() { + if image.terminal_id.is_none() + && image.image.protocol != crate::pets::ImageProtocol::Sixel + && let Some(previous) = retained_images.iter().position(|old| { + std::sync::Arc::ptr_eq(&old.image, &image.image) + && old.first_row == image.first_row + && old.area.as_size() == image.area.as_size() + }) + { + let mut old = retained_images.swap_remove(previous); + old.area = image.area; + *image = old; + moved.push(index); } } } + for old in retained_images.drain(..) { + match old.image.protocol { + crate::pets::ImageProtocol::Kitty | crate::pets::ImageProtocol::KittyLocalFile => { + if let Some(id) = old.terminal_id { + self.backend + .write_all(crate::pets::kitty_delete_image(id).as_bytes())?; + } + } + crate::pets::ImageProtocol::Sixel => self.invalidate_viewport(), + } + } - // Not every terminal or multiplexer hides intermediate cursor moves inside a - // synchronized update, especially when the frame spans multiple writes. + // A text clear, focus highlight, or full repaint can erase an otherwise stable image. let updates = diff_buffers(self.previous_buffer(), self.current_buffer()); - // A text clear, focus highlight, or full repaint may erase an otherwise stable image. - let repaint_images = changed - || updates.iter().any(|update| { - images.iter().any(|placement| match update { - DrawCommand::Put { x, y, .. } => placement.area.contains((*x, *y).into()), + let mut redraw = Vec::new(); + let mut next_id = first_id; + for index in 0..images.len() { + let image = &images[index]; + if image.terminal_id.is_none() { + while images + .iter() + .any(|other| other.terminal_id == Some(next_id)) + { + next_id += 1; + } + images[index].terminal_id = Some(next_id); + redraw.push(index); + next_id += 1; + } else if !moved.contains(&index) + && updates.iter().any(|update| match update { + DrawCommand::Put { x, y, .. } => image.area.contains((*x, *y).into()), DrawCommand::ClearToEnd { x, y, .. } => { - *y >= placement.area.y - && *y < placement.area.bottom() - && *x < placement.area.right() + (image.area.y..image.area.bottom()).contains(y) && *x < image.area.right() } }) - }); + { + redraw.push(index); + } + } if !updates.is_empty() && !self.hidden_cursor { self.hide_cursor()?; } self.flush_updates(updates)?; - if repaint_images { - *retained_images = images; - for (index, placement) in retained_images.iter_mut().enumerate() { - let id = first_id + index as u32; - let kitty_placement = if self.tmux_kitty_image_base.is_some() { - crate::pets::KittyPlacement::Virtual(id) - } else { - crate::pets::KittyPlacement::Cursor(id) - }; - placement.file = placement - .image - .draw( - &mut self.backend, - placement.area, - placement.first_row, - kitty_placement, - ) - .map_err(io::Error::other)?; - } + *retained_images = images; + for index in moved { + let image = &retained_images[index]; + let id = image + .terminal_id + .ok_or_else(|| io::Error::other("missing terminal ID for image"))?; + queue!( + self.backend, + SavePosition, + MoveTo(image.area.x, image.area.y) + )?; + self.backend + .write_all(crate::pets::kitty_move_image(id, image.area).as_bytes())?; + queue!(self.backend, RestorePosition)?; + } + for index in redraw { + let placement = &mut retained_images[index]; + let id = placement + .terminal_id + .ok_or_else(|| io::Error::other("missing terminal ID for placement"))?; + let kitty_placement = if tmux { + crate::pets::KittyPlacement::Virtual(id) + } else { + crate::pets::KittyPlacement::Cursor(id) + }; + placement.file = placement + .image + .draw( + &mut self.backend, + placement.area, + placement.first_row, + kitty_placement, + ) + .map_err(io::Error::other)?; } match cursor_position { @@ -958,8 +1014,13 @@ mod tests { (crate::pets::ImageProtocol::Kitty, Some(0x80_0100)), ] { let image = std::sync::Arc::new( - crate::transcript_image::TranscriptImage::load(&path, /*width*/ 40, protocol) - .unwrap(), + crate::transcript_image::TranscriptImage::load( + &path, + /*width*/ 40, + protocol, + crate::transcript_image::PreviewKind::Inspected, + ) + .unwrap(), ); let mut terminal = Terminal::with_options(CaptureBackend::new(/*width*/ 40, /*height*/ 12)).unwrap(); @@ -997,6 +1058,7 @@ mod tests { ), first_row: 0, file: None, + terminal_id: None, }); }) .unwrap(); @@ -1027,6 +1089,75 @@ mod tests { } } + #[test] + fn native_kitty_moves_an_image_without_resending_other_visible_images() { + use crate::pets::ImageProtocol; + use crate::transcript_image::ImagePlacement; + use crate::transcript_image::TranscriptImage; + use std::sync::Arc; + + let mut terminal = + Terminal::with_options(CaptureBackend::new(/*width*/ 40, /*height*/ 12)).unwrap(); + let size = terminal.last_known_screen_size; + terminal.set_viewport_area(size.into()); + let raster = Arc::new(TranscriptImage { + protocol: ImageProtocol::Kitty, + columns: 2, + rows: 2, + pixels: image::RgbaImage::from_pixel(16, 32, image::Rgba([10, 110, 200, 255])), + cell_width: 8, + cell_height: 16, + }); + let fixed = ImagePlacement { + image: Arc::clone(&raster), + area: Rect::new( + /*x*/ 2, /*y*/ 2, /*width*/ 2, /*height*/ 2, + ), + first_row: 0, + file: None, + terminal_id: None, + }; + let mut moving = fixed.registration(); + moving.area.x = 16; + let mut retained = Vec::new(); + terminal + .draw_with_size(size, &mut retained, |frame| { + frame + .image_placements + .extend([fixed.registration(), moving.registration()]); + }) + .unwrap(); + let original_ids: Vec<_> = retained.iter().map(|image| image.terminal_id).collect(); + assert_eq!(terminal.backend().output().matches("_Ga=T").count(), 2); + + moving.area.x += 2; + terminal + .draw_with_size(size, &mut retained, |frame| { + frame + .image_placements + .extend([fixed.registration(), moving.registration()]); + }) + .unwrap(); + let output = terminal.backend().output(); + assert_eq!(output.matches("_Ga=T").count(), 2); + assert_eq!(output.matches("_Ga=p").count(), 1); + assert_eq!( + retained + .iter() + .map(|image| image.terminal_id) + .collect::>(), + original_ids + ); + + terminal + .draw_with_size(size, &mut retained, |frame| { + frame.image_placements.push(moving.registration()); + }) + .unwrap(); + assert_eq!(terminal.backend().output().matches("_Ga=T").count(), 2); + assert_eq!(retained[0].terminal_id, original_ids[1]); + } + impl Backend for CaptureBackend { type Error = io::Error; diff --git a/codex-rs/tui/src/history_cell/notices.rs b/codex-rs/tui/src/history_cell/notices.rs index d5d68e2d22..033ef0659e 100644 --- a/codex-rs/tui/src/history_cell/notices.rs +++ b/codex-rs/tui/src/history_cell/notices.rs @@ -344,11 +344,20 @@ pub(crate) fn new_info_event(message: String, hint: Option) -> PlainHist PlainHistoryCell { lines } } -pub(crate) fn new_interruption_notice() -> PlainHistoryCell { - PlainHistoryCell::new(vec![ - Line::from("■ Conversation interrupted - use /feedback if something went wrong") - .style(crate::style::secondary_text_style()), - ]) +#[derive(Debug)] +pub(crate) struct InterruptionNotice(pub(crate) Option); + +impl HistoryCell for InterruptionNotice { + fn display_lines(&self, _width: u16) -> Vec> { + vec![ + Line::from("■ Conversation interrupted - use /feedback if something went wrong") + .style(crate::style::secondary_text_style()), + ] + } + + fn raw_lines(&self) -> Vec> { + plain_lines(self.display_lines(/*width*/ u16::MAX)) + } } pub(crate) fn new_error_event(message: String) -> PlainHistoryCell { diff --git a/codex-rs/tui/src/history_cell/patches.rs b/codex-rs/tui/src/history_cell/patches.rs index ca4313b137..8191cdce08 100644 --- a/codex-rs/tui/src/history_cell/patches.rs +++ b/codex-rs/tui/src/history_cell/patches.rs @@ -280,7 +280,7 @@ pub(crate) fn new_image_generation_call( revised_prompt: Option, saved_path: Option, ) -> ImageGenerationHistoryCell { - let detail = revised_prompt.unwrap_or(call_id); + let detail = revised_prompt.unwrap_or_else(|| call_id.clone()); let failed = status == "failed"; let heading = if failed { vec!["✗ ".red().bold(), "Image generation failed".bold()].into() @@ -296,21 +296,52 @@ pub(crate) fn new_image_generation_call( } ImageGenerationHistoryCell { + id: call_id, lines, failed, - saved_path: saved_path.filter(|_| !failed), + saved_path: saved_path.filter(|_| status == "completed"), } } #[derive(Debug)] pub(crate) struct ImageGenerationHistoryCell { + pub(crate) id: String, failed: bool, lines: Vec>, saved_path: Option, } +impl ImageGenerationHistoryCell { + /// A completed generation with a saved artifact belongs to the answer. + pub(crate) fn is_turn_output(&self) -> bool { + self.saved_path.is_some() + } +} + +/// Hold saved results until the turn can place them after the answer. +#[derive(Debug, Default)] +pub(crate) struct PendingGeneratedImages(HashMap>); + +impl PendingGeneratedImages { + pub(crate) fn record(&mut self, turn_id: String, image: ImageGenerationHistoryCell) { + let images = self.0.entry(turn_id).or_default(); + if let Some(existing) = images.iter_mut().find(|existing| existing.id == image.id) { + *existing = image; + } else { + images.push(image); + } + } + + pub(crate) fn take(&mut self, turn_id: &str) -> Vec { + self.0.remove(turn_id).unwrap_or_default() + } +} + impl HistoryCell for ImageGenerationHistoryCell { fn tool_call_labels(&self) -> Vec { + if self.is_turn_output() { + return Vec::new(); + } let label = if self.failed { "Image generation failed" } else { @@ -323,6 +354,46 @@ impl HistoryCell for ImageGenerationHistoryCell { self.lines.clone() } + fn display_hyperlink_lines(&self, width: u16) -> Vec { + let mut lines = + crate::terminal_hyperlinks::plain_hyperlink_lines(self.display_lines(width)); + if let Some(path) = &self.saved_path + && let Some(line) = lines.last_mut() + && let Some(span) = line.line.spans.last() + { + let width = crate::line_truncation::span_width(span); + let start = line.line.width().saturating_sub(width); + line.hyperlinks + .push(crate::terminal_hyperlinks::TerminalHyperlink::document( + start..start + width, + crate::markdown_render::FileLink { + path: path.to_string_lossy().into_owned(), + location: None, + }, + )); + } + lines + } + + fn compact_hyperlink_lines(&self, width: u16) -> Vec { + let Some(path) = &self.saved_path else { + return self.display_hyperlink_lines(width); + }; + let mut line = HyperlinkLine::from(Line::from(" Generated image".dim())); + line.hyperlinks.push(TerminalHyperlink::document( + 2..line.line.width(), + crate::markdown_render::FileLink { + path: path.to_string_lossy().into_owned(), + location: None, + }, + )); + vec![line] + } + + fn transcript_hyperlink_lines(&self, width: u16) -> Vec { + self.display_hyperlink_lines(width) + } + fn raw_lines(&self) -> Vec> { plain_lines(self.lines.clone()) } diff --git a/codex-rs/tui/src/history_cell/separators.rs b/codex-rs/tui/src/history_cell/separators.rs index 14b8c93aa6..b916f6bd8e 100644 --- a/codex-rs/tui/src/history_cell/separators.rs +++ b/codex-rs/tui/src/history_cell/separators.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; /// restoring the conversation constructs new cells and refreshes whether the timestamp needs a date. #[derive(Debug)] pub struct FinalMessageSeparator { + pub(crate) turn_id: Option, elapsed_seconds: Option, runtime_metrics: Option, completed_at: Option>, @@ -51,7 +52,10 @@ impl FinalMessageSeparator { return None; } rendered_completion_turn_ids.insert(turn.id.clone(), has_label); - let cell = Self::new(elapsed_seconds, /*runtime_metrics*/ None); + let cell = Self { + turn_id: Some(turn.id.clone()), + ..Self::new(elapsed_seconds, /*runtime_metrics*/ None) + }; Some(match completed_at { Some(completed_at) => cell.with_completed_at(completed_at, ClockFormat::system()), None => cell, @@ -64,6 +68,7 @@ impl FinalMessageSeparator { runtime_metrics: Option, ) -> Self { Self { + turn_id: None, elapsed_seconds, runtime_metrics, completed_at: None, diff --git a/codex-rs/tui/src/history_cell/tests.rs b/codex-rs/tui/src/history_cell/tests.rs index 1c8fc0cce6..6822d4f578 100644 --- a/codex-rs/tui/src/history_cell/tests.rs +++ b/codex-rs/tui/src/history_cell/tests.rs @@ -609,7 +609,7 @@ fn image_generation_call_renders_saved_path() { "call-image-generation".to_string(), "completed", Some("A tiny blue square".to_string()), - Some(saved_path), + Some(saved_path.clone()), ); assert_eq!( @@ -620,6 +620,18 @@ fn image_generation_call_renders_saved_path() { expected_saved_path, ], ); + let lines = cell.transcript_hyperlink_lines(/*width*/ 80); + let link = &lines.last().unwrap().hyperlinks[0]; + assert_eq!( + link.reader_destination(/*documents*/ true), + Some(crate::terminal_hyperlinks::ReaderLink::Document( + crate::markdown_render::FileLink { + path: saved_path.to_string_lossy().into_owned(), + location: None, + }, + )), + ); + assert_eq!(link.reader_destination(/*documents*/ false), None); } fn session_configured_event(model: &str) -> ThreadSessionState { diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index f038d87855..d4d479d9a0 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -219,6 +219,7 @@ mod session_start; mod session_state; mod shortcut_help; mod sidebar_chooser; +mod sidebar_image; mod sidebar_tab_strip; mod skills_helpers; mod slash_command; diff --git a/codex-rs/tui/src/markdown_render/local_links.rs b/codex-rs/tui/src/markdown_render/local_links.rs index 0c6919f047..91d06947ce 100644 --- a/codex-rs/tui/src/markdown_render/local_links.rs +++ b/codex-rs/tui/src/markdown_render/local_links.rs @@ -39,6 +39,12 @@ pub(crate) struct FileLink { pub(crate) location: Option, } +impl FileLink { + pub(crate) fn is_image(&self) -> bool { + self.location.is_none() && crate::sidebar_image::supports_path(&self.path) + } +} + pub(crate) fn is_markdown_file(path: &str) -> bool { path.rsplit_once('.').is_some_and(|(_, ext)| { ext.eq_ignore_ascii_case("md") || ext.eq_ignore_ascii_case("markdown") diff --git a/codex-rs/tui/src/pager_overlay/transcript.rs b/codex-rs/tui/src/pager_overlay/transcript.rs index 1e9a8ed242..aa3b2293a4 100644 --- a/codex-rs/tui/src/pager_overlay/transcript.rs +++ b/codex-rs/tui/src/pager_overlay/transcript.rs @@ -340,6 +340,11 @@ impl TranscriptOverlay { .iter() .rposition(|cell| is_session_history_prefix_cell(cell.as_ref())) .map_or(/*default*/ 0, |index| index + 1); + self.insert_history_page(index, cells); + index + } + + pub(crate) fn insert_history_page(&mut self, index: usize, cells: Vec>) { let added = cells.len(); self.cells.splice(index..index, cells); for highlight in [&mut self.highlight_cell, &mut self.pending_highlight] { @@ -349,7 +354,6 @@ impl TranscriptOverlay { } self.view.set_highlight(self.highlight_cell); self.view.history_loaded(&self.cells, index..index + added); - index } pub(crate) fn replace_cells(&mut self, cells: Vec>) { diff --git a/codex-rs/tui/src/pets/image_protocol.rs b/codex-rs/tui/src/pets/image_protocol.rs index 38f387eb1e..a2cdfe1033 100644 --- a/codex-rs/tui/src/pets/image_protocol.rs +++ b/codex-rs/tui/src/pets/image_protocol.rs @@ -259,6 +259,14 @@ impl KittyPlacement { } } +pub(crate) fn kitty_move_image(id: u32, area: ratatui::layout::Rect) -> String { + let (columns, rows) = (area.width, area.height); + // Lowercase i removes placements but keeps the uploaded pixel data. + wrap_for_tmux_if_needed(&format!( + "{ESC}_Ga=d,d=i,i={id},q=2;{ST}{ESC}_Ga=p,i={id},c={columns},r={rows},C=1,q=2;{ST}" + )) +} + pub fn kitty_transmit_png_with_id( path: &Path, columns: u16, diff --git a/codex-rs/tui/src/pets/mod.rs b/codex-rs/tui/src/pets/mod.rs index 540799f8c6..99e3a7e24f 100644 --- a/codex-rs/tui/src/pets/mod.rs +++ b/codex-rs/tui/src/pets/mod.rs @@ -48,6 +48,7 @@ pub(crate) use image_protocol::PetImageUnsupportedReason; #[cfg(not(test))] pub(crate) use image_protocol::detect_pet_image_support; pub(crate) use image_protocol::kitty_delete_image; +pub(crate) use image_protocol::kitty_move_image; pub(crate) use image_protocol::kitty_transmit_png_file_with_id; pub(crate) use image_protocol::kitty_transmit_png_with_id; pub(crate) use picker::PET_PICKER_VIEW_ID; diff --git a/codex-rs/tui/src/sidebar_image.rs b/codex-rs/tui/src/sidebar_image.rs new file mode 100644 index 0000000000..f2c84e6aad --- /dev/null +++ b/codex-rs/tui/src/sidebar_image.rs @@ -0,0 +1,174 @@ +//! Decode a workspace image once and reuse the transcript's terminal image lifecycle. + +use std::io::Cursor; +use std::sync::Arc; + +use color_eyre::eyre::OptionExt; +use color_eyre::eyre::Result; +use image::DynamicImage; +use image::ImageFormat; +use ratatui::buffer::Buffer; +use ratatui::layout::Rect; +use ratatui::style::Stylize; +use ratatui::text::Line; +use ratatui::widgets::Widget; + +use crate::pets::ImageProtocol; +use crate::transcript_image::ImagePlacement; +use crate::transcript_image::TranscriptImage; +use crate::transcript_image::terminal_cell_size; + +#[derive(Debug)] +pub(crate) enum DocumentContent { + Text(String), + Image(Box), +} + +pub(crate) struct SidebarImage { + label: String, + source_size: (u32, u32), + raster: DynamicImage, + fitted: Option>, +} + +impl std::fmt::Debug for SidebarImage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SidebarImage") + .field("label", &self.label) + .finish_non_exhaustive() + } +} + +pub(crate) fn supports_path(path: &str) -> bool { + path.rsplit_once('.') + .and_then(|(_, extension)| ImageFormat::from_extension(extension)) + .is_some_and(supported_format) +} + +fn supported_format(format: ImageFormat) -> bool { + matches!( + format, + ImageFormat::Png | ImageFormat::Jpeg | ImageFormat::Gif | ImageFormat::WebP + ) +} + +impl SidebarImage { + pub(crate) fn decode(bytes: &[u8]) -> Result { + let reader = image::ImageReader::new(Cursor::new(bytes)).with_guessed_format()?; + let format = reader + .format() + .filter(|format| supported_format(*format)) + .ok_or_eyre("Unsupported image format")?; + let mut limits = image::Limits::default(); + limits.max_alloc = Some(32 * 1024 * 1024); + let image = crate::image_decode::decode_oriented(reader, limits)?; + let (width, height) = (image.width(), image.height()); + Ok(Self { + label: format!( + "{} · {width} × {height}", + format.extensions_str()[0].to_ascii_uppercase() + ), + source_size: (width, height), + raster: if width > 768 || height > 768 { + image.thumbnail(768, 768) + } else { + image + }, + fitted: None, + }) + } + + pub(crate) fn paint( + &mut self, + area: Rect, + buf: &mut Buffer, + protocol: Option, + resize_hint: Option<&str>, + ) -> Option { + Line::from(self.label.as_str()).dim().render(area, buf); + let available_rows = area.height.saturating_sub(2 + u16::from(area.height > 3)); + if area.height > 1 { + let footer = Rect::new(area.x, area.bottom() - 1, area.width, 1); + if let Some(hint) = resize_hint { + crate::sidebar_chooser::resize_hint(hint, footer.width).render(footer, buf); + } else { + crate::footer_hint::HintLine::new( + [ + crate::footer_hint::shortcut("←/→", "tabs"), + crate::footer_hint::shortcut("Esc", "close"), + ], + footer.width, + ) + .line + .render(footer, buf); + } + } + // tmux uses Kitty's virtual placements. The shared placeholder renderer + // currently emits row marks for up to six rows. + let rows = if crate::tui::tmux_fullscreen_image_base().is_some() { + available_rows.min(6) + } else { + available_rows + }; + let body = Rect::new(area.x, area.y.saturating_add(1), area.width, rows); + if body.is_empty() { + return None; + } + let Some((protocol, (cw, ch))) = protocol.and_then(|protocol| { + let size = terminal_cell_size() + .or_else(|| (protocol != ImageProtocol::Sixel).then_some((8, 16)))?; + Some((protocol, size)) + }) else { + Line::from("Image preview unavailable in this terminal").render(body, buf); + return None; + }; + let (width, height) = self.source_size; + let (max_width, max_height) = match protocol { + ImageProtocol::Sixel => (160, 80), + ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (width, height), + }; + let scale = (f64::from((u32::from(body.width) * cw).min(max_width)) / f64::from(width)) + .min(f64::from((u32::from(body.height) * ch).min(max_height)) / f64::from(height)) + .min(1.0); + let pixel_width = (f64::from(width) * scale).round().max(1.0) as u32; + let pixel_height = (f64::from(height) * scale).round().max(1.0) as u32; + let columns = pixel_width.div_ceil(cw) as u16; + let rows = pixel_height.div_ceil(ch) as u16; + if !self.fitted.as_ref().is_some_and(|image| { + image.protocol == protocol + && image.cell_width == cw + && image.cell_height == ch + && image.columns == columns + && image.rows == rows + }) { + let pixels = self + .raster + .thumbnail( + pixel_width.min(self.raster.width()), + pixel_height.min(self.raster.height()), + ) + .to_rgba8(); + self.fitted = Some(Arc::new(TranscriptImage { + protocol, + columns, + rows, + pixels, + cell_width: cw, + cell_height: ch, + })); + } + let image = self.fitted.as_ref()?; + Some(ImagePlacement { + image: Arc::clone(image), + area: Rect::new( + body.x + (body.width - image.columns) / 2, + body.y + (body.height - image.rows) / 2, + image.columns, + image.rows, + ), + first_row: 0, + file: None, + terminal_id: None, + }) + } +} diff --git a/codex-rs/tui/src/startup_draft.rs b/codex-rs/tui/src/startup_draft.rs index e1f6338030..49244cc7ec 100644 --- a/codex-rs/tui/src/startup_draft.rs +++ b/codex-rs/tui/src/startup_draft.rs @@ -138,6 +138,7 @@ impl StartupDraft { ); terminal_restore_guard.events = Some(tui.event_broker.clone()); tui.terminal_app_over_ssh = initialized_terminal.terminal_app_over_ssh; + tui.herdr_kitty_graphics = initialized_terminal.herdr_kitty_graphics; tui.set_alt_screen_enabled(screen.use_alt_screen); let mut pump = StartupDraftPump::new(&tui, initial_screen, session_action); pump.motion = screen.welcome_motion; diff --git a/codex-rs/tui/src/terminal_probe.rs b/codex-rs/tui/src/terminal_probe.rs index e16720c6f6..dbf67fc42c 100644 --- a/codex-rs/tui/src/terminal_probe.rs +++ b/codex-rs/tui/src/terminal_probe.rs @@ -49,7 +49,7 @@ fn alternate_scroll_report(input: &[u8]) -> Option { mod imp { use super::DefaultColors; use super::parse_default_colors; - use super::startup_replay::startup_replay_input; + use super::startup_replay::parse_startup_input; use super::terminal_identity; use std::fs::File; use std::fs::OpenOptions; @@ -73,13 +73,14 @@ mod imp { static DA1_SUPPORTED: AtomicBool = AtomicBool::new(false); /// Results from the TUI's one-shot startup terminal probe. - #[derive(Debug, Clone, Copy, Eq, PartialEq)] + #[derive(Debug, Default, Clone, Copy, Eq, PartialEq)] pub(crate) struct StartupProbe { pub(crate) alternate_scroll_enabled: Option, pub(crate) cursor_position: Option, pub(crate) default_colors: Option, pub(crate) keyboard_enhancement_supported: Option, pub(crate) terminal_app_over_ssh: Option, + pub(crate) kitty_graphics: bool, } /// Whether the startup probe should query keyboard enhancement support. @@ -305,6 +306,7 @@ mod imp { pub(crate) fn startup( timeout: Duration, keyboard_probe: StartupKeyboardEnhancementProbe, + probe_herdr: bool, ) -> io::Result { DA1_SUPPORTED.store(false, Ordering::Relaxed); let mut tty = Tty::open()?; @@ -319,6 +321,10 @@ mod imp { } // Query before DA1 so its reply acts as a barrier even if mode 1007 is unsupported. query.extend_from_slice(b"\x1B[?1007$p"); + if probe_herdr { + // Query a single inline RGB pixel without storing or displaying an image. + query.extend_from_slice(b"\x1b_Gi=31,s=1,v=1,a=q,t=d,f=24;AAAA\x1b\\"); + } // DA1 finishes keyboard detection and establishes whether we can use it at exit. query.extend_from_slice(b"\x1B[c\x1B[6n\x1B]10;?\x1B\\\x1B]11;?\x1B\\"); if query_identity { @@ -332,12 +338,17 @@ mod imp { timeout, keyboard_probe, query_identity, + probe_herdr, &mut buffer, ); - crossterm::event::buffer_input(&startup_replay_input(&buffer))?; + let (replay, kitty_graphics) = parse_startup_input(&buffer, probe_herdr); + crossterm::event::buffer_input(&replay)?; let saw_da1 = find_primary_device_attributes(&buffer).is_some(); DA1_SUPPORTED.store(saw_da1, Ordering::Relaxed); - result + result.map(|probe| StartupProbe { + kitty_graphics, + ..probe + }) } /// Reads available terminal bytes until `parse` recognizes a probe response or time expires. @@ -373,15 +384,13 @@ mod imp { timeout: Duration, keyboard_probe: StartupKeyboardEnhancementProbe, query_identity: bool, + probe_herdr: bool, buffer: &mut Vec, ) -> io::Result { let deadline = Instant::now() + timeout; let mut probe = StartupProbe { - alternate_scroll_enabled: None, - cursor_position: None, - default_colors: None, - keyboard_enhancement_supported: None, terminal_app_over_ssh: (!query_identity).then_some(/*t*/ false), + ..Default::default() }; let mut saw_supported_keyboard = false; loop { @@ -392,7 +401,9 @@ mod imp { buffer, keyboard_probe, ); - if startup_probe_complete(&probe, keyboard_probe) { + if startup_probe_complete(&probe, keyboard_probe) + && (!probe_herdr || find_primary_device_attributes(buffer).is_some()) + { return Ok(probe); } let now = Instant::now(); @@ -698,6 +709,7 @@ mod imp { default_colors: None, keyboard_enhancement_supported: None, terminal_app_over_ssh: Some(false), + kitty_graphics: false, }; let mut saw_supported_keyboard = false; update_startup_probe( @@ -718,6 +730,7 @@ mod imp { }), keyboard_enhancement_supported: Some(true), terminal_app_over_ssh: Some(false), + kitty_graphics: false, } ); assert!(startup_probe_complete( @@ -734,6 +747,7 @@ mod imp { default_colors: None, keyboard_enhancement_supported: None, terminal_app_over_ssh: None, + kitty_graphics: false, }; let mut supported_keyboard = false; let input = @@ -772,6 +786,7 @@ mod imp { Duration::from_millis(/*millis*/ 5), StartupKeyboardEnhancementProbe::Skip, /*query_identity*/ true, + /*probe_herdr*/ false, &mut buffer, ) .expect("finish within probe deadline"); @@ -792,6 +807,7 @@ mod imp { Duration::from_secs(/*secs*/ 1), StartupKeyboardEnhancementProbe::Query, /*query_identity*/ false, + /*probe_herdr*/ false, &mut buffer, ) .expect("read terminal responses after buffered typeahead"); @@ -807,11 +823,15 @@ mod imp { }), keyboard_enhancement_supported: Some(true), terminal_app_over_ssh: Some(false), + kitty_graphics: false, } ); let mut expected_replay = vec![b'x'; 2_048]; expected_replay.extend_from_slice(b"\x1B[20;10R\x1B[?64;1;2c\x1B[?7u"); - assert_eq!(startup_replay_input(&buffer), expected_replay); + assert_eq!( + parse_startup_input(&buffer, /*probe_herdr*/ false), + (expected_replay, false) + ); } #[test] diff --git a/codex-rs/tui/src/terminal_probe/startup_replay.rs b/codex-rs/tui/src/terminal_probe/startup_replay.rs index 1c3d4b4c8b..9f1666a080 100644 --- a/codex-rs/tui/src/terminal_probe/startup_replay.rs +++ b/codex-rs/tui/src/terminal_probe/startup_replay.rs @@ -1,15 +1,17 @@ -//! Preserve startup terminal input while filtering completed color and scroll-mode replies. +//! Preserve startup terminal input while filtering replies to terminal probes. const MAX_REPLAYED_OSC_BYTES: usize = 1024; -/// Preserve input and bracketed paste while omitting completed terminal-color replies. +/// Return the input to replay and whether Herdr acknowledged the Kitty graphics query. /// /// Keep bounded incomplete OSC prefixes so unread continuations stay parser-framed. -pub(super) fn startup_replay_input(input: &[u8]) -> Vec { +pub(super) fn parse_startup_input(input: &[u8], probe_herdr: bool) -> (Vec, bool) { const PASTE_START: &[u8] = b"\x1b[200~"; const PASTE_END: &[u8] = b"\x1b[201~"; + const KITTY_REPLY: &[u8] = b"\x1b_Gi=31;"; let mut replay = Vec::with_capacity(input.len()); + let mut kitty_graphics = false; let mut cursor = 0; let mut inside_paste = false; while cursor < input.len() { @@ -52,6 +54,17 @@ pub(super) fn startup_replay_input(input: &[u8]) -> Vec { continue; } + // Only consume replies to our query. Bracketed paste above remains user input. + if probe_herdr + && let Some(message) = input[cursor..].strip_prefix(KITTY_REPLY) + && let Some(end) = message.iter().position(|b| !(0x20..=0x7e).contains(b)) + && message[end..].starts_with(b"\x1b\\") + { + kitty_graphics = &message[..end] == b"OK"; + cursor += KITTY_REPLY.len() + end + 2; + continue; + } + if input[cursor..].starts_with(b"\x1b]") { if let Some((end, terminator_len)) = super::osc_payload_end(&input[cursor + 2..]) { cursor += 2 + end + terminator_len; @@ -65,7 +78,7 @@ pub(super) fn startup_replay_input(input: &[u8]) -> Vec { replay.push(input[cursor]); cursor += 1; } - replay + (replay, kitty_graphics) } #[cfg(test)] diff --git a/codex-rs/tui/src/terminal_probe/startup_replay_tests.rs b/codex-rs/tui/src/terminal_probe/startup_replay_tests.rs index 0c3500aeed..075dfd9ef6 100644 --- a/codex-rs/tui/src/terminal_probe/startup_replay_tests.rs +++ b/codex-rs/tui/src/terminal_probe/startup_replay_tests.rs @@ -1,7 +1,13 @@ +#![cfg(test)] + use super::MAX_REPLAYED_OSC_BYTES; -use super::startup_replay_input; +use super::parse_startup_input; use pretty_assertions::assert_eq; +fn startup_replay_input(input: &[u8]) -> Vec { + parse_startup_input(input, /*probe_herdr*/ false).0 +} + #[test] fn startup_replay_bounds_oversized_osc_without_exposing_its_payload() { let mut input = b"typed\x1b]10;".to_vec(); @@ -56,3 +62,19 @@ fn startup_replay_omits_terminal_color_replies_interleaved_with_bracketed_paste( response_after_paste ); } + +#[test] +fn herdr_graphics_reply_selects_previews_without_replaying_the_response_as_keys() { + let paste = b"\x1b[200~\x1b_Gi=31;OK\x1b\\\x1b[201~"; + for (reply, expected) in [ + (b"\x1b_Gi=31;OK\x1b\\".as_slice(), true), + (b"\x1b_Gi=31;EINVAL: unsupported\x1b\\", false), + (b"", false), + ] { + let input = [paste.as_slice(), reply, b"typed"].concat(); + assert_eq!( + parse_startup_input(&input, /*probe_herdr*/ true), + ([paste.as_slice(), b"typed"].concat(), expected) + ); + } +} diff --git a/codex-rs/tui/src/transcript_image.rs b/codex-rs/tui/src/transcript_image.rs index 4a6d6950f1..9c42a25058 100644 --- a/codex-rs/tui/src/transcript_image.rs +++ b/codex-rs/tui/src/transcript_image.rs @@ -30,6 +30,7 @@ pub(crate) struct ImagePlacement { pub(crate) area: Rect, pub(crate) first_row: usize, pub(crate) file: Option, + pub(crate) terminal_id: Option, } impl ImagePlacement { @@ -40,6 +41,7 @@ impl ImagePlacement { area: self.area, first_row: self.first_row, file: None, + terminal_id: None, } } } @@ -73,10 +75,29 @@ pub(crate) fn terminal_cell_size() -> Option<(u32, u32)> { }) } +/// Generated output is larger than an image shown as a tool detail. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum PreviewKind { + Inspected, + Generated, +} + impl TranscriptImage { fn load_pixels(path: &Path, max_width: u32, max_height: u32) -> Option { + if max_width == 0 || max_height == 0 { + return None; + } + let image = Self::decode(path)?; + Some( + image + .thumbnail(max_width.min(image.width()), max_height.min(image.height())) + .to_rgba8(), + ) + } + + fn decode(path: &Path) -> Option { let metadata = path.metadata().ok()?; - if max_width == 0 || !metadata.is_file() || metadata.len() > 32 * 1024 * 1024 { + if !metadata.is_file() || metadata.len() > 32 * 1024 * 1024 { return None; } let reader = image::ImageReader::open(path) @@ -87,32 +108,53 @@ impl TranscriptImage { // screenshots usable, but never allocate hundreds of MB for an optional thumbnail. let mut limits = image::Limits::default(); limits.max_alloc = Some(32 * 1024 * 1024); - let image = crate::image_decode::decode_oriented(reader, limits).ok()?; - Some( - image - .thumbnail(max_width.min(image.width()), max_height.min(image.height())) - .to_rgba8(), - ) + crate::image_decode::decode_oriented(reader, limits).ok() } - pub(crate) fn load(path: &Path, width: u16, protocol: ImageProtocol) -> Option { - let columns = width.saturating_sub(2).min(48); + pub(crate) fn load( + path: &Path, + width: u16, + protocol: ImageProtocol, + kind: PreviewKind, + ) -> Option { + let columns = width.saturating_sub(2); + if columns == 0 { + return None; + } // Sixel paints physical pixels. Guessing could paint outside the reserved rows. let (cell_width, cell_height) = match protocol { ImageProtocol::Sixel => terminal_cell_size()?, - ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (8, 16), + ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => match kind { + PreviewKind::Inspected => (8, 16), + PreviewKind::Generated => terminal_cell_size().unwrap_or((8, 16)), + }, }; - // The sprite encoder scans each active color per Sixel band. Bound both its CPU - // work and worst-case output even with large terminal cells or high-entropy images. - let (max_width, max_height) = match protocol { - ImageProtocol::Sixel => (160, 80), - ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (384, 160), + let image = Self::decode(path)?; + // Keep the current Sixel encoding and tmux placeholder budgets. + let (columns, max_width, max_height) = match (protocol, kind) { + (ImageProtocol::Sixel, _) => (columns.min(48), 160, 80.min(6 * cell_height)), + (_, PreviewKind::Inspected) => (columns.min(48), 384, 6 * cell_height), + (_, PreviewKind::Generated) => { + let ratio = f64::from(image.width()) / f64::from(image.height()); + let max_width = if ratio >= 2.0 { + 768 + } else if ratio >= 1.01 { + 480 + } else { + 400 + }; + let max_height = if crate::tui::tmux_fullscreen_image_base().is_some() { + 6 * cell_height + } else { + 400 + }; + (columns, max_width, max_height) + } }; - let pixels = Self::load_pixels( - path, - (u32::from(columns) * cell_width).min(max_width), - (6 * cell_height).min(max_height), - )?; + let max_width = (u32::from(columns) * cell_width).min(max_width); + let pixels = image + .thumbnail(max_width.min(image.width()), max_height.min(image.height())) + .to_rgba8(); Some(Self { protocol, columns: pixels.width().div_ceil(cell_width) as u16, diff --git a/codex-rs/tui/src/transcript_image/composer.rs b/codex-rs/tui/src/transcript_image/composer.rs index ee3088141f..c0a010d8c0 100644 --- a/codex-rs/tui/src/transcript_image/composer.rs +++ b/codex-rs/tui/src/transcript_image/composer.rs @@ -125,6 +125,7 @@ impl ComposerImages { area: tile, first_row: 0, file: None, + terminal_id: None, }); } else { Paragraph::new(attachment.placeholder.as_str()) diff --git a/codex-rs/tui/src/transcript_view.rs b/codex-rs/tui/src/transcript_view.rs index 51565637b0..dbd5b9fdef 100644 --- a/codex-rs/tui/src/transcript_view.rs +++ b/codex-rs/tui/src/transcript_view.rs @@ -439,8 +439,14 @@ impl TranscriptView { if skip >= rows.len() { continue; } + let image_indent = if visible.layout.image_left_aligned { + // Match the two-column prefix the assistant's Markdown uses. + 2 + } else { + self.area.width.saturating_sub(image.columns) / 2 + }; let area = Rect::new( - self.area.x + self.area.width.saturating_sub(image.columns) / 2, + self.area.x + image_indent, entry_y + skip as u16, image.columns, (rows.len() - skip) as u16, @@ -450,6 +456,7 @@ impl TranscriptView { area, first_row: visible.row.saturating_sub(image_start), file: None, + terminal_id: None, }); } placements diff --git a/codex-rs/tui/src/transcript_view/layout.rs b/codex-rs/tui/src/transcript_view/layout.rs index 6a371b2731..3155d62e7e 100644 --- a/codex-rs/tui/src/transcript_view/layout.rs +++ b/codex-rs/tui/src/transcript_view/layout.rs @@ -89,9 +89,19 @@ fn cell_content_layout( if let Some(protocol) = presentation.image_protocol && let Some(path) = cell.image_path() { + use crate::transcript_image::PreviewKind; + let kind = if cell + .as_any() + .is::() + { + PreviewKind::Generated + } else { + PreviewKind::Inspected + }; layout.image = - crate::transcript_image::TranscriptImage::load(path.as_path(), width, protocol) + crate::transcript_image::TranscriptImage::load(path.as_path(), width, protocol, kind) .map(Arc::new); + layout.image_left_aligned = kind == PreviewKind::Generated; } layout } diff --git a/codex-rs/tui/src/transcript_view/layout_tests.rs b/codex-rs/tui/src/transcript_view/layout_tests.rs index 6900994893..6f9f0d4e7d 100644 --- a/codex-rs/tui/src/transcript_view/layout_tests.rs +++ b/codex-rs/tui/src/transcript_view/layout_tests.rs @@ -164,16 +164,29 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() { image::RgbaImage::from_pixel(480, 160, image::Rgba([255, 0, 0, 255])) .save(&path) .unwrap(); + let square_path = dir.path().join("generated.png"); + image::RgbImage::new(768, 768).save(&square_path).unwrap(); let cells: Vec> = vec![ Arc::new(new_view_image_tool_call( "view-call".to_string(), LegacyAppPathString::from_string(path.to_string_lossy().into_owned()), )), + Arc::new( + crate::history_cell::AgentMarkdownCell::new( + "Here is the image.".to_string(), + dir.path(), + ) + .with_phase(Some(codex_protocol::models::MessagePhase::FinalAnswer)), + ), Arc::new(new_image_generation_call( "image-call".to_string(), "completed", - Some("A red rectangle".to_string()), - Some(AbsolutePathBuf::try_from(path.clone()).unwrap()), + Some("A square".to_string()), + Some(AbsolutePathBuf::try_from(square_path.clone()).unwrap()), + )), + Arc::new(crate::history_cell::FinalMessageSeparator::new( + Some(2), + /*runtime_metrics*/ None, )), ]; let area = Rect::new( @@ -193,8 +206,18 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() { let text_only = buf.clone(); view.image_protocol = Some(ImageProtocol::Kitty); view.render(area, &mut buf, &cells); + view.sync_turn_details(&cells, /*enabled*/ true, /*active*/ None); + view.render(area, &mut buf, &cells); let collapsed = frame(&buf); let collapsed_areas: Vec<_> = view.image_placements().iter().map(|p| p.area).collect(); + assert_eq!(collapsed_areas.len(), 1); + assert!( + collapsed.find("Here is the image.").unwrap() < collapsed.find("Generated image").unwrap() + ); + assert_eq!(collapsed_areas[0].x, area.x + 2); + assert!(collapsed_areas[0].height > 6); + view.sync_turn_details(&cells, /*enabled*/ false, /*active*/ None); + view.render(area, &mut buf, &cells); // Find may request full text, but only manual expansion may reveal inspected pixels. assert!( view.entry_layout(&cells, /*index*/ 0, /*full_content*/ true) @@ -220,7 +243,8 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() { frame(&buf) ) ); - for (index, expected) in text.iter().enumerate() { + for index in [0, 2] { + let expected = &text[index]; let layout = view.layout(&cells, index).unwrap(); assert_eq!(layout.text(), expected); let pinned_narrow = layout.rewrap(/*width*/ 8); @@ -231,7 +255,7 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() { } // Short, inset viewports may begin in an image. Its placement stays inside the viewport. let short = Rect::new( - /*x*/ 7, /*y*/ 5, /*width*/ 16, /*height*/ 3, + /*x*/ 7, /*y*/ 5, /*width*/ 16, /*height*/ 5, ); view.jump_to_latest(); view.render(short, &mut Buffer::empty(short), &cells); @@ -266,6 +290,7 @@ fn fullscreen_image_previews_follow_layout_without_entering_text_surfaces() { // Resumed sessions with a deleted file fall back to the original text entry. std::fs::remove_file(&path).unwrap(); + std::fs::remove_file(&square_path).unwrap(); view.image_protocol = Some(ImageProtocol::Kitty); view.render(area, &mut buf, &cells); assert_eq!(buf, text_only); diff --git a/codex-rs/tui/src/transcript_view/search.rs b/codex-rs/tui/src/transcript_view/search.rs index c320557e1b..037207c98d 100644 --- a/codex-rs/tui/src/transcript_view/search.rs +++ b/codex-rs/tui/src/transcript_view/search.rs @@ -595,6 +595,15 @@ impl TranscriptView { &mut self, cells: &[Arc], inserted: Range, + ) { + self.history_page_loaded(cells, inserted, /*inserted_images*/ None); + } + + pub(crate) fn history_page_loaded( + &mut self, + cells: &[Arc], + inserted: Range, + inserted_images: Option>, ) { self.invalidate_turn_details_from(/*index*/ 0); // With only a header and live output, the first older page extends the canonical tail. @@ -604,7 +613,17 @@ impl TranscriptView { { self.last_tail = cells.last().map(EntryKey::cell); } + let image_span = inserted_images.map(|images| { + // The older-page prepend shifts the separately inserted images. + let len = inserted.len(); + self.prepend_snapshot_history(cells, images.start + len..images.end + len) + }); let inserted = self.prepend_snapshot_history(cells, inserted); + let inserted = if inserted.is_empty() { + image_span.unwrap_or(inserted) + } else { + inserted + }; if let Position::Reading(anchor) = self.position { let index = self.resolve(cells, anchor); self.position = Position::Reading(Anchor { diff --git a/codex-rs/tui/src/transcript_view/search_tests.rs b/codex-rs/tui/src/transcript_view/search_tests.rs index 7e031bcd3b..d43b15245a 100644 --- a/codex-rs/tui/src/transcript_view/search_tests.rs +++ b/codex-rs/tui/src/transcript_view/search_tests.rs @@ -918,6 +918,33 @@ fn search_scans_a_page_inserted_after_the_retained_session_header() { assert!(Arc::ptr_eq(&cells[0], &header)); } +#[test] +fn search_resumes_from_the_text_in_a_page_with_a_relocated_image() { + let mut cells = vec![cell("answer"), cell("completion")]; + let mut view = TranscriptView { + area: Rect::new( + /*x*/ 0, /*y*/ 0, /*width*/ 80, /*height*/ 6, + ), + history: TranscriptHistoryState::Partial, + ..TranscriptView::default() + }; + view.begin_search(); + view.paste_search("needle"); + finish_scan(&mut view, &cells); + assert!(view.search.needs_history(view.history)); + + let match_cell = cell("older needle"); + cells.insert(/*index*/ 1, cell("Generated image")); + cells.insert(/*index*/ 0, Arc::clone(&match_cell)); + view.history = TranscriptHistoryState::Complete; + view.history_page_loaded(&cells, 0..1, Some(1..2)); + finish_scan(&mut view, &cells); + assert_eq!( + view.search.current.as_ref().unwrap().anchor.key, + EntryKey::cell(&match_cell) + ); +} + #[test] fn search_finds_the_first_older_entry_after_replaying_the_session_header() { let header = || -> Arc { diff --git a/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__layout__tests__fullscreen_image_artifact_rows.snap b/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__layout__tests__fullscreen_image_artifact_rows.snap index 7827815393..704649a10b 100644 --- a/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__layout__tests__fullscreen_image_artifact_rows.snap +++ b/codex-rs/tui/src/transcript_view/snapshots/codex_tui__transcript_view__layout__tests__fullscreen_image_artifact_rows.snap @@ -1,16 +1,42 @@ --- source: tui/src/transcript_view/layout_tests.rs -assertion_line: 210 +assertion_line: 239 expression: "format!(\"Collapsed inspection / generated output\\n{collapsed}\\nGraphics: {collapsed_areas:?}\\n\\nExpanded inspection / generated output\\n{}\\nGraphics: {expanded_areas:?}\",\nframe(&buf))" --- Collapsed inspection / generated output -• Viewed image artifact.png - + Show details + ▸ Viewed image /workspace/artifact.png -• Generated Image: - └ A red rectangle - └ Saved to: file:///workspace/artifact.png -Graphics: [Rect { x: 52, y: 6, width: 36, height: 6 }] +• Here is the image. + + Generated image + + + + + + + + + + + + + + + + + + + + + + + + + + + Worked for 2s +Graphics: [Rect { x: 2, y: 5, width: 50, height: 25 }] Expanded inspection / generated output • Viewed image /workspace/artifact.png @@ -22,7 +48,34 @@ Expanded inspection / generated output -• Generated Image: - └ A red rectangle - └ Saved to: file:///workspace/artifact.png -Graphics: [Rect { x: 52, y: 2, width: 36, height: 6 }, Rect { x: 52, y: 12, width: 36, height: 6 }] +• Here is the image. + + Generated image + + + + + + + + + + + + + + + + + + + + + + + + + + + Worked for 2s +Graphics: [Rect { x: 52, y: 2, width: 36, height: 6 }, Rect { x: 2, y: 12, width: 50, height: 25 }] diff --git a/codex-rs/tui/src/transcript_view/text.rs b/codex-rs/tui/src/transcript_view/text.rs index 3ab2157758..30944944b2 100644 --- a/codex-rs/tui/src/transcript_view/text.rs +++ b/codex-rs/tui/src/transcript_view/text.rs @@ -47,6 +47,7 @@ use ratatui::text::Line; /// One cell's logical text and its width-dependent display rows. pub(crate) struct TextLayout { pub(super) image: Option>, + pub(super) image_left_aligned: bool, logical: Vec, text: String, rows: Vec, @@ -212,6 +213,7 @@ impl TextLayout { layout = layout.with_control(control.label.clone(), control.anchor); } layout.image = self.image.clone(); + layout.image_left_aligned = self.image_left_aligned; if self.leading_blank_hidden { layout = layout.without_leading_blank_row(); } @@ -941,6 +943,7 @@ impl TextLayout { } Self { image: None, + image_left_aligned: false, logical, text, rows, diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index e6cd10f4fe..e32f831d55 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -112,6 +112,7 @@ pub(crate) struct InitializedTerminal { pub(crate) terminal: Terminal, pub(crate) enhanced_keys_supported: bool, pub(crate) terminal_app_over_ssh: bool, + pub(crate) herdr_kitty_graphics: Option, pub(crate) stderr_guard: terminal_stderr::TerminalStderrGuard, } @@ -516,6 +517,12 @@ pub(crate) fn init() -> Result { #[cfg(unix)] let backend = CrosstermBackend::new(stdout()); + #[cfg(unix)] + let probe_herdr = { + let info = codex_terminal_detection::terminal_info(); + info.multiplexer.is_none() && info.term_program.as_deref() == Some("herdr") + }; + #[cfg(unix)] let startup_probe = { use crate::terminal_probe::StartupKeyboardEnhancementProbe; @@ -526,8 +533,11 @@ pub(crate) fn init() -> Result { } else { StartupKeyboardEnhancementProbe::Query }; - match crate::terminal_probe::startup(crate::terminal_probe::DEFAULT_TIMEOUT, keyboard_probe) - { + match crate::terminal_probe::startup( + crate::terminal_probe::DEFAULT_TIMEOUT, + keyboard_probe, + probe_herdr, + ) { Ok(probe) => { tracing::info!( duration_ms = %started_at.elapsed().as_millis(), @@ -543,13 +553,7 @@ pub(crate) fn init() -> Result { duration_ms = %started_at.elapsed().as_millis(), "terminal startup probes failed: {err}" ); - crate::terminal_probe::StartupProbe { - alternate_scroll_enabled: None, - cursor_position: None, - default_colors: None, - keyboard_enhancement_supported: None, - terminal_app_over_ssh: None, - } + crate::terminal_probe::StartupProbe::default() } } }; @@ -613,6 +617,10 @@ pub(crate) fn init() -> Result { .unwrap_or(/*default*/ false), #[cfg(not(unix))] terminal_app_over_ssh: false, + #[cfg(unix)] + herdr_kitty_graphics: probe_herdr.then_some(startup_probe.kitty_graphics), + #[cfg(not(unix))] + herdr_kitty_graphics: None, stderr_guard, }; restore_guard.active = false; @@ -744,6 +752,7 @@ pub struct Tui { enhanced_keys_supported: bool, // Cache the startup result so later configuration loads use the same terminal policy. pub(crate) terminal_app_over_ssh: bool, + pub(crate) herdr_kitty_graphics: Option, notification_backend: Option, notification_condition: NotificationCondition, scrollback: ScrollbackStrategy, @@ -822,6 +831,7 @@ impl Tui { terminal_focused: Arc::new(AtomicBool::new(true)), enhanced_keys_supported, terminal_app_over_ssh: false, + herdr_kitty_graphics: None, notification_backend: Some(detect_backend(NotificationMethod::default())), notification_condition: NotificationCondition::default(), scrollback, diff --git a/codex-rs/tui/src/tui/inline_image_history_tests.rs b/codex-rs/tui/src/tui/inline_image_history_tests.rs index 9caaf5b7ab..6137eb6fd9 100644 --- a/codex-rs/tui/src/tui/inline_image_history_tests.rs +++ b/codex-rs/tui/src/tui/inline_image_history_tests.rs @@ -49,6 +49,7 @@ fn preview_at(protocol: ImageProtocol, y: u16) -> ImagePlacement { area: Rect::new(/*x*/ 0, y, /*width*/ 2, /*height*/ 1), first_row: 0, file: None, + terminal_id: None, } }