diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 6c158f14b6..f3eb880312 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -1036,16 +1036,18 @@ fn main() -> anyhow::Result<()> { codex_build_info::initialize!(); let remote_control_disabled = codex_app_server::take_remote_control_disabled_env(); arg0_dispatch_or_else(move |arg0_paths: Arg0DispatchPaths| async move { - // Keep the CLI dispatcher off the runtime's stack while the TUI rebuilds a thread. - Box::pin(cli_main(arg0_paths, remote_control_disabled)).await?; + // Finish polling the large subcommand dispatcher before polling interactive TUI events. + if let Some(tui) = Box::pin(dispatch_cli(arg0_paths, remote_control_disabled)).await? { + tui.await?; + } Ok(()) }) } -async fn cli_main( +async fn dispatch_cli( arg0_paths: Arg0DispatchPaths, remote_control_disabled: bool, -) -> anyhow::Result<()> { +) -> anyhow::Result>>> { let MultitoolCli { config_overrides: mut root_config_overrides, feature_toggles, @@ -1097,6 +1099,11 @@ async fn cli_main( profile_v2_for_subcommand(&interactive, subcommand)?; } + let launch_tui = move |interactive, remote, auth_token_env, arg0_paths| async move { + let exit_info = + run_interactive_tui(interactive, remote, auth_token_env, arg0_paths).await?; + handle_app_exit(exit_info, daemon_cli_executable.as_deref()) + }; let open_agents_overview = matches!(&subcommand, Some(Subcommand::Agents(_))); match subcommand { None | Some(Subcommand::Agents(_)) => { @@ -1129,17 +1136,15 @@ async fn cli_main( } interactive.agents_overview = true; } - let exit_info = run_interactive_tui( + return Ok(Some(launch_tui( interactive, - root_remote.clone(), - root_remote_auth_token_env.clone(), - arg0_paths.clone(), - ) - .await?; - handle_app_exit(exit_info, daemon_cli_executable.as_deref())?; + root_remote, + root_remote_auth_token_env, + arg0_paths, + ))); } Some(Subcommand::TcpTunnel(args)) => { - return codex_tcp_tunnel::run(args).await; + codex_tcp_tunnel::run(args).await?; } Some(Subcommand::Exec(mut exec_cli)) => { reject_remote_mode_for_subcommand( @@ -1252,15 +1257,15 @@ async fn cli_main( subcommand.is_none(), "--print-startup-config cannot run a subcommand" ); - return codex_app_server::print_startup_config( + codex_app_server::print_startup_config( arg0_paths, root_config_overrides, LoaderOverrides::default(), strict_config, startup_config, ) - .await - .map_err(Into::into); + .await?; + return Ok(None); } match subcommand { None => { @@ -1405,7 +1410,7 @@ async fn cli_main( AppServerDaemonSubcommand::PidUpdateLoop { check_package_ownership: true, .. - } => return Ok(()), + } => return Ok(None), AppServerDaemonSubcommand::Update { from_cli: false, .. } @@ -1528,16 +1533,12 @@ async fn cli_main( include_non_interactive, config_overrides, ); - let exit_info = run_interactive_tui( + return Ok(Some(launch_tui( interactive, - remote.remote.or(root_remote.clone()), - remote - .remote_auth_token_env - .or(root_remote_auth_token_env.clone()), - arg0_paths.clone(), - ) - .await?; - handle_app_exit(exit_info, daemon_cli_executable.as_deref())?; + remote.remote.or(root_remote), + remote.remote_auth_token_env.or(root_remote_auth_token_env), + arg0_paths, + ))); } Some(Subcommand::Archive(cmd)) => { let output = run_session_archive_cli_command( @@ -1628,16 +1629,12 @@ async fn cli_main( all, config_overrides, ); - let exit_info = run_interactive_tui( + return Ok(Some(launch_tui( interactive, - remote.remote.or(root_remote.clone()), - remote - .remote_auth_token_env - .or(root_remote_auth_token_env.clone()), - arg0_paths.clone(), - ) - .await?; - handle_app_exit(exit_info, daemon_cli_executable.as_deref())?; + remote.remote.or(root_remote), + remote.remote_auth_token_env.or(root_remote_auth_token_env), + arg0_paths, + ))); } Some(Subcommand::Login(mut login_cli)) => { reject_remote_mode_for_subcommand( @@ -1762,7 +1759,7 @@ async fn cli_main( .parse_overrides() .map_err(anyhow::Error::msg)?; sandbox_setup::run(setup_cli, config_profile.cloned(), cli_overrides).await?; - return Ok(()); + return Ok(None); } reject_remote_mode_for_subcommand( root_remote.as_deref(), @@ -1770,7 +1767,8 @@ async fn cli_main( "sandbox", )?; if let Some(SandboxSubcommand::Uninstall(command)) = subcommand { - return command.run(); + command.run()?; + return Ok(None); } let loader_overrides = loader_overrides_for_profile(config_profile)?; #[cfg(target_os = "macos")] @@ -1950,7 +1948,7 @@ async fn cli_main( }, } - Ok(()) + Ok(None) } fn profile_v2_for_subcommand<'a>( diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 205674fbb1..cef1a77f0b 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -1247,12 +1247,22 @@ impl App { Ok(()) } + fn prepare_frame_images(&mut self) -> Option { + let protocol = self.chat_widget.pet_image_support().protocol().filter(|_| { + !self.app_server_target.uses_remote_workspace() + && crate::projectless::has_only_local_environments(&self.environment_manager) + }); + self.chat_widget.prepare_composer_images(protocol); + protocol + } + fn render_chat_widget_frame(&mut self, tui: &mut tui::Tui, screen_size: Size) -> Result { self.chat_widget .set_sparkle_terminal_focus(tui.is_terminal_focused()); if tui.is_owned_screen() { return self.render_owned_transcript(tui, screen_size); } + self.prepare_frame_images(); self.chat_widget .empty_state_animation .borrow_mut() @@ -1287,6 +1297,9 @@ impl App { let area = frame.area(); rendered_area = area; chat_widget.render(area, frame.buffer); + if let Some(previews) = &self.chat_widget.composer_images { + frame.image_placements = previews.placements.take(); + } self.chat_widget.note_rendered_width(area.width); if let Some((x, y)) = chat_widget.cursor_pos(area) { frame.set_cursor_style(chat_widget.cursor_style(area)); diff --git a/codex-rs/tui/src/app/owned_transcript.rs b/codex-rs/tui/src/app/owned_transcript.rs index f87102e1c7..76432f6ec9 100644 --- a/codex-rs/tui/src/app/owned_transcript.rs +++ b/codex-rs/tui/src/app/owned_transcript.rs @@ -77,13 +77,10 @@ impl App { } fn sync_owned_transcript(&mut self, width: u16) -> bool { + self.transcript_view.image_protocol = self.prepare_frame_images(); let chat_widget = &self.chat_widget; let transcript_width = chat_widget.history_wrap_width(width); let view = &mut self.transcript_view; - view.image_protocol = chat_widget.pet_image_support().protocol().filter(|_| { - !self.app_server_target.uses_remote_workspace() - && crate::projectless::has_only_local_environments(&self.environment_manager) - }); view.mouse_scroll_speed = self.local_settings.tui.mouse_scroll_speed.unwrap_or(1.0); view.primary_selection = self.right_click_paste_environment.primary; view.copy_on_select = self @@ -367,7 +364,10 @@ impl App { frame.set_cursor_position(position); } if chat_widget.no_modal_or_popup_active() { - frame.transcript_images = view.image_placements(); + frame.image_placements = view.image_placements(); + if let Some(previews) = &chat_widget.composer_images { + frame.image_placements.extend(previews.placements.take()); + } } })?; if let Some((surface, tip)) = &turn_tip diff --git a/codex-rs/tui/src/app/owned_transcript_tests.rs b/codex-rs/tui/src/app/owned_transcript_tests.rs index c3ec487f6d..2317862c29 100644 --- a/codex-rs/tui/src/app/owned_transcript_tests.rs +++ b/codex-rs/tui/src/app/owned_transcript_tests.rs @@ -1864,6 +1864,106 @@ async fn fullscreen_composer_scrolls_without_moving_caret_or_transcript() -> Res Ok(()) } +#[tokio::test] +async fn composer_thumbnails_follow_draft_deletion_and_resize() -> Result<()> { + use crate::custom_terminal::test_support::last_rendered_buffer; + use crate::pets::ImageProtocol; + use crate::pets::PetImageSupport; + use crate::pets::PetImageUnsupportedReason; + + let dir = tempfile::tempdir()?; + let banner = dir.path().join("banner.png"); + let portrait = dir.path().join("portrait.png"); + let red = image::Rgba([255, 0, 0, 255]); + image::RgbaImage::from_pixel(/*width*/ 400, /*height*/ 1, red).save(&banner)?; + + for owned in [false, true] { + image::RgbaImage::new(/*width*/ 4, /*height*/ 400).save(&portrait)?; + let mut app = crate::app::test_support::make_test_app().await; + let mut tui = crate::tui::test_support::make_test_tui()?; + tui.set_owned_screen(owned)?; + app.chat_widget + .set_pet_image_support_for_tests(PetImageSupport::Supported(ImageProtocol::Kitty)); + app.chat_widget.attach_image(portrait.clone()); + app.chat_widget.handle_key_event(KeyCode::Home.into()); + app.chat_widget.attach_image(banner.clone()); + app.chat_widget.handle_key_event(KeyCode::End.into()); + app.chat_widget.attach_image(banner.clone()); + + let narrow = Size::new(/*width*/ 20, /*height*/ 24); + app.render_chat_widget_frame(&mut tui, narrow)?; + let placements = &tui.retained_images; + assert_eq!(placements.len(), 2); + let pixels = &placements[0].image.pixels; + // The whole panorama fits in a single pixel row inside the outlined tile. + assert_eq!( + pixels.pixels().filter(|pixel| **pixel == red).count(), + (pixels.width() - 2) as usize + ); + let clear = &placements[1].image.pixels; + assert_eq!( + clear.get_pixel(clear.width() / 2, clear.height() / 2), + clear.get_pixel(/*x*/ 1, /*y*/ 1) + ); + assert_ne!( + clear.get_pixel(/*x*/ 0, /*y*/ 0), + clear.get_pixel(/*x*/ 1, /*y*/ 1) + ); + + let buffer = last_rendered_buffer(&tui.terminal); + assert!(buffer_text(buffer).contains("+1")); + if owned { + // The terminal paints graphics after its text buffer. Mark their footprints + // for this snapshot of the strip and the first editable line. + let mut strip = buffer.clone(); + for placement in placements { + for position in placement.area.positions() { + strip[position].set_symbol("░"); + } + } + let snapshot = buffer_text(&strip) + .lines() + .skip(usize::from(placements[0].area.y - strip.area.y)) + .take(6) + .collect::>() + .join("\n"); + insta::assert_snapshot!("composer_thumbnail_strip", snapshot); + } + + let wide = Size::new(/*width*/ 60, /*height*/ 24); + app.render_chat_widget_frame(&mut tui, wide)?; + assert_eq!(tui.retained_images.len(), 3); + let portrait_image = tui.retained_images[1].image.clone(); + app.chat_widget.handle_key_event(KeyCode::Home.into()); + app.chat_widget.handle_key_event(KeyCode::Delete.into()); + app.render_chat_widget_frame(&mut tui, wide)?; + assert_eq!(tui.retained_images.len(), 2); + assert_eq!(tui.retained_images[0].image.pixels, portrait_image.pixels); + // A draft submits the current file, even if it was replaced since attachment. + image::RgbaImage::from_pixel(/*width*/ 2, /*height*/ 2, red).save(&portrait)?; + app.render_chat_widget_frame(&mut tui, wide)?; + let updated = &tui.retained_images[0].image.pixels; + assert!(updated.pixels().any(|pixel| *pixel == red)); + // Both insufficient height and an unsupported terminal leave the references editable. + for (height, support) in [ + (5, PetImageSupport::Supported(ImageProtocol::Kitty)), + ( + 24, + PetImageSupport::Unsupported(PetImageUnsupportedReason::Terminal), + ), + ] { + app.chat_widget.set_pet_image_support_for_tests(support); + app.render_chat_widget_frame(&mut tui, Size::new(wide.width, height))?; + assert!(tui.retained_images.is_empty()); + assert!( + buffer_text(last_rendered_buffer(&tui.terminal)).contains("[Image #1][Image #2]") + ); + } + tui.set_owned_screen(/*owned*/ false)?; + } + Ok(()) +} + #[tokio::test] async fn fullscreen_composer_remote_images_leave_an_editable_prompt_row() -> Result<()> { let mut app = crate::app::test_support::make_test_app().await; diff --git a/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_transcript__tests__composer_thumbnail_strip.snap b/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_transcript__tests__composer_thumbnail_strip.snap new file mode 100644 index 0000000000..4e0453597b --- /dev/null +++ b/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_transcript__tests__composer_thumbnail_strip.snap @@ -0,0 +1,10 @@ +--- +source: tui/src/app/owned_transcript_tests.rs +expression: snapshot +--- + ░░░░░░ ░░░░░░ +1 + ░░░░░░ ░░░░░░ + ░░░░░░ ░░░░░░ + +› [Image #2][Image + #1][Image #3] diff --git a/codex-rs/tui/src/app/startup.rs b/codex-rs/tui/src/app/startup.rs index fe37a84c62..c47633c83c 100644 --- a/codex-rs/tui/src/app/startup.rs +++ b/codex-rs/tui/src/app/startup.rs @@ -1694,7 +1694,8 @@ See the Codex keymap documentation for supported actions and examples." tracing::warn!(error = %err, "failed to shut down embedded app server"); } let clear_pet_result = tui.clear_ambient_pet_image(); - let clear_result = tui.terminal.clear(); + let clear_images = tui.terminal.clear_frame_images(&mut tui.retained_images); + let clear_result = clear_images.and(tui.terminal.clear()); if let Some(task) = app.daemon_restart.take() && let Err(err) = task.await.unwrap_or_else(|err| Err(err.into())) { diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 73d779e69a..7e194ed628 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -230,6 +230,10 @@ //! - When all placeholders for a size are cleared or deleted, the next paste of that size reuses //! the base label without a suffix. //! +//! On supported terminals the local draft references also have a strip of thumbnails. +//! It shows the first attachments that fit and a count of those hidden. Removing a +//! reference removes its preview through the same attachment bookkeeping used for submission. +//! //! # Remote Image Rows (Up/Down/Delete) //! //! Remote image URLs are rendered as non-editable `[Image #N]` rows above the textarea (inside the @@ -4591,6 +4595,7 @@ impl ChatComposer { status, composer: composer_rect, remote_images: remote_images_rect, + thumbnails, textarea: textarea_rect, popup: popup_rect, footer: footer_rect, @@ -4885,6 +4890,14 @@ impl ChatComposer { } let style = user_message_style(); Block::default().style(style).render(composer_rect, buf); + if let Some(previews) = options.image_previews { + previews.render( + thumbnails, + buf, + &self.attachments.local_images, + &self.draft.textarea, + ); + } if !remote_images_rect.is_empty() { let first = self .attachments diff --git a/codex-rs/tui/src/bottom_pane/chat_composer/composer_layout.rs b/codex-rs/tui/src/bottom_pane/chat_composer/composer_layout.rs index 949edcf916..d687ed06a8 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer/composer_layout.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer/composer_layout.rs @@ -125,9 +125,26 @@ impl ChatComposer { width: textarea_rect.width, height: remote_images_height, }; - textarea_rect.y = textarea_rect.y.saturating_add(consumed); - textarea_rect.height = textarea_rect.height.saturating_sub(consumed); + let mut thumbnails = Rect { + y: textarea_rect.y.saturating_add(consumed), + height: self.thumbnail_height(textarea_rect.width, options), + ..textarea_rect + }; + let available = textarea_rect.height.saturating_sub(consumed); + if thumbnails.height > 0 + && available.saturating_sub(thumbnails.height) + < self + .draft + .textarea + .desired_height(textarea_rect.width) + .max(1) + { + thumbnails.height = 0; + } + textarea_rect.y = thumbnails.bottom(); + textarea_rect.height = available - thumbnails.height; ComposerLayout { + thumbnails, status, composer: composer_rect, remote_images: remote_images_rect, @@ -137,6 +154,12 @@ impl ChatComposer { } } + fn thumbnail_height(&self, width: u16, options: ComposerRenderOptions<'_>) -> u16 { + options.image_previews.map_or(0, |previews| { + previews.height(width, self.attachments.local_images.len()) + }) + } + pub(crate) fn cursor_pos_with_options( &self, area: Rect, @@ -185,6 +208,7 @@ impl ChatComposer { let remote_images_separator = u16::from(remote_images_height > 0); self.draft.textarea.desired_height(inner_width) + remote_images_height + + self.thumbnail_height(inner_width, options) + remote_images_separator + self.status_surface_height(options) + if self.shortcuts_above_composer(options) { diff --git a/codex-rs/tui/src/bottom_pane/chat_composer/footer_state.rs b/codex-rs/tui/src/bottom_pane/chat_composer/footer_state.rs index c0bee40bdf..a130b91b7e 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer/footer_state.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer/footer_state.rs @@ -44,6 +44,7 @@ pub(super) struct ComposerLayout { pub(super) status: Rect, pub(super) composer: Rect, pub(super) remote_images: Rect, + pub(super) thumbnails: Rect, pub(super) textarea: Rect, pub(super) popup: Rect, pub(super) footer: Rect, @@ -71,6 +72,7 @@ pub(crate) enum CommandPopupPlacement { pub(crate) struct ComposerRenderOptions<'a> { /// Fullscreen limits the composer independently of activity and modal views. pub(crate) max_height: Option, + pub(crate) image_previews: Option<&'a crate::transcript_image::composer::ComposerImages>, pub(crate) rendered_selection: Option<&'a std::cell::RefCell>, /// Reserve a shared hint row independently of whether it currently contains a notice. diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 16f0aa4c5c..d38dd154ac 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -2555,6 +2555,7 @@ impl BottomPane { let composer: RenderableItem<'_> = if let Some(questions) = question_editor { RenderableItem::Borrowed(questions.as_ref()) } else if options.max_height.is_none() + && options.image_previews.is_none() && options.textarea_right_reserve == 0 && options.warning_count == 0 && options.footer.is_none() diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index f9c05e0349..d2962e8001 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -599,6 +599,7 @@ pub(crate) struct ChatWidget { // Remember the account's Reserve entry notice across chats and transient banner refreshes. luna_reserve_notice_account_id: Option, pub(crate) warning_display_state: WarningDisplayState, + pub(crate) composer_images: Option, pub(crate) rendered_selection: std::cell::RefCell, rate_limit_switch_prompt: RateLimitSwitchPromptState, add_credits_nudge_email_in_flight: Option, diff --git a/codex-rs/tui/src/chatwidget/constructor.rs b/codex-rs/tui/src/chatwidget/constructor.rs index 71c4f23aba..6e405d201e 100644 --- a/codex-rs/tui/src/chatwidget/constructor.rs +++ b/codex-rs/tui/src/chatwidget/constructor.rs @@ -178,6 +178,7 @@ impl ChatWidget { backend_banner_notice_model: None, luna_reserve_notice_account_id: None, warning_display_state: WarningDisplayState::default(), + composer_images: None, rendered_selection: Default::default(), rate_limit_switch_prompt: RateLimitSwitchPromptState::default(), add_credits_nudge_email_in_flight: None, diff --git a/codex-rs/tui/src/chatwidget/rendering.rs b/codex-rs/tui/src/chatwidget/rendering.rs index 97a518cd8a..8ab45414e5 100644 --- a/codex-rs/tui/src/chatwidget/rendering.rs +++ b/codex-rs/tui/src/chatwidget/rendering.rs @@ -122,6 +122,20 @@ impl ExternalWriterNotice { } impl ChatWidget { + pub(crate) fn prepare_composer_images(&mut self, protocol: Option) { + self.composer_images = protocol + .filter(|_| { + !self.bottom_pane.composer_local_images().is_empty() + && self.no_modal_or_popup_active() + }) + .and_then(|protocol| { + crate::transcript_image::composer::ComposerImages::for_terminal( + protocol, + self.composer_images.take(), + ) + }); + } + pub(crate) fn as_renderable(&self) -> RenderableItem<'_> { let backdrop = self.as_backdrop_renderable(); match self.centered_dialog() { @@ -270,6 +284,7 @@ impl ChatWidget { self.ambient_pet_wrap_reserved_cols() }; options.rendered_selection = Some(&self.rendered_selection); + options.image_previews = self.composer_images.as_ref(); options.warning_count = self.warning_display_state.count; options.textarea_right_reserve = right_reserve; options.separate_status_line = options.command_popup_placement diff --git a/codex-rs/tui/src/custom_terminal.rs b/codex-rs/tui/src/custom_terminal.rs index 0607a8ca6a..1bd2968e7a 100644 --- a/codex-rs/tui/src/custom_terminal.rs +++ b/codex-rs/tui/src/custom_terminal.rs @@ -82,8 +82,8 @@ pub struct Frame<'a> { /// The buffer that is used to draw the current frame pub(crate) buffer: &'a mut Buffer, - /// Fullscreen graphics to send after text, before the synchronized frame is committed. - pub(crate) transcript_images: Vec, + /// Frame graphics to send after text, before the synchronized frame is committed. + pub(crate) image_placements: Vec, } impl Frame<'_> { @@ -257,7 +257,7 @@ where /// Get a Frame object which provides a consistent view into the terminal state for rendering. pub fn get_frame(&mut self) -> Frame<'_> { Frame { - transcript_images: Vec::new(), + image_placements: Vec::new(), cursor_position: None, cursor_style: SetCursorStyle::DefaultUserShape, viewport_area: self.viewport_area, @@ -353,6 +353,28 @@ where self.draw_with_size(screen_size, &mut Vec::new(), render_callback) } + /// Remove graphics on the screen that owns them before switching buffers or yielding the terminal. + pub(crate) fn clear_frame_images( + &mut self, + images: &mut Vec, + ) -> io::Result<()> { + if images.is_empty() { + return Ok(()); + } + let text = self.previous_buffer().clone(); + let cursor = (!self.hidden_cursor).then_some(self.last_known_cursor_pos); + let style = self + .last_cursor_style + .unwrap_or(SetCursorStyle::DefaultUserShape); + self.draw_with_size(self.last_known_screen_size, images, |frame| { + *frame.buffer = text; + if let Some(cursor) = cursor { + frame.set_cursor_position(cursor); + frame.set_cursor_style(style); + } + }) + } + /// Draws a single frame using a screen size already obtained by the caller. pub(crate) fn draw_with_size( &mut self, @@ -375,7 +397,7 @@ where // Buffer. Thus, we're taking the important data out of the Frame and dropping it. let cursor_position = frame.cursor_position; let cursor_style = frame.cursor_style; - let images = std::mem::take(&mut frame.transcript_images); + let images = std::mem::take(&mut frame.image_placements); let changed = *retained_images != images; if changed { // Remove old graphics before text painting, including when switching to overlays. @@ -931,7 +953,7 @@ mod tests { frame.buffer_mut(), ); frame - .transcript_images + .image_placements .push(crate::transcript_image::ImagePlacement { image: std::sync::Arc::clone(&image), area: Rect::new( diff --git a/codex-rs/tui/src/transcript_image.rs b/codex-rs/tui/src/transcript_image.rs index 9aa9159603..d3f91705d3 100644 --- a/codex-rs/tui/src/transcript_image.rs +++ b/codex-rs/tui/src/transcript_image.rs @@ -1,9 +1,11 @@ -//! Bounded image previews for the owned fullscreen transcript. +//! Bounded local image previews for the transcript and composer. //! //! Only decoded local image pixels reach the terminal encoders. Previews are never source text; //! their reserved rows belong to layout. The terminal retains visible placements and any local //! transport files until those placements change. +pub(crate) mod composer; + use std::io::Write; use std::path::Path; use std::sync::Arc; @@ -14,6 +16,7 @@ use crossterm::cursor::RestorePosition; use crossterm::cursor::SavePosition; use crossterm::queue; use image::DynamicImage; +use image::ImageDecoder; use image::ImageFormat; use image::RgbaImage; use ratatui::layout::Rect; @@ -60,20 +63,11 @@ pub(crate) fn terminal_cell_size() -> Option<(u32, u32)> { } impl TranscriptImage { - pub(crate) fn load(path: &Path, width: u16, protocol: ImageProtocol) -> Option { - let columns = width.saturating_sub(2).min(48); + fn load_pixels(path: &Path, max_width: u32, max_height: u32) -> Option { let metadata = path.metadata().ok()?; - // Bound encoded input as well as the raster: metadata alone can dwarf a tiny image. - if columns == 0 || !metadata.is_file() || metadata.len() > 32 * 1024 * 1024 { + if max_width == 0 || !metadata.is_file() || metadata.len() > 32 * 1024 * 1024 { return None; } - // Sixel paints physical pixels rather than sizing to a Kitty cell rectangle. Guessing - // the cell size could paint beyond the reserved rows and cover the composer. - let (cell_width, cell_height) = match protocol { - ImageProtocol::Sixel => terminal_cell_size()?, - // Kitty scales to cell coordinates itself. Use a stable, typical cell aspect ratio. - ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (8, 16), - }; let mut reader = image::ImageReader::open(path) .ok()? .with_guessed_format() @@ -82,22 +76,40 @@ impl TranscriptImage { // screenshots usable, but never allocate hundreds of MB for an optional thumbnail. let mut limits = image::Limits::default(); limits.max_alloc = Some(32 * 1024 * 1024); - reader.limits(limits); - let image = reader.decode().ok()?; + reader.limits(limits.clone()); + let mut decoder = reader.into_decoder().ok()?; + limits.reserve(decoder.total_bytes()).ok()?; + decoder.set_limits(limits).ok()?; + let orientation = decoder.orientation().ok(); + let mut image = DynamicImage::from_decoder(decoder).ok()?; + if let Some(orientation) = orientation { + image.apply_orientation(orientation); + } + Some( + image + .thumbnail(max_width.min(image.width()), max_height.min(image.height())) + .to_rgba8(), + ) + } + + pub(crate) fn load(path: &Path, width: u16, protocol: ImageProtocol) -> Option { + let columns = width.saturating_sub(2).min(48); + // Sixel paints physical pixels. Guessing could paint outside the reserved rows. + let (cell_width, cell_height) = match protocol { + ImageProtocol::Sixel => terminal_cell_size()?, + ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (8, 16), + }; // The sprite encoder scans each active color per Sixel band. Bound both its CPU // work and worst-case output even with large terminal cells or high-entropy images. let (max_width, max_height) = match protocol { ImageProtocol::Sixel => (160, 80), ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (384, 160), }; - let pixels = image - .thumbnail( - (u32::from(columns) * cell_width) - .min(image.width()) - .min(max_width), - (6 * cell_height).min(image.height()).min(max_height), - ) - .to_rgba8(); + let pixels = Self::load_pixels( + path, + (u32::from(columns) * cell_width).min(max_width), + (6 * cell_height).min(max_height), + )?; Some(Self { protocol, columns: pixels.width().div_ceil(cell_width) as u16, diff --git a/codex-rs/tui/src/transcript_image/composer.rs b/codex-rs/tui/src/transcript_image/composer.rs new file mode 100644 index 0000000000..ee3088141f --- /dev/null +++ b/codex-rs/tui/src/transcript_image/composer.rs @@ -0,0 +1,200 @@ +//! The draft thumbnail strip keeps only visible rasters between draws and places them +//! in textarea reference order. Attachment numbering and submission order are independent. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use image::Rgba; +use image::RgbaImage; +use ratatui::buffer::Buffer; +use ratatui::layout::Rect; +use ratatui::widgets::Paragraph; +use ratatui::widgets::Widget; +use ratatui::widgets::Wrap; + +use super::ImagePlacement; +use super::TranscriptImage; +use super::terminal_cell_size; +use crate::bottom_pane::LocalImageAttachment; +use crate::bottom_pane::TextArea; +use crate::pets::ImageProtocol; + +// Six columns by three rows is square with typical 8 × 16 pixel cells. +const WIDTH: u16 = 6; +const HEIGHT: u16 = 3; + +#[derive(Hash, PartialEq, Eq)] +struct TileKey { + path: PathBuf, + version: Option<(u64, std::time::SystemTime)>, +} + +pub(crate) struct ComposerImages { + protocol: ImageProtocol, + cell_size: (u32, u32), + background: (u8, u8, u8), + cache: RefCell>>>, + pub(crate) placements: RefCell>, +} + +impl ComposerImages { + pub(crate) fn for_terminal(protocol: ImageProtocol, previous: Option) -> Option { + // Kitty can scale a nominal raster into its cell placement. Sixel needs physical pixels. + let cell_size = terminal_cell_size() + .or_else(|| (protocol != ImageProtocol::Sixel).then_some((8, 16)))?; + let background = crate::terminal_palette::default_bg() + .map(crate::style::user_message_bg_rgb) + .unwrap_or((48, 48, 48)); + if previous.as_ref().is_some_and(|old| { + (old.protocol, old.cell_size, old.background) == (protocol, cell_size, background) + }) { + return previous; + } + Some(Self { + protocol, + cell_size, + background, + cache: RefCell::default(), + placements: RefCell::default(), + }) + } + + /// Reserve a blank row between any visible strip and the draft. + pub(crate) fn height(&self, width: u16, count: usize) -> u16 { + let (visible, hidden) = visible_thumbnails(width, count); + if visible > 0 { + HEIGHT + 1 + } else if hidden > 0 && u32::from(width) >= 2 + hidden.ilog10() { + 2 + } else { + 0 + } + } + + pub(crate) fn render( + &self, + area: Rect, + buf: &mut Buffer, + attachments: &[LocalImageAttachment], + textarea: &TextArea, + ) { + let (visible, hidden) = visible_thumbnails(area.width, attachments.len()); + let mut cache = self.cache.borrow_mut(); + let mut previous = std::mem::take(&mut *cache); + let mut placements = self.placements.borrow_mut(); + placements.clear(); + let style = crate::style::secondary_text_style(); + let mut x = area.x; + if area.is_empty() { + return; + } + let mut ordered: Vec<_> = attachments.iter().collect(); + if ordered.len() > 1 { + let positions: HashMap<_, _> = textarea + .text_element_ranges() + .map(|range| (&textarea.text()[range.clone()], range.start)) + .collect(); + ordered.sort_by_key(|image| { + positions + .get(image.placeholder.as_str()) + .copied() + .unwrap_or(usize::MAX) + }); + } + for attachment in ordered.into_iter().take(visible) { + let path = &attachment.path; + let key = TileKey { + path: path.clone(), + version: path + .metadata() + .ok() + .and_then(|m| Some((m.len(), m.modified().ok()?))), + }; + let image = cache.entry(key).or_insert_with_key(|key| { + previous + .remove(key) + .unwrap_or_else(|| self.load_tile(path).map(Arc::new)) + }); + let tile = Rect::new(x, area.y, WIDTH, HEIGHT); + if let Some(image) = image { + placements.push(ImagePlacement { + image: image.clone(), + area: tile, + first_row: 0, + file: None, + }); + } else { + Paragraph::new(attachment.placeholder.as_str()) + .style(style) + .wrap(Wrap { trim: false }) + .render(tile, buf); + } + x = x.saturating_add(WIDTH + 1); + } + if hidden > 0 { + buf.set_string(x, area.y, format!("+{hidden}"), style); + } + } + + fn load_tile(&self, path: &Path) -> Option { + let (cell_width, cell_height) = self.cell_size; + let width = u32::from(WIDTH) * cell_width; + let height = u32::from(HEIGHT) * cell_height; + let scale = width.div_ceil(384).max(height.div_ceil(160)); + // Kitty scales the raster to the tile. Sixel must paint at the physical cell size. + if scale > 1 && self.protocol == ImageProtocol::Sixel { + return None; + } + let width = (width / scale).max(3); + let height = (height / scale).max(3); + let pixels = TranscriptImage::load_pixels(path, width - 2, height - 2)?; + let (r, g, b) = self.background; + let contrast = if crate::color::is_light(self.background) { + 0 + } else { + 255 + }; + let (red, green, blue) = crate::color::blend( + (contrast, contrast, contrast), + self.background, + /*alpha*/ 0.45, + ); + let edge = Rgba([red, green, blue, 255]); + let mut tile = RgbaImage::from_fn(width, height, |x, y| { + if x == 0 || y == 0 || x == width - 1 || y == height - 1 { + edge + } else { + Rgba([r, g, b, 255]) + } + }); + image::imageops::overlay( + &mut tile, + &pixels, + i64::from((width - pixels.width()) / 2), + i64::from((height - pixels.height()) / 2), + ); + Some(TranscriptImage { + protocol: self.protocol, + columns: WIDTH, + rows: HEIGHT, + pixels: tile, + cell_width, + cell_height, + }) + } +} + +fn visible_thumbnails(width: u16, count: usize) -> (usize, usize) { + let mut visible = ((usize::from(width) + 1) / usize::from(WIDTH + 1)).min(count); + while visible > 0 && visible < count { + let badge = 2 + (count - visible).ilog10() as usize; + if visible * usize::from(WIDTH + 1) + badge <= usize::from(width) { + break; + } + visible -= 1; + } + (visible, count - visible) +} diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 019f820411..25c70c4e51 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -700,7 +700,7 @@ pub struct Tui { screen_size: ScreenSizePolicy, ambient_pet_image_state: crate::pets::PetImageRenderState, pet_picker_preview_image_state: crate::pets::PetImageRenderState, - pub(crate) transcript_images: Vec, + pub(crate) retained_images: Vec, alt_saved_viewport: Option, #[cfg(unix)] suspend_context: SuspendContext, @@ -779,7 +779,7 @@ impl Tui { screen_size: ScreenSizePolicy::default(), ambient_pet_image_state: crate::pets::PetImageRenderState::default(), pet_picker_preview_image_state: crate::pets::PetImageRenderState::default(), - transcript_images: Vec::new(), + retained_images: Vec::new(), alt_saved_viewport: None, #[cfg(unix)] suspend_context: SuspendContext::new(), @@ -973,6 +973,9 @@ impl Tui { // Restore both screens' input modes before giving the terminal to an editor. Repaint the // last frame on the alternate screen so separate-window editors leave Codex visible. let was_alt_screen = self.is_alt_screen_active(); + if let Err(err) = self.terminal.clear_frame_images(&mut self.retained_images) { + tracing::warn!(%err, "failed to clear images before terminal handoff"); + } let visible_frame = (was_alt_screen && matches!(handoff, TerminalHandoff::KeepScreen)) .then(|| self.terminal.previous_buffer().clone()); let restore_result = if let Some(visible_frame) = visible_frame { @@ -1112,6 +1115,8 @@ impl Tui { if !self.alt_screen_enabled { return Ok(()); } + self.terminal + .clear_frame_images(&mut self.retained_images)?; // Entry can change buffers before later input-mode configuration fails. self.terminal.invalidate_cursor_state(); if self.owned_screen { @@ -1162,6 +1167,8 @@ impl Tui { if !self.alt_screen_enabled || !self.is_alt_screen_active() { return Ok(()); } + self.terminal + .clear_frame_images(&mut self.retained_images)?; let result = ALTERNATE_SCREEN.leave(self.terminal.backend_mut()); if ALTERNATE_SCREEN.is_active() { return result; @@ -1398,7 +1405,7 @@ impl Tui { self.suspend_context.set_cursor_y(inline_area_bottom); } - terminal.draw_with_size(screen_size, &mut self.transcript_images, draw_fn) + terminal.draw_with_size(screen_size, &mut self.retained_images, draw_fn) })??; Ok(()) } @@ -1534,7 +1541,7 @@ impl Tui { self.suspend_context.set_cursor_y(inline_area_bottom); } - terminal.draw_with_size(screen_size, &mut self.transcript_images, |frame| { + terminal.draw_with_size(screen_size, &mut self.retained_images, |frame| { draw_fn(frame); }) })??;