openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-30 19:18:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add local image thumbnails to the TUI composer (#58261)

Browse Source 
## What changed

- Show draft attachment thumbnails in reference order in inline and fullscreen views on supported terminals with local workspaces. Display a `+N` count for attachments that do not fit, and preserve editable image references when previews are unavailable or space is limited.
- Refresh previews when attached files change, apply image orientation metadata, and clear frame graphics before terminal handoffs, screen switches, and shutdown.
- Finish CLI subcommand dispatch before polling interactive TUI events.

## Testing

Add a regression test and snapshot covering thumbnail order, overflow, resizing, attachment deletion, file replacement, and fallback behavior in both inline and fullscreen views.

GitOrigin-RevId: 1ea7c89e089cecc390e145c1790b6ede8bad8d5e
This commit is contained in:
Eric Traut
2026-10-30 06:37:32 +00:00
committed by copyberry
parent d5cbb90b72
commit 8b5ce2fd61
17 changed files with 496 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/cli/src/main.rs
+35 -37
View File
@@ -1036,16 +1036,18 @@ fn main() -> anyhow::Result<()> {
codex_build_info::initialize!();
let remote_control_disabled = codex_app_server::take_remote_control_disabled_env();
arg0_dispatch_or_else(move |arg0_paths: Arg0DispatchPaths| async move {
// Keep the CLI dispatcher off the runtime's stack while the TUI rebuilds a thread.
Box::pin(cli_main(arg0_paths, remote_control_disabled)).await?;
// Finish polling the large subcommand dispatcher before polling interactive TUI events.
if let Some(tui) = Box::pin(dispatch_cli(arg0_paths, remote_control_disabled)).await? {
tui.await?;
}
Ok(())
})
}
async fn cli_main(
async fn dispatch_cli(
arg0_paths: Arg0DispatchPaths,
remote_control_disabled: bool,
) -> anyhow::Result<()> {
) -> anyhow::Result<Option<impl Future<Output = anyhow::Result<()>>>> {
let MultitoolCli {
config_overrides: mut root_config_overrides,
feature_toggles,
@@ -1097,6 +1099,11 @@ async fn cli_main(
profile_v2_for_subcommand(&interactive, subcommand)?;
}
let launch_tui = move |interactive, remote, auth_token_env, arg0_paths| async move {
let exit_info =
run_interactive_tui(interactive, remote, auth_token_env, arg0_paths).await?;
handle_app_exit(exit_info, daemon_cli_executable.as_deref())
};
let open_agents_overview = matches!(&subcommand, Some(Subcommand::Agents(_)));
match subcommand {
None | Some(Subcommand::Agents(_)) => {
@@ -1129,17 +1136,15 @@ async fn cli_main(
}
interactive.agents_overview = true;
}
let exit_info = run_interactive_tui(
return Ok(Some(launch_tui(
interactive,
root_remote.clone(),
root_remote_auth_token_env.clone(),
arg0_paths.clone(),
)
.await?;
handle_app_exit(exit_info, daemon_cli_executable.as_deref())?;
root_remote,
root_remote_auth_token_env,
arg0_paths,
)));
}
Some(Subcommand::TcpTunnel(args)) => {
return codex_tcp_tunnel::run(args).await;
codex_tcp_tunnel::run(args).await?;
}
Some(Subcommand::Exec(mut exec_cli)) => {
reject_remote_mode_for_subcommand(
@@ -1252,15 +1257,15 @@ async fn cli_main(
subcommand.is_none(),
"--print-startup-config cannot run a subcommand"
);
return codex_app_server::print_startup_config(
codex_app_server::print_startup_config(
arg0_paths,
root_config_overrides,
LoaderOverrides::default(),
strict_config,
startup_config,
)
.await
.map_err(Into::into);
.await?;
return Ok(None);
}
match subcommand {
None => {
@@ -1405,7 +1410,7 @@ async fn cli_main(
AppServerDaemonSubcommand::PidUpdateLoop {
check_package_ownership: true,
..
} => return Ok(()),
} => return Ok(None),
AppServerDaemonSubcommand::Update {
from_cli: false, ..
}
@@ -1528,16 +1533,12 @@ async fn cli_main(
include_non_interactive,
config_overrides,
);
let exit_info = run_interactive_tui(
return Ok(Some(launch_tui(
interactive,
remote.remote.or(root_remote.clone()),
remote
.remote_auth_token_env
.or(root_remote_auth_token_env.clone()),
arg0_paths.clone(),
)
.await?;
handle_app_exit(exit_info, daemon_cli_executable.as_deref())?;
remote.remote.or(root_remote),
remote.remote_auth_token_env.or(root_remote_auth_token_env),
arg0_paths,
)));
}
Some(Subcommand::Archive(cmd)) => {
let output = run_session_archive_cli_command(
@@ -1628,16 +1629,12 @@ async fn cli_main(
all,
config_overrides,
);
let exit_info = run_interactive_tui(
return Ok(Some(launch_tui(
interactive,
remote.remote.or(root_remote.clone()),
remote
.remote_auth_token_env
.or(root_remote_auth_token_env.clone()),
arg0_paths.clone(),
)
.await?;
handle_app_exit(exit_info, daemon_cli_executable.as_deref())?;
remote.remote.or(root_remote),
remote.remote_auth_token_env.or(root_remote_auth_token_env),
arg0_paths,
)));
}
Some(Subcommand::Login(mut login_cli)) => {
reject_remote_mode_for_subcommand(
@@ -1762,7 +1759,7 @@ async fn cli_main(
.parse_overrides()
.map_err(anyhow::Error::msg)?;
sandbox_setup::run(setup_cli, config_profile.cloned(), cli_overrides).await?;
return Ok(());
return Ok(None);
}
reject_remote_mode_for_subcommand(
root_remote.as_deref(),
@@ -1770,7 +1767,8 @@ async fn cli_main(
"sandbox",
)?;
if let Some(SandboxSubcommand::Uninstall(command)) = subcommand {
return command.run();
command.run()?;
return Ok(None);
}
let loader_overrides = loader_overrides_for_profile(config_profile)?;
#[cfg(target_os = "macos")]
@@ -1950,7 +1948,7 @@ async fn cli_main(
},
}
Ok(())
Ok(None)
}
fn profile_v2_for_subcommand<'a>(
codex-rs/tui/src/app.rs
+13
View File
@@ -1247,12 +1247,22 @@ impl App {
Ok(())
}
fn prepare_frame_images(&mut self) -> Option<crate::pets::ImageProtocol> {
let protocol = self.chat_widget.pet_image_support().protocol().filter(|_| {
!self.app_server_target.uses_remote_workspace()
&& crate::projectless::has_only_local_environments(&self.environment_manager)
});
self.chat_widget.prepare_composer_images(protocol);
protocol
}
fn render_chat_widget_frame(&mut self, tui: &mut tui::Tui, screen_size: Size) -> Result<Rect> {
self.chat_widget
.set_sparkle_terminal_focus(tui.is_terminal_focused());
if tui.is_owned_screen() {
return self.render_owned_transcript(tui, screen_size);
}
self.prepare_frame_images();
self.chat_widget
.empty_state_animation
.borrow_mut()
@@ -1287,6 +1297,9 @@ impl App {
let area = frame.area();
rendered_area = area;
chat_widget.render(area, frame.buffer);
if let Some(previews) = &self.chat_widget.composer_images {
frame.image_placements = previews.placements.take();
}
self.chat_widget.note_rendered_width(area.width);
if let Some((x, y)) = chat_widget.cursor_pos(area) {
frame.set_cursor_style(chat_widget.cursor_style(area));
codex-rs/tui/src/app/owned_transcript.rs
+5 -5
View File
@@ -77,13 +77,10 @@ impl App {
}
fn sync_owned_transcript(&mut self, width: u16) -> bool {
self.transcript_view.image_protocol = self.prepare_frame_images();
let chat_widget = &self.chat_widget;
let transcript_width = chat_widget.history_wrap_width(width);
let view = &mut self.transcript_view;
view.image_protocol = chat_widget.pet_image_support().protocol().filter(|_| {
!self.app_server_target.uses_remote_workspace()
&& crate::projectless::has_only_local_environments(&self.environment_manager)
});
view.mouse_scroll_speed = self.local_settings.tui.mouse_scroll_speed.unwrap_or(1.0);
view.primary_selection = self.right_click_paste_environment.primary;
view.copy_on_select = self
@@ -367,7 +364,10 @@ impl App {
frame.set_cursor_position(position);
}
if chat_widget.no_modal_or_popup_active() {
frame.transcript_images = view.image_placements();
frame.image_placements = view.image_placements();
if let Some(previews) = &chat_widget.composer_images {
frame.image_placements.extend(previews.placements.take());
}
}
})?;
if let Some((surface, tip)) = &turn_tip
codex-rs/tui/src/app/owned_transcript_tests.rs
+100
View File
@@ -1864,6 +1864,106 @@ async fn fullscreen_composer_scrolls_without_moving_caret_or_transcript() -> Res
Ok(())
}
#[tokio::test]
async fn composer_thumbnails_follow_draft_deletion_and_resize() -> Result<()> {
use crate::custom_terminal::test_support::last_rendered_buffer;
use crate::pets::ImageProtocol;
use crate::pets::PetImageSupport;
use crate::pets::PetImageUnsupportedReason;
let dir = tempfile::tempdir()?;
let banner = dir.path().join("banner.png");
let portrait = dir.path().join("portrait.png");
let red = image::Rgba([255, 0, 0, 255]);
image::RgbaImage::from_pixel(/*width*/ 400, /*height*/ 1, red).save(&banner)?;
for owned in [false, true] {
image::RgbaImage::new(/*width*/ 4, /*height*/ 400).save(&portrait)?;
let mut app = crate::app::test_support::make_test_app().await;
let mut tui = crate::tui::test_support::make_test_tui()?;
tui.set_owned_screen(owned)?;
app.chat_widget
.set_pet_image_support_for_tests(PetImageSupport::Supported(ImageProtocol::Kitty));
app.chat_widget.attach_image(portrait.clone());
app.chat_widget.handle_key_event(KeyCode::Home.into());
app.chat_widget.attach_image(banner.clone());
app.chat_widget.handle_key_event(KeyCode::End.into());
app.chat_widget.attach_image(banner.clone());
let narrow = Size::new(/*width*/ 20, /*height*/ 24);
app.render_chat_widget_frame(&mut tui, narrow)?;
let placements = &tui.retained_images;
assert_eq!(placements.len(), 2);
let pixels = &placements[0].image.pixels;
// The whole panorama fits in a single pixel row inside the outlined tile.
assert_eq!(
pixels.pixels().filter(|pixel| **pixel == red).count(),
(pixels.width() - 2) as usize
);
let clear = &placements[1].image.pixels;
assert_eq!(
clear.get_pixel(clear.width() / 2, clear.height() / 2),
clear.get_pixel(/*x*/ 1, /*y*/ 1)
);
assert_ne!(
clear.get_pixel(/*x*/ 0, /*y*/ 0),
clear.get_pixel(/*x*/ 1, /*y*/ 1)
);
let buffer = last_rendered_buffer(&tui.terminal);
assert!(buffer_text(buffer).contains("+1"));
if owned {
// The terminal paints graphics after its text buffer. Mark their footprints
// for this snapshot of the strip and the first editable line.
let mut strip = buffer.clone();
for placement in placements {
for position in placement.area.positions() {
strip[position].set_symbol("░");
}
}
let snapshot = buffer_text(&strip)
.lines()
.skip(usize::from(placements[0].area.y - strip.area.y))
.take(6)
.collect::<Vec<_>>()
.join("\n");
insta::assert_snapshot!("composer_thumbnail_strip", snapshot);
}
let wide = Size::new(/*width*/ 60, /*height*/ 24);
app.render_chat_widget_frame(&mut tui, wide)?;
assert_eq!(tui.retained_images.len(), 3);
let portrait_image = tui.retained_images[1].image.clone();
app.chat_widget.handle_key_event(KeyCode::Home.into());
app.chat_widget.handle_key_event(KeyCode::Delete.into());
app.render_chat_widget_frame(&mut tui, wide)?;
assert_eq!(tui.retained_images.len(), 2);
assert_eq!(tui.retained_images[0].image.pixels, portrait_image.pixels);
// A draft submits the current file, even if it was replaced since attachment.
image::RgbaImage::from_pixel(/*width*/ 2, /*height*/ 2, red).save(&portrait)?;
app.render_chat_widget_frame(&mut tui, wide)?;
let updated = &tui.retained_images[0].image.pixels;
assert!(updated.pixels().any(|pixel| *pixel == red));
// Both insufficient height and an unsupported terminal leave the references editable.
for (height, support) in [
(5, PetImageSupport::Supported(ImageProtocol::Kitty)),
(
24,
PetImageSupport::Unsupported(PetImageUnsupportedReason::Terminal),
),
] {
app.chat_widget.set_pet_image_support_for_tests(support);
app.render_chat_widget_frame(&mut tui, Size::new(wide.width, height))?;
assert!(tui.retained_images.is_empty());
assert!(
buffer_text(last_rendered_buffer(&tui.terminal)).contains("[Image #1][Image #2]")
);
}
tui.set_owned_screen(/*owned*/ false)?;
}
Ok(())
}
#[tokio::test]
async fn fullscreen_composer_remote_images_leave_an_editable_prompt_row() -> Result<()> {
let mut app = crate::app::test_support::make_test_app().await;
codex-rs/tui/src/app/snapshots/codex_tui__app__owned_transcript__tests__composer_thumbnail_strip.snap
+10
View File
@@ -0,0 +1,10 @@
---
source: tui/src/app/owned_transcript_tests.rs
expression: snapshot
---
░░░░░░ ░░░░░░ +1
░░░░░░ ░░░░░░
░░░░░░ ░░░░░░
› [Image #2][Image
#1][Image #3]
codex-rs/tui/src/app/startup.rs
+2 -1
View File
@@ -1694,7 +1694,8 @@ See the Codex keymap documentation for supported actions and examples."
tracing::warn!(error = %err, "failed to shut down embedded app server");
}
let clear_pet_result = tui.clear_ambient_pet_image();
let clear_result = tui.terminal.clear();
let clear_images = tui.terminal.clear_frame_images(&mut tui.retained_images);
let clear_result = clear_images.and(tui.terminal.clear());
if let Some(task) = app.daemon_restart.take()
&& let Err(err) = task.await.unwrap_or_else(|err| Err(err.into()))
{
codex-rs/tui/src/bottom_pane/chat_composer.rs
+13
View File
@@ -230,6 +230,10 @@
//! - When all placeholders for a size are cleared or deleted, the next paste of that size reuses
//! the base label without a suffix.
//!
//! On supported terminals the local draft references also have a strip of thumbnails.
//! It shows the first attachments that fit and a count of those hidden. Removing a
//! reference removes its preview through the same attachment bookkeeping used for submission.
//!
//! # Remote Image Rows (Up/Down/Delete)
//!
//! Remote image URLs are rendered as non-editable `[Image #N]` rows above the textarea (inside the
@@ -4591,6 +4595,7 @@ impl ChatComposer {
status,
composer: composer_rect,
remote_images: remote_images_rect,
thumbnails,
textarea: textarea_rect,
popup: popup_rect,
footer: footer_rect,
@@ -4885,6 +4890,14 @@ impl ChatComposer {
}
let style = user_message_style();
Block::default().style(style).render(composer_rect, buf);
if let Some(previews) = options.image_previews {
previews.render(
thumbnails,
buf,
&self.attachments.local_images,
&self.draft.textarea,
);
}
if !remote_images_rect.is_empty() {
let first = self
.attachments
codex-rs/tui/src/bottom_pane/chat_composer/composer_layout.rs
+26 -2
View File
@@ -125,9 +125,26 @@ impl ChatComposer {
width: textarea_rect.width,
height: remote_images_height,
};
textarea_rect.y = textarea_rect.y.saturating_add(consumed);
textarea_rect.height = textarea_rect.height.saturating_sub(consumed);
let mut thumbnails = Rect {
y: textarea_rect.y.saturating_add(consumed),
height: self.thumbnail_height(textarea_rect.width, options),
..textarea_rect
};
let available = textarea_rect.height.saturating_sub(consumed);
if thumbnails.height > 0
&& available.saturating_sub(thumbnails.height)
< self
.draft
.textarea
.desired_height(textarea_rect.width)
.max(1)
{
thumbnails.height = 0;
}
textarea_rect.y = thumbnails.bottom();
textarea_rect.height = available - thumbnails.height;
ComposerLayout {
thumbnails,
status,
composer: composer_rect,
remote_images: remote_images_rect,
@@ -137,6 +154,12 @@ impl ChatComposer {
}
}
fn thumbnail_height(&self, width: u16, options: ComposerRenderOptions<'_>) -> u16 {
options.image_previews.map_or(0, |previews| {
previews.height(width, self.attachments.local_images.len())
})
}
pub(crate) fn cursor_pos_with_options(
&self,
area: Rect,
@@ -185,6 +208,7 @@ impl ChatComposer {
let remote_images_separator = u16::from(remote_images_height > 0);
self.draft.textarea.desired_height(inner_width)
+ remote_images_height
+ self.thumbnail_height(inner_width, options)
+ remote_images_separator
+ self.status_surface_height(options)
+ if self.shortcuts_above_composer(options) {
codex-rs/tui/src/bottom_pane/chat_composer/footer_state.rs
+2
View File
@@ -44,6 +44,7 @@ pub(super) struct ComposerLayout {
pub(super) status: Rect,
pub(super) composer: Rect,
pub(super) remote_images: Rect,
pub(super) thumbnails: Rect,
pub(super) textarea: Rect,
pub(super) popup: Rect,
pub(super) footer: Rect,
@@ -71,6 +72,7 @@ pub(crate) enum CommandPopupPlacement {
pub(crate) struct ComposerRenderOptions<'a> {
/// Fullscreen limits the composer independently of activity and modal views.
pub(crate) max_height: Option<u16>,
pub(crate) image_previews: Option<&'a crate::transcript_image::composer::ComposerImages>,
pub(crate) rendered_selection:
Option<&'a std::cell::RefCell<crate::rendered_selection::RenderedSelection>>,
/// Reserve a shared hint row independently of whether it currently contains a notice.
codex-rs/tui/src/bottom_pane/mod.rs
+1
View File
@@ -2555,6 +2555,7 @@ impl BottomPane {
let composer: RenderableItem<'_> = if let Some(questions) = question_editor {
RenderableItem::Borrowed(questions.as_ref())
} else if options.max_height.is_none()
&& options.image_previews.is_none()
&& options.textarea_right_reserve == 0
&& options.warning_count == 0
&& options.footer.is_none()
codex-rs/tui/src/chatwidget.rs
+1
View File
@@ -599,6 +599,7 @@ pub(crate) struct ChatWidget {
// Remember the account's Reserve entry notice across chats and transient banner refreshes.
luna_reserve_notice_account_id: Option<String>,
pub(crate) warning_display_state: WarningDisplayState,
pub(crate) composer_images: Option<crate::transcript_image::composer::ComposerImages>,
pub(crate) rendered_selection: std::cell::RefCell<crate::rendered_selection::RenderedSelection>,
rate_limit_switch_prompt: RateLimitSwitchPromptState,
add_credits_nudge_email_in_flight: Option<rate_limits::PendingCreditsNudge>,
codex-rs/tui/src/chatwidget/constructor.rs
+1
View File
@@ -178,6 +178,7 @@ impl ChatWidget {
backend_banner_notice_model: None,
luna_reserve_notice_account_id: None,
warning_display_state: WarningDisplayState::default(),
composer_images: None,
rendered_selection: Default::default(),
rate_limit_switch_prompt: RateLimitSwitchPromptState::default(),
add_credits_nudge_email_in_flight: None,
codex-rs/tui/src/chatwidget/rendering.rs
+15
View File
@@ -122,6 +122,20 @@ impl ExternalWriterNotice {
}
impl ChatWidget {
pub(crate) fn prepare_composer_images(&mut self, protocol: Option<crate::pets::ImageProtocol>) {
self.composer_images = protocol
.filter(|_| {
!self.bottom_pane.composer_local_images().is_empty()
&& self.no_modal_or_popup_active()
})
.and_then(|protocol| {
crate::transcript_image::composer::ComposerImages::for_terminal(
protocol,
self.composer_images.take(),
)
});
}
pub(crate) fn as_renderable(&self) -> RenderableItem<'_> {
let backdrop = self.as_backdrop_renderable();
match self.centered_dialog() {
@@ -270,6 +284,7 @@ impl ChatWidget {
self.ambient_pet_wrap_reserved_cols()
};
options.rendered_selection = Some(&self.rendered_selection);
options.image_previews = self.composer_images.as_ref();
options.warning_count = self.warning_display_state.count;
options.textarea_right_reserve = right_reserve;
options.separate_status_line = options.command_popup_placement
codex-rs/tui/src/custom_terminal.rs
+27 -5
View File
@@ -82,8 +82,8 @@ pub struct Frame<'a> {
/// The buffer that is used to draw the current frame
pub(crate) buffer: &'a mut Buffer,
/// Fullscreen graphics to send after text, before the synchronized frame is committed.
pub(crate) transcript_images: Vec<crate::transcript_image::ImagePlacement>,
/// Frame graphics to send after text, before the synchronized frame is committed.
pub(crate) image_placements: Vec<crate::transcript_image::ImagePlacement>,
}
impl Frame<'_> {
@@ -257,7 +257,7 @@ where
/// Get a Frame object which provides a consistent view into the terminal state for rendering.
pub fn get_frame(&mut self) -> Frame<'_> {
Frame {
transcript_images: Vec::new(),
image_placements: Vec::new(),
cursor_position: None,
cursor_style: SetCursorStyle::DefaultUserShape,
viewport_area: self.viewport_area,
@@ -353,6 +353,28 @@ where
self.draw_with_size(screen_size, &mut Vec::new(), render_callback)
}
/// Remove graphics on the screen that owns them before switching buffers or yielding the terminal.
pub(crate) fn clear_frame_images(
&mut self,
images: &mut Vec<crate::transcript_image::ImagePlacement>,
) -> io::Result<()> {
if images.is_empty() {
return Ok(());
}
let text = self.previous_buffer().clone();
let cursor = (!self.hidden_cursor).then_some(self.last_known_cursor_pos);
let style = self
.last_cursor_style
.unwrap_or(SetCursorStyle::DefaultUserShape);
self.draw_with_size(self.last_known_screen_size, images, |frame| {
*frame.buffer = text;
if let Some(cursor) = cursor {
frame.set_cursor_position(cursor);
frame.set_cursor_style(style);
}
})
}
/// Draws a single frame using a screen size already obtained by the caller.
pub(crate) fn draw_with_size<F>(
&mut self,
@@ -375,7 +397,7 @@ where
// Buffer. Thus, we're taking the important data out of the Frame and dropping it.
let cursor_position = frame.cursor_position;
let cursor_style = frame.cursor_style;
let images = std::mem::take(&mut frame.transcript_images);
let images = std::mem::take(&mut frame.image_placements);
let changed = *retained_images != images;
if changed {
// Remove old graphics before text painting, including when switching to overlays.
@@ -931,7 +953,7 @@ mod tests {
frame.buffer_mut(),
);
frame
.transcript_images
.image_placements
.push(crate::transcript_image::ImagePlacement {
image: std::sync::Arc::clone(&image),
area: Rect::new(
codex-rs/tui/src/transcript_image.rs
+34 -22
View File
@@ -1,9 +1,11 @@
//! Bounded image previews for the owned fullscreen transcript.
//! Bounded local image previews for the transcript and composer.
//!
//! Only decoded local image pixels reach the terminal encoders. Previews are never source text;
//! their reserved rows belong to layout. The terminal retains visible placements and any local
//! transport files until those placements change.
pub(crate) mod composer;
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
@@ -14,6 +16,7 @@ use crossterm::cursor::RestorePosition;
use crossterm::cursor::SavePosition;
use crossterm::queue;
use image::DynamicImage;
use image::ImageDecoder;
use image::ImageFormat;
use image::RgbaImage;
use ratatui::layout::Rect;
@@ -60,20 +63,11 @@ pub(crate) fn terminal_cell_size() -> Option<(u32, u32)> {
}
impl TranscriptImage {
pub(crate) fn load(path: &Path, width: u16, protocol: ImageProtocol) -> Option<Self> {
let columns = width.saturating_sub(2).min(48);
fn load_pixels(path: &Path, max_width: u32, max_height: u32) -> Option<RgbaImage> {
let metadata = path.metadata().ok()?;
// Bound encoded input as well as the raster: metadata alone can dwarf a tiny image.
if columns == 0 || !metadata.is_file() || metadata.len() > 32 * 1024 * 1024 {
if max_width == 0 || !metadata.is_file() || metadata.len() > 32 * 1024 * 1024 {
return None;
}
// Sixel paints physical pixels rather than sizing to a Kitty cell rectangle. Guessing
// the cell size could paint beyond the reserved rows and cover the composer.
let (cell_width, cell_height) = match protocol {
ImageProtocol::Sixel => terminal_cell_size()?,
// Kitty scales to cell coordinates itself. Use a stable, typical cell aspect ratio.
ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (8, 16),
};
let mut reader = image::ImageReader::open(path)
.ok()?
.with_guessed_format()
@@ -82,22 +76,40 @@ impl TranscriptImage {
// screenshots usable, but never allocate hundreds of MB for an optional thumbnail.
let mut limits = image::Limits::default();
limits.max_alloc = Some(32 * 1024 * 1024);
reader.limits(limits);
let image = reader.decode().ok()?;
reader.limits(limits.clone());
let mut decoder = reader.into_decoder().ok()?;
limits.reserve(decoder.total_bytes()).ok()?;
decoder.set_limits(limits).ok()?;
let orientation = decoder.orientation().ok();
let mut image = DynamicImage::from_decoder(decoder).ok()?;
if let Some(orientation) = orientation {
image.apply_orientation(orientation);
}
Some(
image
.thumbnail(max_width.min(image.width()), max_height.min(image.height()))
.to_rgba8(),
)
}
pub(crate) fn load(path: &Path, width: u16, protocol: ImageProtocol) -> Option<Self> {
let columns = width.saturating_sub(2).min(48);
// Sixel paints physical pixels. Guessing could paint outside the reserved rows.
let (cell_width, cell_height) = match protocol {
ImageProtocol::Sixel => terminal_cell_size()?,
ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (8, 16),
};
// The sprite encoder scans each active color per Sixel band. Bound both its CPU
// work and worst-case output even with large terminal cells or high-entropy images.
let (max_width, max_height) = match protocol {
ImageProtocol::Sixel => (160, 80),
ImageProtocol::Kitty | ImageProtocol::KittyLocalFile => (384, 160),
};
let pixels = image
.thumbnail(
(u32::from(columns) * cell_width)
.min(image.width())
.min(max_width),
(6 * cell_height).min(image.height()).min(max_height),
)
.to_rgba8();
let pixels = Self::load_pixels(
path,
(u32::from(columns) * cell_width).min(max_width),
(6 * cell_height).min(max_height),
)?;
Some(Self {
protocol,
columns: pixels.width().div_ceil(cell_width) as u16,
codex-rs/tui/src/transcript_image/composer.rs
+200
View File
@@ -0,0 +1,200 @@
//! The draft thumbnail strip keeps only visible rasters between draws and places them
//! in textarea reference order. Attachment numbering and submission order are independent.
use std::cell::RefCell;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use image::Rgba;
use image::RgbaImage;
use ratatui::buffer::Buffer;
use ratatui::layout::Rect;
use ratatui::widgets::Paragraph;
use ratatui::widgets::Widget;
use ratatui::widgets::Wrap;
use super::ImagePlacement;
use super::TranscriptImage;
use super::terminal_cell_size;
use crate::bottom_pane::LocalImageAttachment;
use crate::bottom_pane::TextArea;
use crate::pets::ImageProtocol;
// Six columns by three rows is square with typical 8 × 16 pixel cells.
const WIDTH: u16 = 6;
const HEIGHT: u16 = 3;
#[derive(Hash, PartialEq, Eq)]
struct TileKey {
path: PathBuf,
version: Option<(u64, std::time::SystemTime)>,
}
pub(crate) struct ComposerImages {
protocol: ImageProtocol,
cell_size: (u32, u32),
background: (u8, u8, u8),
cache: RefCell<HashMap<TileKey, Option<Arc<TranscriptImage>>>>,
pub(crate) placements: RefCell<Vec<ImagePlacement>>,
}
impl ComposerImages {
pub(crate) fn for_terminal(protocol: ImageProtocol, previous: Option<Self>) -> Option<Self> {
// Kitty can scale a nominal raster into its cell placement. Sixel needs physical pixels.
let cell_size = terminal_cell_size()
.or_else(|| (protocol != ImageProtocol::Sixel).then_some((8, 16)))?;
let background = crate::terminal_palette::default_bg()
.map(crate::style::user_message_bg_rgb)
.unwrap_or((48, 48, 48));
if previous.as_ref().is_some_and(|old| {
(old.protocol, old.cell_size, old.background) == (protocol, cell_size, background)
}) {
return previous;
}
Some(Self {
protocol,
cell_size,
background,
cache: RefCell::default(),
placements: RefCell::default(),
})
}
/// Reserve a blank row between any visible strip and the draft.
pub(crate) fn height(&self, width: u16, count: usize) -> u16 {
let (visible, hidden) = visible_thumbnails(width, count);
if visible > 0 {
HEIGHT + 1
} else if hidden > 0 && u32::from(width) >= 2 + hidden.ilog10() {
2
} else {
0
}
}
pub(crate) fn render(
&self,
area: Rect,
buf: &mut Buffer,
attachments: &[LocalImageAttachment],
textarea: &TextArea,
) {
let (visible, hidden) = visible_thumbnails(area.width, attachments.len());
let mut cache = self.cache.borrow_mut();
let mut previous = std::mem::take(&mut *cache);
let mut placements = self.placements.borrow_mut();
placements.clear();
let style = crate::style::secondary_text_style();
let mut x = area.x;
if area.is_empty() {
return;
}
let mut ordered: Vec<_> = attachments.iter().collect();
if ordered.len() > 1 {
let positions: HashMap<_, _> = textarea
.text_element_ranges()
.map(|range| (&textarea.text()[range.clone()], range.start))
.collect();
ordered.sort_by_key(|image| {
positions
.get(image.placeholder.as_str())
.copied()
.unwrap_or(usize::MAX)
});
}
for attachment in ordered.into_iter().take(visible) {
let path = &attachment.path;
let key = TileKey {
path: path.clone(),
version: path
.metadata()
.ok()
.and_then(|m| Some((m.len(), m.modified().ok()?))),
};
let image = cache.entry(key).or_insert_with_key(|key| {
previous
.remove(key)
.unwrap_or_else(|| self.load_tile(path).map(Arc::new))
});
let tile = Rect::new(x, area.y, WIDTH, HEIGHT);
if let Some(image) = image {
placements.push(ImagePlacement {
image: image.clone(),
area: tile,
first_row: 0,
file: None,
});
} else {
Paragraph::new(attachment.placeholder.as_str())
.style(style)
.wrap(Wrap { trim: false })
.render(tile, buf);
}
x = x.saturating_add(WIDTH + 1);
}
if hidden > 0 {
buf.set_string(x, area.y, format!("+{hidden}"), style);
}
}
fn load_tile(&self, path: &Path) -> Option<TranscriptImage> {
let (cell_width, cell_height) = self.cell_size;
let width = u32::from(WIDTH) * cell_width;
let height = u32::from(HEIGHT) * cell_height;
let scale = width.div_ceil(384).max(height.div_ceil(160));
// Kitty scales the raster to the tile. Sixel must paint at the physical cell size.
if scale > 1 && self.protocol == ImageProtocol::Sixel {
return None;
}
let width = (width / scale).max(3);
let height = (height / scale).max(3);
let pixels = TranscriptImage::load_pixels(path, width - 2, height - 2)?;
let (r, g, b) = self.background;
let contrast = if crate::color::is_light(self.background) {
0
} else {
255
};
let (red, green, blue) = crate::color::blend(
(contrast, contrast, contrast),
self.background,
/*alpha*/ 0.45,
);
let edge = Rgba([red, green, blue, 255]);
let mut tile = RgbaImage::from_fn(width, height, |x, y| {
if x == 0 || y == 0 || x == width - 1 || y == height - 1 {
edge
} else {
Rgba([r, g, b, 255])
}
});
image::imageops::overlay(
&mut tile,
&pixels,
i64::from((width - pixels.width()) / 2),
i64::from((height - pixels.height()) / 2),
);
Some(TranscriptImage {
protocol: self.protocol,
columns: WIDTH,
rows: HEIGHT,
pixels: tile,
cell_width,
cell_height,
})
}
}
fn visible_thumbnails(width: u16, count: usize) -> (usize, usize) {
let mut visible = ((usize::from(width) + 1) / usize::from(WIDTH + 1)).min(count);
while visible > 0 && visible < count {
let badge = 2 + (count - visible).ilog10() as usize;
if visible * usize::from(WIDTH + 1) + badge <= usize::from(width) {
break;
}
visible -= 1;
}
(visible, count - visible)
}
codex-rs/tui/src/tui.rs
+11 -4
View File
@@ -700,7 +700,7 @@ pub struct Tui {
screen_size: ScreenSizePolicy,
ambient_pet_image_state: crate::pets::PetImageRenderState,
pet_picker_preview_image_state: crate::pets::PetImageRenderState,
pub(crate) transcript_images: Vec<crate::transcript_image::ImagePlacement>,
pub(crate) retained_images: Vec<crate::transcript_image::ImagePlacement>,
alt_saved_viewport: Option<ratatui::layout::Rect>,
#[cfg(unix)]
suspend_context: SuspendContext,
@@ -779,7 +779,7 @@ impl Tui {
screen_size: ScreenSizePolicy::default(),
ambient_pet_image_state: crate::pets::PetImageRenderState::default(),
pet_picker_preview_image_state: crate::pets::PetImageRenderState::default(),
transcript_images: Vec::new(),
retained_images: Vec::new(),
alt_saved_viewport: None,
#[cfg(unix)]
suspend_context: SuspendContext::new(),
@@ -973,6 +973,9 @@ impl Tui {
// Restore both screens' input modes before giving the terminal to an editor. Repaint the
// last frame on the alternate screen so separate-window editors leave Codex visible.
let was_alt_screen = self.is_alt_screen_active();
if let Err(err) = self.terminal.clear_frame_images(&mut self.retained_images) {
tracing::warn!(%err, "failed to clear images before terminal handoff");
}
let visible_frame = (was_alt_screen && matches!(handoff, TerminalHandoff::KeepScreen))
.then(|| self.terminal.previous_buffer().clone());
let restore_result = if let Some(visible_frame) = visible_frame {
@@ -1112,6 +1115,8 @@ impl Tui {
if !self.alt_screen_enabled {
return Ok(());
}
self.terminal
.clear_frame_images(&mut self.retained_images)?;
// Entry can change buffers before later input-mode configuration fails.
self.terminal.invalidate_cursor_state();
if self.owned_screen {
@@ -1162,6 +1167,8 @@ impl Tui {
if !self.alt_screen_enabled || !self.is_alt_screen_active() {
return Ok(());
}
self.terminal
.clear_frame_images(&mut self.retained_images)?;
let result = ALTERNATE_SCREEN.leave(self.terminal.backend_mut());
if ALTERNATE_SCREEN.is_active() {
return result;
@@ -1398,7 +1405,7 @@ impl Tui {
self.suspend_context.set_cursor_y(inline_area_bottom);
}
terminal.draw_with_size(screen_size, &mut self.transcript_images, draw_fn)
terminal.draw_with_size(screen_size, &mut self.retained_images, draw_fn)
})??;
Ok(())
}
@@ -1534,7 +1541,7 @@ impl Tui {
self.suspend_context.set_cursor_y(inline_area_bottom);
}
terminal.draw_with_size(screen_size, &mut self.transcript_images, |frame| {
terminal.draw_with_size(screen_size, &mut self.retained_images, |frame| {
draw_fn(frame);
})
})??;