diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 56631966d5..af9c81b1fb 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2482,6 +2482,7 @@ dependencies = [ "codex-utils-cli", "crossterm", "http 1.4.0", + "insta", "owo-colors", "pretty_assertions", "ratatui", @@ -2492,6 +2493,7 @@ dependencies = [ "tokio-stream", "tracing", "tracing-subscriber", + "unicode-segmentation", "unicode-width 0.2.1", ] diff --git a/codex-rs/cloud-tasks/Cargo.toml b/codex-rs/cloud-tasks/Cargo.toml index be3ec552c8..12bd6fdf7f 100644 --- a/codex-rs/cloud-tasks/Cargo.toml +++ b/codex-rs/cloud-tasks/Cargo.toml @@ -37,7 +37,9 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } tokio-stream = { workspace = true } tracing = { workspace = true, features = ["log"] } tracing-subscriber = { workspace = true, features = ["env-filter"] } +unicode-segmentation = { workspace = true } unicode-width = { workspace = true } [dev-dependencies] +insta = { workspace = true } pretty_assertions = { workspace = true } diff --git a/codex-rs/cloud-tasks/src/scrollable_diff.rs b/codex-rs/cloud-tasks/src/scrollable_diff.rs index 59dd076b36..a88fe1bf6e 100644 --- a/codex-rs/cloud-tasks/src/scrollable_diff.rs +++ b/codex-rs/cloud-tasks/src/scrollable_diff.rs @@ -1,4 +1,4 @@ -use unicode_width::UnicodeWidthChar; +use unicode_segmentation::UnicodeSegmentation; use unicode_width::UnicodeWidthStr; /// Scroll position and geometry for a vertical scroll view. @@ -131,38 +131,57 @@ impl ScrollableDiff { let mut line = String::new(); let mut line_cols = 0usize; let mut last_soft_idx: Option = None; // last whitespace or punctuation break - for (_i, ch) in raw.char_indices() { - if ch == '\n' { + for grapheme in raw + .graphemes(/*is_extended*/ true) + .flat_map(|grapheme| grapheme.split_inclusive(char::is_whitespace)) + { + if grapheme == "\n" { out.push(std::mem::take(&mut line)); out_idx.push(raw_idx); line_cols = 0; last_soft_idx = None; continue; } - let w = UnicodeWidthChar::width(ch).unwrap_or(0); - if line_cols.saturating_add(w) > max_cols { + let grapheme_width = display_width(grapheme); + if line_cols.saturating_add(grapheme_width) > max_cols { if let Some(split) = last_soft_idx { let (prefix, rest) = line.split_at(split); - out.push(prefix.trim_end().to_string()); - out_idx.push(raw_idx); + let prefix = prefix.trim_end(); + if !prefix.is_empty() { + out.push(prefix.to_string()); + out_idx.push(raw_idx); + } line = rest.trim_start().to_string(); last_soft_idx = None; - // retry add current ch now that line may be shorter + // Retry adding the current grapheme now that the line may be shorter. } else if !line.is_empty() { out.push(std::mem::take(&mut line)); out_idx.push(raw_idx); } } - if ch.is_whitespace() - || matches!( - ch, - ',' | ';' | '.' | ':' | ')' | ']' | '}' | '|' | '/' | '?' | '!' | '-' | '_' - ) + if grapheme.chars().all(char::is_whitespace) + || grapheme.chars().any(|ch| { + matches!( + ch, + ',' | ';' + | '.' + | ':' + | ')' + | ']' + | '}' + | '|' + | '/' + | '?' + | '!' + | '-' + | '_' + ) + }) { last_soft_idx = Some(line.len()); } - line.push(ch); - line_cols = UnicodeWidthStr::width(line.as_str()); + line.push_str(grapheme); + line_cols = display_width(&line); } if !line.is_empty() { out.push(line); @@ -174,3 +193,16 @@ impl ScrollableDiff { self.state.content_h = self.wrapped.len() as u16; } } + +/// Counts terminal cells, including the halfwidth sound marks omitted by `unicode-width`. +fn display_width(text: &str) -> usize { + UnicodeWidthStr::width(text) + + text + .chars() + .filter(|ch| matches!(ch, '\u{FF9E}' | '\u{FF9F}')) + .count() +} + +#[cfg(test)] +#[path = "scrollable_diff_tests.rs"] +mod tests; diff --git a/codex-rs/cloud-tasks/src/scrollable_diff_tests.rs b/codex-rs/cloud-tasks/src/scrollable_diff_tests.rs new file mode 100644 index 0000000000..4db0a4f33f --- /dev/null +++ b/codex-rs/cloud-tasks/src/scrollable_diff_tests.rs @@ -0,0 +1,48 @@ +use super::ScrollableDiff; +use pretty_assertions::assert_eq; + +#[test] +fn wraps_halfwidth_sound_marks_using_ratatui_width() { + let mut diff = ScrollableDiff::new(); + diff.set_content(vec![ + "aｶﾞbﾊﾟc".to_string(), + "a ﾞb".to_string(), + "a ﾟb".to_string(), + ]); + diff.set_width(/*width*/ 2); + + insta::assert_snapshot!( + diff.wrapped_lines().join("\n"), + @r" + a + ｶﾞ + b + ﾊﾟ + c + a + ﾞb + a + ﾟb + " + ); + assert_eq!(diff.wrapped_src_indices(), [0, 0, 0, 0, 0, 1, 1, 2, 2]); +} + +#[test] +fn does_not_emit_empty_rows_after_full_width_sound_mark_graphemes() { + for grapheme in ["ｶﾞ", "ﾊﾟ"] { + let mut diff = ScrollableDiff::new(); + diff.set_content(vec![format!("a {grapheme} tail")]); + diff.set_width(/*width*/ 2); + + let expected = ["a", grapheme, "ta", "il"]; + let actual = diff + .wrapped_lines() + .iter() + .map(String::as_str) + .collect::>(); + assert_eq!(actual, expected); + assert_eq!(diff.wrapped_src_indices(), [0, 0, 0, 0]); + assert_eq!(diff.state.content_h, 4); + } +} diff --git a/codex-rs/tui/src/bottom_pane/list_selection_view.rs b/codex-rs/tui/src/bottom_pane/list_selection_view.rs index 347c1c3d9a..7ef6f281f0 100644 --- a/codex-rs/tui/src/bottom_pane/list_selection_view.rs +++ b/codex-rs/tui/src/bottom_pane/list_selection_view.rs @@ -2569,6 +2569,41 @@ mod tests { ); } + #[test] + fn snapshot_narrow_width_counts_halfwidth_sound_marks() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let items = vec![ + SelectionItem { + name: "abｶﾞc".to_string(), + description: Some("dakuten description".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: "aﾊﾟc".to_string(), + description: Some("handakuten description".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + ]; + let view = new_view( + SelectionViewParams { + title: Some("Halfwidth sound marks".to_string()), + items, + ..Default::default() + }, + tx, + ); + + let rendered = format!( + "width 20:\n{}\n\nwidth 24:\n{}", + render_lines_with_width(&view, /*width*/ 20), + render_lines_with_width(&view, /*width*/ 24) + ); + assert_snapshot!("list_selection_halfwidth_sound_marks_narrow", rendered); + } + #[test] fn snapshot_auto_visible_col_width_mode_scroll_behavior() { assert_snapshot!( diff --git a/codex-rs/tui/src/bottom_pane/request_user_input/render.rs b/codex-rs/tui/src/bottom_pane/request_user_input/render.rs index 773f9548bb..973e3feef3 100644 --- a/codex-rs/tui/src/bottom_pane/request_user_input/render.rs +++ b/codex-rs/tui/src/bottom_pane/request_user_input/render.rs @@ -7,8 +7,7 @@ use ratatui::widgets::Paragraph; use ratatui::widgets::Widget; use std::borrow::Cow; use std::time::Instant; -use unicode_width::UnicodeWidthChar; -use unicode_width::UnicodeWidthStr; +use unicode_segmentation::UnicodeSegmentation; use crate::bottom_pane::popup_consts::standard_popup_hint_line; use crate::bottom_pane::scroll_state::ScrollState; @@ -18,7 +17,9 @@ use crate::bottom_pane::selection_popup_common::menu_surface_padding_height; use crate::bottom_pane::selection_popup_common::render_menu_surface; use crate::bottom_pane::selection_popup_common::render_rows; use crate::bottom_pane::selection_popup_common::wrap_styled_line; +use crate::line_truncation::line_width; use crate::render::renderable::Renderable; +use crate::width::display_width; use super::DESIRED_SPACERS_BETWEEN_SECTIONS; use super::RequestUserInputOverlay; @@ -421,12 +422,6 @@ impl RequestUserInputOverlay { } } -fn line_width(line: &Line<'_>) -> usize { - line.iter() - .map(|span| UnicodeWidthStr::width(span.content.as_ref())) - .sum() -} - /// Render rows into `area`, bottom-aligning the visible rows when fewer than /// `area.height` lines are produced. /// @@ -489,7 +484,7 @@ fn truncate_line_word_boundary_with_ellipsis( } let ellipsis = "…"; - let ellipsis_width = UnicodeWidthStr::width(ellipsis); + let ellipsis_width = display_width(ellipsis); if ellipsis_width >= max_width { return Line::from(ellipsis); } @@ -509,19 +504,19 @@ fn truncate_line_word_boundary_with_ellipsis( 'outer: for (span_idx, span) in line.spans.iter().enumerate() { let text = span.content.as_ref(); - for (byte_idx, ch) in text.char_indices() { - let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0); - if used.saturating_add(ch_width) > limit { + for (byte_idx, grapheme) in text.grapheme_indices(/*is_extended*/ true) { + let grapheme_width = display_width(grapheme); + if used.saturating_add(grapheme_width) > limit { overflowed = true; break 'outer; } - used = used.saturating_add(ch_width); + used = used.saturating_add(grapheme_width); let bp = BreakPoint { span_idx, - byte_end: byte_idx + ch.len_utf8(), + byte_end: byte_idx + grapheme.len(), }; last_fit = Some(bp); - if ch.is_whitespace() { + if grapheme.chars().all(char::is_whitespace) { last_word_break = Some(bp); } } @@ -576,3 +571,7 @@ fn truncate_line_word_boundary_with_ellipsis( Line::from(spans_out).style(line_style) } + +#[cfg(test)] +#[path = "render_tests.rs"] +mod tests; diff --git a/codex-rs/tui/src/bottom_pane/request_user_input/render_tests.rs b/codex-rs/tui/src/bottom_pane/request_user_input/render_tests.rs new file mode 100644 index 0000000000..2bf87d342b --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/request_user_input/render_tests.rs @@ -0,0 +1,37 @@ +use super::truncate_line_word_boundary_with_ellipsis; +use pretty_assertions::assert_eq; +use ratatui::buffer::Buffer; +use ratatui::layout::Rect; +use ratatui::style::Stylize; +use ratatui::text::Line; +use ratatui::text::Span; +use ratatui::widgets::Paragraph; +use ratatui::widgets::Widget; + +#[test] +fn halfwidth_sound_marks_are_truncated_at_a_grapheme_boundary() { + let line = Line::from("abｶﾞ tail"); + + assert_eq!( + truncate_line_word_boundary_with_ellipsis(line, /*max_width*/ 4), + Line::from(vec![Span::raw("ab"), Span::raw("…")]) + ); +} + +#[test] +fn halfwidth_sound_marks_are_truncated_and_rendered_at_a_grapheme_boundary() { + let lines = [ + Line::from(vec!["ab".bold(), "ｶﾞ".cyan(), " tail".dim()]), + Line::from(vec!["xy".bold(), "ﾊﾟ".magenta(), " tail".dim()]), + ] + .map(|line| truncate_line_word_boundary_with_ellipsis(line, /*max_width*/ 5)); + let area = Rect::new(0, 0, 5, 2); + let mut buf = Buffer::empty(area); + + Paragraph::new(lines.to_vec()).render(area, &mut buf); + + insta::assert_snapshot!( + "request_user_input_halfwidth_sound_marks_truncation", + format!("{buf:?}\n{lines:#?}") + ); +} diff --git a/codex-rs/tui/src/bottom_pane/request_user_input/snapshots/codex_tui__bottom_pane__request_user_input__render__tests__request_user_input_halfwidth_sound_marks_truncation.snap b/codex-rs/tui/src/bottom_pane/request_user_input/snapshots/codex_tui__bottom_pane__request_user_input__render__tests__request_user_input_halfwidth_sound_marks_truncation.snap new file mode 100644 index 0000000000..4dc0fc13b7 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/request_user_input/snapshots/codex_tui__bottom_pane__request_user_input__render__tests__request_user_input_halfwidth_sound_marks_truncation.snap @@ -0,0 +1,33 @@ +--- +source: tui/src/bottom_pane/request_user_input/render_tests.rs +expression: "format!(\"{buf:?}\\n{lines:#?}\")" +--- +Buffer { + area: Rect { x: 0, y: 0, width: 5, height: 2 }, + content: [ + "abｶﾞ…", // hidden by multi-width symbols: [(3, " ")] + "xyﾊﾟ…", // hidden by multi-width symbols: [(3, " ")] + ], + styles: [ + x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD, + x: 2, y: 0, fg: Cyan, bg: Reset, underline: Reset, modifier: NONE, + x: 3, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 4, y: 0, fg: Cyan, bg: Reset, underline: Reset, modifier: NONE, + x: 0, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD, + x: 2, y: 1, fg: Magenta, bg: Reset, underline: Reset, modifier: NONE, + x: 3, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 4, y: 1, fg: Magenta, bg: Reset, underline: Reset, modifier: NONE, + ] +} +[ + Line::from_iter([ + Span::from("ab").bold(), + Span::from("ｶ\u{ff9e}").cyan(), + Span::from("…").cyan(), + ]), + Line::from_iter([ + Span::from("xy").bold(), + Span::from("ﾊ\u{ff9f}").magenta(), + Span::from("…").magenta(), + ]), +] diff --git a/codex-rs/tui/src/bottom_pane/selection_popup_common.rs b/codex-rs/tui/src/bottom_pane/selection_popup_common.rs index 5964e47277..e1841286cf 100644 --- a/codex-rs/tui/src/bottom_pane/selection_popup_common.rs +++ b/codex-rs/tui/src/bottom_pane/selection_popup_common.rs @@ -7,14 +7,15 @@ use ratatui::text::Line; use ratatui::text::Span; use ratatui::widgets::Block; use ratatui::widgets::Widget; -use unicode_width::UnicodeWidthStr; use crate::key_hint::KeyBinding; +use crate::line_truncation::line_width; use crate::line_truncation::truncate_line_with_ellipsis_if_overflow; use crate::render::Insets; use crate::render::RectExt as _; use crate::style::accent_style; use crate::style::user_message_style; +use crate::width::display_width; use super::scroll_state::ScrollState; use super::selection_row_layout::SelectionDescriptionLayout; @@ -170,7 +171,7 @@ fn compute_desc_col( if row.disabled_reason.is_some() { spans.push(" (disabled)".dim()); } - Line::from(spans).width() + line_width(&Line::from(spans)) }) .max() .unwrap_or(0), @@ -182,7 +183,7 @@ fn compute_desc_col( if row.disabled_reason.is_some() { spans.push(" (disabled)".dim()); } - Line::from(spans).width() + line_width(&Line::from(spans)) }) .max() .unwrap_or(0), @@ -225,6 +226,9 @@ fn should_wrap_name_in_column(row: &GenericDisplayRow) -> bool { } fn wrap_two_column_row(row: &GenericDisplayRow, desc_col: usize, width: u16) -> Vec> { + use crate::wrapping::RtOptions; + use crate::wrapping::word_wrap_lines; + let Some(description) = row.description.as_deref() else { return Vec::new(); }; @@ -245,14 +249,13 @@ fn wrap_two_column_row(row: &GenericDisplayRow, desc_col: usize, width: u16) -> .unwrap_or(0) .min(left_width.saturating_sub(1)); - let name_subsequent_indent = " ".repeat(name_wrap_indent); - let name_options = textwrap::Options::new(left_width) - .initial_indent("") - .subsequent_indent(name_subsequent_indent.as_str()); - let name_lines = textwrap::wrap(row.name.as_str(), name_options); + let name_options = RtOptions::new(left_width) + .initial_indent(Line::from("")) + .subsequent_indent(Line::from(" ".repeat(name_wrap_indent))); + let name_lines = word_wrap_lines(row.name.lines(), name_options); - let desc_options = textwrap::Options::new(right_width).initial_indent(""); - let desc_lines = textwrap::wrap(description, desc_options); + let desc_options = RtOptions::new(right_width).initial_indent(Line::from("")); + let desc_lines = word_wrap_lines(description.lines(), desc_options); let rows = name_lines.len().max(desc_lines.len()).max(1); let mut out = Vec::with_capacity(rows); @@ -265,7 +268,7 @@ fn wrap_two_column_row(row: &GenericDisplayRow, desc_col: usize, width: u16) -> if let Some(desc) = desc_lines.get(idx) { let left_used = spans .iter() - .map(|span| UnicodeWidthStr::width(span.content.as_ref())) + .map(|span| display_width(span.content.as_ref())) .sum::(); let gap = if left_used == 0 { desc_col @@ -797,6 +800,123 @@ mod tests { assert_eq!(two_col.len(), 0); } + #[test] + fn popup_name_truncation_counts_halfwidth_sound_marks() { + for (name, desc_col, match_index, expected) in + [("abｶﾞc", 6, None, "abｶﾞ…"), ("aｶﾞc", 4, Some(1), "a…")] + { + let row = GenericDisplayRow { + name: name.to_string(), + description: Some("description".to_string()), + match_indices: match_index.map(|index| vec![index]), + ..Default::default() + }; + let text = + build_full_line(&row, desc_col, SelectionDescriptionLayout::Columns).to_string(); + + assert!(text.starts_with(expected), "unexpected row: {text:?}"); + } + } + + #[test] + fn fuzzy_matched_emoji_graphemes_keep_description_alignment() { + let rows = [ + GenericDisplayRow { + name: "👍🏻".to_string(), + match_indices: Some(vec![1]), + description: Some("description".to_string()), + ..Default::default() + }, + GenericDisplayRow { + name: "👨‍👩‍👧‍👦".to_string(), + match_indices: Some(vec![2]), + description: Some("description".to_string()), + ..Default::default() + }, + ]; + let area = Rect::new(0, 0, /*width*/ 20, /*height*/ 2); + let mut buf = Buffer::empty(area); + + for (row_index, row) in rows.iter().enumerate() { + let line = build_full_line( + row, + /*desc_col*/ 4, + SelectionDescriptionLayout::Columns, + ); + let name = line.spans.first().expect("fuzzy-matched name span"); + assert_eq!(name.content.as_ref(), row.name); + assert!(name.style.add_modifier.contains(Modifier::BOLD)); + + let row_area = Rect::new(area.x, area.y + row_index as u16, area.width, 1); + ratatui::widgets::Widget::render( + ratatui::widgets::Paragraph::new(line), + row_area, + &mut buf, + ); + assert_eq!(buf[(4, row_index as u16)].symbol(), "d"); + } + + insta::assert_snapshot!("popup_fuzzy_matched_emoji_graphemes", format!("{buf:?}")); + } + + #[test] + fn wrapped_two_column_rows_count_halfwidth_sound_marks() { + let rows = vec![GenericDisplayRow { + name: "abｶﾞc".to_string(), + description: Some("abﾊﾟc".to_string()), + wrap_indent: Some(0), + ..Default::default() + }]; + let area = Rect::new(0, 0, 9, 2); + let mut buf = Buffer::empty(area); + + let rendered_lines = render_rows( + area, + &mut buf, + &rows, + &ScrollState::default(), + /*max_results*/ 1, + "no rows", + ); + let rendered = (0..area.height) + .map(|y| { + (0..area.width) + .map(|x| buf[(x, y)].symbol()) + .collect::() + .trim_end() + .to_string() + }) + .collect::>() + .join("\n"); + + assert_eq!(rendered_lines, 2); + insta::assert_snapshot!(rendered, @r" + abｶﾞ ab + c ﾊﾟ c + "); + } + + #[test] + fn wrapped_two_column_rows_preserve_hard_line_breaks() { + let row = GenericDisplayRow { + name: "first\nsecond".to_string(), + description: Some("alpha\nbeta".to_string()), + wrap_indent: Some(0), + ..Default::default() + }; + + let rendered = wrap_two_column_row(&row, /*desc_col*/ 8, /*width*/ 24) + .into_iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n"); + + insta::assert_snapshot!(rendered, @r" + first alpha + second beta + "); + } + #[test] fn selected_rows_use_the_shared_accent_style() { let rows = vec![GenericDisplayRow { diff --git a/codex-rs/tui/src/bottom_pane/selection_row_layout.rs b/codex-rs/tui/src/bottom_pane/selection_row_layout.rs index 692e086171..eace3095bf 100644 --- a/codex-rs/tui/src/bottom_pane/selection_row_layout.rs +++ b/codex-rs/tui/src/bottom_pane/selection_row_layout.rs @@ -3,9 +3,11 @@ use std::borrow::Cow; use ratatui::style::Stylize; use ratatui::text::Line; use ratatui::text::Span; -use unicode_width::UnicodeWidthChar; +use unicode_segmentation::UnicodeSegmentation; use super::selection_popup_common::GenericDisplayRow; +use crate::line_truncation::line_width; +use crate::width::display_width; use crate::wrapping::RtOptions; use crate::wrapping::word_wrap_line; @@ -82,35 +84,28 @@ fn build_name_spans(row: &GenericDisplayRow, name_limit: usize) -> Vec name_limit { - truncated = true; - break; - } - used_width = next_width; + let mut match_indices = row.match_indices.iter().flatten().peekable(); + let mut char_idx = 0usize; + for grapheme in row.name.graphemes(/*is_extended*/ true) { + let next_width = used_width.saturating_add(display_width(grapheme)); + if next_width > name_limit { + truncated = true; + break; + } + used_width = next_width; - if idx_iter.peek().is_some_and(|next| **next == char_idx) { - idx_iter.next(); - name_spans.push(ch.to_string().bold()); - } else { - name_spans.push(ch.to_string().into()); - } - } - } else { - for ch in row.name.chars() { - let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0); - let next_width = used_width.saturating_add(ch_width); - if next_width > name_limit { - truncated = true; - break; - } - used_width = next_width; - name_spans.push(ch.to_string().into()); + let mut matched = false; + for _ in grapheme.chars() { + matched |= match_indices.next_if(|next| **next == char_idx).is_some(); + char_idx += 1; } + + let grapheme = grapheme.to_string(); + name_spans.push(if matched { + grapheme.bold() + } else { + grapheme.into() + }); } if truncated { @@ -145,13 +140,13 @@ pub(super) fn build_full_line( description_layout: SelectionDescriptionLayout, ) -> Line<'static> { let description = combined_description(row, description_layout); - let name_prefix_width = Line::from(row.name_prefix_spans.clone()).width(); + let name_prefix_width = line_width(&Line::from(row.name_prefix_spans.clone())); let name_limit = description .as_ref() .map(|_| desc_col.saturating_sub(2).saturating_sub(name_prefix_width)) .unwrap_or(usize::MAX); let name_spans = build_name_spans(row, name_limit); - let name_width = name_prefix_width + Line::from(name_spans.clone()).width(); + let name_width = name_prefix_width + line_width(&Line::from(name_spans.clone())); let mut spans = row.name_prefix_spans.clone(); spans.extend(name_spans); @@ -170,8 +165,7 @@ pub(super) fn build_full_line( /// Render a row as a full-width label followed by an indented description. pub(super) fn wrap_stacked_row(row: &GenericDisplayRow, width: u16) -> Vec> { let width = width.max(1); - let prefix_width = Line::from(row.name_prefix_spans.clone()) - .width() + let prefix_width = line_width(&Line::from(row.name_prefix_spans.clone())) .min(width.saturating_sub(1) as usize); let indent = " ".repeat(prefix_width); diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__list_selection_view__tests__list_selection_halfwidth_sound_marks_narrow.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__list_selection_view__tests__list_selection_halfwidth_sound_marks_narrow.snap new file mode 100644 index 0000000000..81badbe243 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__list_selection_view__tests__list_selection_halfwidth_sound_marks_narrow.snap @@ -0,0 +1,30 @@ +--- +source: tui/src/bottom_pane/list_selection_view.rs +expression: rendered +--- +width 20: + + Halfwidth sound + +› 1. abｶﾞc + dakute + n + descri + ption + 2. aﾊﾟc + handak + uten + descri + ption + + +width 24: + + Halfwidth sound mark + +› 1. abｶﾞc dakuten + descriptio + n + 2. aﾊﾟc handakuten + descriptio + n diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__selection_popup_common__tests__popup_fuzzy_matched_emoji_graphemes.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__selection_popup_common__tests__popup_fuzzy_matched_emoji_graphemes.snap new file mode 100644 index 0000000000..22bd1bfdcb --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__selection_popup_common__tests__popup_fuzzy_matched_emoji_graphemes.snap @@ -0,0 +1,21 @@ +--- +source: tui/src/bottom_pane/selection_popup_common.rs +expression: "format!(\"{buf:?}\")" +--- +Buffer { + area: Rect { x: 0, y: 0, width: 20, height: 2 }, + content: [ + "👍🏻 description ", // hidden by multi-width symbols: [(1, " ")] + "👨‍👩‍👧‍👦 description ", // hidden by multi-width symbols: [(1, " ")] + ], + styles: [ + x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD, + x: 1, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 4, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 15, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 0, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD, + x: 1, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 4, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 15, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + ] +} diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__textarea__tests__textarea_halfwidth_sound_marks_wrap_and_align_with_cursor.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__textarea__tests__textarea_halfwidth_sound_marks_wrap_and_align_with_cursor.snap new file mode 100644 index 0000000000..7ea4a38fba --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__textarea__tests__textarea_halfwidth_sound_marks_wrap_and_align_with_cursor.snap @@ -0,0 +1,20 @@ +--- +source: tui/src/bottom_pane/textarea.rs +expression: "format!(\"dakuten\\n{dakuten}\\n\\nhandakuten\\n{handakuten}\\n\\nstandalone\\n{standalone}\")" +--- +dakuten +cursor: Some((1, 1)) +"12ｶﾞ" Hidden by multi-width symbols: [(3, " ")] +"x " + + +handakuten +cursor: Some((3, 1)) +"ab " +"ﾊﾟc" Hidden by multi-width symbols: [(1, " ")] + + +standalone +cursor: Some((2, 1)) +"a " +"ﾞb" diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__textarea__tests__textarea_masked_graphemes_align_with_cursor.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__textarea__tests__textarea_masked_graphemes_align_with_cursor.snap new file mode 100644 index 0000000000..b82038f930 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__textarea__tests__textarea_masked_graphemes_align_with_cursor.snap @@ -0,0 +1,6 @@ +--- +source: tui/src/bottom_pane/textarea.rs +expression: "format!(\"cursor: {:?}\\n{}\", t.cursor_pos(area), terminal.backend())" +--- +cursor: Some((4, 0)) +"****" diff --git a/codex-rs/tui/src/bottom_pane/textarea.rs b/codex-rs/tui/src/bottom_pane/textarea.rs index 62421acb29..ed65662f86 100644 --- a/codex-rs/tui/src/bottom_pane/textarea.rs +++ b/codex-rs/tui/src/bottom_pane/textarea.rs @@ -17,6 +17,7 @@ use crate::keymap::RuntimeKeymap; use crate::keymap::VimNormalKeymap; use crate::keymap::VimOperatorKeymap; use crate::keymap::VimTextObjectKeymap; +use crate::width::display_width; use codex_protocol::user_input::ByteRange; use codex_protocol::user_input::TextElement as UserTextElement; use crossterm::event::KeyCode; @@ -35,7 +36,6 @@ use std::cell::RefCell; use std::ops::Range; use textwrap::Options; use unicode_segmentation::UnicodeSegmentation; -use unicode_width::UnicodeWidthStr; mod vim; use self::vim::VimMode; @@ -430,7 +430,7 @@ impl TextArea { let effective_scroll = self.effective_scroll(area.height, &lines, state.scroll); let i = Self::wrapped_line_index_by_start(&lines, self.cursor_pos)?; let ls = &lines[i]; - let col = self.text[ls.start..self.cursor_pos].width() as u16; + let col = display_width(&self.text[ls.start..self.cursor_pos]) as u16; let screen_row = i .saturating_sub(effective_scroll as usize) .try_into() @@ -444,7 +444,7 @@ impl TextArea { fn current_display_col(&self) -> usize { let bol = self.beginning_of_current_line(); - self.text[bol..self.cursor_pos].width() + display_width(&self.text[bol..self.cursor_pos]) } fn wrapped_line_index_by_start(lines: &[Range], pos: usize) -> Option { @@ -462,7 +462,7 @@ impl TextArea { ) { let mut width_so_far = 0usize; for (i, g) in self.text[line_start..line_end].grapheme_indices(true) { - width_so_far += g.width(); + width_so_far += display_width(g); if width_so_far > target_col { self.cursor_pos = line_start + i; // Avoid landing inside an element; round to nearest boundary @@ -1212,9 +1212,9 @@ impl TextArea { let lines = &cache.lines; if let Some(idx) = Self::wrapped_line_index_by_start(lines, self.cursor_pos) { let cur_range = &lines[idx]; - let target_col = self - .preferred_col - .unwrap_or_else(|| self.text[cur_range.start..self.cursor_pos].width()); + let target_col = self.preferred_col.unwrap_or_else(|| { + display_width(&self.text[cur_range.start..self.cursor_pos]) + }); if idx > 0 { let prev = &lines[idx - 1]; let line_start = prev.start; @@ -1275,9 +1275,9 @@ impl TextArea { let lines = &cache.lines; if let Some(idx) = Self::wrapped_line_index_by_start(lines, self.cursor_pos) { let cur_range = &lines[idx]; - let target_col = self - .preferred_col - .unwrap_or_else(|| self.text[cur_range.start..self.cursor_pos].width()); + let target_col = self.preferred_col.unwrap_or_else(|| { + display_width(&self.text[cur_range.start..self.cursor_pos]) + }); if idx + 1 < lines.len() { let next = &lines[idx + 1]; let line_start = next.start; @@ -1994,7 +1994,7 @@ impl TextArea { continue; } let styled = &self.text[overlap_start..overlap_end]; - let x_off = self.text[line_range.start..overlap_start].width() as u16; + let x_off = display_width(&self.text[line_range.start..overlap_start]) as u16; let style = base_style.fg(Color::Cyan); buf.set_string(area.x + x_off, y, text_for_display(styled), style); } @@ -2008,7 +2008,7 @@ impl TextArea { continue; } let highlighted = &self.text[overlap_start..overlap_end]; - let x_off = self.text[line_range.start..overlap_start].width() as u16; + let x_off = display_width(&self.text[line_range.start..overlap_start]) as u16; buf.set_string(area.x + x_off, y, text_for_display(highlighted), *style); } } @@ -2027,8 +2027,8 @@ impl TextArea { let y = area.y + row as u16; let line_range = r.start..r.end - 1; let masked = self.text[line_range.clone()] - .chars() - .map(|_| mask_char) + .graphemes(/*is_extended*/ true) + .flat_map(|grapheme| std::iter::repeat_n(mask_char, display_width(grapheme))) .collect::(); buf.set_string(area.x, y, &masked, Style::default()); } @@ -3518,6 +3518,90 @@ mod tests { assert_eq!(buf[(0, 1)].symbol(), "5"); } + #[test] + fn halfwidth_sound_marks_wrap_and_align_with_the_cursor() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let mut snapshots = Vec::new(); + for (label, text, width, cursor, cells) in [ + ("dakuten", "12ｶﾞx", 4, (1, 1), [(2, 0, "ｶﾞ"), (0, 1, "x")]), + ("handakuten", "abﾊﾟc", 3, (3, 1), [(0, 1, "ﾊﾟ"), (2, 1, "c")]), + ("standalone", "a ﾞb", 2, (2, 1), [(0, 1, "ﾞ"), (1, 1, "b")]), + ] { + let mut t = ta_with(text); + t.set_cursor(text.len()); + let area = Rect::new(0, 0, width, /*height*/ 2); + let mut terminal = Terminal::new(TestBackend::new(area.width, area.height)).unwrap(); + terminal + .draw(|frame| { + ratatui::widgets::WidgetRef::render_ref( + &(&t), + frame.area(), + frame.buffer_mut(), + ); + }) + .unwrap(); + + assert_eq!(t.desired_height(area.width), 2); + assert_eq!(t.cursor_pos(area), Some(cursor)); + for (x, y, expected) in cells { + assert_eq!(terminal.backend().buffer()[(x, y)].symbol(), expected); + } + snapshots.push(format!( + "{label}\ncursor: {:?}\n{}", + t.cursor_pos(area), + terminal.backend() + )); + } + + insta::assert_snapshot!( + "textarea_halfwidth_sound_marks_wrap_and_align_with_cursor", + snapshots.join("\n\n") + ); + } + + #[test] + fn masked_graphemes_align_with_the_cursor() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let text = "界ﾞa"; + let mut t = ta_with(text); + t.set_cursor(text.len()); + let area = Rect::new(0, 0, /*width*/ 4, /*height*/ 1); + let mut terminal = Terminal::new(TestBackend::new(area.width, area.height)).unwrap(); + let mut state = TextAreaState::default(); + terminal + .draw(|frame| { + t.render_ref_masked(frame.area(), frame.buffer_mut(), &mut state, '*'); + }) + .unwrap(); + + assert_eq!(t.cursor_pos(area), Some((4, 0))); + insta::assert_snapshot!( + "textarea_masked_graphemes_align_with_cursor", + format!("cursor: {:?}\n{}", t.cursor_pos(area), terminal.backend()) + ); + } + + #[test] + fn overwide_halfwidth_sound_marks_do_not_add_a_phantom_cursor_row() { + let area = Rect::new(0, 0, /*width*/ 2, /*height*/ 2); + + for grapheme in ["ｶﾞﾞ", "界ﾞ"] { + let text = format!("{grapheme}ab"); + let mut t = ta_with(&text); + t.set_cursor(text.len()); + + assert_eq!(t.desired_height(area.width), 2); + assert_eq!(t.cursor_pos(area), Some((2, 1))); + + t.set_cursor(grapheme.len()); + assert_eq!(t.cursor_pos(area), Some((0, 1))); + } + } + #[test] fn cursor_pos_with_state_basic_and_scroll_behaviors() { // Case 1: No wrapping needed, height fits — scroll ignored, y maps directly. diff --git a/codex-rs/tui/src/history_cell/exec.rs b/codex-rs/tui/src/history_cell/exec.rs index b1a3a52508..365558eb30 100644 --- a/codex-rs/tui/src/history_cell/exec.rs +++ b/codex-rs/tui/src/history_cell/exec.rs @@ -1,6 +1,7 @@ //! Background terminal interaction and process-summary history cells. use super::*; +use crate::width::display_width; #[derive(Debug)] pub(crate) struct UnifiedExecInteractionCell { @@ -137,9 +138,9 @@ impl HistoryCell for UnifiedExecProcessesCell { } let prefix = " • "; - let prefix_width = UnicodeWidthStr::width(prefix); + let prefix_width = display_width(prefix); let truncation_suffix = " [...]"; - let truncation_suffix_width = UnicodeWidthStr::width(truncation_suffix); + let truncation_suffix_width = display_width(truncation_suffix); let mut shown = 0usize; for process in &self.processes { if shown >= max_processes { @@ -189,7 +190,7 @@ impl HistoryCell for UnifiedExecProcessesCell { } else { chunk_prefix_next }; - let chunk_prefix_width = UnicodeWidthStr::width(chunk_prefix); + let chunk_prefix_width = display_width(chunk_prefix); if wrap_width <= chunk_prefix_width { out.push(Line::from(chunk_prefix.dim())); continue; diff --git a/codex-rs/tui/src/history_cell/mod.rs b/codex-rs/tui/src/history_cell/mod.rs index 06c5cdfea5..10bdd7f6ef 100644 --- a/codex-rs/tui/src/history_cell/mod.rs +++ b/codex-rs/tui/src/history_cell/mod.rs @@ -103,7 +103,6 @@ use std::time::Duration; use std::time::Instant; use tracing::error; use unicode_segmentation::UnicodeSegmentation; -use unicode_width::UnicodeWidthStr; use url::Url; const RAW_DIFF_SUMMARY_WIDTH: usize = 10_000; diff --git a/codex-rs/tui/src/history_cell/session.rs b/codex-rs/tui/src/history_cell/session.rs index b4a2ca050d..03df0141f7 100644 --- a/codex-rs/tui/src/history_cell/session.rs +++ b/codex-rs/tui/src/history_cell/session.rs @@ -1,7 +1,9 @@ //! Session headers, onboarding guidance, and transcript cards. use super::*; +use crate::line_truncation::line_width; use crate::line_truncation::truncate_line_with_ellipsis_if_overflow; +use crate::width::display_width; pub(crate) const SESSION_HEADER_MAX_INNER_WIDTH: usize = 56; // Just an eyeballed value @@ -34,15 +36,7 @@ fn with_border_internal( lines: Vec>, forced_inner_width: Option, ) -> Vec> { - let max_line_width = lines - .iter() - .map(|line| { - line.iter() - .map(|span| UnicodeWidthStr::width(span.content.as_ref())) - .sum::() - }) - .max() - .unwrap_or(0); + let max_line_width = lines.iter().map(line_width).max().unwrap_or(0); let content_width = forced_inner_width .unwrap_or(max_line_width) .max(max_line_width); @@ -52,10 +46,7 @@ fn with_border_internal( out.push(vec![format!("╭{}╮", "─".repeat(border_inner_width)).dim()].into()); for line in lines.into_iter() { - let used_width: usize = line - .iter() - .map(|span| UnicodeWidthStr::width(span.content.as_ref())) - .sum(); + let used_width = line_width(&line); let span_count = line.spans.len(); let mut spans: Vec> = Vec::with_capacity(span_count + 4); spans.push(Span::from("│ ").dim()); @@ -90,7 +81,7 @@ impl TooltipHistoryCell { impl HistoryCell for TooltipHistoryCell { fn display_lines(&self, width: u16) -> Vec> { let indent = " "; - let indent_width = UnicodeWidthStr::width(indent); + let indent_width = display_width(indent); let wrap_width = usize::from(width.max(1)) .saturating_sub(indent_width) .max(1); @@ -303,7 +294,7 @@ impl SessionHeaderHistoryCell { if max_width == 0 { return String::new(); } - if UnicodeWidthStr::width(formatted.as_str()) > max_width { + if display_width(formatted.as_str()) > max_width { return crate::text_formatting::center_truncate_path(&formatted, max_width); } } @@ -371,7 +362,7 @@ impl HistoryCell for SessionHeaderHistoryCell { let dir_label = format!("{DIR_LABEL:_ OpenAI Codex (vtest) │", + "│ │", + "│ model: gpt-5 /model to change │", + "│ directory: …ﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟ-project │", // hidden by multi-width symbols: [(15, " "), (17, " "), (19, " "), (21, " "), (23, " "), (25, " "), (27, " "), (29, " "), (31, " ")] + "╰────────────────────────────────────────╯", + ], + styles: [ + x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 5, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD, + x: 17, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 13, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 18, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 21, y: 3, fg: Cyan, bg: Reset, underline: Reset, modifier: NONE, + x: 27, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 13, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 40, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + ] +} diff --git a/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__session_header_halfwidth_sound_marks.snap b/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__session_header_halfwidth_sound_marks.snap new file mode 100644 index 0000000000..0fb17271fa --- /dev/null +++ b/codex-rs/tui/src/history_cell/snapshots/codex_tui__history_cell__tests__session_header_halfwidth_sound_marks.snap @@ -0,0 +1,37 @@ +--- +source: tui/src/history_cell/tests.rs +expression: "format!(\"{buf:?}\")" +--- +Buffer { + area: Rect { x: 0, y: 0, width: 80, height: 6 }, + content: [ + "╭───────────────────────────────────────────╮ ", + "│ >_ OpenAI Codex (vtest) │ ", + "│ │ ", + "│ model: gpt-5-ｶﾞ-ﾊﾟ /model to change │ ", // hidden by multi-width symbols: [(20, " "), (23, " ")] + "│ directory: project │ ", + "╰───────────────────────────────────────────╯ ", + ], + styles: [ + x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 45, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 0, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 5, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: BOLD, + x: 17, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 45, y: 1, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 0, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 45, y: 2, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 0, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 13, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 24, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 27, y: 3, fg: Cyan, bg: Reset, underline: Reset, modifier: NONE, + x: 33, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 45, y: 3, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 0, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 13, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 20, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 45, y: 4, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 0, y: 5, fg: Reset, bg: Reset, underline: Reset, modifier: DIM, + x: 45, y: 5, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + ] +} diff --git a/codex-rs/tui/src/history_cell/tests.rs b/codex-rs/tui/src/history_cell/tests.rs index 0eacf63ab8..2402a6a595 100644 --- a/codex-rs/tui/src/history_cell/tests.rs +++ b/codex-rs/tui/src/history_cell/tests.rs @@ -783,6 +783,16 @@ fn ps_output_long_command_snapshot() { insta::assert_snapshot!(rendered); } +#[test] +fn ps_output_halfwidth_sound_marks_snapshot() { + let cell = new_unified_exec_processes_output(vec![UnifiedExecProcessDetails { + command_display: "echo ｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟ".to_string(), + recent_chunks: vec!["output ｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟ".to_string()], + }]); + let rendered = render_lines(&cell.display_lines(/*width*/ 24)).join("\n"); + insta::assert_snapshot!(rendered); +} + #[test] fn ps_output_many_sessions_snapshot() { let cell = new_unified_exec_processes_output( @@ -1653,6 +1663,44 @@ fn session_header_indicates_yolo_mode() { insta::assert_snapshot!(rendered); } +#[test] +fn session_header_aligns_halfwidth_sound_marks() { + let cell: Box = Box::new(SessionHeaderHistoryCell::new( + "gpt-5-ｶﾞ-ﾊﾟ".to_string(), + /*reasoning_effort*/ None, + /*show_fast_status*/ false, + PathBuf::from("project"), + "test", + )); + + let width = 80; + let height = cell.desired_height(width); + let area = Rect::new(0, 0, width, height); + let mut buf = Buffer::empty(area); + cell.render(area, &mut buf); + + insta::assert_snapshot!("session_header_halfwidth_sound_marks", format!("{buf:?}")); +} + +#[test] +fn session_header_truncates_halfwidth_directory() { + let cell: Box = Box::new(SessionHeaderHistoryCell::new( + "gpt-5".to_string(), + /*reasoning_effort*/ None, + /*show_fast_status*/ false, + PathBuf::from("ｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟ-project"), + "test", + )); + + let width = 42; + let height = cell.desired_height(width); + let area = Rect::new(0, 0, width, height); + let mut buf = Buffer::empty(area); + cell.render(area, &mut buf); + + insta::assert_snapshot!("session_header_halfwidth_directory", format!("{buf:?}")); +} + #[test] fn yolo_mode_includes_managed_full_access_profiles() { let permission_profile: PermissionProfile = PermissionProfile::Managed { diff --git a/codex-rs/tui/src/line_truncation.rs b/codex-rs/tui/src/line_truncation.rs index d8a9408e37..fe1f7bb6f3 100644 --- a/codex-rs/tui/src/line_truncation.rs +++ b/codex-rs/tui/src/line_truncation.rs @@ -1,11 +1,12 @@ use ratatui::text::Line; use ratatui::text::Span; -use unicode_width::UnicodeWidthChar; -use unicode_width::UnicodeWidthStr; +use unicode_segmentation::UnicodeSegmentation; + +use crate::width::display_width; pub(crate) fn line_width(line: &Line<'_>) -> usize { line.iter() - .map(|span| UnicodeWidthStr::width(span.content.as_ref())) + .map(|span| display_width(span.content.as_ref())) .sum() } @@ -23,7 +24,7 @@ pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> L let mut spans_out: Vec> = Vec::with_capacity(spans.len()); for span in spans { - let span_width = UnicodeWidthStr::width(span.content.as_ref()); + let span_width = display_width(span.content.as_ref()); if span_width == 0 { spans_out.push(span); @@ -43,13 +44,13 @@ pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> L let style = span.style; let text = span.content.as_ref(); let mut end_idx = 0usize; - for (idx, ch) in text.char_indices() { - let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0); - if used + ch_width > max_width { + for (idx, grapheme) in text.grapheme_indices(/*is_extended*/ true) { + let grapheme_width = display_width(grapheme); + if used + grapheme_width > max_width { break; } - end_idx = idx + ch.len_utf8(); - used += ch_width; + end_idx = idx + grapheme.len(); + used += grapheme_width; } if end_idx > 0 { @@ -98,3 +99,7 @@ pub(crate) fn truncate_line_with_ellipsis_if_overflow( spans, } } + +#[cfg(test)] +#[path = "line_truncation_tests.rs"] +mod tests; diff --git a/codex-rs/tui/src/line_truncation_tests.rs b/codex-rs/tui/src/line_truncation_tests.rs new file mode 100644 index 0000000000..cb27c005f1 --- /dev/null +++ b/codex-rs/tui/src/line_truncation_tests.rs @@ -0,0 +1,19 @@ +use super::line_width; +use super::truncate_line_to_width; +use pretty_assertions::assert_eq; +use ratatui::text::Line; + +#[test] +fn halfwidth_sound_marks_stay_with_their_grapheme_when_truncated() { + let line = Line::from("abｶﾞc"); + + assert_eq!(line_width(&line), 5); + assert_eq!( + truncate_line_to_width(line.clone(), /*max_width*/ 3), + Line::from("ab") + ); + assert_eq!( + truncate_line_to_width(line, /*max_width*/ 4), + Line::from("abｶﾞ") + ); +} diff --git a/codex-rs/tui/src/live_wrap.rs b/codex-rs/tui/src/live_wrap.rs index 53c63b108e..a0ad15ba66 100644 --- a/codex-rs/tui/src/live_wrap.rs +++ b/codex-rs/tui/src/live_wrap.rs @@ -1,5 +1,5 @@ -use unicode_width::UnicodeWidthChar; -use unicode_width::UnicodeWidthStr; +use crate::width::display_width; +use unicode_segmentation::UnicodeSegmentation; /// A single visual row produced by RowBuilder. #[derive(Debug, Clone, PartialEq, Eq)] @@ -11,7 +11,7 @@ pub struct Row { impl Row { pub fn width(&self) -> usize { - self.text.width() + display_width(&self.text) } } @@ -140,9 +140,9 @@ impl RowBuilder { let (prefix, suffix, taken) = take_prefix_by_width(&self.current_line, self.target_width); if taken == 0 { - // Avoid infinite loop on pathological inputs; take one scalar and continue. - if let Some((i, ch)) = self.current_line.char_indices().next() { - let len = i + ch.len_utf8(); + // Avoid an infinite loop on an indivisible grapheme wider than the target. + if let Some(grapheme) = self.current_line.graphemes(/*is_extended*/ true).next() { + let len = grapheme.len(); let p = self.current_line[..len].to_string(); self.rows.push(Row { text: p, @@ -176,13 +176,13 @@ pub fn take_prefix_by_width(text: &str, max_cols: usize) -> (String, &str, usize } let mut cols = 0usize; let mut end_idx = 0usize; - for (i, ch) in text.char_indices() { - let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0); - if cols.saturating_add(ch_width) > max_cols { + for (i, grapheme) in text.grapheme_indices(/*is_extended*/ true) { + let grapheme_width = display_width(grapheme); + if cols.saturating_add(grapheme_width) > max_cols { break; } - cols += ch_width; - end_idx = i + ch.len_utf8(); + cols += grapheme_width; + end_idx = i + grapheme.len(); if cols == max_cols { break; } @@ -235,6 +235,22 @@ mod tests { ); } + #[test] + fn halfwidth_sound_marks_stay_with_their_grapheme_when_wrapping() { + assert_eq!( + take_prefix_by_width("abｶﾞc", /*max_cols*/ 3), + ("ab".to_string(), "ｶﾞc", 2) + ); + assert_eq!( + take_prefix_by_width("abｶﾞc", /*max_cols*/ 4), + ("abｶﾞ".to_string(), "c", 4) + ); + + let mut row_builder = RowBuilder::new(/*target_width*/ 1); + row_builder.push_fragment("ｶﾞx"); + assert_eq!(row_builder.rows()[0].text, "ｶﾞ"); + } + #[test] fn fragmentation_invariance_long_token() { let s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; // 26 chars diff --git a/codex-rs/tui/src/markdown_render.rs b/codex-rs/tui/src/markdown_render.rs index 9d644b8376..7a7cc4cad4 100644 --- a/codex-rs/tui/src/markdown_render.rs +++ b/codex-rs/tui/src/markdown_render.rs @@ -49,6 +49,8 @@ use crate::terminal_hyperlinks::annotate_web_urls_in_line; use crate::terminal_hyperlinks::remap_wrapped_line; use crate::terminal_hyperlinks::visible_lines; use crate::terminal_hyperlinks::web_destination; +use crate::width::char_width; +use crate::width::display_width; use crate::wrapping::RtOptions; use crate::wrapping::adaptive_wrap_line; use crate::wrapping::word_wrap_line; @@ -72,8 +74,6 @@ use std::ops::Range; use std::path::Path; use std::path::PathBuf; use std::sync::LazyLock; -use unicode_width::UnicodeWidthChar; -use unicode_width::UnicodeWidthStr; use url::Url; mod streaming; @@ -1302,7 +1302,7 @@ where let plain = cell.plain_text(); let mut word_count = 0usize; for token in plain.split_whitespace() { - let token_width = token.width(); + let token_width = display_width(token); body_token_width = body_token_width.max(token_width); long_body_token_count += usize::from(token_width >= 20); word_count += 1; @@ -1311,7 +1311,7 @@ where body_token_count += word_count; total_words += word_count; total_cells += 1; - total_cell_width += plain.width(); + total_cell_width += display_width(&plain); } } @@ -1321,7 +1321,7 @@ where total_words as f64 / total_cells as f64 }; let avg_cell_width = if total_cells == 0 { - header_plain.width() as f64 + display_width(&header_plain) as f64 } else { total_cell_width as f64 / total_cells as f64 }; @@ -1607,7 +1607,7 @@ where } else { current_text.push(ch); } - column += UnicodeWidthChar::width(ch).unwrap_or(/*default*/ 0); + column += char_width(ch); } flush(&mut out, &mut current_text, current_destination); } @@ -1756,7 +1756,10 @@ where #[inline] fn spans_display_width(spans: &[Span<'_>]) -> usize { - spans.iter().map(|span| span.content.width()).sum() + spans + .iter() + .map(|span| display_width(span.content.as_ref())) + .sum() } #[inline] @@ -1775,7 +1778,10 @@ where #[inline] fn longest_token_width(text: &str) -> usize { - text.split_whitespace().map(str::width).max().unwrap_or(0) + text.split_whitespace() + .map(display_width) + .max() + .unwrap_or(0) } fn push_inline_style(&mut self, style: Style) { @@ -1887,7 +1893,7 @@ where } } else { let mut spans = self.current_initial_indent.clone(); - let shift = spans.iter().map(|span| span.content.width()).sum::(); + let shift = Self::spans_display_width(&spans); spans.append(&mut line.line.spans); for hyperlink in &mut line.hyperlinks { hyperlink.columns = @@ -1926,7 +1932,7 @@ where }; let mut spans = self.prefix_spans(pending_marker_line); - let shift = spans.iter().map(|span| span.content.width()).sum::(); + let shift = Self::spans_display_width(&spans); spans.append(&mut line.line.spans); for hyperlink in &mut line.hyperlinks { hyperlink.columns = hyperlink.columns.start + shift..hyperlink.columns.end + shift; @@ -2844,6 +2850,13 @@ mod tests { })); } + #[test] + fn table_widths_count_halfwidth_sound_marks() { + let cell = make_cell("ｶﾞﾊﾟ"); + assert_eq!(W::cell_display_width(&cell), 4); + assert_eq!(W::longest_token_width("ｶﾞﾊﾟtail"), 8); + } + #[test] fn key_value_table_keeps_web_annotations() { let destination = "https://example.com/a/very/long/path"; diff --git a/codex-rs/tui/src/markdown_render/table_key_value.rs b/codex-rs/tui/src/markdown_render/table_key_value.rs index e071f242bc..29c0559f77 100644 --- a/codex-rs/tui/src/markdown_render/table_key_value.rs +++ b/codex-rs/tui/src/markdown_render/table_key_value.rs @@ -8,12 +8,12 @@ use crate::render::line_utils::line_to_static; use crate::render::line_utils::push_owned_lines; use crate::terminal_hyperlinks::HyperlinkLine; use crate::terminal_hyperlinks::remap_wrapped_line; +use crate::width::display_width; use crate::wrapping::RtOptions; use crate::wrapping::word_wrap_line; use ratatui::style::Style; use ratatui::text::Line; use ratatui::text::Span; -use unicode_width::UnicodeWidthStr; const FIELD_LEADING_PADDING: usize = 1; const FIELD_GAP: usize = 2; @@ -48,7 +48,7 @@ pub(super) fn should_render_records( let has_fragmented_token = cell .plain_text() .split_whitespace() - .any(|token| token.width() > *width); + .any(|token| display_width(token) > *width); match metrics.kind { TableColumnKind::Compact => has_fragmented_token, TableColumnKind::TokenHeavy => { @@ -105,7 +105,7 @@ pub(super) fn render_records( ) -> Vec { let label_width = headers .iter() - .map(|header| header.plain_text().width()) + .map(|header| display_width(&header.plain_text())) .max() .unwrap_or(0); let minimum_value_width = if metrics @@ -167,9 +167,9 @@ fn render_aligned_field( let label = header.plain_text(); spans.push(Span::raw(" ".repeat(FIELD_LEADING_PADDING))); spans.push(Span::styled(label.clone(), label_style)); - spans.push(Span::raw( - " ".repeat(label_width.saturating_sub(label.width()) + FIELD_GAP), - )); + spans.push(Span::raw(" ".repeat( + label_width.saturating_sub(display_width(&label)) + FIELD_GAP, + ))); } else { spans.push(Span::raw(" ".repeat(value_indent))); } @@ -186,7 +186,7 @@ fn render_stacked_field( ) { let label_width = available_width .map(|width| width.saturating_sub(FIELD_LEADING_PADDING).max(1)) - .unwrap_or_else(|| header.plain_text().width().max(1)); + .unwrap_or_else(|| display_width(&header.plain_text()).max(1)); let label = Line::from(Span::styled(header.plain_text(), label_style)); let mut wrapped_labels = Vec::new(); push_owned_lines( @@ -218,7 +218,7 @@ fn push_prefixed_value_line( ) { let shift = prefix .iter() - .map(|span| span.content.width()) + .map(|span| display_width(span.content.as_ref())) .sum::(); prefix.append(&mut value_line.line.spans); let mut output_line = HyperlinkLine::new(Line::from(prefix)); @@ -257,27 +257,11 @@ fn wrap_cell(cell: &TableCell, width: usize) -> Vec { fn cell_width(cell: &TableCell) -> usize { cell.lines .iter() - .map(|line| { - line.line - .spans - .iter() - .map(|span| span.content.width()) - .sum::() - }) + .map(HyperlinkLine::width) .max() .unwrap_or(0) } fn widest_line_width(lines: &[HyperlinkLine]) -> usize { - lines - .iter() - .map(|line| { - line.line - .spans - .iter() - .map(|span| span.content.width()) - .sum::() - }) - .max() - .unwrap_or(0) + lines.iter().map(HyperlinkLine::width).max().unwrap_or(0) } diff --git a/codex-rs/tui/src/markdown_render_tests.rs b/codex-rs/tui/src/markdown_render_tests.rs index e0e498e69a..62fa55ca99 100644 --- a/codex-rs/tui/src/markdown_render_tests.rs +++ b/codex-rs/tui/src/markdown_render_tests.rs @@ -1724,6 +1724,24 @@ fn table_renders_key_value_records_when_compact_fragmentation_is_systemic_snapsh assert_snapshot!(plain_lines(&text).join("\n")); } +#[test] +fn table_renders_halfwidth_sound_marks_at_constrained_width_snapshot() { + let md = r#"| Key | Notes | +| --- | --- | +| ｶﾞﾊﾟtail | First ｶﾞ row with an escaped \| pipe. | +| ﾊﾟｶﾞtail | Second ﾊﾟ row with an escaped \| pipe. | +| short | Final ｶﾞﾊﾟ row. | +"#; + let grid = render_markdown_text_with_width(md, Some(/*width*/ 23)); + let records = render_markdown_text_with_width(md, Some(/*width*/ 17)); + + assert_snapshot!(format!( + "grid (23 cells):\n{}\n\nrecords (17 cells):\n{}", + plain_lines(&grid).join("\n"), + plain_lines(&records).join("\n") + )); +} + #[test] fn table_inside_blockquote_has_quote_prefix() { let md = "> | A | B |\n> |---|---|\n> | 1 | 2 |\n"; diff --git a/codex-rs/tui/src/snapshots/codex_tui__markdown_render__markdown_render_tests__table_renders_halfwidth_sound_marks_at_constrained_width_snapshot.snap b/codex-rs/tui/src/snapshots/codex_tui__markdown_render__markdown_render_tests__table_renders_halfwidth_sound_marks_at_constrained_width_snapshot.snap new file mode 100644 index 0000000000..f57506498e --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__markdown_render__markdown_render_tests__table_renders_halfwidth_sound_marks_at_constrained_width_snapshot.snap @@ -0,0 +1,41 @@ +--- +source: tui/src/markdown_render_tests.rs +expression: "format!(\"grid (23 cells):\\n{}\\n\\nrecords (17 cells):\\n{}\", plain_lines(&grid).join(\"\\n\"), plain_lines(&records).join(\"\\n\"))" +--- +grid (23 cells): + Key Notes +━━━━━━━━━━ ━━━━━━━━━━━ + ｶﾞﾊﾟtail First ｶﾞ + row with + an + escaped | + pipe. +────────── ─────────── + ﾊﾟｶﾞtail Second ﾊﾟ + row with + an + escaped | + pipe. +────────── ─────────── + short Final ｶﾞ + ﾊﾟ row. + +records (17 cells): + Key + ｶﾞﾊﾟtail + Notes + First ｶﾞ row + with an escaped + | pipe. +───────────────── + Key + ﾊﾟｶﾞtail + Notes + Second ﾊﾟ row + with an escaped + | pipe. +───────────────── + Key + short + Notes + Final ｶﾞﾊﾟ row. diff --git a/codex-rs/tui/src/status/card.rs b/codex-rs/tui/src/status/card.rs index 91027f8103..382a9e3c6f 100644 --- a/codex-rs/tui/src/status/card.rs +++ b/codex-rs/tui/src/status/card.rs @@ -4,9 +4,11 @@ use crate::history_cell::PlainHistoryCell; use crate::history_cell::plain_lines; use crate::history_cell::with_border_with_inner_width; use crate::legacy_core::config::Config; +use crate::line_truncation::line_width; use crate::token_usage::TokenUsage; use crate::token_usage::TokenUsageInfo; use crate::version::CODEX_CLI_VERSION; +use crate::width::display_width; use chrono::DateTime; use chrono::Local; use codex_app_server_protocol::AskForApproval; @@ -26,12 +28,10 @@ use ratatui::prelude::*; use ratatui::style::Stylize; use std::collections::BTreeSet; use std::path::PathBuf; -use unicode_width::UnicodeWidthStr; use url::Url; use super::account::StatusAccountDisplay; use super::format::FieldFormatter; -use super::format::line_display_width; use super::format::push_label; use super::format::truncate_line_to_width; use super::helpers::compose_account_display; @@ -482,7 +482,7 @@ impl StatusHistoryCell { ]; // On narrow terminals, keep the percentage visible rather than // letting the fixed-width progress bar crowd out the reset time. - let value_spans = if line_display_width(&Line::from(full_value_spans.clone())) + let value_spans = if line_width(&Line::from(full_value_spans.clone())) <= formatter.value_width(available_inner_width) { full_value_spans @@ -498,9 +498,7 @@ impl StatusHistoryCell { inline_spans.push(Span::from(" ").dim()); inline_spans.push(resets_span.clone()); - if line_display_width(&Line::from(inline_spans.clone())) - <= available_inner_width - { + if line_width(&Line::from(inline_spans.clone())) <= available_inner_width { lines.push(Line::from(inline_spans)); } else { lines.push(base_line); @@ -863,7 +861,7 @@ impl HistoryCell for StatusHistoryCell { lines.extend(self.rate_limit_lines(&rate_limit_state, available_inner_width, &formatter)); - let content_width = lines.iter().map(line_display_width).max().unwrap_or(0); + let content_width = lines.iter().map(line_width).max().unwrap_or(0); let inner_width = content_width.min(available_inner_width); let truncated_lines: Vec> = lines .into_iter() @@ -891,10 +889,10 @@ impl HistoryCell for StatusHistoryCell { .map(|span| span.content.as_ref()) .collect::(); if let Some(start_byte) = visible.find(CHATGPT_USAGE_URL) { - let start = visible[..start_byte].width(); + let start = display_width(&visible[..start_byte]); line.hyperlinks .push(crate::terminal_hyperlinks::TerminalHyperlink::web( - start..start + CHATGPT_USAGE_URL.width(), + start..start + display_width(CHATGPT_USAGE_URL), CHATGPT_USAGE_URL.to_string(), )); } diff --git a/codex-rs/tui/src/status/format.rs b/codex-rs/tui/src/status/format.rs index fda729040a..e7ff9fd138 100644 --- a/codex-rs/tui/src/status/format.rs +++ b/codex-rs/tui/src/status/format.rs @@ -1,8 +1,9 @@ use ratatui::prelude::*; use ratatui::style::Stylize; use std::collections::BTreeSet; -use unicode_width::UnicodeWidthChar; -use unicode_width::UnicodeWidthStr; +use unicode_segmentation::UnicodeSegmentation; + +use crate::width::display_width; #[derive(Debug, Clone)] pub(crate) struct FieldFormatter { @@ -21,10 +22,10 @@ impl FieldFormatter { { let label_width = labels .into_iter() - .map(|label| UnicodeWidthStr::width(label.as_ref())) + .map(|label| display_width(label.as_ref())) .max() .unwrap_or(0); - let indent_width = UnicodeWidthStr::width(Self::INDENT); + let indent_width = display_width(Self::INDENT); let value_offset = indent_width + label_width + 1 + 3; Self { @@ -72,7 +73,7 @@ impl FieldFormatter { buf.push_str(label); buf.push(':'); - let label_width = UnicodeWidthStr::width(label); + let label_width = display_width(label); let padding = 3 + self.label_width.saturating_sub(label_width); for _ in 0..padding { buf.push(' '); @@ -92,12 +93,6 @@ pub(crate) fn push_label(labels: &mut Vec, seen: &mut BTreeSet, labels.push(owned); } -pub(crate) fn line_display_width(line: &Line<'static>) -> usize { - line.iter() - .map(|span| UnicodeWidthStr::width(span.content.as_ref())) - .sum() -} - pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> Line<'static> { if max_width == 0 { return Line::from(Vec::>::new()); @@ -109,7 +104,7 @@ pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> L for span in line.spans { let text = span.content.into_owned(); let style = span.style; - let span_width = UnicodeWidthStr::width(text.as_str()); + let span_width = display_width(text.as_str()); if span_width == 0 { spans_out.push(Span::styled(text, style)); @@ -127,13 +122,13 @@ pub(crate) fn truncate_line_to_width(line: Line<'static>, max_width: usize) -> L } let mut truncated = String::new(); - for ch in text.chars() { - let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0); - if used + ch_width > max_width { + for grapheme in text.graphemes(/*is_extended*/ true) { + let grapheme_width = display_width(grapheme); + if used + grapheme_width > max_width { break; } - truncated.push(ch); - used += ch_width; + truncated.push_str(grapheme); + used += grapheme_width; } if !truncated.is_empty() { diff --git a/codex-rs/tui/src/status/helpers.rs b/codex-rs/tui/src/status/helpers.rs index e779883b18..4f6f926525 100644 --- a/codex-rs/tui/src/status/helpers.rs +++ b/codex-rs/tui/src/status/helpers.rs @@ -2,13 +2,13 @@ use crate::exec_command::relativize_to_home; use crate::legacy_core::config::Config; use crate::status::StatusAccountDisplay; use crate::text_formatting; +use crate::width::display_width; use chrono::DateTime; use chrono::Local; use codex_protocol::account::PlanType; use codex_utils_path_uri::PathConvention; use codex_utils_path_uri::PathUri; use std::path::Path; -use unicode_width::UnicodeWidthStr; fn normalize_agents_display_path(path: &Path) -> String { dunce::simplified(path).display().to_string() @@ -164,7 +164,7 @@ pub(crate) fn format_directory_display(directory: &Path, max_width: Option max_width { + if display_width(&formatted) > max_width { return text_formatting::center_truncate_path(&formatted, max_width); } } @@ -233,6 +233,15 @@ mod tests { } } + #[test] + fn format_directory_display_truncates_halfwidth_sound_marks() { + let directory = Path::new("workspace").join("ｶﾞ").join("project"); + let max_width = display_width(directory.to_string_lossy().as_ref()) - 1; + let formatted = format_directory_display(&directory, Some(max_width)); + + insta::assert_snapshot!(formatted.replace('\\', "/"), @"workspace/…/project"); + } + #[tokio::test] async fn compose_agents_summary_includes_global_agents_path() { let codex_home = TempDir::new().expect("temp codex home"); diff --git a/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_truncates_halfwidth_kana_in_narrow_terminal.snap b/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_truncates_halfwidth_kana_in_narrow_terminal.snap new file mode 100644 index 0000000000..17442d7076 --- /dev/null +++ b/codex-rs/tui/src/status/snapshots/codex_tui__status__tests__status_snapshot_truncates_halfwidth_kana_in_narrow_terminal.snap @@ -0,0 +1,24 @@ +--- +source: tui/src/status/tests.rs +expression: sanitized +--- +/status + +╭────────────────────────────────────────╮ +│ >_ OpenAI Codex (v0.0.0) │ +│ │ +│ Visit │ +│ https://chatgpt.com/codex/settings/usa │ +│ for up-to-date │ +│ information on rate limits and credits │ +│ │ +│ Model: ｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞ │ +│ Directory: [[workspace]] │ +│ Permissions: Custom (workspa │ +│ Agents.md: │ +│ Account: ｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟ@ex │ +│ Thread name: ｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞ │ +│ Collaboration mode: ｶﾞﾊﾟ collaborat │ +│ │ +│ Limits: data not availa │ +╰────────────────────────────────────────╯ diff --git a/codex-rs/tui/src/status/tests.rs b/codex-rs/tui/src/status/tests.rs index d95bc217f5..e8032a4486 100644 --- a/codex-rs/tui/src/status/tests.rs +++ b/codex-rs/tui/src/status/tests.rs @@ -1525,6 +1525,42 @@ async fn status_snapshot_truncates_in_narrow_terminal() { assert_snapshot!(sanitized); } +#[tokio::test] +async fn status_snapshot_truncates_halfwidth_kana_in_narrow_terminal() { + let temp_home = TempDir::new().expect("temp home"); + let mut config = test_config(&temp_home).await; + set_workspace_cwd(&mut config, test_path_buf("/workspace/tests").abs()); + + let account = StatusAccountDisplay::ChatGpt { + email: Some("ｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟ@example.com".to_string()), + plan: Some("ｶﾞﾊﾟ plan".to_string()), + }; + let usage = TokenUsage::default(); + let now = chrono::Local + .with_ymd_and_hms(2024, 1, 2, 3, 4, 5) + .single() + .expect("timestamp"); + let composite = new_status_output( + &config, + Some(&account), + /*token_info*/ None, + &usage, + &None, + Some("ｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟ thread".to_string()), + /*forked_from*/ None, + /*rate_limits*/ None, + /*plan_type*/ None, + now, + "ｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟｶﾞﾊﾟ-model", + Some("ｶﾞﾊﾟ collaboration mode"), + /*reasoning_effort_override*/ None, + ); + let rendered_lines = render_lines(&composite.display_lines(/*width*/ 42)); + let sanitized = sanitize_directory(rendered_lines).join("\n"); + + assert_snapshot!(sanitized); +} + #[tokio::test] async fn status_snapshot_shows_missing_limits_message() { let temp_home = TempDir::new().expect("temp home"); diff --git a/codex-rs/tui/src/terminal_hyperlinks.rs b/codex-rs/tui/src/terminal_hyperlinks.rs index b86402fd3a..3e415fa14a 100644 --- a/codex-rs/tui/src/terminal_hyperlinks.rs +++ b/codex-rs/tui/src/terminal_hyperlinks.rs @@ -20,8 +20,11 @@ use ratatui::widgets::Widget; use ratatui::widgets::Wrap; use url::Url; +use crate::line_truncation::line_width; use crate::render::line_utils::line_to_borrowed; use crate::render::line_utils::line_to_static; +use crate::width::char_width; +use crate::width::display_width; use crate::wrapping::RtOptions; use crate::wrapping::adaptive_wrap_line; @@ -86,16 +89,12 @@ impl HyperlinkLine { } pub(crate) fn width(&self) -> usize { - self.line - .spans - .iter() - .map(|span| usize::from(span.content.as_ref().cell_width())) - .sum() + line_width(&self.line) } pub(crate) fn push_span(&mut self, span: Span<'static>, destination: Option<&str>) { let start = self.width(); - let end = start + usize::from(span.content.as_ref().cell_width()); + let end = start + display_width(span.content.as_ref()); self.line.push_span(span); if end > start && let Some(destination) = destination.and_then(web_destination) @@ -158,7 +157,7 @@ pub(crate) fn prefix_hyperlink_lines( } else { subsequent_prefix.clone() }; - let shift = usize::from(prefix.content.as_ref().cell_width()); + let shift = display_width(prefix.content.as_ref()); let mut spans = Vec::with_capacity(line.line.spans.len() + 1); spans.push(prefix); spans.extend(line.line.spans); @@ -245,7 +244,7 @@ pub(crate) fn remap_wrapped_line( continue; }; let mapped = &rendered[rendered_start..]; - let mut output_column = usize::from(rendered[..rendered_start].cell_width()); + let mut output_column = display_width(&rendered[..rendered_start]); for ch in mapped.chars() { let width = char_cell_width(ch); while source @@ -320,8 +319,8 @@ pub(crate) fn web_links_in_text(text: &str) -> Vec { let Some(destination) = web_destination(candidate) else { continue; }; - let start = usize::from(text[..raw_start + trimmed_start].cell_width()); - let end = start + usize::from(candidate.cell_width()); + let start = display_width(&text[..raw_start + trimmed_start]); + let end = start + display_width(candidate); links.push(TerminalHyperlink::web(start..end, destination)); } links @@ -492,8 +491,7 @@ fn char_cell_width(ch: char) -> usize { if ch.is_control() { return 0; } - let mut encoded = [0; 4]; - usize::from(ch.encode_utf8(&mut encoded).cell_width()) + char_width(ch) } pub(crate) fn mark_buffer_hyperlinks( @@ -644,6 +642,25 @@ mod tests { ); } + #[test] + fn hyperlink_columns_follow_a_long_prefix_without_wrapping() { + let prefix = "a".repeat(65_536); + let destination = "https://example.com/long-prefix"; + let text = format!("{prefix} {destination}"); + + assert_eq!( + HyperlinkLine::new(Line::from(text.clone())).width(), + text.len() + ); + assert_eq!( + web_links_in_text(&text), + vec![TerminalHyperlink::web( + /*columns*/ 65_537..65_537 + destination.len(), + destination.to_string(), + )] + ); + } + #[test] fn preserves_balanced_parentheses_in_bare_web_urls() { let destination = "https://en.wikipedia.org/wiki/Function_(mathematics)"; diff --git a/codex-rs/tui/src/text_formatting.rs b/codex-rs/tui/src/text_formatting.rs index a89f392ebb..51b60c8f74 100644 --- a/codex-rs/tui/src/text_formatting.rs +++ b/codex-rs/tui/src/text_formatting.rs @@ -1,6 +1,6 @@ use unicode_segmentation::UnicodeSegmentation; -use unicode_width::UnicodeWidthChar; -use unicode_width::UnicodeWidthStr; + +use crate::width::display_width; pub(crate) fn capitalize_first(input: &str) -> String { let mut chars = input.chars(); @@ -33,14 +33,12 @@ pub(crate) fn format_and_truncate_tool_result( } } -/// Format JSON text in a compact single-line format with spaces for better Ratatui wrapping. -/// Ex: `{"a":"b",c:["d","e"]}` -> `{"a": "b", "c": ["d", "e"]}` -/// Returns the formatted JSON string if the input is valid JSON, otherwise returns None. -/// This is a little complicated, but it's necessary because Ratatui's wrapping is *very* limited -/// and can only do line breaks at whitespace. If we use the default serde_json format, we get lines -/// without spaces that Ratatui can't wrap nicely. If we use the serde_json pretty format as-is, -/// it's much too sparse and uses too many terminal rows. -/// Relevant issue: https://github.com/ratatui/ratatui/issues/293 +/// Formats JSON on one line with spaces after separators for readability and natural wrap points. +/// +/// Compact JSON is hard to scan, while pretty-printed JSON consumes unnecessary terminal rows. +/// Returns the formatted JSON string for valid input, or `None` otherwise. +/// +/// Example: `{"a":"b","c":["d","e"]}` becomes `{"a": "b", "c": ["d", "e"]}`. pub(crate) fn format_json_compact(text: &str) -> Option { let json = serde_json::from_str::(text).ok()?; let json_pretty = serde_json::to_string_pretty(&json).unwrap_or_else(|_| json.to_string()); @@ -121,7 +119,7 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String { if max_width == 0 { return String::new(); } - if UnicodeWidthStr::width(path) <= max_width { + if display_width(path) <= max_width { return path.to_string(); } @@ -142,7 +140,7 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String { if raw_segments.is_empty() { if has_leading_sep { let root = sep.to_string(); - if UnicodeWidthStr::width(root.as_str()) <= max_width { + if display_width(root.as_str()) <= max_width { return root; } } @@ -174,27 +172,27 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String { if allowed_width == 0 { return String::new(); } - if UnicodeWidthStr::width(original) <= allowed_width { + if display_width(original) <= allowed_width { return original.to_string(); } if allowed_width == 1 { return "…".to_string(); } - let mut kept: Vec = Vec::new(); + let mut kept = Vec::new(); let mut used_width = 1; // reserve space for leading ellipsis - for ch in original.chars().rev() { - let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0); - if used_width + ch_width > allowed_width { + for grapheme in original.graphemes(/*is_extended*/ true).rev() { + let grapheme_width = display_width(grapheme); + if used_width + grapheme_width > allowed_width { break; } - used_width += ch_width; - kept.push(ch); + used_width += grapheme_width; + kept.push(grapheme); } kept.reverse(); let mut truncated = String::from("…"); - for ch in kept { - truncated.push(ch); + for grapheme in kept { + truncated.push_str(grapheme); } truncated }; @@ -236,7 +234,7 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String { |segments: &mut Vec>, allow_front_truncate: bool| -> Option { loop { let candidate = assemble(has_leading_sep, segments); - let width = UnicodeWidthStr::width(candidate.as_str()); + let width = display_width(candidate.as_str()); if width <= max_width { return Some(candidate); } @@ -263,11 +261,11 @@ pub(crate) fn center_truncate_path(path: &str, max_width: usize) -> String { let mut changed = false; for idx in indices { - let original_width = UnicodeWidthStr::width(segments[idx].original); + let original_width = display_width(segments[idx].original); if original_width <= max_width && segment_count > 2 { continue; } - let seg_width = UnicodeWidthStr::width(segments[idx].text.as_str()); + let seg_width = display_width(segments[idx].text.as_str()); let other_width = width.saturating_sub(seg_width); let allowed_width = max_width.saturating_sub(other_width).max(1); let new_text = front_truncate(segments[idx].original, allowed_width); diff --git a/codex-rs/tui/src/width.rs b/codex-rs/tui/src/width.rs index a69cddb27b..9c9fe3e55d 100644 --- a/codex-rs/tui/src/width.rs +++ b/codex-rs/tui/src/width.rs @@ -1,16 +1,38 @@ -//! Width guards for transcript rendering with fixed prefix columns. +//! Terminal display-width helpers and guards for fixed prefix columns. //! //! Several rendering paths reserve a fixed number of columns for bullets, //! gutters, or labels before laying out content. When the terminal is very //! narrow, those reserved columns can consume the entire width, leaving zero //! or negative space for content. //! -//! These helpers centralise the subtraction and enforce a strict-positive +//! The display-width helpers match Ratatui's terminal-cell semantics while retaining `usize` +//! precision for long lines. The guards centralise subtraction and enforce a strict-positive //! contract: they return `Some(n)` where `n > 0`, or `None` when no usable //! content width remains. Callers treat `None` as "render prefix-only //! fallback" rather than attempting wrapped rendering at zero width, which //! would produce empty or unstable output. +use unicode_width::UnicodeWidthChar; +use unicode_width::UnicodeWidthStr; + +/// Returns the display width Ratatui uses for terminal text without its `u16` limit. +pub(crate) fn display_width(text: &str) -> usize { + UnicodeWidthStr::width(text) + + text + .chars() + .filter(|ch| matches!(ch, '\u{FF9E}' | '\u{FF9F}')) + .count() +} + +/// Returns a scalar's terminal width, treating halfwidth sound marks as visible cells. +pub(crate) fn char_width(ch: char) -> usize { + if matches!(ch, '\u{FF9E}' | '\u{FF9F}') { + 1 + } else { + UnicodeWidthChar::width(ch).unwrap_or(0) + } +} + /// Returns usable content width after reserving fixed columns. /// /// Guarantees a strict positive width (`Some(n)` where `n > 0`) or `None` when @@ -36,7 +58,22 @@ pub(crate) fn usable_content_width_u16(total_width: u16, reserved_cols: u16) -> #[cfg(test)] mod tests { use super::*; + use crate::line_truncation::line_width; use pretty_assertions::assert_eq; + use ratatui::text::Line; + + #[test] + fn display_width_matches_ratatui_halfwidth_sound_marks_without_overflow() { + assert_eq!(display_width("ｶﾞﾊﾟ"), 4); + assert_eq!(display_width("ｶﾞﾞ"), 3); + assert_eq!(display_width("界ﾞ"), 3); + assert_eq!(char_width('\u{FF9E}'), 1); + assert_eq!(char_width('\u{FF9F}'), 1); + + let text = "a".repeat(65_536); + assert_eq!(display_width(&text), 65_536); + assert_eq!(line_width(&Line::from(text)), 65_536); + } #[test] fn usable_content_width_returns_none_when_reserved_exhausts_width() { diff --git a/codex-rs/tui/src/wrapping.rs b/codex-rs/tui/src/wrapping.rs index 1636360def..d65c6b88aa 100644 --- a/codex-rs/tui/src/wrapping.rs +++ b/codex-rs/tui/src/wrapping.rs @@ -32,9 +32,198 @@ use std::ops::Range; use textwrap::Options; use textwrap::WordSeparator; use textwrap::core::Word; -use textwrap::core::display_width; +use textwrap::word_splitters::split_words; +use unicode_segmentation::UnicodeSegmentation; +use crate::line_truncation::line_width; use crate::render::line_utils::push_owned_lines; +use crate::width::display_width; + +/// Projected text keeps source-offset lookup separate from legal grapheme split points. +struct ProjectedText { + text: String, + source_boundaries: Vec<(usize, usize)>, + grapheme_boundaries: Vec, +} + +/// Replaces halfwidth sound-mark graphemes with equally wide, textwrap-safe placeholders. +/// +/// Source boundaries recover original byte offsets, while grapheme boundaries keep placeholders +/// indivisible and preserve leading whitespace as a wrapping opportunity. +fn project_halfwidth_sound_marks(text: &str) -> Option { + if !text.contains(['\u{FF9E}', '\u{FF9F}']) { + return None; + } + + let mut projected = String::with_capacity(text.len()); + let mut source_boundaries = vec![(0, 0)]; + let mut grapheme_boundaries = vec![0]; + for (source_start, grapheme) in text.grapheme_indices(/*is_extended*/ true) { + if grapheme.contains(['\u{FF9E}', '\u{FF9F}']) { + let source_end = source_start + grapheme.len(); + let content_start = grapheme + .find(|ch: char| !ch.is_whitespace()) + .unwrap_or(grapheme.len()); + let (whitespace, content) = grapheme.split_at(content_start); + for (offset, ch) in whitespace.char_indices() { + projected.push(ch); + source_boundaries.push((projected.len(), source_start + offset + ch.len_utf8())); + grapheme_boundaries.push(projected.len()); + } + + let width = display_width(content); + let projected_start = projected.len(); + for _ in 0..width / 2 { + if projected.len() > projected_start { + projected.push('\u{2060}'); + } + projected.push('界'); + source_boundaries.push((projected.len(), source_end)); + } + if width % 2 == 1 { + if projected.len() > projected_start { + projected.push('\u{2060}'); + } + projected.push('a'); + source_boundaries.push((projected.len(), source_end)); + } + } else { + for (offset, ch) in grapheme.char_indices() { + projected.push(ch); + source_boundaries.push((projected.len(), source_start + offset + ch.len_utf8())); + } + } + grapheme_boundaries.push(projected.len()); + } + + Some(ProjectedText { + text: projected, + source_boundaries, + grapheme_boundaries, + }) +} + +/// Maps a projected byte offset back to the corresponding original-text boundary. +fn source_offset(boundaries: &[(usize, usize)], projected_offset: usize) -> usize { + boundaries + .binary_search_by_key(&projected_offset, |(offset, _)| *offset) + .map(|index| boundaries[index].1) + .unwrap_or(projected_offset) +} + +/// Splits oversized projected words without separating placeholders for one source grapheme. +fn break_projected_words<'a>( + words: impl Iterator>, + projected: &'a ProjectedText, + line_width: usize, +) -> Vec> { + let projected_start = projected.text.as_ptr() as usize; + let mut pieces = Vec::new(); + + for word in words { + if display_width(word.word) <= line_width { + pieces.push(word); + continue; + } + + let word_start = word.word.as_ptr() as usize - projected_start; + let word_end = word_start + word.word.len(); + let mut piece_start = word_start; + let mut piece_width = 0; + let mut atom_start = word_start; + let boundary_start = projected + .grapheme_boundaries + .partition_point(|atom_end| *atom_end <= word_start); + + for atom_end in projected + .grapheme_boundaries + .iter() + .copied() + .skip(boundary_start) + { + if atom_end > word_end { + break; + } + + let atom_width = display_width(&projected.text[atom_start..atom_end]); + if piece_width > 0 && piece_width + atom_width > line_width { + pieces.push(Word::from(&projected.text[piece_start..atom_start])); + piece_start = atom_start; + piece_width = 0; + } + piece_width += atom_width; + atom_start = atom_end; + } + + let mut last = Word::from(&projected.text[piece_start..word_end]); + last.whitespace = word.whitespace; + last.penalty = word.penalty; + pieces.push(last); + } + + pieces +} + +/// Wraps projected text and translates the resulting ranges back to source byte offsets. +fn wrap_projected_ranges( + projected: &ProjectedText, + opts: &Options<'_>, + include_trailing_spaces: bool, +) -> Vec> { + let line_widths = [ + opts.width + .saturating_sub(display_width(opts.initial_indent)), + opts.width + .saturating_sub(display_width(opts.subsequent_indent)), + ]; + let line_ending = opts.line_ending.as_str(); + let mut ranges = Vec::new(); + let mut line_start = 0; + + for line in projected.text.split(line_ending) { + let words = opts.word_separator.find_words(line); + let split_words = split_words(words, &opts.word_splitter); + let mut broken_words = if opts.break_words { + break_projected_words(split_words, projected, line_widths[1]) + } else { + split_words.collect() + }; + if opts.break_words && !opts.initial_indent.is_empty() { + broken_words.insert(0, Word::from("")); + } + + let wrapped_words = opts.wrap_algorithm.wrap(&broken_words, &line_widths); + let mut cursor = line_start; + for words in wrapped_words { + let Some(last_word) = words.last() else { + let source = source_offset(&projected.source_boundaries, cursor); + ranges.push(source..source + usize::from(include_trailing_spaces)); + continue; + }; + let len = words + .iter() + .map(|word| word.word.len() + word.whitespace.len()) + .sum::() + - last_word.whitespace.len(); + let end = cursor + len; + let trailing_spaces = if include_trailing_spaces { + projected.text[end..] + .chars() + .take_while(|ch| *ch == ' ') + .count() + } else { + 0 + }; + let source_start = source_offset(&projected.source_boundaries, cursor); + let source_end = source_offset(&projected.source_boundaries, end + trailing_spaces); + ranges.push(source_start..source_end + usize::from(include_trailing_spaces)); + cursor = end + last_word.whitespace.len(); + } + line_start += line.len() + line_ending.len(); + } + + ranges +} /// Returns byte-ranges into `text` for each wrapped line, including /// trailing whitespace and a +1 sentinel byte. Used by the textarea @@ -44,6 +233,9 @@ where O: Into>, { let opts = width_or_options.into(); + if let Some(projected) = project_halfwidth_sound_marks(text) { + return wrap_projected_ranges(&projected, &opts, /*include_trailing_spaces*/ true); + } let mut lines: Vec> = Vec::new(); let mut cursor = 0usize; for (line_index, line) in textwrap::wrap(text, &opts).iter().enumerate() { @@ -87,6 +279,9 @@ where O: Into>, { let opts = width_or_options.into(); + if let Some(projected) = project_halfwidth_sound_marks(text) { + return wrap_projected_ranges(&projected, &opts, /*include_trailing_spaces*/ false); + } let mut lines: Vec> = Vec::new(); let mut cursor = 0usize; for (line_index, line) in textwrap::wrap(text, &opts).iter().enumerate() { @@ -684,7 +879,7 @@ fn word_wrap_flattened_line<'a>( // Compute first line range with reduced width due to initial indent. let initial_width_available = opts .width - .saturating_sub(rt_opts.initial_indent.width()) + .saturating_sub(line_width(&rt_opts.initial_indent)) .max(1); let initial_wrapped = wrap_ranges_trim(flat, opts.clone().width(initial_width_available)); let Some(first_line_range) = initial_wrapped.first() else { @@ -713,7 +908,7 @@ fn word_wrap_flattened_line<'a>( let base = base + skip_leading_spaces; let subsequent_width_available = opts .width - .saturating_sub(rt_opts.subsequent_indent.width()) + .saturating_sub(line_width(&rt_opts.subsequent_indent)) .max(1); let remaining_wrapped = wrap_ranges_trim(&flat[base..], opts.width(subsequent_width_available)); for r in &remaining_wrapped { @@ -758,11 +953,11 @@ fn mixed_url_wrap_line<'a>( ) -> Vec> { let initial_width_available = rt_opts .width - .saturating_sub(rt_opts.initial_indent.width()) + .saturating_sub(line_width(&rt_opts.initial_indent)) .max(1); let subsequent_width_available = rt_opts .width - .saturating_sub(rt_opts.subsequent_indent.width()) + .saturating_sub(line_width(&rt_opts.subsequent_indent)) .max(1); let ranges = mixed_url_wrap_ranges(flat, initial_width_available, subsequent_width_available); @@ -831,12 +1026,14 @@ fn mixed_url_wrap_ranges( 0 }; let empty_line_piece_limit = line_limit.saturating_sub(empty_line_prefix_width).max(1); + let mut indivisible = false; if line_start.is_none() && !piece.is_url && piece.width(text) > empty_line_piece_limit { - pending.splice( - pending_idx..=pending_idx, - split_mixed_url_word(text, piece, empty_line_piece_limit), - ); - continue; + let split = split_mixed_url_word(text, piece.clone(), empty_line_piece_limit); + if split.len() > 1 { + pending.splice(pending_idx..=pending_idx, split); + continue; + } + indivisible = true; } let piece_width = piece.width(text); @@ -845,6 +1042,7 @@ fn mixed_url_wrap_ranges( .unwrap_or(0); let fits = if line_start.is_none() { piece.is_url + || indivisible || empty_line_prefix_width + piece_width <= line_limit || empty_line_prefix_width >= line_limit } else { @@ -894,16 +1092,27 @@ fn split_mixed_url_word(text: &str, word: MixedUrlWord, line_limit: usize) -> Ve return vec![word]; } - let source = Word::from(&text[word.range.clone()]); - let mut offset = word.range.start; let mut pieces = Vec::new(); - for piece in source.break_apart(line_limit.max(1)) { - let end = offset + piece.word.len(); + let mut start = word.range.start; + let mut width = 0usize; + for (offset, grapheme) in text[word.range.clone()].grapheme_indices(/*is_extended*/ true) { + let grapheme_width = display_width(grapheme); + if width > 0 && width + grapheme_width > line_limit.max(1) { + let end = word.range.start + offset; + pieces.push(MixedUrlWord { + range: start..end, + is_url: false, + }); + start = end; + width = 0; + } + width += grapheme_width; + } + if start < word.range.end { pieces.push(MixedUrlWord { - range: offset..end, + range: start..word.range.end, is_url: false, }); - offset = end; } pieces } @@ -1490,6 +1699,24 @@ them."# ); } + #[test] + fn adaptive_wrap_line_mixed_url_counts_halfwidth_sound_marks() { + let line = Line::from("ｶﾞﾊﾟtail https://x.co"); + let out = adaptive_wrap_line(&line, RtOptions::new(/*width*/ 4)); + let rendered = out.iter().map(concat_line).collect_vec(); + + assert_eq!(rendered, ["ｶﾞﾊﾟ", "tail", "https://x.co"]); + } + + #[test] + fn adaptive_wrap_line_mixed_url_makes_progress_for_an_indivisible_grapheme() { + let line = Line::from("ｶﾞ https://x.co"); + let out = adaptive_wrap_line(&line, RtOptions::new(/*width*/ 1)); + let rendered = out.iter().map(concat_line).collect_vec(); + + assert_eq!(rendered, ["ｶﾞ", "https://x.co"]); + } + #[test] fn map_owned_wrapped_line_to_range_recovers_on_non_prefix_mismatch() { // Match source chars first, then introduce a non-penalty mismatch. @@ -1553,6 +1780,84 @@ them."# assert_eq!(rebuilt, text); } + #[test] + fn wrap_ranges_count_halfwidth_sound_marks_without_changing_byte_offsets() { + for (text, width, expected) in [ + ("abｶﾞc", 4, &["abｶﾞ", "c"][..]), + ("abｶﾞc", 3, &["ab", "ｶﾞc"][..]), + ("ﾞab", 2, &["ﾞa", "b"][..]), + ("a ﾞb", 2, &["a", "ﾞb"][..]), + ("a ﾟb", 2, &["a", "ﾟb"][..]), + ("ｶﾞﾞx", 3, &["ｶﾞﾞ", "x"][..]), + ("界ﾞx", 3, &["界ﾞ", "x"][..]), + ("ｶﾞﾞab", 2, &["ｶﾞﾞ", "ab"][..]), + ("界ﾞab", 2, &["界ﾞ", "ab"][..]), + ("abｶﾞﾞcd", 2, &["ab", "ｶﾞﾞ", "cd"][..]), + ("ab界ﾞcd", 2, &["ab", "界ﾞ", "cd"][..]), + ] { + let ranges = wrap_ranges_trim(text, Options::new(width)); + let wrapped = ranges + .iter() + .map(|range| &text[range.clone()]) + .collect_vec(); + assert_eq!(wrapped, expected); + } + + for (text, emoji, sound_mark) in [("a👨‍👩 ﾞ", "👨‍👩", "ﾞ"), ("a👍🏻 ﾟ", "👍🏻", "ﾟ")] + { + for options in [ + Options::new(/*width*/ 2), + Options::new(/*width*/ 2).word_separator(WordSeparator::AsciiSpace), + ] { + let ranges = wrap_ranges_trim(text, options); + let wrapped = ranges + .iter() + .map(|range| &text[range.clone()]) + .collect_vec(); + + assert_eq!(wrapped, ["a", emoji, sound_mark]); + } + } + + for grapheme in ["ｶﾞﾞ", "界ﾞ"] { + for width in [1, 2] { + let ranges = wrap_ranges(grapheme, Options::new(width)); + assert_eq!(ranges, std::iter::once(0..grapheme.len() + 1).collect_vec()); + } + } + } + + #[test] + fn wrap_ranges_preserve_crlf_source_boundaries_without_splitting_graphemes() { + for prefix in ["ｶﾞ", "ﾊﾟ"] { + let text = format!("{prefix}\r\nnext"); + + for word_separator in [ + WordSeparator::UnicodeBreakProperties, + WordSeparator::AsciiSpace, + ] { + for line_ending in [textwrap::LineEnding::LF, textwrap::LineEnding::CRLF] { + let options = Options::new(/*width*/ 4) + .line_ending(line_ending) + .word_separator(word_separator) + .wrap_algorithm(textwrap::WrapAlgorithm::FirstFit); + let first_end = + prefix.len() + usize::from(line_ending == textwrap::LineEnding::LF); + let second_start = prefix.len() + "\r\n".len(); + + let ranges = wrap_ranges(&text, options.clone()); + assert_eq!(ranges, [0..first_end + 1, second_start..text.len() + 1]); + for range in &ranges { + assert!(text.get(range.start..range.end - 1).is_some()); + } + + let trimmed = wrap_ranges_trim(&text, options); + assert_eq!(trimmed, [0..first_end, second_start..text.len()]); + } + } + } + } + #[test] fn map_owned_wrapped_line_to_range_repro_overconsumes_repeated_prefix_patterns() { let text = "- - foo";