mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Support Cloud image messages and editable draft handoffs (#60061)
## Why Cloud messages previously accepted only plain text, and switching a draft to Cloud waited for session preparation without processing composer input. ## What changed - Support ordered text and image parts in Cloud submissions and user-message summaries, preserving attachment placeholders, image detail, and client message IDs. - Accept inline data URLs and uploaded file IDs, including image-only messages. Reject invalid references and text spans locally, and cap encoded image requests at 3.75 MiB while retaining the existing plain-text limit. - Keep the composer editable and redraw the TUI while Cloud prepares a conversation. Carry the latest draft, attachments, and Vim mode into the destination, retain drafts on failure, and prevent submission to the outgoing session during the handoff. - Preserve buffered paste input and pending clipboard reads across the handoff, with connection and failure hints. ## Testing Add regression coverage for image wire encoding, receipt identity, validation and size boundaries, draft editing during successful and failed handoffs, clipboard completion, Vim state, and attachment recovery after rejected submissions. GitOrigin-RevId: 0cbcde0ab89ba961780b5f88456057e6b6a19cd5
This commit is contained in:
committed by
copyberry
parent
d4a341bd52
commit
e6bf88ae02
@@ -3,9 +3,16 @@
|
||||
|
||||
use super::*;
|
||||
use crate::clipboard_copy::worker::CopyResult;
|
||||
use crate::clipboard_paste::PasteImageError;
|
||||
|
||||
const IMAGE_PASTE_REPEAT_WINDOW: Duration = Duration::from_millis(/*millis*/ 250);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ClipboardPasteContext {
|
||||
Composer,
|
||||
DraftHandoff,
|
||||
}
|
||||
|
||||
pub(super) enum PendingCopy {
|
||||
Message(u64, String),
|
||||
Selection(u64),
|
||||
@@ -13,7 +20,11 @@ pub(super) enum PendingCopy {
|
||||
|
||||
impl ChatWidget {
|
||||
/// Recognize image paste shortcuts, including presses consumed by repeat suppression.
|
||||
pub(super) fn image_paste_key_action(&mut self, key: KeyEvent) -> Option<KeyEventAction> {
|
||||
pub(super) fn image_paste_key_action(
|
||||
&mut self,
|
||||
key: KeyEvent,
|
||||
context: ClipboardPasteContext,
|
||||
) -> Option<KeyEventAction> {
|
||||
let shift_insert = cfg!(windows) && key_hint::shift(KeyCode::Insert).is_press(key);
|
||||
let image_paste = key.kind == KeyEventKind::Press
|
||||
&& ((matches!(key.code, KeyCode::Char(c) if c.eq_ignore_ascii_case(&'v'))
|
||||
@@ -27,23 +38,38 @@ impl ChatWidget {
|
||||
}
|
||||
return None;
|
||||
}
|
||||
if Instant::now() < self.suppress_image_paste_until {
|
||||
self.suppress_image_paste_until = Instant::now() + IMAGE_PASTE_REPEAT_WINDOW;
|
||||
let now = Instant::now();
|
||||
let suppressed = now < self.suppress_image_paste_until;
|
||||
self.suppress_image_paste_until = now + IMAGE_PASTE_REPEAT_WINDOW;
|
||||
if suppressed {
|
||||
return Some(KeyEventAction::None);
|
||||
}
|
||||
self.suppress_image_paste_until = Instant::now() + IMAGE_PASTE_REPEAT_WINDOW;
|
||||
let text_fallback = (cfg!(windows)
|
||||
&& (key.modifiers == KeyModifiers::CONTROL || shift_insert))
|
||||
.then_some(key.code);
|
||||
if text_fallback.is_some() {
|
||||
if text_fallback.is_some() && context == ClipboardPasteContext::Composer {
|
||||
self.bottom_pane.prepare_composer_clipboard_paste();
|
||||
}
|
||||
Some(KeyEventAction::PasteImage { text_fallback })
|
||||
Some(KeyEventAction::PasteImage {
|
||||
text_fallback,
|
||||
context,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn paste_image(&mut self) -> Result<(), crate::clipboard_paste::PasteImageError> {
|
||||
pub(crate) fn paste_image(
|
||||
&mut self,
|
||||
context: ClipboardPasteContext,
|
||||
text_fallback: Option<KeyCode>,
|
||||
) -> Result<(), PasteImageError> {
|
||||
let result = paste_image_to_temp_png();
|
||||
self.suppress_image_paste_until = Instant::now() + IMAGE_PASTE_REPEAT_WINDOW;
|
||||
if context == ClipboardPasteContext::DraftHandoff
|
||||
&& text_fallback.is_some()
|
||||
&& matches!(&result, Err(PasteImageError::NoImage(_)))
|
||||
{
|
||||
// Settle typing before capturing the asynchronous text target.
|
||||
self.bottom_pane.prepare_composer_clipboard_paste();
|
||||
}
|
||||
let (path, info) = result?;
|
||||
tracing::debug!(
|
||||
"pasted image size={}x{} format={}",
|
||||
@@ -51,7 +77,7 @@ impl ChatWidget {
|
||||
info.height,
|
||||
info.encoded_format.label()
|
||||
);
|
||||
self.attach_image(path);
|
||||
self.attach_image(path, context);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -35,6 +35,15 @@ impl ChatWidget {
|
||||
draft
|
||||
}
|
||||
|
||||
/// Continue an active Cloud-switch draft without undoing an accepted Vim Escape.
|
||||
pub(crate) fn restore_cloud_switch_draft(&mut self, draft: ComposerDraftSnapshot) {
|
||||
let continue_vim_normal_mode = draft.vim_normal_mode;
|
||||
self.restore_startup_draft(draft);
|
||||
if continue_vim_normal_mode {
|
||||
self.bottom_pane.set_vim_enabled(/*enabled*/ true);
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore the exact draft entered before the fully initialized composer became available.
|
||||
pub(crate) fn restore_startup_draft(&mut self, draft: ComposerDraftSnapshot) {
|
||||
self.bottom_pane
|
||||
|
||||
@@ -12,7 +12,10 @@ use crate::clipboard_copy::worker::CopyResult;
|
||||
pub(crate) enum KeyEventAction {
|
||||
None,
|
||||
CopyLastResponse(Arc<str>),
|
||||
PasteImage { text_fallback: Option<KeyCode> },
|
||||
PasteImage {
|
||||
text_fallback: Option<KeyCode>,
|
||||
context: ClipboardPasteContext,
|
||||
},
|
||||
}
|
||||
|
||||
impl ChatWidget {
|
||||
@@ -250,12 +253,7 @@ impl ChatWidget {
|
||||
self.on_ctrl_c();
|
||||
return KeyEventAction::None;
|
||||
}
|
||||
KeyEvent {
|
||||
code: KeyCode::Char(c),
|
||||
modifiers,
|
||||
kind: KeyEventKind::Press,
|
||||
..
|
||||
} if modifiers.contains(KeyModifiers::CONTROL) && c.eq_ignore_ascii_case(&'d') => {
|
||||
key if Self::is_ctrl_d_press(key) => {
|
||||
if self.on_ctrl_d() {
|
||||
return KeyEventAction::None;
|
||||
}
|
||||
@@ -264,7 +262,9 @@ impl ChatWidget {
|
||||
self.quit_shortcut_key = None;
|
||||
}
|
||||
_ => {
|
||||
if let Some(action) = self.image_paste_key_action(key_event) {
|
||||
if let Some(action) =
|
||||
self.image_paste_key_action(key_event, ClipboardPasteContext::Composer)
|
||||
{
|
||||
return action;
|
||||
}
|
||||
if key_event.kind == KeyEventKind::Press {
|
||||
@@ -363,12 +363,12 @@ impl ChatWidget {
|
||||
KeyEventAction::None
|
||||
}
|
||||
|
||||
/// Attach a local image to the composer when the active model supports image inputs.
|
||||
/// Attach a local image, deferring model validation while the draft awaits its destination.
|
||||
///
|
||||
/// When the model does not advertise image support, we keep the draft unchanged and surface a
|
||||
/// warning event so users can switch models or remove attachments.
|
||||
pub(crate) fn attach_image(&mut self, path: PathBuf) {
|
||||
if !self.current_model_supports_images() {
|
||||
/// Ordinary paste still checks the active model. Handoff only retains the attachment locally;
|
||||
/// submission validates the destination model and backend before admission.
|
||||
pub(crate) fn attach_image(&mut self, path: PathBuf, context: ClipboardPasteContext) {
|
||||
if context == ClipboardPasteContext::Composer && !self.current_model_supports_images() {
|
||||
self.add_to_history(history_cell::new_warning_event(
|
||||
self.image_inputs_not_supported_message(),
|
||||
));
|
||||
@@ -376,6 +376,9 @@ impl ChatWidget {
|
||||
return;
|
||||
}
|
||||
tracing::info!("attach_image path={path:?}");
|
||||
if context == ClipboardPasteContext::DraftHandoff && !self.has_active_view() {
|
||||
self.bottom_pane.settle_before_handoff_image();
|
||||
}
|
||||
self.bottom_pane.attach_image(path);
|
||||
self.refresh_startup_recovery();
|
||||
self.request_redraw();
|
||||
@@ -608,6 +611,11 @@ impl ChatWidget {
|
||||
self.finish_paste_burst_tick(frame_requester, flushed)
|
||||
}
|
||||
|
||||
pub(crate) fn handle_handoff_paste_tick(&mut self, frame_requester: FrameRequester) -> bool {
|
||||
let flushed = self.bottom_pane.flush_handoff_paste_if_due();
|
||||
self.finish_paste_burst_tick(frame_requester, flushed)
|
||||
}
|
||||
|
||||
fn finish_paste_burst_tick(&mut self, frame_requester: FrameRequester, flushed: bool) -> bool {
|
||||
if flushed {
|
||||
self.refresh_startup_recovery();
|
||||
@@ -700,11 +708,23 @@ impl ChatWidget {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_ctrl_d_press(key: KeyEvent) -> bool {
|
||||
matches!(
|
||||
key,
|
||||
KeyEvent {
|
||||
code: KeyCode::Char(c),
|
||||
modifiers,
|
||||
kind: KeyEventKind::Press,
|
||||
..
|
||||
} if modifiers.contains(KeyModifiers::CONTROL) && c.eq_ignore_ascii_case(&'d')
|
||||
)
|
||||
}
|
||||
|
||||
/// Handles a Ctrl+D press at the chat-widget layer.
|
||||
///
|
||||
/// Ctrl-D only participates in quit when the composer is empty and no modal/popup is active.
|
||||
/// Otherwise it should be routed to the active view and not attempt to quit.
|
||||
fn on_ctrl_d(&mut self) -> bool {
|
||||
pub(super) fn on_ctrl_d(&mut self) -> bool {
|
||||
let key = key_hint::ctrl(KeyCode::Char('d'));
|
||||
if !DOUBLE_PRESS_QUIT_SHORTCUT_ENABLED {
|
||||
if !self.bottom_pane.composer_is_empty() || !self.bottom_pane.no_modal_or_popup_active()
|
||||
|
||||
@@ -196,9 +196,39 @@ impl ChatWidget {
|
||||
self.bottom_pane.active_view_id() == Some(view_id)
|
||||
}
|
||||
|
||||
pub(crate) fn handle_restricted_key(&mut self, key: KeyEvent, mode: RestrictedInputMode) {
|
||||
pub(crate) fn composer_paste_deadline(&self) -> Option<Instant> {
|
||||
self.bottom_pane.composer_paste_deadline()
|
||||
}
|
||||
|
||||
pub(crate) fn handle_restricted_key(
|
||||
&mut self,
|
||||
key: KeyEvent,
|
||||
mode: RestrictedInputMode,
|
||||
) -> KeyEventAction {
|
||||
if self.external_writer_view {
|
||||
return;
|
||||
return KeyEventAction::None;
|
||||
}
|
||||
if mode == RestrictedInputMode::DraftHandoff {
|
||||
// Buffered typing also belongs to the draft. Queue the usual quit only when
|
||||
// Ctrl+D has no draft or active input surface to edit.
|
||||
if Self::is_ctrl_d_press(key)
|
||||
&& self.composer_is_empty()
|
||||
&& self.bottom_pane.no_modal_or_popup_active()
|
||||
&& self.on_ctrl_d()
|
||||
{
|
||||
return KeyEventAction::None;
|
||||
}
|
||||
if !self.startup_submission_has_protected_input()
|
||||
&& !self.blocks_direct_input
|
||||
&& self.bottom_pane.composer_input_enabled()
|
||||
&& let Some(action) =
|
||||
self.image_paste_key_action(key, ClipboardPasteContext::DraftHandoff)
|
||||
{
|
||||
return action;
|
||||
}
|
||||
self.bottom_pane.handle_restricted_key(key, mode);
|
||||
self.refresh_startup_recovery();
|
||||
return KeyEventAction::None;
|
||||
}
|
||||
let startup_key_handled = self.input_queue.startup_submission.is_some()
|
||||
&& !self.startup_submission_has_protected_input()
|
||||
@@ -210,7 +240,7 @@ impl ChatWidget {
|
||||
)]));
|
||||
}
|
||||
if startup_key_handled {
|
||||
return;
|
||||
return KeyEventAction::None;
|
||||
}
|
||||
if self.input_queue.startup_submission.is_some()
|
||||
&& !self.startup_submission_has_protected_input()
|
||||
@@ -218,10 +248,10 @@ impl ChatWidget {
|
||||
// Cursor movement must preserve the confirmed draft's paste provenance.
|
||||
let result = self.bottom_pane.handle_key_event(key);
|
||||
self.handle_composer_input_result(result, /*had_modal_or_popup*/ false);
|
||||
return;
|
||||
return KeyEventAction::None;
|
||||
}
|
||||
if self.handle_question_key(key) {
|
||||
return;
|
||||
return KeyEventAction::None;
|
||||
}
|
||||
if key.kind == KeyEventKind::Press && self.chat_keymap.edit_queued_message.is_pressed(key) {
|
||||
if let Some(composer) = self.pop_latest_queued_composer_state() {
|
||||
@@ -243,6 +273,7 @@ impl ChatWidget {
|
||||
let result = self.bottom_pane.handle_restricted_key(key, mode);
|
||||
self.handle_composer_input_result(result, /*had_modal_or_popup*/ false);
|
||||
}
|
||||
KeyEventAction::None
|
||||
}
|
||||
|
||||
pub(crate) fn reconnect_failed(&mut self) {
|
||||
|
||||
@@ -417,6 +417,99 @@ async fn parent_owned_thread_preserves_queued_input_before_draining() {
|
||||
assert_no_submit_op(&mut op_rx);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handoff_image_survives_model_rejection_and_submission_failure() {
|
||||
let (mut source, _events, mut source_ops) =
|
||||
make_chatwidget_manual(/*model_override*/ None).await;
|
||||
source.thread_id = Some(ThreadId::new());
|
||||
let mut models = source.model_catalog.try_list_models().unwrap();
|
||||
for model in &mut models {
|
||||
model
|
||||
.input_modalities
|
||||
.retain(|modality| *modality != InputModality::Image);
|
||||
}
|
||||
source.model_catalog = Arc::new(ModelCatalog::new(models));
|
||||
source.sync_image_paste_enabled();
|
||||
let path = PathBuf::from("image.png");
|
||||
let action = source.handle_restricted_key(
|
||||
KeyEvent::new(KeyCode::Char('v'), KeyModifiers::ALT),
|
||||
RestrictedInputMode::DraftHandoff,
|
||||
);
|
||||
let KeyEventAction::PasteImage { context, .. } = action else {
|
||||
panic!("expected handoff image paste");
|
||||
};
|
||||
source.attach_image(path.clone(), context);
|
||||
let draft = source.composer_draft_snapshot();
|
||||
assert_eq!(
|
||||
draft.local_images,
|
||||
vec![LocalImageAttachment {
|
||||
placeholder: "[Image #1]".into(),
|
||||
path,
|
||||
}],
|
||||
);
|
||||
insta::assert_snapshot!(source.bottom_pane.composer_text(), @"[Image #1]");
|
||||
source.handle_restricted_key(KeyCode::Enter.into(), RestrictedInputMode::DraftHandoff);
|
||||
assert_no_submit_op(&mut source_ops);
|
||||
|
||||
// A failed switch leaves the source active: submission must reject but retain the image.
|
||||
source.handle_key_event(KeyCode::Enter.into());
|
||||
assert_no_submit_op(&mut source_ops);
|
||||
assert_eq!(
|
||||
source.bottom_pane.composer_draft_snapshot().local_images,
|
||||
draft.local_images
|
||||
);
|
||||
|
||||
let (mut destination, _events, mut destination_ops) =
|
||||
make_chatwidget_manual(/*model_override*/ None).await;
|
||||
destination.thread_id = Some(ThreadId::new());
|
||||
destination.restore_startup_draft(draft.clone());
|
||||
let mut supported_models = destination.model_catalog.try_list_models().unwrap();
|
||||
let current_model = destination.current_model().to_string();
|
||||
let model = supported_models
|
||||
.iter_mut()
|
||||
.find(|model| model.model == current_model)
|
||||
.expect("current model");
|
||||
if !model.input_modalities.contains(&InputModality::Image) {
|
||||
model.input_modalities.push(InputModality::Image);
|
||||
}
|
||||
let mut unsupported_models = supported_models.clone();
|
||||
for model in &mut unsupported_models {
|
||||
model
|
||||
.input_modalities
|
||||
.retain(|modality| *modality != InputModality::Image);
|
||||
}
|
||||
destination.model_catalog = Arc::new(ModelCatalog::new(unsupported_models));
|
||||
destination.handle_key_event(KeyCode::Enter.into());
|
||||
assert_no_submit_op(&mut destination_ops);
|
||||
assert_eq!(
|
||||
destination
|
||||
.bottom_pane
|
||||
.composer_draft_snapshot()
|
||||
.local_images,
|
||||
draft.local_images
|
||||
);
|
||||
|
||||
// A capable destination admits the image; a subsequent submission failure restores it.
|
||||
destination.model_catalog = Arc::new(ModelCatalog::new(supported_models));
|
||||
destination.handle_key_event(KeyCode::Enter.into());
|
||||
assert_matches!(next_submit_op(&mut destination_ops), Op::UserTurn { .. });
|
||||
let client_id = destination
|
||||
.input_queue
|
||||
.pending_user_message_client_id
|
||||
.clone()
|
||||
.expect("pending submission");
|
||||
destination.restore_rejected_user_message(&client_id, "submission failed".into());
|
||||
assert_eq!(
|
||||
destination
|
||||
.bottom_pane
|
||||
.composer_draft_snapshot()
|
||||
.local_images,
|
||||
draft.local_images
|
||||
);
|
||||
insta::assert_snapshot!(destination.bottom_pane.composer_text(), @"[Image #1]");
|
||||
assert_no_submit_op(&mut destination_ops);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn local_clipboard_image_does_not_turn_a_held_bang_into_a_shell_command() {
|
||||
let (mut chat, _events, mut operations) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||
@@ -427,11 +520,11 @@ async fn local_clipboard_image_does_not_turn_a_held_bang_into_a_shell_command()
|
||||
assert_eq!(chat.bottom_pane.composer_recovery_snapshot().text, "!");
|
||||
|
||||
let paste = chat.handle_key_event(KeyEvent::new(KeyCode::Char('v'), KeyModifiers::ALT));
|
||||
let KeyEventAction::PasteImage { .. } = paste else {
|
||||
let KeyEventAction::PasteImage { context, .. } = paste else {
|
||||
panic!("expected a clipboard image request");
|
||||
};
|
||||
let image_path = PathBuf::from("image.png");
|
||||
chat.attach_image(image_path.clone());
|
||||
chat.attach_image(image_path.clone(), context);
|
||||
chat.handle_key_event(KeyCode::Enter.into());
|
||||
|
||||
let draft = chat.bottom_pane.composer_recovery_snapshot();
|
||||
|
||||
@@ -2592,7 +2592,7 @@ async fn copy_shortcut_can_be_remapped() {
|
||||
let paste = KeyEvent::new(code, modifiers);
|
||||
assert_matches!(
|
||||
chat.handle_key_event(paste),
|
||||
crate::chatwidget::KeyEventAction::PasteImage { text_fallback }
|
||||
crate::chatwidget::KeyEventAction::PasteImage { text_fallback, .. }
|
||||
if text_fallback == (cfg!(windows) && modifiers == KeyModifiers::CONTROL)
|
||||
.then_some(code)
|
||||
);
|
||||
@@ -2609,7 +2609,8 @@ async fn copy_shortcut_can_be_remapped() {
|
||||
assert_matches!(
|
||||
chat.handle_key_event(shift_insert),
|
||||
KeyEventAction::PasteImage {
|
||||
text_fallback: Some(KeyCode::Insert)
|
||||
text_fallback: Some(KeyCode::Insert),
|
||||
..
|
||||
}
|
||||
);
|
||||
assert_eq!(chat.bottom_pane.composer_text(), "q");
|
||||
|
||||
Reference in New Issue
Block a user