openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-08 20:40:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support Cloud image messages and editable draft handoffs (#60061)

Browse Source 
## Why

Cloud messages previously accepted only plain text, and switching a draft to Cloud waited for session preparation without processing composer input.

## What changed

- Support ordered text and image parts in Cloud submissions and user-message summaries, preserving attachment placeholders, image detail, and client message IDs.
- Accept inline data URLs and uploaded file IDs, including image-only messages. Reject invalid references and text spans locally, and cap encoded image requests at 3.75 MiB while retaining the existing plain-text limit.
- Keep the composer editable and redraw the TUI while Cloud prepares a conversation. Carry the latest draft, attachments, and Vim mode into the destination, retain drafts on failure, and prevent submission to the outgoing session during the handoff.
- Preserve buffered paste input and pending clipboard reads across the handoff, with connection and failure hints.

## Testing

Add regression coverage for image wire encoding, receipt identity, validation and size boundaries, draft editing during successful and failed handoffs, clipboard completion, Vim state, and attachment recovery after rejected submissions.

GitOrigin-RevId: 0cbcde0ab89ba961780b5f88456057e6b6a19cd5
This commit is contained in:
Benjamin Carlsson
2026-11-07 04:48:30 +00:00
committed by copyberry
parent d4a341bd52
commit e6bf88ae02
46 changed files with 1918 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/app-server-client/src/cloud_backend/events.rs
+40 -5
View File
@@ -2,6 +2,7 @@
use codex_app_server_protocol as protocol;
use codex_cloud_client as cloud;
use codex_protocol::models::ImageDetail;
use color_eyre::eyre::Result;
use color_eyre::eyre::bail;
@@ -92,15 +93,49 @@ pub(super) fn item_summary(item: cloud::ItemSummary) -> Option<protocol::ThreadI
cloud::ItemSummary::UserMessage {
id,
client_id,
text,
content,
} => protocol::ThreadItemPayload::UserMessage {
id,
client_id,
content: text
content: content
.into_iter()
.map(|text| protocol::UserInput::Text {
text,
text_elements: Vec::new(),
.map(|item| match item {
cloud::UserMessageInput::Text {
text,
text_elements,
} => protocol::UserInput::Text {
text,
text_elements: text_elements
.into_iter()
.map(|element| {
protocol::TextElement::new(
protocol::ByteRange {
start: element.start,
end: element.end,
},
element.placeholder,
)
})
.collect(),
},
cloud::UserMessageInput::Image { image, detail } => {
protocol::UserInput::Image {
image: match image {
cloud::ImageReference::Url(url) => {
protocol::ImageReference::Inline { url }
}
cloud::ImageReference::FileId(file_id) => {
protocol::ImageReference::File { file_id }
}
},
detail: detail.map(|detail| match detail {
cloud::ImageDetail::Auto => ImageDetail::Auto,
cloud::ImageDetail::Low => ImageDetail::Low,
cloud::ImageDetail::High => ImageDetail::High,
cloud::ImageDetail::Original => ImageDetail::Original,
}),
}
}
})
.collect(),
},
codex-rs/app-server-client/src/cloud_backend/events_tests.rs
+29 -7
View File
@@ -4,11 +4,25 @@ use super::*;
use pretty_assertions::assert_eq;
#[test]
fn text_receipt_preserves_submission_identity() {
fn image_receipt_preserves_submission_identity_and_attachment() {
let label = "[Image #1]";
let actual = item_summary(cloud::ItemSummary::UserMessage {
id: "item-1".into(),
client_id: Some("submission-1".into()),
text: vec!["hello".into(), "world".into()],
content: vec![
cloud::UserMessageInput::Image {
image: cloud::ImageReference::FileId("file-synthetic".into()),
detail: Some(cloud::ImageDetail::High),
},
cloud::UserMessageInput::Text {
text: format!("{label} describe"),
text_elements: vec![cloud::UserMessageTextElement {
start: 0,
end: label.len(),
placeholder: Some(label.into()),
}],
},
],
});
assert_eq!(
actual,
@@ -17,13 +31,21 @@ fn text_receipt_preserves_submission_identity() {
id: "item-1".into(),
client_id: Some("submission-1".into()),
content: vec![
protocol::UserInput::Text {
text: "hello".into(),
text_elements: Vec::new()
protocol::UserInput::Image {
image: protocol::ImageReference::File {
file_id: "file-synthetic".into(),
},
detail: Some(ImageDetail::High),
},
protocol::UserInput::Text {
text: "world".into(),
text_elements: Vec::new()
text: format!("{label} describe"),
text_elements: vec![protocol::TextElement::new(
protocol::ByteRange {
start: 0,
end: label.len(),
},
Some(label.into()),
)],
},
],
},
codex-rs/app-server-client/src/cloud_backend/mod.rs
+1 -6
View File
@@ -225,12 +225,7 @@ async fn run_worker(
{
-32601
} else if error.is::<PreflightRejection>()
|| user_message_submission
&& (error.is::<MissingReplyAttachment>()
|| matches!(
error.downcast_ref::<cloud::MutationError>(),
Some(cloud::MutationError::InvalidRequest)
))
|| user_message_submission && error.is::<MissingReplyAttachment>()
{
-32602
} else {
codex-rs/app-server-client/src/cloud_backend/request_tests.rs
+52 -1
View File
@@ -63,7 +63,13 @@ fn user_input_uses_stable_id_and_the_existing_cloud_configuration() {
let request = cloud_message(message()).unwrap();
assert_eq!(request.thread_id, "thread-a");
assert_eq!(request.client_user_message_id, "submission-a");
assert_eq!(request.text, "one\ntwo");
assert_eq!(
request.input,
vec![cloud::UserMessageInput::Text {
text: "one\ntwo".into(),
text_elements: Vec::new(),
}]
);
assert_eq!(
request.turn_trigger.as_deref(),
Some("safety_recourse_follow_up")
@@ -89,6 +95,51 @@ fn user_input_uses_stable_id_and_the_existing_cloud_configuration() {
);
}
#[test]
fn image_submission_keeps_attachment_markers_and_input_order() {
let label = "[Image #1]";
let request = cloud_message(protocol::TurnStartParams {
input: vec![
protocol::UserInput::Image {
image: protocol::ImageReference::File {
file_id: "file-synthetic".into(),
},
detail: Some(ImageDetail::High),
},
protocol::UserInput::Text {
text: format!("{label} describe"),
text_elements: vec![protocol::TextElement::new(
protocol::ByteRange {
start: 0,
end: label.len(),
},
Some(label.into()),
)],
},
],
..message()
})
.unwrap();
assert_eq!(
request.input,
vec![
cloud::UserMessageInput::Image {
image: cloud::ImageReference::FileId("file-synthetic".into()),
detail: Some(cloud::ImageDetail::High),
},
cloud::UserMessageInput::Text {
text: format!("{label} describe"),
text_elements: vec![cloud::UserMessageTextElement {
start: 0,
end: label.len(),
placeholder: Some(label.into()),
}],
},
]
);
}
#[test]
fn duplicate_and_queued_admissions_never_invent_a_running_turn() {
for outcome in [
codex-rs/app-server-client/src/cloud_backend/requests.rs
+69 -12
View File
@@ -6,6 +6,7 @@ use super::send_notification;
use crate::CloudUserMessageReceipt;
use codex_app_server_protocol as protocol;
use codex_cloud_client as cloud;
use codex_protocol::models::ImageDetail;
use color_eyre::eyre::Result;
use color_eyre::eyre::bail;
use color_eyre::eyre::eyre;
@@ -194,7 +195,15 @@ impl Requests {
Request::TurnStart { params, .. } => {
let mut message = cloud_message(params)?;
message.mode = mode;
let admission = self.client.add_user_message(&message).await?;
let admission = match self.client.add_user_message(&message).await {
Err(cloud::MutationError::InvalidRequest) => {
return Err(PreflightRejection(
"The Cloud message is empty, invalid, or too large.",
)
.into());
}
result => result?,
};
if matches!(mode, cloud::SubmissionMode::Steer) {
serde_json::to_value(admitted_message(admission.outcome)?)?
} else {
@@ -299,23 +308,71 @@ fn cloud_message(
"Changing cloud settings or sending structured input is not supported yet.",
));
}
let text = input
.into_iter()
.map(|item| match item {
let mut content = Vec::with_capacity(input.len());
for item in input {
let item = match item {
protocol::UserInput::Text {
text,
text_elements,
} if text_elements.is_empty() => Ok(text),
_ => Err(PreflightRejection(
"Cloud conversations currently accept plain text only.",
)),
})
.collect::<Result<Vec<_>, _>>()?
.join("\n");
} => {
// Keep the existing line separation for consecutive plain text parts.
if text_elements.is_empty()
&& let Some(cloud::UserMessageInput::Text {
text: previous,
text_elements: previous_elements,
}) = content.last_mut()
&& previous_elements.is_empty()
{
previous.push('\n');
previous.push_str(&text);
continue;
}
cloud::UserMessageInput::Text {
text,
text_elements: text_elements
.into_iter()
.map(|element| cloud::UserMessageTextElement {
placeholder: element.placeholder().map(str::to_owned),
start: element.byte_range.start,
end: element.byte_range.end,
})
.collect(),
}
}
protocol::UserInput::Image { image, detail } => cloud::UserMessageInput::Image {
image: match image {
protocol::ImageReference::Inline { url } => cloud::ImageReference::Url(url),
protocol::ImageReference::File { file_id } => {
cloud::ImageReference::FileId(file_id)
}
},
detail: detail.map(|detail| match detail {
ImageDetail::Auto => cloud::ImageDetail::Auto,
ImageDetail::Low => cloud::ImageDetail::Low,
ImageDetail::High => cloud::ImageDetail::High,
ImageDetail::Original => cloud::ImageDetail::Original,
}),
},
protocol::UserInput::LocalImage { .. } => {
return Err(PreflightRejection(
"Prepare local images before sending them to Cloud.",
));
}
protocol::UserInput::Audio { .. }
| protocol::UserInput::LocalAudio { .. }
| protocol::UserInput::Skill { .. }
| protocol::UserInput::Mention { .. } => {
return Err(PreflightRejection(
"Cloud conversations currently accept text and images.",
));
}
};
content.push(item);
}
Ok(cloud::AddUserMessageRequest {
thread_id,
client_user_message_id,
text,
input: content,
turn_trigger,
mode: cloud::SubmissionMode::StartIfIdle,
shared_access: false,
codex-rs/app-server-client/src/cloud_backend/submission_tests.rs
+42 -4
View File
@@ -74,8 +74,17 @@ async fn native_submission_uses_one_admission_and_keeps_preflight_guards() -> Re
.expect(2)
.mount(&server)
.await;
let Err(crate::TypedRequestError::Server { source, .. }) =
cloud_handle.submit_user_message(submission("x")).await
let url = "data:image/png;base64,iVBORw0KGgo=";
let image = protocol::UserInput::Image {
image: protocol::ImageReference::Inline { url: url.into() },
detail: None,
};
let Err(crate::TypedRequestError::Server { source, .. }) = cloud_handle
.submit_user_message(protocol::TurnStartParams {
input: vec![image.clone()],
..submission("x")
})
.await
else {
panic!("a lost receipt must remain uncertain");
};
@@ -85,7 +94,16 @@ async fn native_submission_uses_one_admission_and_keeps_preflight_guards() -> Re
let Err(crate::TypedRequestError::Server { source, .. }) = handle
.request_typed::<protocol::TurnStartResponse>(protocol::ClientRequest::TurnStart {
request_id: protocol::RequestId::Integer(1),
params: submission("m"),
params: protocol::TurnStartParams {
input: vec![
image,
protocol::UserInput::Text {
text: "describe the image".into(),
text_elements: Vec::new(),
},
],
..submission("m")
},
})
.await
else {
@@ -135,8 +153,22 @@ async fn native_submission_uses_one_admission_and_keeps_preflight_guards() -> Re
assert_eq!(queued.code, -32602);
assert_eq!(
queued.message,
"Cloud conversations currently accept plain text only."
"Prepare local images before sending them to Cloud."
);
let invalid_image = handle
.request(protocol::ClientRequest::TurnStart {
request_id: protocol::RequestId::Integer(4),
params: protocol::TurnStartParams {
input: vec![protocol::UserInput::Image {
image: protocol::ImageReference::Inline { url: String::new() },
detail: None,
}],
..submission("invalid-image")
},
})
.await?
.expect_err("an invalid image must fail before admission");
assert_eq!(invalid_image.code, -32602);
drop(reply_sender);
let Err(crate::TypedRequestError::Server { source, .. }) =
@@ -157,6 +189,12 @@ async fn native_submission_uses_one_admission_and_keeps_preflight_guards() -> Re
client.shutdown().await?;
worker.await?;
let calls = server.received_requests().await.unwrap();
assert!(calls.iter().all(|request| {
request
.body
.windows(url.len())
.any(|bytes| bytes == url.as_bytes())
}));
assert_eq!(
calls
.iter()