openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-10 21:00:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support Cloud image messages and editable draft handoffs (#60061)

Browse Source 
## Why

Cloud messages previously accepted only plain text, and switching a draft to Cloud waited for session preparation without processing composer input.

## What changed

- Support ordered text and image parts in Cloud submissions and user-message summaries, preserving attachment placeholders, image detail, and client message IDs.
- Accept inline data URLs and uploaded file IDs, including image-only messages. Reject invalid references and text spans locally, and cap encoded image requests at 3.75 MiB while retaining the existing plain-text limit.
- Keep the composer editable and redraw the TUI while Cloud prepares a conversation. Carry the latest draft, attachments, and Vim mode into the destination, retain drafts on failure, and prevent submission to the outgoing session during the handoff.
- Preserve buffered paste input and pending clipboard reads across the handoff, with connection and failure hints.

## Testing

Add regression coverage for image wire encoding, receipt identity, validation and size boundaries, draft editing during successful and failed handoffs, clipboard completion, Vim state, and attachment recovery after rejected submissions.

GitOrigin-RevId: 0cbcde0ab89ba961780b5f88456057e6b6a19cd5
This commit is contained in:
Benjamin Carlsson
2026-11-07 04:48:30 +00:00
committed by copyberry
parent d4a341bd52
commit e6bf88ae02
46 changed files with 1918 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/cloud-client/src/item_tests.rs
+109 -1
View File
@@ -88,7 +88,75 @@ fn user_message_preserves_client_id_and_text() {
Some(ItemSummary::UserMessage {
id: "user-item".into(),
client_id: Some("client-id".into()),
text: vec!["hello".into()],
content: vec![UserMessageInput::Text {
text: "hello".into(),
text_elements: vec![]
}],
})
);
}
#[test]
fn user_message_preserves_images_spans_and_client_id() {
let url = "data:image/png;base64,AA==";
let label = "[Image #1]";
let element = [
message(/*tag*/ 1, b"\x08\x0d\x10\x03"),
message(/*tag*/ 2, label.as_bytes()),
]
.concat();
let text = [
message(/*tag*/ 1, "é [Image #1]".as_bytes()),
message(/*tag*/ 2, &message(/*tag*/ 1, &element)),
]
.concat();
let image_bytes = [vec![0x08, 99], message(/*tag*/ 2, url.as_bytes())].concat();
let content = [
message(/*tag*/ 1, &message(/*tag*/ 1, &text)),
message(
/*tag*/ 1,
&message(/*tag*/ 2, &message(/*tag*/ 1, &image_bytes)),
),
]
.concat();
let user = message(
/*tag*/ 1,
&[
message(/*tag*/ 1, b"client-id"),
message(/*tag*/ 2, &content),
message(/*tag*/ 3, b"user-item"),
]
.concat(),
);
let expected = ItemSummary::UserMessage {
id: "user-item".into(),
client_id: Some("client-id".into()),
content: vec![
UserMessageInput::Text {
text: "é [Image #1]".into(),
text_elements: vec![UserMessageTextElement {
start: 3,
end: 13,
placeholder: Some(label.into()),
}],
},
UserMessageInput::Image {
image: ImageReference::Url(url.into()),
detail: None,
},
],
};
assert_eq!(summary(&user), Some(expected.clone()));
assert_eq!(format!("{expected:?}"), "ItemSummary([redacted])");
let event = TranscriptEvent::decode("t", &lifecycle(/*tag*/ 31, &user))
.unwrap()
.unwrap();
assert_eq!(
event.update,
TranscriptUpdate::ItemCompleted(ItemLifecycle {
turn_id: "u".into(),
at_ms: 5,
item: expected,
})
);
}
@@ -261,6 +329,46 @@ fn unsupported_and_incomplete_items_stay_opaque() {
.is_none()
);
}
for input in [
message(/*tag*/ 8, b""), // Future input variant.
message(/*tag*/ 2, &message(/*tag*/ 3, b"")), // Future image reference.
message(/*tag*/ 2, &message(/*tag*/ 1, b"\x08\x01")), // Missing URL.
message(/*tag*/ 2, &message(/*tag*/ 2, b"")), // Missing file ID.
message(
/*tag*/ 1,
&[
message(/*tag*/ 1, b"hello"),
message(/*tag*/ 2, &message(/*tag*/ 1, b"")), // Missing byte range.
]
.concat(),
),
message(
/*tag*/ 1,
&[
message(/*tag*/ 1, b"hello"),
message(
/*tag*/ 2,
&message(/*tag*/ 1, &message(/*tag*/ 1, b"\x08\x01")),
), // Missing start.
]
.concat(),
),
] {
let user = message(
/*tag*/ 1,
&[
message(/*tag*/ 1, b"client-id"),
message(/*tag*/ 2, &message(/*tag*/ 1, &input)),
message(/*tag*/ 3, b"user-item"),
]
.concat(),
);
assert!(
TranscriptEvent::decode("t", &lifecycle(/*tag*/ 27, &user))
.unwrap()
.is_none()
);
}
assert!(
TranscriptEvent::decode("t", &notification(/*tag*/ 27, &absent_time))
.unwrap()
codex-rs/cloud-client/src/items.rs
+55 -6
View File
@@ -1,6 +1,10 @@
//! Selected item summaries for live start/completion events, without interpreting executor paths.
use crate::FileUpdate;
use crate::ImageDetail;
use crate::ImageReference;
use crate::TranscriptUpdate;
use crate::UserMessageInput;
use crate::UserMessageTextElement;
use crate::item_wire as wire;
use crate::transcript::DecodeFailure;
use crate::transcript::ITEM_STARTED;
@@ -24,7 +28,7 @@ pub enum ItemSummary {
UserMessage {
id: String,
client_id: Option<String>,
text: Vec<String>,
content: Vec<UserMessageInput>,
},
AgentMessage {
id: String,
@@ -169,17 +173,62 @@ pub(crate) fn decode(
pub(crate) fn summary(item: wire::Item) -> Result<Option<ItemSummary>, DecodeFailure> {
let summary = match item.value {
Some(wire::Value::UserMessage(value)) => {
let mut text = Vec::new();
let mut content = Vec::new();
for input in required(value.content)?.values {
let Some(crate::turns_wire::Value::Text(input)) = input.value else {
return Ok(None);
let input = match input.value {
Some(crate::turns_wire::Value::Text(input)) => {
let text = required(input.text)?;
let text_elements = input
.text_elements
.map_or_else(Vec::new, |elements| elements.values)
.into_iter()
.map(|element| {
let range = required(element.byte_range)?;
let start = usize::try_from(required(range.start)?)
.map_err(|_| DecodeFailure::MissingField)?;
let end = usize::try_from(required(range.end)?)
.map_err(|_| DecodeFailure::MissingField)?;
text.get(start..end).ok_or(DecodeFailure::MissingField)?;
Ok(UserMessageTextElement {
start,
end,
placeholder: element.placeholder,
})
})
.collect::<Result<_, DecodeFailure>>()?;
UserMessageInput::Text {
text,
text_elements,
}
}
Some(crate::turns_wire::Value::Image(input)) => {
let (image, detail) = match input.value {
Some(crate::turns_wire::ImageValue::Url(image)) => {
(ImageReference::Url(required(image.value)?), image.detail)
}
Some(crate::turns_wire::ImageValue::FileId(image)) => {
(ImageReference::FileId(required(image.value)?), image.detail)
}
None => return Ok(None),
};
let detail = match detail {
Some(1) => Some(ImageDetail::Auto),
Some(2) => Some(ImageDetail::Low),
Some(3) => Some(ImageDetail::High),
Some(4) => Some(ImageDetail::Original),
// Detail is display metadata. Keep the receipt identity if it is newer.
None | Some(_) => None,
};
UserMessageInput::Image { image, detail }
}
_ => return Ok(None),
};
text.push(required(input.text)?);
content.push(input);
}
ItemSummary::UserMessage {
id: required(value.id)?,
client_id: value.client_id,
text,
content,
}
}
Some(wire::Value::AgentMessage(value)) => ItemSummary::AgentMessage {
codex-rs/cloud-client/src/lib.rs
+4
View File
@@ -69,9 +69,13 @@ pub use transcript::TurnSummary;
pub use turns::AddUserMessageRequest;
pub use turns::Admission;
pub use turns::AdmissionOutcome;
pub use turns::ImageDetail;
pub use turns::ImageReference;
pub use turns::InterruptTurnRequest;
pub use turns::MutationError;
pub use turns::SubmissionMode;
pub use turns::UserMessageInput;
pub use turns::UserMessageTextElement;
pub use types::Credentials;
pub use types::Error;
pub use types::Event;
codex-rs/cloud-client/src/turns.rs
+122 -8
View File
@@ -9,24 +9,59 @@ use bytes::Bytes;
use prost::Message;
use std::fmt;
// CCA forwards the envelope through a 4 MiB boundary; reserve 256 KiB for forwarding.
const MAX_IMAGE_MESSAGE_BYTES: usize = 4 * 1024 * 1024 - 256 * 1024;
#[derive(Clone, Copy, Debug)]
pub enum SubmissionMode {
Steer,
Queue,
StartIfIdle,
}
/// Plain text using the existing thread configuration. Persist the message ID with
/// Text and images using the existing thread configuration. Persist the message ID with
/// the input before calling; never reuse it for a different logical submission.
#[derive(Clone)]
pub struct AddUserMessageRequest {
pub thread_id: String,
pub client_user_message_id: String,
pub text: String,
pub input: Vec<UserMessageInput>,
/// Optional classification supplied by the caller for this turn.
pub turn_trigger: Option<String>,
pub mode: SubmissionMode,
pub shared_access: bool,
}
/// Ordered input parts for one logical user message.
#[derive(Clone, PartialEq, Eq)]
pub enum UserMessageInput {
Text {
text: String,
text_elements: Vec<UserMessageTextElement>,
},
Image {
image: ImageReference,
detail: Option<ImageDetail>,
},
}
/// A placeholder span in the UTF-8 bytes of a text input part.
#[derive(Clone, PartialEq, Eq)]
pub struct UserMessageTextElement {
pub start: usize,
pub end: usize,
pub placeholder: Option<String>,
}
/// An inline data URL or an uploaded file ID. HTTP image URLs are not supported.
#[derive(Clone, PartialEq, Eq)]
pub enum ImageReference {
Url(String),
FileId(String),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ImageDetail {
Auto,
Low,
High,
Original,
}
#[derive(Clone)]
pub struct InterruptTurnRequest {
pub thread_id: String,
@@ -73,6 +108,9 @@ macro_rules! redacted_debug {
}
redacted_debug!(
AddUserMessageRequest,
UserMessageInput,
UserMessageTextElement,
ImageReference,
InterruptTurnRequest,
Admission,
AdmissionOutcome
@@ -87,18 +125,86 @@ impl Client {
) -> Result<Admission, MutationError> {
if request.thread_id.trim().is_empty()
|| request.client_user_message_id.trim().is_empty()
|| request.text.trim().is_empty()
|| !request.input.iter().any(|input| match input {
UserMessageInput::Text { text, .. } => !text.trim().is_empty(),
UserMessageInput::Image { .. } => true,
})
|| request.input.iter().any(|input| match input {
UserMessageInput::Text {
text,
text_elements,
} => text_elements
.iter()
.any(|element| text.get(element.start..element.end).is_none()),
UserMessageInput::Image { image, .. } => match image {
ImageReference::Url(value) => {
value.split_once(',').is_none_or(|(prefix, payload)| {
!prefix
.get(..5)
.is_some_and(|scheme| scheme.eq_ignore_ascii_case("data:"))
|| payload.trim().is_empty()
})
}
ImageReference::FileId(value) => value.trim().is_empty(),
},
})
{
return Err(MutationError::InvalidRequest);
}
let value = wire::AddRequest {
client_user_message_id: request.client_user_message_id.clone(),
input: Some(wire::Input {
values: vec![wire::UserInput {
value: Some(wire::Value::Text(wire::Text {
text: Some(request.text.clone()),
})),
}],
values: request
.input
.iter()
.map(|input| wire::UserInput {
value: Some(match input {
UserMessageInput::Text {
text,
text_elements,
} => wire::Value::Text(wire::Text {
text: Some(text.clone()),
text_elements: (!text_elements.is_empty()).then(|| {
wire::TextElements {
values: text_elements
.iter()
.map(|element| wire::TextElement {
byte_range: Some(wire::ByteRange {
end: Some(element.end as u64),
start: Some(element.start as u64),
}),
placeholder: element.placeholder.clone(),
})
.collect(),
}
}),
}),
UserMessageInput::Image { image, detail } => {
let detail = detail.map(|detail| match detail {
ImageDetail::Auto => 1,
ImageDetail::Low => 2,
ImageDetail::High => 3,
ImageDetail::Original => 4,
});
let value = match image {
ImageReference::Url(url) => {
wire::ImageValue::Url(wire::ImageReference {
detail,
value: Some(url.clone()),
})
}
ImageReference::FileId(file_id) => {
wire::ImageValue::FileId(wire::ImageReference {
detail,
value: Some(file_id.clone()),
})
}
};
wire::Value::Image(wire::Image { value: Some(value) })
}
}),
})
.collect(),
}),
thread_id: request.thread_id.clone(),
turn_trigger: request.turn_trigger.clone(),
@@ -109,6 +215,14 @@ impl Client {
SubmissionMode::StartIfIdle => 3,
}),
};
if request
.input
.iter()
.any(|input| matches!(input, UserMessageInput::Image { .. }))
&& value.encoded_len() > MAX_IMAGE_MESSAGE_BYTES
{
return Err(MutationError::InvalidRequest);
}
let payload = self
.mutate(
value,
codex-rs/cloud-client/src/turns_tests.rs
+221 -2
View File
@@ -6,7 +6,10 @@ fn request() -> AddUserMessageRequest {
AddUserMessageRequest {
thread_id: "t".into(),
client_user_message_id: "m".into(),
text: "hello".into(),
input: vec![UserMessageInput::Text {
text: "hello".into(),
text_elements: vec![],
}],
turn_trigger: None,
mode: SubmissionMode::StartIfIdle,
shared_access: true,
@@ -106,6 +109,80 @@ async fn submission_preserves_admission_variants_and_wire_identity() {
}
}
#[tokio::test]
async fn image_submission_preserves_order_references_and_optional_detail() {
let url = "DaTa:image/png;base64,AA==";
let label = "[Image #1]";
let text = format!("é {label}");
let input = vec![
UserMessageInput::Image {
image: ImageReference::Url(url.into()),
detail: None,
},
UserMessageInput::Text {
text: text.clone(),
text_elements: vec![UserMessageTextElement {
start: 3,
end: text.len(),
placeholder: Some(label.into()),
}],
},
UserMessageInput::Image {
image: ImageReference::FileId("file-private".into()),
detail: Some(ImageDetail::High),
},
];
for part in &input {
assert_eq!(format!("{part:?}"), "UserMessageInput([redacted])");
}
let mut fixture = fixture(response(
vec![frame(&admitted(/*outcome*/ 3))],
tonic::Status::ok(""),
))
.await;
let result = fixture
.client
.add_user_message(&AddUserMessageRequest { input, ..request() })
.await
.unwrap();
assert_eq!(
result.outcome,
AdmissionOutcome::Duplicate {
turn_id: Some("u".into())
}
);
let span = [
message(/*tag*/ 1, b"\x08\x0d\x10\x03"),
message(/*tag*/ 2, label.as_bytes()),
]
.concat();
let text = [
message(/*tag*/ 1, text.as_bytes()),
message(/*tag*/ 2, &message(/*tag*/ 1, &span)),
]
.concat();
let url = message(/*tag*/ 1, &message(/*tag*/ 2, url.as_bytes()));
let file = message(
/*tag*/ 2,
&[vec![0x08, 3], message(/*tag*/ 2, b"file-private")].concat(),
);
let values = [
message(/*tag*/ 1, &message(/*tag*/ 2, &url)),
message(/*tag*/ 1, &message(/*tag*/ 1, &text)),
message(/*tag*/ 1, &message(/*tag*/ 2, &file)),
]
.concat();
let expected = [
message(/*tag*/ 4, b"m"),
message(/*tag*/ 11, &values),
message(/*tag*/ 23, b"t"),
vec![0xd0, 1, 1, 0xf0, 1, 3],
]
.concat();
assert_request(&mut fixture, "AddUserMessage", &expected).await;
assert!(fixture.requests.try_recv().is_err());
}
#[tokio::test]
async fn duplicate_can_omit_turn_but_malformed_admission_is_unknown() {
let payloads = vec![
vec![],
@@ -203,7 +280,51 @@ async fn invalid_input_never_sends_and_diagnostics_redact() {
let mut fixture = fixture(response(vec![frame(&[])], tonic::Status::ok(""))).await;
for input in [
AddUserMessageRequest {
text: " ".into(),
input: vec![],
..request()
},
AddUserMessageRequest {
input: vec![UserMessageInput::Text {
text: "x".repeat(MAX_MESSAGE_BYTES),
text_elements: vec![],
}],
..request()
},
AddUserMessageRequest {
input: vec![UserMessageInput::Image {
image: ImageReference::Url(format!(
"data:,{}",
"A".repeat(4 * 1024 * 1024 - 256 * 1024)
)),
detail: None,
}],
..request()
},
AddUserMessageRequest {
input: vec![UserMessageInput::Image {
image: ImageReference::Url("https://example.com/image.png".into()),
detail: None,
}],
..request()
},
AddUserMessageRequest {
input: vec![
UserMessageInput::Text {
text: "hello".into(),
text_elements: vec![],
},
UserMessageInput::Image {
image: ImageReference::FileId("".into()),
detail: Some(ImageDetail::Auto),
},
],
..request()
},
AddUserMessageRequest {
input: vec![UserMessageInput::Text {
text: " ".into(),
text_elements: vec![],
}],
..request()
},
AddUserMessageRequest {
@@ -221,6 +342,19 @@ async fn invalid_input_never_sends_and_diagnostics_redact() {
Err(MutationError::InvalidRequest)
));
}
for url in ["data:image/png;base64,", "data:,", "data:no-comma"] {
let input = AddUserMessageRequest {
input: vec![UserMessageInput::Image {
image: ImageReference::Url(url.into()),
detail: None,
}],
..request()
};
assert!(matches!(
fixture.client.add_user_message(&input).await,
Err(MutationError::InvalidRequest)
));
}
let input = InterruptTurnRequest {
thread_id: "private".into(),
turn_id: "".into(),
@@ -239,6 +373,91 @@ async fn invalid_input_never_sends_and_diagnostics_redact() {
}
assert!(fixture.requests.try_recv().is_err());
}
#[tokio::test]
async fn image_preflight_applies_exact_encoded_envelope_boundary() {
let cap = 4 * 1024 * 1024 - 256 * 1024;
for extra in [0, 1] {
// With these IDs and settings, the nested wire envelope adds 38 bytes.
let url = format!("data:,{}", "A".repeat(cap - 38 - "data:,".len() + extra));
let expected = [
message(/*tag*/ 4, b"m"),
message(
/*tag*/ 11,
&message(
/*tag*/ 1,
&message(
/*tag*/ 2,
&message(/*tag*/ 1, &message(/*tag*/ 2, url.as_bytes())),
),
),
),
message(/*tag*/ 23, b"t"),
vec![0xd0, 1, 1, 0xf0, 1, 3],
]
.concat();
assert_eq!(expected.len(), cap + extra);
let mut fixture = fixture(response(
vec![frame(&admitted(/*outcome*/ 3))],
tonic::Status::ok(""),
))
.await;
let input = vec![UserMessageInput::Image {
image: ImageReference::Url(url),
detail: None,
}];
let result = fixture
.client
.add_user_message(&AddUserMessageRequest { input, ..request() })
.await;
if extra == 0 {
result.unwrap();
assert_request(&mut fixture, "AddUserMessage", &expected).await;
} else {
assert!(matches!(result, Err(MutationError::InvalidRequest)));
assert!(fixture.requests.try_recv().is_err());
}
}
}
#[tokio::test]
async fn plain_text_retains_limit_above_image_forwarding_cap() {
let mut fixture = fixture(response(
vec![frame(&admitted(/*outcome*/ 3))],
tonic::Status::ok(""),
))
.await;
let text = "x".repeat(4 * 1024 * 1024);
let input = vec![UserMessageInput::Text {
text: text.clone(),
text_elements: vec![],
}];
let result = fixture
.client
.add_user_message(&AddUserMessageRequest { input, ..request() })
.await
.unwrap();
assert_eq!(
result.outcome,
AdmissionOutcome::Duplicate {
turn_id: Some("u".into())
}
);
let expected = [
message(/*tag*/ 4, b"m"),
message(
/*tag*/ 11,
&message(
/*tag*/ 1,
&message(/*tag*/ 1, &message(/*tag*/ 1, text.as_bytes())),
),
),
message(/*tag*/ 23, b"t"),
vec![0xd0, 1, 1, 0xf0, 1, 3],
]
.concat();
assert_request(&mut fixture, "AddUserMessage", &expected).await;
}
#[tokio::test]
async fn dropping_pending_submission_does_not_send_a_retry_or_interrupt() {
let (sender, dropped) = tokio::sync::oneshot::channel::<()>();
codex-rs/cloud-client/src/turns_wire.rs
+44 -3
View File
@@ -1,4 +1,4 @@
//! Native plain-text submission and interruption envelopes; omitted settings inherit thread config.
//! Native text/image submission and interruption envelopes; omitted settings inherit thread config.
use prost::Message;
#[derive(Clone, Message)]
@@ -30,8 +30,8 @@ pub(super) struct UserInput {
pub(super) enum Value {
#[prost(message, tag = "1")]
Text(Text),
#[prost(bytes, tag = "2")]
Image(Vec<u8>),
#[prost(message, tag = "2")]
Image(Image),
#[prost(bytes, tag = "3")]
LocalImage(Vec<u8>),
#[prost(bytes, tag = "4")]
@@ -47,6 +47,47 @@ pub(super) enum Value {
pub(super) struct Text {
#[prost(string, optional, tag = "1")]
pub text: Option<String>,
#[prost(message, optional, tag = "2")]
pub text_elements: Option<TextElements>,
}
#[derive(Clone, Message)]
pub(super) struct TextElements {
#[prost(message, repeated, tag = "1")]
pub values: Vec<TextElement>,
}
#[derive(Clone, Message)]
pub(super) struct TextElement {
#[prost(message, optional, tag = "1")]
pub byte_range: Option<ByteRange>,
#[prost(string, optional, tag = "2")]
pub placeholder: Option<String>,
}
#[derive(Clone, Message)]
pub(super) struct ByteRange {
#[prost(uint64, optional, tag = "1")]
pub end: Option<u64>,
#[prost(uint64, optional, tag = "2")]
pub start: Option<u64>,
}
#[derive(Clone, Message)]
pub(super) struct Image {
#[prost(oneof = "ImageValue", tags = "1, 2")]
pub value: Option<ImageValue>,
}
#[derive(Clone, prost::Oneof)]
pub(super) enum ImageValue {
#[prost(message, tag = "1")]
Url(ImageReference),
#[prost(message, tag = "2")]
FileId(ImageReference),
}
// Both native image alternatives have the same field layout.
#[derive(Clone, Message)]
pub(super) struct ImageReference {
#[prost(int32, optional, tag = "1")]
pub detail: Option<i32>,
#[prost(string, optional, tag = "2")]
pub value: Option<String>,
}
#[derive(Clone, Message)]
pub(super) struct AddResponse {