From b05eaf31f023ff085724d8ab413b4156cb399360 Mon Sep 17 00:00:00 2001 From: Curtis 'Fjord' Hawthorne Date: Sat, 21 Mar 2026 15:30:51 -0700 Subject: [PATCH] Handle inline image request caps gracefully git-stack-id: fjord/original_image_size_warn git-stack-title: Handle inline image request caps gracefully --- .../app-server/tests/common/models_cache.rs | 2 + .../codex-api/tests/models_integration.rs | 2 + codex-rs/core/src/client.rs | 4 + codex-rs/core/src/codex.rs | 64 ++++++- codex-rs/core/src/codex_tests.rs | 96 +++++++++++ codex-rs/core/src/context_manager/history.rs | 157 +++++++++--------- .../core/src/context_manager/history_tests.rs | 136 +++++++++++++++ codex-rs/core/src/error.rs | 99 +++++++++++ codex-rs/core/src/error_tests.rs | 24 +++ .../core/src/inline_image_request_limit.rs | 141 ++++++++++++++++ .../src/inline_image_request_limit_tests.rs | 145 ++++++++++++++++ codex-rs/core/src/lib.rs | 1 + .../core/src/models_manager/model_info.rs | 2 + codex-rs/core/tests/suite/model_switching.rs | 4 + codex-rs/core/tests/suite/models_cache_ttl.rs | 2 + codex-rs/core/tests/suite/personality.rs | 4 + codex-rs/core/tests/suite/remote_models.rs | 47 ++++++ codex-rs/core/tests/suite/rmcp_client.rs | 2 + .../tests/suite/spawn_agent_description.rs | 2 + codex-rs/core/tests/suite/view_image.rs | 142 ++++++++++++++++ codex-rs/protocol/src/openai_models.rs | 10 ++ 21 files changed, 1002 insertions(+), 84 deletions(-) create mode 100644 codex-rs/core/src/inline_image_request_limit.rs create mode 100644 codex-rs/core/src/inline_image_request_limit_tests.rs diff --git a/codex-rs/app-server/tests/common/models_cache.rs b/codex-rs/app-server/tests/common/models_cache.rs index 427f6cc1f2..3f3017e67f 100644 --- a/codex-rs/app-server/tests/common/models_cache.rs +++ b/codex-rs/app-server/tests/common/models_cache.rs @@ -40,6 +40,8 @@ fn preset_to_info(preset: &ModelPreset, priority: i32) -> ModelInfo { truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, diff --git a/codex-rs/codex-api/tests/models_integration.rs b/codex-rs/codex-api/tests/models_integration.rs index 3ffb2496b4..0324f03699 100644 --- a/codex-rs/codex-api/tests/models_integration.rs +++ b/codex-rs/codex-api/tests/models_integration.rs @@ -88,6 +88,8 @@ async fn models_client_hits_models_endpoint() { truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 7649d926fb..5fd45fbe92 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -104,6 +104,7 @@ use crate::default_client::build_reqwest_client; use crate::error::CodexErr; use crate::error::Result; use crate::flags::CODEX_RS_SSE_FIXTURE; +use crate::inline_image_request_limit::inline_image_request_limit_error; use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::WireApi; use crate::response_debug_context::extract_response_debug_context; @@ -692,6 +693,9 @@ impl ModelClientSession { ) -> Result { let instructions = &prompt.base_instructions.text; let input = prompt.get_formatted_input(); + if let Some(error) = inline_image_request_limit_error(&input, model_info) { + return Err(CodexErr::InlineImageRequestLimitExceeded(error)); + } let tools = create_tools_json_for_responses_api(&prompt.tools)?; let default_reasoning_effort = model_info.default_reasoning_level; let reasoning = if model_info.supports_reasoning_summaries { diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 2c70aad7a7..265480477c 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -182,6 +182,7 @@ use crate::error::CodexErr; use crate::error::Result as CodexResult; #[cfg(test)] use crate::exec::StreamOutput; +use crate::inline_image_request_limit::inline_image_request_limit_error; use codex_config::CONFIG_TOML_FILE; mod rollout_reconstruction; @@ -3326,16 +3327,69 @@ impl Session { .await } - /// Records input items: always append to conversation history and - /// persist these response items to rollout. + /// Records input items, sanitizing inline tool images before persistence when + /// they would overflow the model request. pub(crate) async fn record_conversation_items( &self, turn_context: &TurnContext, items: &[ResponseItem], ) { - self.record_into_history(items, turn_context).await; - self.persist_rollout_response_items(items).await; - self.send_raw_response_items(turn_context, items).await; + let items_to_record = { + let mut state = self.state.lock().await; + let items_to_record = Self::prepare_items_for_recording(&state, turn_context, items); + state.record_items(items_to_record.iter(), turn_context.truncation_policy); + items_to_record + }; + + self.persist_rollout_response_items(&items_to_record).await; + self.send_raw_response_items(turn_context, &items_to_record) + .await; + } + + fn prepare_items_for_recording( + state: &SessionState, + turn_context: &TurnContext, + items: &[ResponseItem], + ) -> Vec { + let mut items_to_record = items.to_vec(); + let should_check_image_request_limits = items_to_record.iter().any(|item| match item { + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + output.content_items().is_some_and(|content_items| { + content_items.iter().any(|content_item| { + matches!( + content_item, + codex_protocol::models::FunctionCallOutputContentItem::InputImage { .. } + ) + }) + }) + } + _ => false, + }); + if !should_check_image_request_limits { + return items_to_record; + } + + let mut candidate_history = state.clone_history(); + candidate_history.record_items(items_to_record.iter(), turn_context.truncation_policy); + let prompt_input = candidate_history.for_prompt(&turn_context.model_info.input_modalities); + let Some(error) = inline_image_request_limit_error(&prompt_input, &turn_context.model_info) + else { + return items_to_record; + }; + + let mut pending_items = ContextManager::new(); + pending_items.replace(items_to_record.clone()); + if pending_items.replace_last_turn_tool_outputs_with_failure_message( + &error.tool_output_recovery_message(), + ) { + warn!( + "inline image request limit would be exceeded before upload; sanitizing tool image output before persistence" + ); + items_to_record = pending_items.raw_items().to_vec(); + } + + items_to_record } /// Append ResponseItems to the in-memory conversation history only. diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index f5e5d4463e..edce72c2d1 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -20,6 +20,7 @@ use crate::tools::format_exec_output_str; use codex_features::Features; use codex_protocol::ThreadId; use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; @@ -925,6 +926,101 @@ async fn reconstruct_history_uses_replacement_history_verbatim() { assert_eq!(reconstructed.history, replacement_history); } +#[tokio::test] +async fn record_conversation_items_sanitizes_inline_tool_images_before_persistence() { + let (session, mut turn_context) = make_session_and_context().await; + let session = Arc::new(session); + let rollout_path = attach_rollout_recorder(&session).await; + let user = user_message("describe the image"); + let image_url = "data:image/png;base64,AAAA".to_string(); + turn_context.model_info.inline_image_request_limit_bytes = Some(10); + let tool_call = ResponseItem::FunctionCall { + id: None, + name: "view_image".to_string(), + namespace: None, + arguments: "{}".to_string(), + call_id: "view-image".to_string(), + }; + let original_output = ResponseItem::FunctionCallOutput { + call_id: "view-image".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputText { + text: "captured".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: image_url.clone(), + detail: None, + }, + ]), + }; + let recovery_message = + crate::error::InlineImageRequestLimitExceededError::local_preflight_bytes( + image_url.len(), + 10, + ) + .tool_output_recovery_message(); + let recovered_output = ResponseItem::FunctionCallOutput { + call_id: "view-image".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "captured".to_string(), + }, + FunctionCallOutputContentItem::InputText { + text: recovery_message, + }, + ]), + success: Some(false), + }, + }; + + session + .record_conversation_items(&turn_context, std::slice::from_ref(&user)) + .await; + session + .record_conversation_items(&turn_context, std::slice::from_ref(&tool_call)) + .await; + session + .record_conversation_items(&turn_context, std::slice::from_ref(&original_output)) + .await; + + assert_eq!( + session.clone_history().await.raw_items().to_vec(), + vec![user.clone(), tool_call.clone(), recovered_output.clone()] + ); + + session.flush_rollout().await; + let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("read rollout history") + else { + panic!("expected resumed rollout history"); + }; + + let persisted_response_items = resumed + .history + .into_iter() + .filter_map(|item| match item { + RolloutItem::ResponseItem(item) => Some(item), + _ => None, + }) + .collect::>(); + let ResponseItem::FunctionCallOutput { output, .. } = recovered_output else { + panic!("expected function call output"); + }; + let persisted_output = ResponseItem::FunctionCallOutput { + call_id: "view-image".to_string(), + output: FunctionCallOutputPayload { + body: output.body, + success: None, + }, + }; + assert_eq!( + persisted_response_items, + vec![user, tool_call, persisted_output] + ); +} + #[tokio::test] async fn record_initial_history_reconstructs_resumed_transcript() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index ec2df30d3b..dc188da8e4 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -3,6 +3,8 @@ use crate::context_manager::normalize; use crate::event_mapping::has_non_contextual_dev_message_content; use crate::event_mapping::is_contextual_dev_message_content; use crate::event_mapping::is_contextual_user_message_content; +use crate::inline_image_request_limit::parse_base64_image_data_url; +use crate::inline_image_request_limit::visit_response_item_input_images; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_protocol::models::BaseInstructions; @@ -181,32 +183,60 @@ impl ContextManager { /// Replace image content in the last turn if it originated from a tool output. /// Returns true when a tool image was replaced, false otherwise. pub(crate) fn replace_last_turn_images(&mut self, placeholder: &str) -> bool { - let Some(index) = self.items.iter().rposition(|item| { - matches!(item, ResponseItem::FunctionCallOutput { .. }) || is_user_turn_boundary(item) - }) else { - return false; - }; + self.visit_last_turn_tool_outputs_mut(|output| { + let Some(content_items) = output.content_items_mut() else { + return false; + }; + let mut replaced = false; + let placeholder = placeholder.to_string(); + for item in content_items.iter_mut() { + if matches!(item, FunctionCallOutputContentItem::InputImage { .. }) { + *item = FunctionCallOutputContentItem::InputText { + text: placeholder.clone(), + }; + replaced = true; + } + } + replaced + }) + } - match &mut self.items[index] { - ResponseItem::FunctionCallOutput { output, .. } => { - let Some(content_items) = output.content_items_mut() else { - return false; - }; - let mut replaced = false; - let placeholder = placeholder.to_string(); - for item in content_items.iter_mut() { - if matches!(item, FunctionCallOutputContentItem::InputImage { .. }) { - *item = FunctionCallOutputContentItem::InputText { - text: placeholder.clone(), - }; - replaced = true; + /// Replace image-bearing tool outputs from the current turn with a textual failure so the + /// model can adapt and continue in the same turn. + pub(crate) fn replace_last_turn_tool_outputs_with_failure_message( + &mut self, + message: &str, + ) -> bool { + self.visit_last_turn_tool_outputs_mut(|output| { + let Some(content_items) = output.content_items() else { + return false; + }; + let mut preserved_items = Vec::with_capacity(content_items.len().saturating_add(1)); + let mut removed_images = false; + for item in content_items { + match item { + FunctionCallOutputContentItem::InputText { text } => { + preserved_items + .push(FunctionCallOutputContentItem::InputText { text: text.clone() }); + } + FunctionCallOutputContentItem::InputImage { .. } => { + removed_images = true; } } - replaced } - ResponseItem::Message { .. } => false, - _ => false, - } + if !removed_images { + return false; + } + let message = message.to_string(); + output.body = if preserved_items.is_empty() { + FunctionCallOutputBody::Text(message) + } else { + preserved_items.push(FunctionCallOutputContentItem::InputText { text: message }); + FunctionCallOutputBody::ContentItems(preserved_items) + }; + output.success = Some(false); + true + }) } /// Drop the last `num_turns` instruction turns from this history. @@ -544,40 +574,6 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> } } -/// Returns the base64 payload byte length for inline image data URLs that are -/// eligible for token-estimation discounting. -/// -/// We only discount payloads for `data:image/...;base64,...` URLs (case -/// insensitive markers) and leave everything else at raw serialized size. -fn parse_base64_image_data_url(url: &str) -> Option<&str> { - if !url - .get(.."data:".len()) - .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) - { - return None; - } - let comma_index = url.find(',')?; - let metadata = &url[..comma_index]; - let payload = &url[comma_index + 1..]; - // Parse the media type and parameters without decoding. This keeps the - // estimator cheap while ensuring we only apply the fixed-cost image - // heuristic to image-typed base64 data URLs. - let metadata_without_scheme = &metadata["data:".len()..]; - let mut metadata_parts = metadata_without_scheme.split(';'); - let mime_type = metadata_parts.next().unwrap_or_default(); - let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64")); - if !mime_type - .get(.."image/".len()) - .is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/")) - { - return None; - } - if !has_base64_marker { - return None; - } - Some(payload) -} - fn estimate_original_image_bytes(image_url: &str) -> Option { let key = sha1_digest(image_url.as_bytes()); ORIGINAL_IMAGE_ESTIMATE_CACHE.get_or_insert_with(key, || { @@ -621,7 +617,7 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { let mut payload_bytes = 0i64; let mut replacement_bytes = 0i64; - let mut accumulate = |image_url: &str, detail: Option| { + let accumulate = |image_url: &str, detail: Option| { if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) { payload_bytes = payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX)); @@ -634,32 +630,35 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { } }; - match item { - ResponseItem::Message { content, .. } => { - for content_item in content { - if let ContentItem::InputImage { image_url } = content_item { - accumulate(image_url, None); - } - } - } - ResponseItem::FunctionCallOutput { output, .. } - | ResponseItem::CustomToolCallOutput { output, .. } => { - if let FunctionCallOutputBody::ContentItems(items) = &output.body { - for content_item in items { - if let FunctionCallOutputContentItem::InputImage { image_url, detail } = - content_item - { - accumulate(image_url, *detail); - } - } - } - } - _ => {} - } + visit_response_item_input_images(item, accumulate); (payload_bytes, replacement_bytes) } +impl ContextManager { + fn visit_last_turn_tool_outputs_mut( + &mut self, + mut visitor: impl FnMut(&mut FunctionCallOutputPayload) -> bool, + ) -> bool { + let start = self + .items + .iter() + .rposition(is_user_turn_boundary) + .map_or(0, |index| index.saturating_add(1)); + let mut changed = false; + for item in &mut self.items[start..] { + match item { + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + changed |= visitor(output); + } + _ => {} + } + } + changed + } +} + fn is_model_generated_item(item: &ResponseItem) -> bool { match item { ResponseItem::Message { role, .. } => role == "assistant", diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 9720492c1f..0c54b4ce71 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -761,6 +761,142 @@ fn replace_last_turn_images_does_not_touch_user_images() { assert_eq!(history.raw_items(), items); } +#[test] +fn replace_last_turn_images_replaces_custom_tool_output_images() { + let items = vec![ + user_input_text_msg("hi"), + ResponseItem::CustomToolCallOutput { + call_id: "call-custom".to_string(), + name: None, + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + detail: None, + }, + ]), + success: Some(true), + }, + }, + ]; + let mut history = create_history_with_items(items); + + assert!(history.replace_last_turn_images("Invalid image")); + + assert_eq!( + history.raw_items(), + vec![ + user_input_text_msg("hi"), + ResponseItem::CustomToolCallOutput { + call_id: "call-custom".to_string(), + name: None, + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "Invalid image".to_string(), + }, + ]), + success: Some(true), + }, + }, + ] + ); +} + +#[test] +fn replace_last_turn_tool_outputs_with_failure_message_replaces_only_latest_turn_images() { + let items = vec![ + user_input_text_msg("first"), + ResponseItem::FunctionCallOutput { + call_id: "call-old".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,OLD".to_string(), + detail: None, + }, + ]), + success: Some(true), + }, + }, + user_input_text_msg("second"), + ResponseItem::FunctionCallOutput { + call_id: "call-new".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "captured".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,NEW".to_string(), + detail: None, + }, + ]), + success: Some(true), + }, + }, + ResponseItem::CustomToolCallOutput { + call_id: "call-custom".to_string(), + name: None, + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,CUSTOM".to_string(), + detail: Some(ImageDetail::Original), + }, + ]), + success: Some(true), + }, + }, + ]; + let mut history = create_history_with_items(items); + let message = "Tool image output omitted because total inline image data in this request is 99 bytes, which exceeds the 10 byte limit for a single Responses API request. Retry with fewer images, smaller images, lower detail, or JPEG compression."; + + assert!(history.replace_last_turn_tool_outputs_with_failure_message(message)); + + assert_eq!( + history.raw_items(), + vec![ + user_input_text_msg("first"), + ResponseItem::FunctionCallOutput { + call_id: "call-old".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,OLD".to_string(), + detail: None, + }, + ]), + success: Some(true), + }, + }, + user_input_text_msg("second"), + ResponseItem::FunctionCallOutput { + call_id: "call-new".to_string(), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::ContentItems(vec![ + FunctionCallOutputContentItem::InputText { + text: "captured".to_string(), + }, + FunctionCallOutputContentItem::InputText { + text: message.to_string(), + }, + ]), + success: Some(false), + }, + }, + ResponseItem::CustomToolCallOutput { + call_id: "call-custom".to_string(), + name: None, + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::Text(message.to_string()), + success: Some(false), + }, + }, + ] + ); +} + #[test] fn remove_first_item_handles_local_shell_pair() { let items = vec![ diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index f8058915b8..bb258254b2 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -27,6 +27,100 @@ pub type Result = std::result::Result; /// Limit UI error messages to a reasonable size while keeping useful context. const ERROR_MESSAGE_UI_MAX_BYTES: usize = 2 * 1024; // 2 KiB +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InlineImageRequestLimitExceededError { + total_bytes: Option, + limit_bytes: Option, + total_images: Option, + limit_images: Option, +} + +impl InlineImageRequestLimitExceededError { + pub fn local_preflight_bytes(total_bytes: usize, limit_bytes: usize) -> Self { + Self { + total_bytes: Some(total_bytes), + limit_bytes: Some(limit_bytes), + total_images: None, + limit_images: None, + } + } + + pub fn local_preflight_images(total_images: usize, limit_images: usize) -> Self { + Self { + total_bytes: None, + limit_bytes: None, + total_images: Some(total_images), + limit_images: Some(limit_images), + } + } + + pub fn local_preflight_bytes_and_images( + total_bytes: usize, + limit_bytes: usize, + total_images: usize, + limit_images: usize, + ) -> Self { + Self { + total_bytes: Some(total_bytes), + limit_bytes: Some(limit_bytes), + total_images: Some(total_images), + limit_images: Some(limit_images), + } + } + + pub fn tool_output_recovery_message(&self) -> String { + format!( + "Tool image output omitted because {} Retry with fewer images, smaller images, lower detail, or JPEG compression.", + self.message_body() + ) + } + + fn message_body(&self) -> String { + match ( + self.total_images, + self.limit_images, + self.total_bytes, + self.limit_bytes, + ) { + (Some(total_images), Some(limit_images), Some(total_bytes), Some(limit_bytes)) => { + format!( + "this request contains {total_images} images, which exceeds the {limit_images} image limit, and total inline image data is {total_bytes} bytes, which exceeds the {limit_bytes} byte limit for a single Responses API request." + ) + } + (Some(total_images), Some(limit_images), _, _) => format!( + "this request contains {total_images} images, which exceeds the {limit_images} image limit for a single Responses API request." + ), + (None, None, Some(total_bytes), Some(limit_bytes)) => format!( + "total inline image data in this request is {total_bytes} bytes, which exceeds the {limit_bytes} byte limit for a single Responses API request." + ), + (None, None, None, Some(limit_bytes)) => format!( + "total inline image data in this request exceeds the {limit_bytes} byte limit for a single Responses API request." + ), + (None, None, Some(total_bytes), None) => format!( + "total inline image data in this request is {total_bytes} bytes, which exceeds the single-request limit for the Responses API." + ), + _ => { + "total inline image data in this request exceeds the single-request limit for the Responses API.".to_string() + } + } + } +} + +impl std::fmt::Display for InlineImageRequestLimitExceededError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let message = self.message_body(); + let mut chars = message.chars(); + let message = match chars.next() { + Some(first) => format!("{}{}", first.to_ascii_uppercase(), chars.as_str()), + None => String::new(), + }; + write!( + f, + "{message} Use fewer images, smaller images, lower detail, or JPEG compression and try again." + ) + } +} + #[derive(Error, Debug)] pub enum SandboxErr { /// Error from sandbox execution @@ -116,6 +210,9 @@ pub enum CodexErr { #[error("Image poisoning")] InvalidImageRequest(), + #[error("{0}")] + InlineImageRequestLimitExceeded(InlineImageRequestLimitExceededError), + #[error("{0}")] UsageLimitReached(UsageLimitReachedError), @@ -203,6 +300,7 @@ impl CodexErr { | CodexErr::UsageNotIncluded | CodexErr::QuotaExceeded | CodexErr::InvalidImageRequest() + | CodexErr::InlineImageRequestLimitExceeded(_) | CodexErr::InvalidRequest(_) | CodexErr::RefreshTokenFailed(_) | CodexErr::UnsupportedOperation(_) @@ -584,6 +682,7 @@ impl CodexErr { | CodexErr::InternalServerError | CodexErr::InternalAgentDied => CodexErrorInfo::InternalServerError, CodexErr::UnsupportedOperation(_) + | CodexErr::InlineImageRequestLimitExceeded(_) | CodexErr::ThreadNotFound(_) | CodexErr::AgentLimitReached { .. } => CodexErrorInfo::BadRequest, CodexErr::Sandbox(_) => CodexErrorInfo::SandboxError, diff --git a/codex-rs/core/src/error_tests.rs b/codex-rs/core/src/error_tests.rs index 623831b2cf..53b2891c37 100644 --- a/codex-rs/core/src/error_tests.rs +++ b/codex-rs/core/src/error_tests.rs @@ -70,6 +70,30 @@ fn server_overloaded_maps_to_protocol() { ); } +#[test] +fn inline_image_request_limit_maps_to_bad_request() { + let err = CodexErr::InlineImageRequestLimitExceeded( + InlineImageRequestLimitExceededError::local_preflight_bytes(123, 45), + ); + assert_eq!(err.to_codex_protocol_error(), CodexErrorInfo::BadRequest); + assert_eq!( + err.to_string(), + "Total inline image data in this request is 123 bytes, which exceeds the 45 byte limit for a single Responses API request. Use fewer images, smaller images, lower detail, or JPEG compression and try again." + ); +} + +#[test] +fn inline_image_request_count_limit_maps_to_bad_request() { + let err = CodexErr::InlineImageRequestLimitExceeded( + InlineImageRequestLimitExceededError::local_preflight_images(1_501, 1_500), + ); + assert_eq!(err.to_codex_protocol_error(), CodexErrorInfo::BadRequest); + assert_eq!( + err.to_string(), + "This request contains 1501 images, which exceeds the 1500 image limit for a single Responses API request. Use fewer images, smaller images, lower detail, or JPEG compression and try again." + ); +} + #[test] fn sandbox_denied_uses_aggregated_output_when_stderr_empty() { let output = ExecToolCallOutput { diff --git a/codex-rs/core/src/inline_image_request_limit.rs b/codex-rs/core/src/inline_image_request_limit.rs new file mode 100644 index 0000000000..5136dae75e --- /dev/null +++ b/codex-rs/core/src/inline_image_request_limit.rs @@ -0,0 +1,141 @@ +use crate::error::InlineImageRequestLimitExceededError; +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ImageDetail; +use codex_protocol::models::ResponseItem; +use codex_protocol::openai_models::ModelInfo; + +pub(crate) const DEFAULT_INLINE_IMAGE_REQUEST_LIMIT_BYTES: i64 = 512 * 1024 * 1024; +pub(crate) const DEFAULT_INLINE_IMAGE_REQUEST_LIMIT_IMAGE_COUNT: i64 = 1_500; + +pub(crate) fn inline_image_request_limit_bytes(model_info: &ModelInfo) -> usize { + model_info + .inline_image_request_limit_bytes + .filter(|limit| *limit > 0) + .and_then(|limit| usize::try_from(limit).ok()) + .unwrap_or(usize::try_from(DEFAULT_INLINE_IMAGE_REQUEST_LIMIT_BYTES).unwrap_or(usize::MAX)) +} + +pub(crate) fn inline_image_request_limit_image_count(model_info: &ModelInfo) -> usize { + model_info + .inline_image_request_limit_image_count + .filter(|limit| *limit > 0) + .and_then(|limit| usize::try_from(limit).ok()) + .unwrap_or( + usize::try_from(DEFAULT_INLINE_IMAGE_REQUEST_LIMIT_IMAGE_COUNT).unwrap_or(usize::MAX), + ) +} + +pub(crate) fn visit_response_item_input_images( + item: &ResponseItem, + mut visitor: impl FnMut(&str, Option), +) { + match item { + ResponseItem::Message { content, .. } => { + for content_item in content { + if let ContentItem::InputImage { image_url } = content_item { + visitor(image_url, None); + } + } + } + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { + if let FunctionCallOutputBody::ContentItems(items) = &output.body { + for content_item in items { + if let FunctionCallOutputContentItem::InputImage { image_url, detail } = + content_item + { + visitor(image_url, *detail); + } + } + } + } + _ => {} + } +} + +pub(crate) fn is_inline_data_url(url: &str) -> bool { + url.get(.."data:".len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) +} + +pub(crate) fn parse_base64_image_data_url(url: &str) -> Option<&str> { + if !is_inline_data_url(url) { + return None; + } + let comma_index = url.find(',')?; + let metadata = &url[..comma_index]; + let payload = &url[comma_index + 1..]; + let metadata_without_scheme = &metadata["data:".len()..]; + let mut metadata_parts = metadata_without_scheme.split(';'); + let mime_type = metadata_parts.next().unwrap_or_default(); + let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64")); + if !mime_type + .get(.."image/".len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/")) + { + return None; + } + if !has_base64_marker { + return None; + } + Some(payload) +} + +pub(crate) fn total_inline_image_request_bytes(items: &[ResponseItem]) -> usize { + let mut total = 0usize; + for item in items { + visit_response_item_input_images(item, |image_url, _detail| { + if is_inline_data_url(image_url) { + total = total.saturating_add(image_url.len()); + } + }); + } + total +} + +pub(crate) fn total_image_request_count(items: &[ResponseItem]) -> usize { + let mut total = 0usize; + for item in items { + visit_response_item_input_images(item, |_image_url, _detail| { + total = total.saturating_add(1); + }); + } + total +} + +pub(crate) fn inline_image_request_limit_error( + items: &[ResponseItem], + model_info: &ModelInfo, +) -> Option { + let total_inline_image_bytes = total_inline_image_request_bytes(items); + let limit_bytes = inline_image_request_limit_bytes(model_info); + let total_images = total_image_request_count(items); + let limit_images = inline_image_request_limit_image_count(model_info); + let exceeds_bytes = total_inline_image_bytes > limit_bytes; + let exceeds_images = total_images > limit_images; + if !exceeds_bytes && !exceeds_images { + return None; + } + + Some(if exceeds_bytes && exceeds_images { + InlineImageRequestLimitExceededError::local_preflight_bytes_and_images( + total_inline_image_bytes, + limit_bytes, + total_images, + limit_images, + ) + } else if exceeds_bytes { + InlineImageRequestLimitExceededError::local_preflight_bytes( + total_inline_image_bytes, + limit_bytes, + ) + } else { + InlineImageRequestLimitExceededError::local_preflight_images(total_images, limit_images) + }) +} + +#[cfg(test)] +#[path = "inline_image_request_limit_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/inline_image_request_limit_tests.rs b/codex-rs/core/src/inline_image_request_limit_tests.rs new file mode 100644 index 0000000000..154600663e --- /dev/null +++ b/codex-rs/core/src/inline_image_request_limit_tests.rs @@ -0,0 +1,145 @@ +use super::*; +use codex_protocol::models::FunctionCallOutputPayload; +use pretty_assertions::assert_eq; + +#[test] +fn total_inline_image_request_bytes_counts_across_messages_and_tool_outputs() { + let first = "data:image/png;base64,AAA".to_string(); + let second = "data:image/jpeg;base64,BBBB".to_string(); + let third = "data:image/gif;base64,CCCCC".to_string(); + let items = vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image_url: first.clone(), + }], + end_turn: None, + phase: None, + }, + ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: second.clone(), + detail: Some(ImageDetail::Original), + }, + ]), + }, + ResponseItem::CustomToolCallOutput { + call_id: "call-2".to_string(), + name: None, + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: third.clone(), + detail: None, + }, + ]), + }, + ]; + + assert_eq!( + total_inline_image_request_bytes(&items), + first.len() + second.len() + third.len() + ); +} + +#[test] +fn total_inline_image_request_bytes_ignores_remote_and_file_backed_images() { + let items = vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + }], + end_turn: None, + phase: None, + }, + ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "file:///tmp/image.png".to_string(), + detail: None, + }, + ]), + }, + ]; + + assert_eq!(total_inline_image_request_bytes(&items), 0); +} + +#[test] +fn total_inline_image_request_bytes_uses_utf8_byte_length() { + let image_url = "data:image/svg+xml,😀".to_string(); + let items = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image_url: image_url.clone(), + }], + end_turn: None, + phase: None, + }]; + + assert_eq!(total_inline_image_request_bytes(&items), image_url.len()); +} + +#[test] +fn total_image_request_count_counts_all_request_images() { + let items = vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputImage { + image_url: "file:///tmp/first.png".to_string(), + }, + ContentItem::InputImage { + image_url: "https://example.com/second.png".to_string(), + }, + ], + end_turn: None, + phase: None, + }, + ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,THIRD".to_string(), + detail: None, + }, + ]), + }, + ]; + + assert_eq!(total_image_request_count(&items), 3); +} + +#[test] +fn inline_image_request_limit_error_applies_to_user_images() { + let mut model_info = crate::models_manager::model_info::model_info_from_slug("test-model"); + model_info.inline_image_request_limit_image_count = Some(1); + let items = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputImage { + image_url: "https://example.com/one.png".to_string(), + }, + ContentItem::InputImage { + image_url: "https://example.com/two.png".to_string(), + }, + ], + end_turn: None, + phase: None, + }]; + + assert_eq!( + inline_image_request_limit_error(&items, &model_info) + .expect("count overflow should produce an error") + .to_string(), + "This request contains 2 images, which exceeds the 1 image limit for a single Responses API request. Use fewer images, smaller images, lower detail, or JPEG compression and try again." + ); +} diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 5276b09de3..cdf77e91a2 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -43,6 +43,7 @@ mod flags; mod git_info_tests; mod guardian; mod hook_runtime; +mod inline_image_request_limit; pub mod instructions; pub mod landlock; pub mod mcp; diff --git a/codex-rs/core/src/models_manager/model_info.rs b/codex-rs/core/src/models_manager/model_info.rs index 055a6459d4..b547abe596 100644 --- a/codex-rs/core/src/models_manager/model_info.rs +++ b/codex-rs/core/src/models_manager/model_info.rs @@ -83,6 +83,8 @@ pub(crate) fn model_info_from_slug(slug: &str) -> ModelInfo { truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index 808b04191d..08ac44f994 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -95,6 +95,8 @@ fn test_model_info( truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, @@ -911,6 +913,8 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result< truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(large_context_window), auto_compact_token_limit: None, effective_context_window_percent, diff --git a/codex-rs/core/tests/suite/models_cache_ttl.rs b/codex-rs/core/tests/suite/models_cache_ttl.rs index 7d0ce8046d..50cf7cc078 100644 --- a/codex-rs/core/tests/suite/models_cache_ttl.rs +++ b/codex-rs/core/tests/suite/models_cache_ttl.rs @@ -347,6 +347,8 @@ fn test_remote_model(slug: &str, priority: i32) -> ModelInfo { truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, diff --git a/codex-rs/core/tests/suite/personality.rs b/codex-rs/core/tests/suite/personality.rs index cc68c57156..4c5a78db11 100644 --- a/codex-rs/core/tests/suite/personality.rs +++ b/codex-rs/core/tests/suite/personality.rs @@ -664,6 +664,8 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(128_000), auto_compact_token_limit: None, effective_context_window_percent: 95, @@ -780,6 +782,8 @@ async fn user_turn_personality_remote_model_template_includes_update_message() - truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(128_000), auto_compact_token_limit: None, effective_context_window_percent: 95, diff --git a/codex-rs/core/tests/suite/remote_models.rs b/codex-rs/core/tests/suite/remote_models.rs index 6b2da1116b..2ebb9a512c 100644 --- a/codex-rs/core/tests/suite/remote_models.rs +++ b/codex-rs/core/tests/suite/remote_models.rs @@ -113,6 +113,47 @@ async fn remote_models_get_model_info_uses_longest_matching_prefix() -> Result<( Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remote_models_get_model_info_preserves_inline_image_request_limits() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = MockServer::start().await; + let mut remote_model = test_remote_model("gpt-5.3-codex", ModelVisibility::List, 1_000); + remote_model.inline_image_request_limit_bytes = Some(8); + remote_model.inline_image_request_limit_image_count = Some(9); + mount_models_once( + &server, + ModelsResponse { + models: vec![remote_model], + }, + ) + .await; + + let codex_home = TempDir::new()?; + let config = load_default_config_for_test(&codex_home).await; + + let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); + let provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers(/* openai_base_url */ None)["openai"].clone() + }; + let manager = codex_core::test_support::models_manager_with_provider( + codex_home.path().to_path_buf(), + codex_core::test_support::auth_manager_from_auth(auth), + provider, + ); + + manager.list_models(RefreshStrategy::OnlineIfUncached).await; + + let model_info = manager.get_model_info("gpt-5.3-codex", &config).await; + + assert_eq!(model_info.inline_image_request_limit_bytes, Some(8)); + assert_eq!(model_info.inline_image_request_limit_image_count, Some(9)); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_models_long_model_slug_is_sent_with_high_reasoning() -> Result<()> { skip_if_no_network!(Ok(())); @@ -307,6 +348,8 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, @@ -551,6 +594,8 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> { truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, @@ -1031,6 +1076,8 @@ fn test_remote_model_with_policy( truncation_policy, supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 0335ead7e9..b44d025d50 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -418,6 +418,8 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, diff --git a/codex-rs/core/tests/suite/spawn_agent_description.rs b/codex-rs/core/tests/suite/spawn_agent_description.rs index e27e3466f7..c985a9c20f 100644 --- a/codex-rs/core/tests/suite/spawn_agent_description.rs +++ b/codex-rs/core/tests/suite/spawn_agent_description.rs @@ -80,6 +80,8 @@ fn test_model_info( truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index fab2098ca5..9aa2abc633 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -646,6 +646,146 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_image_tool_over_limit_replaces_output_with_text_and_continues() -> anyhow::Result<()> +{ + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let model_slug = "image-limit-model"; + let model = ModelInfo { + slug: model_slug.to_string(), + display_name: "Image limit model".to_string(), + description: Some("Remote model with a tiny inline image cap".to_string()), + default_reasoning_level: Some(ReasoningEffort::Medium), + supported_reasoning_levels: vec![ReasoningEffortPreset { + effort: ReasoningEffort::Medium, + description: ReasoningEffort::Medium.to_string(), + }], + shell_type: ConfigShellToolType::ShellCommand, + visibility: ModelVisibility::List, + supported_in_api: true, + input_modalities: vec![InputModality::Text, InputModality::Image], + used_fallback_model_metadata: false, + supports_search_tool: false, + priority: 1, + upgrade: None, + base_instructions: "base instructions".to_string(), + model_messages: None, + supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, + support_verbosity: false, + default_verbosity: None, + availability_nux: None, + apply_patch_tool_type: None, + web_search_tool_type: Default::default(), + truncation_policy: TruncationPolicyConfig::bytes(10_000), + supports_parallel_tool_calls: false, + supports_image_detail_original: false, + inline_image_request_limit_bytes: Some(8), + inline_image_request_limit_image_count: None, + context_window: Some(272_000), + auto_compact_token_limit: None, + effective_context_window_percent: 95, + experimental_supported_tools: Vec::new(), + }; + let model_catalog = ModelsResponse { + models: vec![model], + }; + + let mut builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(move |config| { + config.model = Some(model_slug.to_string()); + config.model_catalog = Some(model_catalog); + }); + let test = builder.build_remote_aware(&server).await?; + let TestCodex { + codex, + config, + session_configured, + .. + } = &test; + + let rel_path = "assets/over-limit.png"; + write_workspace_png(&test, rel_path, 20, 20, [0u8, 128, 255, 255]).await?; + + let call_id = "view-image-over-limit"; + let arguments = serde_json::json!({ "path": rel_path }).to_string(); + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "view_image", &arguments), + ev_completed("resp-1"), + ]), + ) + .await; + let second_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "used the fallback text"), + ev_completed("resp-2"), + ]), + ) + .await; + + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "view the image".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + cwd: config.cwd.clone(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model: session_configured.model.clone(), + effort: None, + summary: None, + service_tier: None, + collaboration_mode: None, + personality: None, + approvals_reviewer: None, + }) + .await?; + + wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + let request = second_mock.single_request(); + let body = request.body_json(); + let call_output = request.call_output(call_id, "function_call_output"); + let output_text = request + .function_call_output_text(call_id) + .unwrap_or_else(|| { + panic!( + "expected string tool output after inline-image recovery, got {}", + serde_json::to_string_pretty(&call_output).expect("serialize call output") + ) + }); + + assert!( + output_text.contains( + "Tool image output omitted because total inline image data in this request is" + ), + "unexpected tool output: {output_text}" + ); + assert!( + output_text.contains("8 byte limit"), + "expected limit to be included in tool output: {output_text}" + ); + assert!( + !body.to_string().contains("\"type\":\"input_image\""), + "over-limit request should not include inline images: {body}" + ); + assert!( + find_image_message(&body).is_none(), + "over-limit tool output should not include input_image items" + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -1362,6 +1502,8 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: Some(272_000), auto_compact_token_limit: None, effective_context_window_percent: 95, diff --git a/codex-rs/protocol/src/openai_models.rs b/codex-rs/protocol/src/openai_models.rs index 3c633cbd88..3ffa728c45 100644 --- a/codex-rs/protocol/src/openai_models.rs +++ b/codex-rs/protocol/src/openai_models.rs @@ -270,6 +270,10 @@ pub struct ModelInfo { #[serde(default)] pub supports_image_detail_original: bool, #[serde(default, skip_serializing_if = "Option::is_none")] + pub inline_image_request_limit_bytes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inline_image_request_limit_image_count: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub context_window: Option, /// Token threshold for automatic compaction. When omitted, core derives it /// from `context_window` (90%). When provided, core clamps it to 90% of the @@ -540,6 +544,8 @@ mod tests { truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000), supports_parallel_tool_calls: false, supports_image_detail_original: false, + inline_image_request_limit_bytes: None, + inline_image_request_limit_image_count: None, context_window: None, auto_compact_token_limit: None, effective_context_window_percent: 95, @@ -752,6 +758,8 @@ mod tests { }, "supports_parallel_tool_calls": false, "supports_image_detail_original": false, + "inline_image_request_limit_bytes": null, + "inline_image_request_limit_image_count": null, "context_window": null, "auto_compact_token_limit": null, "effective_context_window_percent": 95, @@ -762,6 +770,8 @@ mod tests { assert_eq!(model.availability_nux, None); assert!(!model.supports_image_detail_original); + assert_eq!(model.inline_image_request_limit_bytes, None); + assert_eq!(model.inline_image_request_limit_image_count, None); assert_eq!(model.web_search_tool_type, WebSearchToolType::Text); assert!(!model.supports_search_tool); }