Handle inline image request caps gracefully

git-stack-id: fjord/original_image_size_warn
git-stack-title: Handle inline image request caps gracefully
This commit is contained in:
Curtis 'Fjord' Hawthorne
2026-03-30 16:44:41 -07:00
parent 0771a56078
commit b05eaf31f0
21 changed files with 1002 additions and 84 deletions
@@ -40,6 +40,8 @@ fn preset_to_info(preset: &ModelPreset, priority: i32) -> ModelInfo {
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
@@ -88,6 +88,8 @@ async fn models_client_hits_models_endpoint() {
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
+4
View File
@@ -104,6 +104,7 @@ use crate::default_client::build_reqwest_client;
use crate::error::CodexErr;
use crate::error::Result;
use crate::flags::CODEX_RS_SSE_FIXTURE;
use crate::inline_image_request_limit::inline_image_request_limit_error;
use crate::model_provider_info::ModelProviderInfo;
use crate::model_provider_info::WireApi;
use crate::response_debug_context::extract_response_debug_context;
@@ -692,6 +693,9 @@ impl ModelClientSession {
) -> Result<ResponsesApiRequest> {
let instructions = &prompt.base_instructions.text;
let input = prompt.get_formatted_input();
if let Some(error) = inline_image_request_limit_error(&input, model_info) {
return Err(CodexErr::InlineImageRequestLimitExceeded(error));
}
let tools = create_tools_json_for_responses_api(&prompt.tools)?;
let default_reasoning_effort = model_info.default_reasoning_level;
let reasoning = if model_info.supports_reasoning_summaries {
+59 -5
View File
@@ -182,6 +182,7 @@ use crate::error::CodexErr;
use crate::error::Result as CodexResult;
#[cfg(test)]
use crate::exec::StreamOutput;
use crate::inline_image_request_limit::inline_image_request_limit_error;
use codex_config::CONFIG_TOML_FILE;
mod rollout_reconstruction;
@@ -3326,16 +3327,69 @@ impl Session {
.await
}
/// Records input items: always append to conversation history and
/// persist these response items to rollout.
/// Records input items, sanitizing inline tool images before persistence when
/// they would overflow the model request.
pub(crate) async fn record_conversation_items(
&self,
turn_context: &TurnContext,
items: &[ResponseItem],
) {
self.record_into_history(items, turn_context).await;
self.persist_rollout_response_items(items).await;
self.send_raw_response_items(turn_context, items).await;
let items_to_record = {
let mut state = self.state.lock().await;
let items_to_record = Self::prepare_items_for_recording(&state, turn_context, items);
state.record_items(items_to_record.iter(), turn_context.truncation_policy);
items_to_record
};
self.persist_rollout_response_items(&items_to_record).await;
self.send_raw_response_items(turn_context, &items_to_record)
.await;
}
fn prepare_items_for_recording(
state: &SessionState,
turn_context: &TurnContext,
items: &[ResponseItem],
) -> Vec<ResponseItem> {
let mut items_to_record = items.to_vec();
let should_check_image_request_limits = items_to_record.iter().any(|item| match item {
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
output.content_items().is_some_and(|content_items| {
content_items.iter().any(|content_item| {
matches!(
content_item,
codex_protocol::models::FunctionCallOutputContentItem::InputImage { .. }
)
})
})
}
_ => false,
});
if !should_check_image_request_limits {
return items_to_record;
}
let mut candidate_history = state.clone_history();
candidate_history.record_items(items_to_record.iter(), turn_context.truncation_policy);
let prompt_input = candidate_history.for_prompt(&turn_context.model_info.input_modalities);
let Some(error) = inline_image_request_limit_error(&prompt_input, &turn_context.model_info)
else {
return items_to_record;
};
let mut pending_items = ContextManager::new();
pending_items.replace(items_to_record.clone());
if pending_items.replace_last_turn_tool_outputs_with_failure_message(
&error.tool_output_recovery_message(),
) {
warn!(
"inline image request limit would be exceeded before upload; sanitizing tool image output before persistence"
);
items_to_record = pending_items.raw_items().to_vec();
}
items_to_record
}
/// Append ResponseItems to the in-memory conversation history only.
+96
View File
@@ -20,6 +20,7 @@ use crate::tools::format_exec_output_str;
use codex_features::Features;
use codex_protocol::ThreadId;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::permissions::FileSystemAccessMode;
use codex_protocol::permissions::FileSystemPath;
@@ -925,6 +926,101 @@ async fn reconstruct_history_uses_replacement_history_verbatim() {
assert_eq!(reconstructed.history, replacement_history);
}
#[tokio::test]
async fn record_conversation_items_sanitizes_inline_tool_images_before_persistence() {
let (session, mut turn_context) = make_session_and_context().await;
let session = Arc::new(session);
let rollout_path = attach_rollout_recorder(&session).await;
let user = user_message("describe the image");
let image_url = "data:image/png;base64,AAAA".to_string();
turn_context.model_info.inline_image_request_limit_bytes = Some(10);
let tool_call = ResponseItem::FunctionCall {
id: None,
name: "view_image".to_string(),
namespace: None,
arguments: "{}".to_string(),
call_id: "view-image".to_string(),
};
let original_output = ResponseItem::FunctionCallOutput {
call_id: "view-image".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputText {
text: "captured".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: image_url.clone(),
detail: None,
},
]),
};
let recovery_message =
crate::error::InlineImageRequestLimitExceededError::local_preflight_bytes(
image_url.len(),
10,
)
.tool_output_recovery_message();
let recovered_output = ResponseItem::FunctionCallOutput {
call_id: "view-image".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "captured".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: recovery_message,
},
]),
success: Some(false),
},
};
session
.record_conversation_items(&turn_context, std::slice::from_ref(&user))
.await;
session
.record_conversation_items(&turn_context, std::slice::from_ref(&tool_call))
.await;
session
.record_conversation_items(&turn_context, std::slice::from_ref(&original_output))
.await;
assert_eq!(
session.clone_history().await.raw_items().to_vec(),
vec![user.clone(), tool_call.clone(), recovered_output.clone()]
);
session.flush_rollout().await;
let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path)
.await
.expect("read rollout history")
else {
panic!("expected resumed rollout history");
};
let persisted_response_items = resumed
.history
.into_iter()
.filter_map(|item| match item {
RolloutItem::ResponseItem(item) => Some(item),
_ => None,
})
.collect::<Vec<_>>();
let ResponseItem::FunctionCallOutput { output, .. } = recovered_output else {
panic!("expected function call output");
};
let persisted_output = ResponseItem::FunctionCallOutput {
call_id: "view-image".to_string(),
output: FunctionCallOutputPayload {
body: output.body,
success: None,
},
};
assert_eq!(
persisted_response_items,
vec![user, tool_call, persisted_output]
);
}
#[tokio::test]
async fn record_initial_history_reconstructs_resumed_transcript() {
let (session, turn_context) = make_session_and_context().await;
+78 -79
View File
@@ -3,6 +3,8 @@ use crate::context_manager::normalize;
use crate::event_mapping::has_non_contextual_dev_message_content;
use crate::event_mapping::is_contextual_dev_message_content;
use crate::event_mapping::is_contextual_user_message_content;
use crate::inline_image_request_limit::parse_base64_image_data_url;
use crate::inline_image_request_limit::visit_response_item_input_images;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use codex_protocol::models::BaseInstructions;
@@ -181,32 +183,60 @@ impl ContextManager {
/// Replace image content in the last turn if it originated from a tool output.
/// Returns true when a tool image was replaced, false otherwise.
pub(crate) fn replace_last_turn_images(&mut self, placeholder: &str) -> bool {
let Some(index) = self.items.iter().rposition(|item| {
matches!(item, ResponseItem::FunctionCallOutput { .. }) || is_user_turn_boundary(item)
}) else {
return false;
};
self.visit_last_turn_tool_outputs_mut(|output| {
let Some(content_items) = output.content_items_mut() else {
return false;
};
let mut replaced = false;
let placeholder = placeholder.to_string();
for item in content_items.iter_mut() {
if matches!(item, FunctionCallOutputContentItem::InputImage { .. }) {
*item = FunctionCallOutputContentItem::InputText {
text: placeholder.clone(),
};
replaced = true;
}
}
replaced
})
}
match &mut self.items[index] {
ResponseItem::FunctionCallOutput { output, .. } => {
let Some(content_items) = output.content_items_mut() else {
return false;
};
let mut replaced = false;
let placeholder = placeholder.to_string();
for item in content_items.iter_mut() {
if matches!(item, FunctionCallOutputContentItem::InputImage { .. }) {
*item = FunctionCallOutputContentItem::InputText {
text: placeholder.clone(),
};
replaced = true;
/// Replace image-bearing tool outputs from the current turn with a textual failure so the
/// model can adapt and continue in the same turn.
pub(crate) fn replace_last_turn_tool_outputs_with_failure_message(
&mut self,
message: &str,
) -> bool {
self.visit_last_turn_tool_outputs_mut(|output| {
let Some(content_items) = output.content_items() else {
return false;
};
let mut preserved_items = Vec::with_capacity(content_items.len().saturating_add(1));
let mut removed_images = false;
for item in content_items {
match item {
FunctionCallOutputContentItem::InputText { text } => {
preserved_items
.push(FunctionCallOutputContentItem::InputText { text: text.clone() });
}
FunctionCallOutputContentItem::InputImage { .. } => {
removed_images = true;
}
}
replaced
}
ResponseItem::Message { .. } => false,
_ => false,
}
if !removed_images {
return false;
}
let message = message.to_string();
output.body = if preserved_items.is_empty() {
FunctionCallOutputBody::Text(message)
} else {
preserved_items.push(FunctionCallOutputContentItem::InputText { text: message });
FunctionCallOutputBody::ContentItems(preserved_items)
};
output.success = Some(false);
true
})
}
/// Drop the last `num_turns` instruction turns from this history.
@@ -544,40 +574,6 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) ->
}
}
/// Returns the base64 payload byte length for inline image data URLs that are
/// eligible for token-estimation discounting.
///
/// We only discount payloads for `data:image/...;base64,...` URLs (case
/// insensitive markers) and leave everything else at raw serialized size.
fn parse_base64_image_data_url(url: &str) -> Option<&str> {
if !url
.get(.."data:".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
{
return None;
}
let comma_index = url.find(',')?;
let metadata = &url[..comma_index];
let payload = &url[comma_index + 1..];
// Parse the media type and parameters without decoding. This keeps the
// estimator cheap while ensuring we only apply the fixed-cost image
// heuristic to image-typed base64 data URLs.
let metadata_without_scheme = &metadata["data:".len()..];
let mut metadata_parts = metadata_without_scheme.split(';');
let mime_type = metadata_parts.next().unwrap_or_default();
let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64"));
if !mime_type
.get(.."image/".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
{
return None;
}
if !has_base64_marker {
return None;
}
Some(payload)
}
fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {
let key = sha1_digest(image_url.as_bytes());
ORIGINAL_IMAGE_ESTIMATE_CACHE.get_or_insert_with(key, || {
@@ -621,7 +617,7 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
let mut payload_bytes = 0i64;
let mut replacement_bytes = 0i64;
let mut accumulate = |image_url: &str, detail: Option<ImageDetail>| {
let accumulate = |image_url: &str, detail: Option<ImageDetail>| {
if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) {
payload_bytes =
payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
@@ -634,32 +630,35 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
}
};
match item {
ResponseItem::Message { content, .. } => {
for content_item in content {
if let ContentItem::InputImage { image_url } = content_item {
accumulate(image_url, None);
}
}
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
for content_item in items {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
content_item
{
accumulate(image_url, *detail);
}
}
}
}
_ => {}
}
visit_response_item_input_images(item, accumulate);
(payload_bytes, replacement_bytes)
}
impl ContextManager {
fn visit_last_turn_tool_outputs_mut(
&mut self,
mut visitor: impl FnMut(&mut FunctionCallOutputPayload) -> bool,
) -> bool {
let start = self
.items
.iter()
.rposition(is_user_turn_boundary)
.map_or(0, |index| index.saturating_add(1));
let mut changed = false;
for item in &mut self.items[start..] {
match item {
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
changed |= visitor(output);
}
_ => {}
}
}
changed
}
}
fn is_model_generated_item(item: &ResponseItem) -> bool {
match item {
ResponseItem::Message { role, .. } => role == "assistant",
@@ -761,6 +761,142 @@ fn replace_last_turn_images_does_not_touch_user_images() {
assert_eq!(history.raw_items(), items);
}
#[test]
fn replace_last_turn_images_replaces_custom_tool_output_images() {
let items = vec![
user_input_text_msg("hi"),
ResponseItem::CustomToolCallOutput {
call_id: "call-custom".to_string(),
name: None,
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,AAA".to_string(),
detail: None,
},
]),
success: Some(true),
},
},
];
let mut history = create_history_with_items(items);
assert!(history.replace_last_turn_images("Invalid image"));
assert_eq!(
history.raw_items(),
vec![
user_input_text_msg("hi"),
ResponseItem::CustomToolCallOutput {
call_id: "call-custom".to_string(),
name: None,
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "Invalid image".to_string(),
},
]),
success: Some(true),
},
},
]
);
}
#[test]
fn replace_last_turn_tool_outputs_with_failure_message_replaces_only_latest_turn_images() {
let items = vec![
user_input_text_msg("first"),
ResponseItem::FunctionCallOutput {
call_id: "call-old".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,OLD".to_string(),
detail: None,
},
]),
success: Some(true),
},
},
user_input_text_msg("second"),
ResponseItem::FunctionCallOutput {
call_id: "call-new".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "captured".to_string(),
},
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,NEW".to_string(),
detail: None,
},
]),
success: Some(true),
},
},
ResponseItem::CustomToolCallOutput {
call_id: "call-custom".to_string(),
name: None,
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,CUSTOM".to_string(),
detail: Some(ImageDetail::Original),
},
]),
success: Some(true),
},
},
];
let mut history = create_history_with_items(items);
let message = "Tool image output omitted because total inline image data in this request is 99 bytes, which exceeds the 10 byte limit for a single Responses API request. Retry with fewer images, smaller images, lower detail, or JPEG compression.";
assert!(history.replace_last_turn_tool_outputs_with_failure_message(message));
assert_eq!(
history.raw_items(),
vec![
user_input_text_msg("first"),
ResponseItem::FunctionCallOutput {
call_id: "call-old".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,OLD".to_string(),
detail: None,
},
]),
success: Some(true),
},
},
user_input_text_msg("second"),
ResponseItem::FunctionCallOutput {
call_id: "call-new".to_string(),
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::ContentItems(vec![
FunctionCallOutputContentItem::InputText {
text: "captured".to_string(),
},
FunctionCallOutputContentItem::InputText {
text: message.to_string(),
},
]),
success: Some(false),
},
},
ResponseItem::CustomToolCallOutput {
call_id: "call-custom".to_string(),
name: None,
output: FunctionCallOutputPayload {
body: FunctionCallOutputBody::Text(message.to_string()),
success: Some(false),
},
},
]
);
}
#[test]
fn remove_first_item_handles_local_shell_pair() {
let items = vec![
+99
View File
@@ -27,6 +27,100 @@ pub type Result<T> = std::result::Result<T, CodexErr>;
/// Limit UI error messages to a reasonable size while keeping useful context.
const ERROR_MESSAGE_UI_MAX_BYTES: usize = 2 * 1024; // 2 KiB
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct InlineImageRequestLimitExceededError {
total_bytes: Option<usize>,
limit_bytes: Option<usize>,
total_images: Option<usize>,
limit_images: Option<usize>,
}
impl InlineImageRequestLimitExceededError {
pub fn local_preflight_bytes(total_bytes: usize, limit_bytes: usize) -> Self {
Self {
total_bytes: Some(total_bytes),
limit_bytes: Some(limit_bytes),
total_images: None,
limit_images: None,
}
}
pub fn local_preflight_images(total_images: usize, limit_images: usize) -> Self {
Self {
total_bytes: None,
limit_bytes: None,
total_images: Some(total_images),
limit_images: Some(limit_images),
}
}
pub fn local_preflight_bytes_and_images(
total_bytes: usize,
limit_bytes: usize,
total_images: usize,
limit_images: usize,
) -> Self {
Self {
total_bytes: Some(total_bytes),
limit_bytes: Some(limit_bytes),
total_images: Some(total_images),
limit_images: Some(limit_images),
}
}
pub fn tool_output_recovery_message(&self) -> String {
format!(
"Tool image output omitted because {} Retry with fewer images, smaller images, lower detail, or JPEG compression.",
self.message_body()
)
}
fn message_body(&self) -> String {
match (
self.total_images,
self.limit_images,
self.total_bytes,
self.limit_bytes,
) {
(Some(total_images), Some(limit_images), Some(total_bytes), Some(limit_bytes)) => {
format!(
"this request contains {total_images} images, which exceeds the {limit_images} image limit, and total inline image data is {total_bytes} bytes, which exceeds the {limit_bytes} byte limit for a single Responses API request."
)
}
(Some(total_images), Some(limit_images), _, _) => format!(
"this request contains {total_images} images, which exceeds the {limit_images} image limit for a single Responses API request."
),
(None, None, Some(total_bytes), Some(limit_bytes)) => format!(
"total inline image data in this request is {total_bytes} bytes, which exceeds the {limit_bytes} byte limit for a single Responses API request."
),
(None, None, None, Some(limit_bytes)) => format!(
"total inline image data in this request exceeds the {limit_bytes} byte limit for a single Responses API request."
),
(None, None, Some(total_bytes), None) => format!(
"total inline image data in this request is {total_bytes} bytes, which exceeds the single-request limit for the Responses API."
),
_ => {
"total inline image data in this request exceeds the single-request limit for the Responses API.".to_string()
}
}
}
}
impl std::fmt::Display for InlineImageRequestLimitExceededError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let message = self.message_body();
let mut chars = message.chars();
let message = match chars.next() {
Some(first) => format!("{}{}", first.to_ascii_uppercase(), chars.as_str()),
None => String::new(),
};
write!(
f,
"{message} Use fewer images, smaller images, lower detail, or JPEG compression and try again."
)
}
}
#[derive(Error, Debug)]
pub enum SandboxErr {
/// Error from sandbox execution
@@ -116,6 +210,9 @@ pub enum CodexErr {
#[error("Image poisoning")]
InvalidImageRequest(),
#[error("{0}")]
InlineImageRequestLimitExceeded(InlineImageRequestLimitExceededError),
#[error("{0}")]
UsageLimitReached(UsageLimitReachedError),
@@ -203,6 +300,7 @@ impl CodexErr {
| CodexErr::UsageNotIncluded
| CodexErr::QuotaExceeded
| CodexErr::InvalidImageRequest()
| CodexErr::InlineImageRequestLimitExceeded(_)
| CodexErr::InvalidRequest(_)
| CodexErr::RefreshTokenFailed(_)
| CodexErr::UnsupportedOperation(_)
@@ -584,6 +682,7 @@ impl CodexErr {
| CodexErr::InternalServerError
| CodexErr::InternalAgentDied => CodexErrorInfo::InternalServerError,
CodexErr::UnsupportedOperation(_)
| CodexErr::InlineImageRequestLimitExceeded(_)
| CodexErr::ThreadNotFound(_)
| CodexErr::AgentLimitReached { .. } => CodexErrorInfo::BadRequest,
CodexErr::Sandbox(_) => CodexErrorInfo::SandboxError,
+24
View File
@@ -70,6 +70,30 @@ fn server_overloaded_maps_to_protocol() {
);
}
#[test]
fn inline_image_request_limit_maps_to_bad_request() {
let err = CodexErr::InlineImageRequestLimitExceeded(
InlineImageRequestLimitExceededError::local_preflight_bytes(123, 45),
);
assert_eq!(err.to_codex_protocol_error(), CodexErrorInfo::BadRequest);
assert_eq!(
err.to_string(),
"Total inline image data in this request is 123 bytes, which exceeds the 45 byte limit for a single Responses API request. Use fewer images, smaller images, lower detail, or JPEG compression and try again."
);
}
#[test]
fn inline_image_request_count_limit_maps_to_bad_request() {
let err = CodexErr::InlineImageRequestLimitExceeded(
InlineImageRequestLimitExceededError::local_preflight_images(1_501, 1_500),
);
assert_eq!(err.to_codex_protocol_error(), CodexErrorInfo::BadRequest);
assert_eq!(
err.to_string(),
"This request contains 1501 images, which exceeds the 1500 image limit for a single Responses API request. Use fewer images, smaller images, lower detail, or JPEG compression and try again."
);
}
#[test]
fn sandbox_denied_uses_aggregated_output_when_stderr_empty() {
let output = ExecToolCallOutput {
@@ -0,0 +1,141 @@
use crate::error::InlineImageRequestLimitExceededError;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ImageDetail;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
pub(crate) const DEFAULT_INLINE_IMAGE_REQUEST_LIMIT_BYTES: i64 = 512 * 1024 * 1024;
pub(crate) const DEFAULT_INLINE_IMAGE_REQUEST_LIMIT_IMAGE_COUNT: i64 = 1_500;
pub(crate) fn inline_image_request_limit_bytes(model_info: &ModelInfo) -> usize {
model_info
.inline_image_request_limit_bytes
.filter(|limit| *limit > 0)
.and_then(|limit| usize::try_from(limit).ok())
.unwrap_or(usize::try_from(DEFAULT_INLINE_IMAGE_REQUEST_LIMIT_BYTES).unwrap_or(usize::MAX))
}
pub(crate) fn inline_image_request_limit_image_count(model_info: &ModelInfo) -> usize {
model_info
.inline_image_request_limit_image_count
.filter(|limit| *limit > 0)
.and_then(|limit| usize::try_from(limit).ok())
.unwrap_or(
usize::try_from(DEFAULT_INLINE_IMAGE_REQUEST_LIMIT_IMAGE_COUNT).unwrap_or(usize::MAX),
)
}
pub(crate) fn visit_response_item_input_images(
item: &ResponseItem,
mut visitor: impl FnMut(&str, Option<ImageDetail>),
) {
match item {
ResponseItem::Message { content, .. } => {
for content_item in content {
if let ContentItem::InputImage { image_url } = content_item {
visitor(image_url, None);
}
}
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
for content_item in items {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
content_item
{
visitor(image_url, *detail);
}
}
}
}
_ => {}
}
}
pub(crate) fn is_inline_data_url(url: &str) -> bool {
url.get(.."data:".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
}
pub(crate) fn parse_base64_image_data_url(url: &str) -> Option<&str> {
if !is_inline_data_url(url) {
return None;
}
let comma_index = url.find(',')?;
let metadata = &url[..comma_index];
let payload = &url[comma_index + 1..];
let metadata_without_scheme = &metadata["data:".len()..];
let mut metadata_parts = metadata_without_scheme.split(';');
let mime_type = metadata_parts.next().unwrap_or_default();
let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64"));
if !mime_type
.get(.."image/".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
{
return None;
}
if !has_base64_marker {
return None;
}
Some(payload)
}
pub(crate) fn total_inline_image_request_bytes(items: &[ResponseItem]) -> usize {
let mut total = 0usize;
for item in items {
visit_response_item_input_images(item, |image_url, _detail| {
if is_inline_data_url(image_url) {
total = total.saturating_add(image_url.len());
}
});
}
total
}
pub(crate) fn total_image_request_count(items: &[ResponseItem]) -> usize {
let mut total = 0usize;
for item in items {
visit_response_item_input_images(item, |_image_url, _detail| {
total = total.saturating_add(1);
});
}
total
}
pub(crate) fn inline_image_request_limit_error(
items: &[ResponseItem],
model_info: &ModelInfo,
) -> Option<InlineImageRequestLimitExceededError> {
let total_inline_image_bytes = total_inline_image_request_bytes(items);
let limit_bytes = inline_image_request_limit_bytes(model_info);
let total_images = total_image_request_count(items);
let limit_images = inline_image_request_limit_image_count(model_info);
let exceeds_bytes = total_inline_image_bytes > limit_bytes;
let exceeds_images = total_images > limit_images;
if !exceeds_bytes && !exceeds_images {
return None;
}
Some(if exceeds_bytes && exceeds_images {
InlineImageRequestLimitExceededError::local_preflight_bytes_and_images(
total_inline_image_bytes,
limit_bytes,
total_images,
limit_images,
)
} else if exceeds_bytes {
InlineImageRequestLimitExceededError::local_preflight_bytes(
total_inline_image_bytes,
limit_bytes,
)
} else {
InlineImageRequestLimitExceededError::local_preflight_images(total_images, limit_images)
})
}
#[cfg(test)]
#[path = "inline_image_request_limit_tests.rs"]
mod tests;
@@ -0,0 +1,145 @@
use super::*;
use codex_protocol::models::FunctionCallOutputPayload;
use pretty_assertions::assert_eq;
#[test]
fn total_inline_image_request_bytes_counts_across_messages_and_tool_outputs() {
let first = "data:image/png;base64,AAA".to_string();
let second = "data:image/jpeg;base64,BBBB".to_string();
let third = "data:image/gif;base64,CCCCC".to_string();
let items = vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: first.clone(),
}],
end_turn: None,
phase: None,
},
ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: second.clone(),
detail: Some(ImageDetail::Original),
},
]),
},
ResponseItem::CustomToolCallOutput {
call_id: "call-2".to_string(),
name: None,
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: third.clone(),
detail: None,
},
]),
},
];
assert_eq!(
total_inline_image_request_bytes(&items),
first.len() + second.len() + third.len()
);
}
#[test]
fn total_inline_image_request_bytes_ignores_remote_and_file_backed_images() {
let items = vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: "https://example.com/image.png".to_string(),
}],
end_turn: None,
phase: None,
},
ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "file:///tmp/image.png".to_string(),
detail: None,
},
]),
},
];
assert_eq!(total_inline_image_request_bytes(&items), 0);
}
#[test]
fn total_inline_image_request_bytes_uses_utf8_byte_length() {
let image_url = "data:image/svg+xml,<svg>😀</svg>".to_string();
let items = vec![ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::InputImage {
image_url: image_url.clone(),
}],
end_turn: None,
phase: None,
}];
assert_eq!(total_inline_image_request_bytes(&items), image_url.len());
}
#[test]
fn total_image_request_count_counts_all_request_images() {
let items = vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url: "file:///tmp/first.png".to_string(),
},
ContentItem::InputImage {
image_url: "https://example.com/second.png".to_string(),
},
],
end_turn: None,
phase: None,
},
ResponseItem::FunctionCallOutput {
call_id: "call-1".to_string(),
output: FunctionCallOutputPayload::from_content_items(vec![
FunctionCallOutputContentItem::InputImage {
image_url: "data:image/png;base64,THIRD".to_string(),
detail: None,
},
]),
},
];
assert_eq!(total_image_request_count(&items), 3);
}
#[test]
fn inline_image_request_limit_error_applies_to_user_images() {
let mut model_info = crate::models_manager::model_info::model_info_from_slug("test-model");
model_info.inline_image_request_limit_image_count = Some(1);
let items = vec![ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![
ContentItem::InputImage {
image_url: "https://example.com/one.png".to_string(),
},
ContentItem::InputImage {
image_url: "https://example.com/two.png".to_string(),
},
],
end_turn: None,
phase: None,
}];
assert_eq!(
inline_image_request_limit_error(&items, &model_info)
.expect("count overflow should produce an error")
.to_string(),
"This request contains 2 images, which exceeds the 1 image limit for a single Responses API request. Use fewer images, smaller images, lower detail, or JPEG compression and try again."
);
}
+1
View File
@@ -43,6 +43,7 @@ mod flags;
mod git_info_tests;
mod guardian;
mod hook_runtime;
mod inline_image_request_limit;
pub mod instructions;
pub mod landlock;
pub mod mcp;
@@ -83,6 +83,8 @@ pub(crate) fn model_info_from_slug(slug: &str) -> ModelInfo {
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
@@ -95,6 +95,8 @@ fn test_model_info(
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
@@ -911,6 +913,8 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result<
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(large_context_window),
auto_compact_token_limit: None,
effective_context_window_percent,
@@ -347,6 +347,8 @@ fn test_remote_model(slug: &str, priority: i32) -> ModelInfo {
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
+4
View File
@@ -664,6 +664,8 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(128_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
@@ -780,6 +782,8 @@ async fn user_turn_personality_remote_model_template_includes_update_message() -
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(128_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
@@ -113,6 +113,47 @@ async fn remote_models_get_model_info_uses_longest_matching_prefix() -> Result<(
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn remote_models_get_model_info_preserves_inline_image_request_limits() -> Result<()> {
skip_if_no_network!(Ok(()));
skip_if_sandbox!(Ok(()));
let server = MockServer::start().await;
let mut remote_model = test_remote_model("gpt-5.3-codex", ModelVisibility::List, 1_000);
remote_model.inline_image_request_limit_bytes = Some(8);
remote_model.inline_image_request_limit_image_count = Some(9);
mount_models_once(
&server,
ModelsResponse {
models: vec![remote_model],
},
)
.await;
let codex_home = TempDir::new()?;
let config = load_default_config_for_test(&codex_home).await;
let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing();
let provider = ModelProviderInfo {
base_url: Some(format!("{}/v1", server.uri())),
..built_in_model_providers(/* openai_base_url */ None)["openai"].clone()
};
let manager = codex_core::test_support::models_manager_with_provider(
codex_home.path().to_path_buf(),
codex_core::test_support::auth_manager_from_auth(auth),
provider,
);
manager.list_models(RefreshStrategy::OnlineIfUncached).await;
let model_info = manager.get_model_info("gpt-5.3-codex", &config).await;
assert_eq!(model_info.inline_image_request_limit_bytes, Some(8));
assert_eq!(model_info.inline_image_request_limit_image_count, Some(9));
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn remote_models_long_model_slug_is_sent_with_high_reasoning() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -307,6 +348,8 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> {
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
@@ -551,6 +594,8 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> {
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
@@ -1031,6 +1076,8 @@ fn test_remote_model_with_policy(
truncation_policy,
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
+2
View File
@@ -418,6 +418,8 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
@@ -80,6 +80,8 @@ fn test_model_info(
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
+142
View File
@@ -646,6 +646,146 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn view_image_tool_over_limit_replaces_output_with_text_and_continues() -> anyhow::Result<()>
{
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let model_slug = "image-limit-model";
let model = ModelInfo {
slug: model_slug.to_string(),
display_name: "Image limit model".to_string(),
description: Some("Remote model with a tiny inline image cap".to_string()),
default_reasoning_level: Some(ReasoningEffort::Medium),
supported_reasoning_levels: vec![ReasoningEffortPreset {
effort: ReasoningEffort::Medium,
description: ReasoningEffort::Medium.to_string(),
}],
shell_type: ConfigShellToolType::ShellCommand,
visibility: ModelVisibility::List,
supported_in_api: true,
input_modalities: vec![InputModality::Text, InputModality::Image],
used_fallback_model_metadata: false,
supports_search_tool: false,
priority: 1,
upgrade: None,
base_instructions: "base instructions".to_string(),
model_messages: None,
supports_reasoning_summaries: false,
default_reasoning_summary: ReasoningSummary::Auto,
support_verbosity: false,
default_verbosity: None,
availability_nux: None,
apply_patch_tool_type: None,
web_search_tool_type: Default::default(),
truncation_policy: TruncationPolicyConfig::bytes(10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: Some(8),
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
experimental_supported_tools: Vec::new(),
};
let model_catalog = ModelsResponse {
models: vec![model],
};
let mut builder = test_codex()
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
.with_config(move |config| {
config.model = Some(model_slug.to_string());
config.model_catalog = Some(model_catalog);
});
let test = builder.build_remote_aware(&server).await?;
let TestCodex {
codex,
config,
session_configured,
..
} = &test;
let rel_path = "assets/over-limit.png";
write_workspace_png(&test, rel_path, 20, 20, [0u8, 128, 255, 255]).await?;
let call_id = "view-image-over-limit";
let arguments = serde_json::json!({ "path": rel_path }).to_string();
responses::mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-1"),
ev_function_call(call_id, "view_image", &arguments),
ev_completed("resp-1"),
]),
)
.await;
let second_mock = responses::mount_sse_once(
&server,
sse(vec![
ev_assistant_message("msg-1", "used the fallback text"),
ev_completed("resp-2"),
]),
)
.await;
codex
.submit(Op::UserTurn {
items: vec![UserInput::Text {
text: "view the image".into(),
text_elements: Vec::new(),
}],
final_output_json_schema: None,
cwd: config.cwd.clone(),
approval_policy: AskForApproval::Never,
sandbox_policy: SandboxPolicy::DangerFullAccess,
model: session_configured.model.clone(),
effort: None,
summary: None,
service_tier: None,
collaboration_mode: None,
personality: None,
approvals_reviewer: None,
})
.await?;
wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
let request = second_mock.single_request();
let body = request.body_json();
let call_output = request.call_output(call_id, "function_call_output");
let output_text = request
.function_call_output_text(call_id)
.unwrap_or_else(|| {
panic!(
"expected string tool output after inline-image recovery, got {}",
serde_json::to_string_pretty(&call_output).expect("serialize call output")
)
});
assert!(
output_text.contains(
"Tool image output omitted because total inline image data in this request is"
),
"unexpected tool output: {output_text}"
);
assert!(
output_text.contains("8 byte limit"),
"expected limit to be included in tool output: {output_text}"
);
assert!(
!body.to_string().contains("\"type\":\"input_image\""),
"over-limit request should not include inline images: {body}"
);
assert!(
find_image_message(&body).is_none(),
"over-limit tool output should not include input_image items"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
@@ -1362,6 +1502,8 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: Some(272_000),
auto_compact_token_limit: None,
effective_context_window_percent: 95,
+10
View File
@@ -270,6 +270,10 @@ pub struct ModelInfo {
#[serde(default)]
pub supports_image_detail_original: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub inline_image_request_limit_bytes: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub inline_image_request_limit_image_count: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub context_window: Option<i64>,
/// Token threshold for automatic compaction. When omitted, core derives it
/// from `context_window` (90%). When provided, core clamps it to 90% of the
@@ -540,6 +544,8 @@ mod tests {
truncation_policy: TruncationPolicyConfig::bytes(/*limit*/ 10_000),
supports_parallel_tool_calls: false,
supports_image_detail_original: false,
inline_image_request_limit_bytes: None,
inline_image_request_limit_image_count: None,
context_window: None,
auto_compact_token_limit: None,
effective_context_window_percent: 95,
@@ -752,6 +758,8 @@ mod tests {
},
"supports_parallel_tool_calls": false,
"supports_image_detail_original": false,
"inline_image_request_limit_bytes": null,
"inline_image_request_limit_image_count": null,
"context_window": null,
"auto_compact_token_limit": null,
"effective_context_window_percent": 95,
@@ -762,6 +770,8 @@ mod tests {
assert_eq!(model.availability_nux, None);
assert!(!model.supports_image_detail_original);
assert_eq!(model.inline_image_request_limit_bytes, None);
assert_eq!(model.inline_image_request_limit_image_count, None);
assert_eq!(model.web_search_tool_type, WebSearchToolType::Text);
assert!(!model.supports_search_tool);
}