From 0a0ebb85355113610dd3f7a3d8b36f68c33465fc Mon Sep 17 00:00:00 2001 From: Curtis 'Fjord' Hawthorne Date: Thu, 6 Aug 2026 03:06:43 +0000 Subject: [PATCH] Add a unified image budget (#37206) ## What changed - Add the gated `unified_image_budget` feature for models that support original image detail or Responses Lite. - Apply one 6,000-pixel, 10,000-patch preprocessing limit regardless of legacy image detail hints. - Hide detail controls from `view_image` and code mode while continuing to accept existing hints for compatibility. - Preserve detail-based resizing and tool contracts for unsupported models. ## Testing - Cover unified resizing limits, Responses Lite, legacy detail hints, unsupported models, and `view_image` integration in code mode. GitOrigin-RevId: cb07bff1669a96599fdfd076b3d9ec80f2b6fff7 --- .../code-mode-protocol/src/description.rs | 23 +++ codex-rs/code-mode-protocol/src/lib.rs | 1 + codex-rs/core/config.schema.json | 6 + codex-rs/core/src/image_preparation.rs | 52 ++++-- codex-rs/core/src/image_preparation_tests.rs | 30 +++- codex-rs/core/src/session/mod.rs | 36 ++-- .../core/src/tools/code_mode/execute_spec.rs | 5 + .../core/src/tools/handlers/view_image.rs | 24 ++- .../src/tools/handlers/view_image_spec.rs | 27 +-- codex-rs/core/src/tools/spec_plan.rs | 15 ++ codex-rs/core/tests/suite/code_mode.rs | 127 +++++++++++++- codex-rs/core/tests/suite/view_image.rs | 164 +++++++++++++++++- codex-rs/features/src/lib.rs | 8 + 13 files changed, 466 insertions(+), 52 deletions(-) diff --git a/codex-rs/code-mode-protocol/src/description.rs b/codex-rs/code-mode-protocol/src/description.rs index f2c787aaaf..dde8ac2980 100644 --- a/codex-rs/code-mode-protocol/src/description.rs +++ b/codex-rs/code-mode-protocol/src/description.rs @@ -9,6 +9,8 @@ use crate::PUBLIC_TOOL_NAME; const MAX_JS_SAFE_INTEGER: u64 = (1_u64 << 53) - 1; const DEFERRED_NESTED_TOOLS_GUIDANCE: &str = r#"Some deferred nested tools may be omitted from this description. They are still available on the global `tools` object and listed in `ALL_TOOLS`. To find one, filter `ALL_TOOLS` by `name` and `description`."#; +const LEGACY_IMAGE_HELPER_DESCRIPTION: &str = r#"`image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument."#; +const UNIFIED_IMAGE_HELPER_DESCRIPTION: &str = r#"`image(imageUrlOrItem: string | { image_url: string } | ImageContent)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`."#; const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/compose tool calls - Evaluates the provided JavaScript code in a fresh V8 isolate as an async module. - All nested tools are available on the global `tools` object, for example `await tools.exec_command(...)`. Tool names are exposed as normalized JavaScript identifiers, for example `await tools.mcp__ologs__get_profile(...)`. @@ -249,18 +251,31 @@ pub fn is_code_mode_nested_tool(tool_name: &str) -> bool { tool_name != crate::PUBLIC_TOOL_NAME && tool_name != crate::WAIT_TOOL_NAME } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ImageDetailVisibility { + Visible, + Hidden, +} + pub fn build_exec_tool_description( enabled_tools: &[ToolDefinition], deferred_tools: &[ToolDefinition], namespace_descriptions: &BTreeMap, default_exec_yield_time_ms: u64, code_mode_only: bool, + image_detail_visibility: ImageDetailVisibility, ) -> String { let mut sections = Vec::new(); sections.push(EXEC_DESCRIPTION_TEMPLATE.replace( "Defaults to 10000 ms.", &format!("Defaults to {default_exec_yield_time_ms} ms."), )); + if image_detail_visibility == ImageDetailVisibility::Hidden { + sections[0] = sections[0].replace( + LEGACY_IMAGE_HELPER_DESCRIPTION, + UNIFIED_IMAGE_HELPER_DESCRIPTION, + ); + } if !deferred_tools.is_empty() { sections.push(DEFERRED_NESTED_TOOLS_GUIDANCE.to_string()); } @@ -714,6 +729,7 @@ fn render_json_schema_literal(value: &JsonValue) -> String { #[cfg(test)] mod tests { use super::CodeModeToolKind; + use super::ImageDetailVisibility; use super::ParsedExecSource; use super::ToolDefinition; use super::ToolNamespaceDescription; @@ -874,6 +890,7 @@ mod tests { &BTreeMap::new(), crate::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ true, + ImageDetailVisibility::Visible, ); assert!(description.contains( "### `foo` @@ -890,6 +907,7 @@ bar" &BTreeMap::new(), crate::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ false, + ImageDetailVisibility::Visible, ); assert!(description.contains("`audio(audioUrlOrItem:")); assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`")); @@ -944,6 +962,7 @@ bar" &namespace_descriptions, crate::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ true, + ImageDetailVisibility::Visible, ); assert_eq!(description.matches("## mcp__sample").count(), 1); assert!(description.contains("## mcp__sample\nShared namespace guidance.")); @@ -985,6 +1004,7 @@ bar" &namespace_descriptions, crate::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ true, + ImageDetailVisibility::Visible, ); assert!(!description.contains("## mcp__sample")); @@ -1085,6 +1105,7 @@ bar" &BTreeMap::new(), crate::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ true, + ImageDetailVisibility::Visible, ); assert_eq!( @@ -1121,6 +1142,7 @@ bar" &BTreeMap::new(), crate::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ true, + ImageDetailVisibility::Visible, ); assert!(description.contains("Some deferred nested tools may be omitted")); @@ -1143,6 +1165,7 @@ bar" &BTreeMap::new(), crate::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ false, + ImageDetailVisibility::Visible, ); assert!(description.contains("Some deferred nested tools may be omitted")); diff --git a/codex-rs/code-mode-protocol/src/lib.rs b/codex-rs/code-mode-protocol/src/lib.rs index c7133f5228..3ec8c9ef1a 100644 --- a/codex-rs/code-mode-protocol/src/lib.rs +++ b/codex-rs/code-mode-protocol/src/lib.rs @@ -7,6 +7,7 @@ mod session; pub use description::CODE_MODE_PRAGMA_PREFIX; pub use description::CodeModeToolKind; pub use description::EnabledToolMetadata; +pub use description::ImageDetailVisibility; pub use description::ToolDefinition; pub use description::ToolNamespaceDescription; pub use description::augment_tool_definition; diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index fd463f4af1..9d67ab7a84 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -756,6 +756,9 @@ "unified_exec_zsh_fork": { "type": "boolean" }, + "unified_image_budget": { + "type": "boolean" + }, "use_agent_identity": { "type": "boolean" }, @@ -5387,6 +5390,9 @@ "unified_exec_zsh_fork": { "type": "boolean" }, + "unified_image_budget": { + "type": "boolean" + }, "use_agent_identity": { "type": "boolean" }, diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs index 36f617b129..9c067565b0 100644 --- a/codex-rs/core/src/image_preparation.rs +++ b/codex-rs/core/src/image_preparation.rs @@ -1,13 +1,17 @@ +use crate::config::ManagedFeatures; use crate::context::ContextualUserFragment; use crate::context::ImageResizeNotice; use crate::context::ImageResizeNoticeSource; use crate::context::ResizedImage; +use crate::original_image_detail::can_request_original_image_detail; use codex_analytics::ImageDetailSetting; use codex_analytics::ImagePreparationMetadata; +use codex_features::Feature; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ImageDetail; use codex_protocol::models::ResponseItem; +use codex_protocol::openai_models::ModelInfo; use codex_utils_image::ImageProcessingError; use codex_utils_image::PromptImageMode; use codex_utils_image::PromptImageResizeLimits; @@ -26,11 +30,25 @@ const HIGH_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits { max_dimension: 2048, max_patches: 2_500, }; -const ORIGINAL_DETAIL_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits { +const UNIFIED_IMAGE_LIMITS: PromptImageResizeLimits = PromptImageResizeLimits { max_dimension: 6000, max_patches: 10_000, }; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ImagePreparationMode { + DetailBased, + UnifiedBudget, +} + +pub(crate) fn unified_image_budget_enabled( + features: &ManagedFeatures, + model_info: &ModelInfo, +) -> bool { + features.enabled(Feature::UnifiedImageBudget) + && (model_info.use_responses_lite || can_request_original_image_detail(model_info)) +} + #[derive(Clone, Copy, Debug)] struct ImageOrigin<'a> { message_role: Option<&'a str>, @@ -76,6 +94,7 @@ impl ImagePreparationError { pub(crate) fn prepare_response_items( items: &mut Vec, + mode: ImagePreparationMode, resize_notice_mode: ImageResizeNoticeMode, ) -> Vec { let mut metadata = Vec::new(); @@ -95,6 +114,7 @@ pub(crate) fn prepare_response_items( ImageResizeNoticeMode::Disabled }, &mut metadata, + mode, ); (!resized_images.is_empty()).then(|| { ImageResizeNotice::new(ImageResizeNoticeSource::UserMessage, resized_images) @@ -114,6 +134,7 @@ pub(crate) fn prepare_response_items( }, resize_notice_mode, &mut metadata, + mode, ); (!resized_images.is_empty()).then(|| { ImageResizeNotice::new(ImageResizeNoticeSource::ToolOutput, resized_images) @@ -148,6 +169,7 @@ fn prepare_message_content( origin: ImageOrigin<'_>, resize_notice_mode: ImageResizeNoticeMode, metadata: &mut Vec, + mode: ImagePreparationMode, ) -> Vec { let image_count = items .iter() @@ -158,7 +180,7 @@ fn prepare_message_content( for item in items { if let ContentItem::InputImage { image_url, detail } = item { image_number += 1; - match prepare_image(image_url, *detail, origin, metadata) { + match prepare_image(image_url, detail, origin, metadata, mode) { Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => { resized_images.push(ResizedImage { image_number, @@ -187,6 +209,7 @@ fn prepare_tool_output_content( origin: ImageOrigin<'_>, resize_notice_mode: ImageResizeNoticeMode, metadata: &mut Vec, + mode: ImagePreparationMode, ) -> Vec { let image_count = items .iter() @@ -197,7 +220,7 @@ fn prepare_tool_output_content( for item in items { if let FunctionCallOutputContentItem::InputImage { image_url, detail } = item { image_number += 1; - match prepare_image(image_url, *detail, origin, metadata) { + match prepare_image(image_url, detail, origin, metadata, mode) { Ok(Some(resize)) if resize_notice_mode == ImageResizeNoticeMode::Enabled => { resized_images.push(ResizedImage { image_number, @@ -235,9 +258,10 @@ fn is_data_url(image_url: &str) -> bool { fn prepare_image( image_url: &mut String, - detail: Option, + detail: &mut Option, origin: ImageOrigin<'_>, metadata: &mut Vec, + mode: ImagePreparationMode, ) -> Result, ImagePreparationError> { if is_remote_image_url(image_url) { return Err(ImagePreparationError::RemoteUrlUnsupported); @@ -246,12 +270,15 @@ fn prepare_image( return Ok(None); } - let (effective_detail, limits) = match detail { - None | Some(ImageDetail::Auto | ImageDetail::High) => { - (ImageDetailSetting::High, HIGH_DETAIL_LIMITS) - } - Some(ImageDetail::Original) => (ImageDetailSetting::Original, ORIGINAL_DETAIL_LIMITS), - Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail), + let (effective_detail, limits) = match mode { + ImagePreparationMode::UnifiedBudget => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS), + ImagePreparationMode::DetailBased => match detail { + None | Some(ImageDetail::Auto | ImageDetail::High) => { + (ImageDetailSetting::High, HIGH_DETAIL_LIMITS) + } + Some(ImageDetail::Original) => (ImageDetailSetting::Original, UNIFIED_IMAGE_LIMITS), + Some(ImageDetail::Low) => return Err(ImagePreparationError::UnsupportedLowDetail), + }, }; let image = load_data_url_for_prompt(image_url, PromptImageMode::ResizeWithLimits(limits))?; metadata.push(ImagePreparationMetadata { @@ -271,6 +298,11 @@ fn prepare_image( prepared_height: image.height, }); *image_url = image.into_data_url(); + if mode == ImagePreparationMode::UnifiedBudget { + // Preserve accurate context-window accounting while older transports still require an + // image detail field. Responses Lite removes this compatibility hint before sending. + *detail = Some(ImageDetail::Original); + } Ok(resize) } diff --git a/codex-rs/core/src/image_preparation_tests.rs b/codex-rs/core/src/image_preparation_tests.rs index 53fad8ff66..9919e41e3e 100644 --- a/codex-rs/core/src/image_preparation_tests.rs +++ b/codex-rs/core/src/image_preparation_tests.rs @@ -51,7 +51,11 @@ fn preparation_preserves_small_image_bytes_and_replaces_remote_urls() { internal_chat_message_metadata_passthrough: None, }]; - prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled); + prepare_response_items( + &mut items, + ImagePreparationMode::DetailBased, + ImageResizeNoticeMode::Disabled, + ); let ResponseItem::Message { content, .. } = &items[0] else { panic!("expected message"); @@ -105,7 +109,11 @@ fn detail_policies_apply_the_expected_budgets() { internal_chat_message_metadata_passthrough: None, }]; - let metadata = prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled); + let metadata = prepare_response_items( + &mut items, + ImagePreparationMode::DetailBased, + ImageResizeNoticeMode::Disabled, + ); let ResponseItem::Message { content, .. } = &items[0] else { panic!("expected message"); @@ -144,7 +152,11 @@ fn preparation_reports_tool_output_item_id() { ]), internal_chat_message_metadata_passthrough: None, }]; - let metadata = prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled); + let metadata = prepare_response_items( + &mut items, + ImagePreparationMode::DetailBased, + ImageResizeNoticeMode::Disabled, + ); assert_eq!( metadata, @@ -202,7 +214,11 @@ fn resize_notices_preserve_original_image_positions_and_skip_failed_images() { }, ]; - prepare_response_items(&mut items, ImageResizeNoticeMode::Enabled); + prepare_response_items( + &mut items, + ImagePreparationMode::DetailBased, + ImageResizeNoticeMode::Enabled, + ); let expected_user_notice = concat!( "\n", "Image 3 of 3 in the preceding user message was resized from 2048x2048 to 1600x1600 pixels.\n", @@ -325,7 +341,11 @@ fn preparation_replaces_only_failed_tool_images_and_preserves_metadata() { internal_chat_message_metadata_passthrough: None, }]; - prepare_response_items(&mut items, ImageResizeNoticeMode::Disabled); + prepare_response_items( + &mut items, + ImagePreparationMode::DetailBased, + ImageResizeNoticeMode::Disabled, + ); assert_eq!( items, diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 01c742be89..96021f37a3 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -31,8 +31,10 @@ use crate::environment_selection::TurnEnvironmentSnapshot; use crate::exec_policy::BANNED_PREFIX_SUGGESTIONS; use crate::exec_policy::ExecPolicyManager; use crate::exec_policy::default_policy_path; +use crate::image_preparation::ImagePreparationMode; use crate::image_preparation::ImageResizeNoticeMode; use crate::image_preparation::prepare_response_items as prepare_image_response_items; +use crate::image_preparation::unified_image_budget_enabled; use crate::parse_turn_item; use crate::realtime_conversation::RealtimeConversationManager; use crate::session::step_context::StepContext; @@ -1445,7 +1447,11 @@ impl Session { // will be processed again if the rollout is reconstructed in a future session. // Never backfill resize notices during replay; only newly recorded items may // emit them, so the historical model prefix remains unchanged. - let _ = prepare_image_response_items(&mut history, ImageResizeNoticeMode::Disabled); + let _ = prepare_image_response_items( + &mut history, + ImagePreparationMode::DetailBased, + ImageResizeNoticeMode::Disabled, + ); prepare_audio_response_items(&mut history); { let mut state = self.state.lock().await; @@ -2911,17 +2917,27 @@ impl Session { items: &'a [ResponseItem], ) -> (Cow<'a, [ResponseItem]>, Vec) { let mut items = items.to_vec(); + let image_preparation_mode = if unified_image_budget_enabled( + &turn_context.config.features, + &turn_context.model_info, + ) { + ImagePreparationMode::UnifiedBudget + } else { + ImagePreparationMode::DetailBased + }; + let image_resize_notice_mode = if turn_context + .config + .features + .enabled(Feature::ImageResizeNotice) + { + ImageResizeNoticeMode::Enabled + } else { + ImageResizeNoticeMode::Disabled + }; let image_preparations = prepare_image_response_items( &mut items, - if turn_context - .config - .features - .enabled(Feature::ImageResizeNotice) - { - ImageResizeNoticeMode::Enabled - } else { - ImageResizeNoticeMode::Disabled - }, + image_preparation_mode, + image_resize_notice_mode, ); prepare_audio_response_items(&mut items); // Most response items get their passthrough turn ID at the durable history boundary. diff --git a/codex-rs/core/src/tools/code_mode/execute_spec.rs b/codex-rs/core/src/tools/code_mode/execute_spec.rs index b3476720c8..3b44117b91 100644 --- a/codex-rs/core/src/tools/code_mode/execute_spec.rs +++ b/codex-rs/core/src/tools/code_mode/execute_spec.rs @@ -1,3 +1,4 @@ +use codex_code_mode::ImageDetailVisibility; use codex_code_mode::ToolDefinition as CodeModeToolDefinition; use codex_tools::FreeformTool; use codex_tools::FreeformToolFormat; @@ -10,6 +11,7 @@ pub(crate) fn create_code_mode_tool( namespace_descriptions: &BTreeMap, default_exec_yield_time_ms: u64, code_mode_only: bool, + image_detail_visibility: ImageDetailVisibility, ) -> ToolSpec { const CODE_MODE_FREEFORM_GRAMMAR: &str = r#" start: pragma_source | plain_source @@ -29,6 +31,7 @@ SOURCE: /[\s\S]+/ namespace_descriptions, default_exec_yield_time_ms, code_mode_only, + image_detail_visibility, ), defer_loading: None, format: FreeformToolFormat { @@ -63,6 +66,7 @@ mod tests { &BTreeMap::new(), codex_code_mode::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ true, + ImageDetailVisibility::Visible, ), ToolSpec::Freeform(FreeformTool { name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(), @@ -72,6 +76,7 @@ mod tests { &BTreeMap::new(), codex_code_mode::DEFAULT_EXEC_YIELD_TIME_MS, /*code_mode_only*/ true, + ImageDetailVisibility::Visible, ), defer_loading: None, format: FreeformToolFormat { diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs index 9511f5d1a3..ca32af5b1a 100644 --- a/codex-rs/core/src/tools/handlers/view_image.rs +++ b/codex-rs/core/src/tools/handlers/view_image.rs @@ -34,6 +34,7 @@ impl Default for ViewImageHandler { Self { options: ViewImageToolOptions { can_request_original_image_detail: false, + unified_image_budget: false, include_environment_id: false, }, } @@ -120,8 +121,7 @@ impl ViewImageHandler { environment_id, detail, } = parse_arguments(&arguments)?; - // `high` is the explicit spelling of the default resized path. - // Other string values remain invalid rather than being silently reinterpreted. + // Keep accepting previously supported detail hints after they disappear from the schema. let detail = match detail.as_deref() { None => None, Some("high") => Some(ViewImageDetail::High), @@ -175,8 +175,8 @@ impl ViewImageHandler { })?; let can_request_original_detail = can_request_original_image_detail(&turn.model_info); - let use_original_detail = - can_request_original_detail && matches!(detail, Some(ViewImageDetail::Original)); + let use_original_detail = self.options.unified_image_budget + || can_request_original_detail && matches!(detail, Some(ViewImageDetail::Original)); let image_detail = if use_original_detail { ImageDetail::Original } else { @@ -196,6 +196,7 @@ impl ViewImageHandler { Ok(boxed_tool_output(ViewImageOutput { image_url, image_detail, + unified_image_budget: self.options.unified_image_budget, })) } } @@ -205,6 +206,7 @@ impl CoreToolRuntime for ViewImageHandler {} pub struct ViewImageOutput { image_url: String, image_detail: ImageDetail, + unified_image_budget: bool, } impl ToolOutput for ViewImageOutput { @@ -234,10 +236,14 @@ impl ToolOutput for ViewImageOutput { } fn code_mode_result(&self, _payload: &ToolPayload) -> serde_json::Value { - serde_json::json!({ - "image_url": self.image_url, - "detail": self.image_detail - }) + if self.unified_image_budget { + serde_json::json!({ "image_url": self.image_url }) + } else { + serde_json::json!({ + "image_url": self.image_url, + "detail": self.image_detail + }) + } } } @@ -283,6 +289,7 @@ mod tests { let output = ViewImageOutput { image_url: "data:image/png;base64,AAA".to_string(), image_detail: DEFAULT_IMAGE_DETAIL, + unified_image_budget: false, }; assert_eq!(output.log_preview(), ""); @@ -293,6 +300,7 @@ mod tests { let output = ViewImageOutput { image_url: "data:image/png;base64,AAA".to_string(), image_detail: DEFAULT_IMAGE_DETAIL, + unified_image_budget: false, }; let result = output.code_mode_result(&ToolPayload::Function { diff --git a/codex-rs/core/src/tools/handlers/view_image_spec.rs b/codex-rs/core/src/tools/handlers/view_image_spec.rs index 5d2b7e0b15..f28d9f8b1a 100644 --- a/codex-rs/core/src/tools/handlers/view_image_spec.rs +++ b/codex-rs/core/src/tools/handlers/view_image_spec.rs @@ -9,6 +9,7 @@ use std::collections::BTreeMap; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ViewImageToolOptions { pub can_request_original_image_detail: bool, + pub unified_image_budget: bool, pub include_environment_id: bool, } @@ -17,7 +18,7 @@ pub fn create_view_image_tool(options: ViewImageToolOptions) -> ToolSpec { "path".to_string(), JsonSchema::string(Some("Local filesystem path to an image file.".to_string())), )]); - if options.can_request_original_image_detail { + if options.can_request_original_image_detail && !options.unified_image_budget { properties.insert( "detail".to_string(), JsonSchema::string_enum( @@ -45,25 +46,29 @@ pub fn create_view_image_tool(options: ViewImageToolOptions) -> ToolSpec { strict: false, defer_loading: None, parameters: JsonSchema::object(properties, Some(vec!["path".to_string()]), Some(false.into())), - output_schema: Some(view_image_output_schema()), + output_schema: Some(view_image_output_schema(options)), }) } -fn view_image_output_schema() -> Value { - json!({ +fn view_image_output_schema(options: ViewImageToolOptions) -> Value { + let mut schema = json!({ "type": "object", "properties": { "image_url": { "type": "string", "description": "Data URL for the loaded image." - }, - "detail": { - "type": "string", - "enum": ["high", "original"], - "description": "Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved." } }, - "required": ["image_url", "detail"], + "required": ["image_url"], "additionalProperties": false - }) + }); + if !options.unified_image_budget { + schema["properties"]["detail"] = json!({ + "type": "string", + "enum": ["high", "original"], + "description": "Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved." + }); + schema["required"] = json!(["image_url", "detail"]); + } + schema } diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index 4c13de5692..0e94988ec1 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -1,6 +1,7 @@ use crate::agent::exceeds_thread_spawn_depth_limit; use crate::agent::next_thread_spawn_depth; use crate::environment_selection::TurnEnvironmentSnapshot; +use crate::image_preparation::unified_image_budget_enabled; use crate::mcp_tool_exposure::append_mcp_tools; use crate::session::session::Session; use crate::session::turn_context::TurnContext; @@ -730,6 +731,12 @@ fn register_code_mode_executors( &namespace_descriptions, default_exec_yield_time_ms, tool_mode == ToolMode::CodeModeOnly, + if unified_image_budget_enabled(&turn_context.config.features, &turn_context.model_info) + { + codex_code_mode::ImageDetailVisibility::Hidden + } else { + codex_code_mode::ImageDetailVisibility::Visible + }, ), code_mode_nested_tool_specs, ); @@ -838,6 +845,10 @@ fn add_core_tool_sources(context: &CoreToolPlanContext<'_>, registry: &mut ToolR can_request_original_image_detail: can_request_original_image_detail( &turn_context.model_info, ), + unified_image_budget: unified_image_budget_enabled( + &turn_context.config.features, + &turn_context.model_info, + ), include_environment_id, })); } @@ -1034,6 +1045,10 @@ fn add_core_utility_tools(context: &CoreToolPlanContext<'_>, registry: &mut Tool can_request_original_image_detail: can_request_original_image_detail( &turn_context.model_info, ), + unified_image_budget: unified_image_budget_enabled( + &turn_context.config.features, + &turn_context.model_info, + ), include_environment_id, })); } diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index dc49f84919..9e3ade1f61 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -82,6 +82,7 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; use std::time::Instant; +use test_case::test_case; use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; @@ -3635,6 +3636,114 @@ async fn code_mode_resizes_explicit_original_image() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_unified_image_budget_accepts_legacy_detail_hints() -> Result<()> { + skip_if_no_network!(Ok(())); + + let image = ImageBuffer::from_pixel( + /*width*/ 2304, + /*height*/ 864, + Rgba([20, 40, 60, 255]), + ); + let mut encoded = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(image).write_to(&mut encoded, image::ImageFormat::Png)?; + let image_data = BASE64_STANDARD.encode(encoded.into_inner()); + let code = format!( + r#" +const data = {}; +const imageUrl = `data:image/png;base64,${{data}}`; +image(imageUrl); +image(imageUrl, "auto"); +image(imageUrl, "high"); +image({{ image_url: imageUrl, detail: "low" }}); +image({{ + type: "image", + data, + mimeType: "image/png", + _meta: {{ "codex/imageDetail": "original" }} +}}); +"#, + serde_json::to_string(&image_data)? + ); + + let server = responses::start_mock_server().await; + let (_test, second_mock) = run_code_mode_turn_with_model_and_config( + &server, + "emit images with legacy detail arguments and MCP metadata", + &code, + "gpt-5.4", + |config| { + let _ = config.features.enable(Feature::UnifiedImageBudget); + }, + ) + .await?; + + let request = second_mock.single_request(); + let items = custom_tool_output_items(&request, "call-1"); + assert_eq!(items.len(), 6, "unexpected code-mode output: {items:?}"); + for item in &items[1..] { + assert_eq!(item["type"], "input_image"); + assert_eq!(item["detail"], "original"); + let image_url = item["image_url"] + .as_str() + .expect("image helper should emit a data URL"); + let (_, payload) = image_url + .split_once(',') + .expect("image data URL should have a base64 payload"); + let image = image::load_from_memory(&BASE64_STANDARD.decode(payload)?)?; + assert_eq!(image.dimensions(), (2304, 864)); + } + + let body = request.body_json(); + let exec_description = body["tools"] + .as_array() + .and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec")) + .and_then(|tool| tool["description"].as_str()) + .expect("the model request should contain the code-mode exec tool"); + assert!( + exec_description + .contains("`image(imageUrlOrItem: string | { image_url: string } | ImageContent)`") + ); + assert!(!exec_description.contains("codex/imageDetail")); + assert!(!exec_description.contains("detail?:")); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_unified_image_budget_preserves_legacy_contract_for_unsupported_model() +-> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let (_test, second_mock) = run_code_mode_turn_with_model_and_config( + &server, + "emit an image on a legacy model", + r#"image("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==");"#, + "gpt-5.2", + |config| { + let _ = config.features.enable(Feature::UnifiedImageBudget); + }, + ) + .await?; + + let request = second_mock.single_request(); + let items = custom_tool_output_items(&request, "call-1"); + assert_eq!(items.len(), 2); + assert_eq!(items[1]["detail"], "high"); + + let body = request.body_json(); + let exec_description = body["tools"] + .as_array() + .and_then(|tools| tools.iter().find(|tool| tool["name"] == "exec")) + .and_then(|tool| tool["description"].as_str()) + .expect("the model request should contain the code-mode exec tool"); + assert!(exec_description.contains("codex/imageDetail")); + assert!(exec_description.contains("detail?:")); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_image_helper_rejects_remote_url() -> Result<()> { skip_if_no_network!(Ok(())); @@ -3718,8 +3827,12 @@ image(s.trim(), "original"); Ok(()) } +#[test_case(false; "legacy detail")] +#[test_case(true; "unified image budget")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> { +async fn code_mode_can_use_view_image_result_with_image_helper( + unified_image_budget: bool, +) -> Result<()> { skip_if_no_network!(Ok(())); let server = responses::start_mock_server().await; @@ -3727,6 +3840,9 @@ async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> { .with_model("gpt-5.4") .with_config(move |config| { let _ = config.features.enable(Feature::CodeMode); + if unified_image_budget { + let _ = config.features.enable(Feature::UnifiedImageBudget); + } }); let test = builder.build(&server).await?; @@ -3737,9 +3853,18 @@ async fn code_mode_can_use_view_image_result_with_image_helper() -> Result<()> { fs::write(&image_path, image_bytes)?; let image_path_json = serde_json::to_string(&image_path.to_string_lossy().to_string())?; + let expected_output_keys = if unified_image_budget { + r#"["image_url"]"# + } else { + r#"["detail","image_url"]"# + }; let code = format!( r#" const out = await tools.view_image({{ path: {image_path_json}, detail: "original" }}); +const keys = Object.keys(out).sort(); +if (JSON.stringify(keys) !== JSON.stringify({expected_output_keys})) {{ + throw new Error(`unexpected view_image output keys: ${{JSON.stringify(keys)}}`); +}} image(out); "# ); diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index be4083c094..ef12dd54c7 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -73,6 +73,13 @@ enum ResizeNoticeExpectation { Enabled, } +#[derive(Clone, Copy, Eq, PartialEq)] +enum ImageBudgetPolicy { + DetailBased, + Unified, + UnifiedResponsesLiteWithoutOriginalSupport, +} + fn disabled_user_turn(test: &TestCodex, items: Vec, model: String) -> Op { let (sandbox_policy, permission_profile) = turn_permission_fields(PermissionProfile::Disabled, test.config.cwd.as_path()); @@ -204,17 +211,27 @@ async fn write_workspace_png( async fn assert_user_turn_local_image_resizes_to( original_dimensions: (u32, u32), expected_dimensions: (u32, u32), + image_budget_policy: ImageBudgetPolicy, resize_notice_expectation: ResizeNoticeExpectation, ) -> anyhow::Result<()> { let server = start_mock_server().await; - let builder = test_codex(); - let mut builder = match resize_notice_expectation { - ResizeNoticeExpectation::Disabled => builder, - ResizeNoticeExpectation::Enabled => builder.with_config(|config| { - let _ = config.features.enable(Feature::ImageResizeNotice); - }), + let builder = match image_budget_policy { + ImageBudgetPolicy::DetailBased | ImageBudgetPolicy::Unified => test_codex(), + ImageBudgetPolicy::UnifiedResponsesLiteWithoutOriginalSupport => test_codex() + .with_model_info_override("gpt-5.4", |model_info| { + model_info.supports_image_detail_original = false; + model_info.use_responses_lite = true; + }), }; + let mut builder = builder.with_config(move |config| { + if image_budget_policy != ImageBudgetPolicy::DetailBased { + let _ = config.features.enable(Feature::UnifiedImageBudget); + } + if matches!(resize_notice_expectation, ResizeNoticeExpectation::Enabled) { + let _ = config.features.enable(Feature::ImageResizeNotice); + } + }); let test = builder.build_with_auto_env(&server).await?; let TestCodex { codex, @@ -329,6 +346,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { assert_user_turn_local_image_resizes_to( (2304, 864), (2048, 768), + ImageBudgetPolicy::DetailBased, ResizeNoticeExpectation::Disabled, ) .await @@ -341,6 +359,7 @@ async fn user_turn_with_vertical_local_image_resizes_to_square_bounds() -> anyho assert_user_turn_local_image_resizes_to( (1024, 4096), (512, 2048), + ImageBudgetPolicy::DetailBased, ResizeNoticeExpectation::Disabled, ) .await @@ -353,11 +372,47 @@ async fn user_turn_local_image_applies_patch_budget_and_reports_resize() -> anyh assert_user_turn_local_image_resizes_to( (2048, 2048), (1600, 1600), + ImageBudgetPolicy::DetailBased, ResizeNoticeExpectation::Enabled, ) .await } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn user_turn_unified_image_budget_enforces_dimension_and_patch_limits() -> anyhow::Result<()> +{ + skip_if_no_network!(Ok(())); + + for (source_dimensions, expected_dimensions, resize_notice_expectation) in [ + ((6401, 100), (6000, 94), ResizeNoticeExpectation::Disabled), + ((3201, 3201), (3200, 3200), ResizeNoticeExpectation::Enabled), + ] { + assert_user_turn_local_image_resizes_to( + source_dimensions, + expected_dimensions, + ImageBudgetPolicy::Unified, + resize_notice_expectation, + ) + .await?; + } + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn user_turn_unified_image_budget_supports_responses_lite_without_original_detail() +-> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + assert_user_turn_local_image_resizes_to( + (2304, 864), + (2304, 864), + ImageBudgetPolicy::UnifiedResponsesLiteWithoutOriginalSupport, + ResizeNoticeExpectation::Disabled, + ) + .await +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -869,6 +924,82 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5 Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_image_unified_budget_hides_detail_but_accepts_legacy_hints() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let mut builder = test_codex().with_model("gpt-5.4").with_config(|config| { + let _ = config.features.enable(Feature::UnifiedImageBudget); + }); + let test = builder.build_with_auto_env(&server).await?; + let rel_path = "assets/unified-example.png"; + write_workspace_png( + &test, + rel_path, + /*width*/ 2304, + /*height*/ 864, + [0u8, 80, 255, 255], + ) + .await?; + + let call_id = "view-image-unified"; + let first_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + call_id, + "view_image", + &serde_json::json!({ "path": rel_path, "detail": "high" }).to_string(), + ), + ev_completed("resp-1"), + ]), + ) + .await; + let second_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("show the screenshot").await?; + + let first_request = first_mock.single_request().body_json(); + let view_image_tool = first_request["tools"] + .as_array() + .and_then(|tools| tools.iter().find(|tool| tool["name"] == "view_image")) + .context("view_image tool should be available")?; + assert!( + view_image_tool["parameters"]["properties"] + .get("detail") + .is_none(), + "the unified image budget should not advertise detail" + ); + + let request = second_mock.single_request(); + let output = request.function_call_output(call_id); + let output_items = output["output"] + .as_array() + .context("view_image should return image content")?; + assert_eq!(output_items.len(), 1); + assert_eq!(output_items[0]["detail"], "original"); + + let image_url = output_items[0]["image_url"] + .as_str() + .context("view_image output should include image_url")?; + let (_, payload) = image_url + .split_once(',') + .context("view_image image_url should include a base64 payload")?; + let image = load_from_memory(&BASE64_STANDARD.decode(payload)?)?; + assert_eq!(image.dimensions(), (2304, 864)); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -1041,8 +1172,27 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> { async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); + assert_view_image_tool_resizes_without_original_support(ImageBudgetPolicy::DetailBased).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_image_unified_budget_stays_disabled_for_unsupported_model() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + assert_view_image_tool_resizes_without_original_support(ImageBudgetPolicy::Unified).await +} + +async fn assert_view_image_tool_resizes_without_original_support( + image_budget_policy: ImageBudgetPolicy, +) -> anyhow::Result<()> { let server = start_mock_server().await; - let mut builder = test_codex().with_model("gpt-5.2"); + let mut builder = test_codex() + .with_model("gpt-5.2") + .with_config(move |config| { + if image_budget_policy == ImageBudgetPolicy::Unified { + let _ = config.features.enable(Feature::UnifiedImageBudget); + } + }); let test = builder.build_with_auto_env(&server).await?; let TestCodex { codex, diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index aaaa5b5a3a..74881e5637 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -224,6 +224,8 @@ pub enum Feature { ImageGeneration, /// Tell the model when a prompt image was resized and include its dimensions. ImageResizeNotice, + /// Apply one shared pixel and token budget to every image, regardless of legacy detail hints. + UnifiedImageBudget, /// Removed compatibility flag for always-on centralized image preparation. ResizeAllImages, /// Removed compatibility flag for always-on response item IDs. @@ -1276,6 +1278,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, + FeatureSpec { + id: Feature::UnifiedImageBudget, + key: "unified_image_budget", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::ResizeAllImages, key: "resize_all_images",