From b04a2c264516ec2e6b3c91dd73ad18a21fd5a88f Mon Sep 17 00:00:00 2001 From: Won Park Date: Sat, 12 Sep 2026 18:30:01 +0000 Subject: [PATCH] Estimate history tokens from content instead of serialized envelopes (#45094) ## Why Serialized response items include message IDs, metadata, and JSON escaping that inflate token estimates without adding model-visible content. ## What changed - Estimate each response item from its content, retaining JSON syntax for structured tool payloads. - Apply image estimates to all image inputs, including non-base64 URLs, and count audio and encrypted content through their modality-specific estimates. - Exclude plaintext reasoning and bookkeeping-only items from replay accounting. ## Testing Update unit expectations for text, images, audio, and encrypted content. Add a remote compaction regression test showing that equal-length text produces identical token usage estimates despite different message IDs, metadata, and JSON escaping, while preserving the submitted messages. GitOrigin-RevId: 1c43d5abbcc1668b4ab413901a2b4b6511ca5892 --- codex-rs/core/src/context_manager/history.rs | 295 ++++++++---------- .../core/src/context_manager/history_tests.rs | 70 ++--- .../tests/suite/compact_remote_trimming.rs | 72 +++++ 3 files changed, 241 insertions(+), 196 deletions(-) diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index f9ad849516..fbd5a428b4 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,6 +1,7 @@ //! Parent model history and bounded host-owned context facts. //! Compaction replaces only the model window. Snapshots include retained facts atomically; //! checkpoint replay and source-call rollback share their live lifecycle. +//! Token estimates charge item content rather than transport metadata. //! Oversized instructions keep an incomplete excerpt for bounded root review, including //! sources recovered from legacy Guardian checkpoints before their raw history is dropped. @@ -34,6 +35,7 @@ use codex_history::ResponseItemEnvelope; use codex_history::RetainedContext; use codex_history::RetainedContextEvent; use codex_history::RetainedInputSource; +use codex_protocol::DEFAULT_FUNCTION_NAMESPACE; use codex_protocol::items::TurnItem; use codex_protocol::models::AgentMessageInputContent; use codex_protocol::models::BaseInstructions; @@ -808,8 +810,7 @@ fn estimate_encrypted_function_output_length(encoded_len: usize) -> usize { /// Returns the same coarse, model-visible token estimate used for full history estimates. /// -/// Ordinary items are JSON-serialized, so callers estimating many items should reuse these -/// results instead of repeatedly estimating the full history. +/// Counts content directly, excluding transport IDs, metadata, and outer JSON escaping. pub(crate) fn estimate_item_token_count(item: &ResponseItem) -> i64 { let model_visible_bytes = estimate_response_item_model_visible_bytes(item); approx_tokens_from_byte_count_i64(model_visible_bytes) @@ -839,6 +840,36 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock i64 { match item { + ResponseItem::Message { content, .. } => content + .iter() + .map(|part| match part { + ContentItem::InputText { text } | ContentItem::OutputText { text } => { + text_bytes(text) + } + ContentItem::InputImage { image_url, detail } => { + estimate_image_bytes(image_url, *detail) + } + ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url), + }) + .fold(0i64, i64::saturating_add), + ResponseItem::AgentMessage { + author, + recipient, + content, + .. + } => content + .iter() + .map(|part| match part { + AgentMessageInputContent::InputText { text } => text_bytes(text), + AgentMessageInputContent::EncryptedContent { encrypted_content } => i64::try_from( + estimate_encrypted_function_output_length(encrypted_content.len()), + ) + .unwrap_or(i64::MAX), + }) + .fold( + text_bytes(author).saturating_add(text_bytes(recipient)), + i64::saturating_add, + ), ResponseItem::Reasoning { encrypted_content: Some(content), .. @@ -851,46 +882,87 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 { encrypted_content: Some(content), .. } => i64::try_from(estimate_reasoning_length(content.len())).unwrap_or(i64::MAX), - item => { - let raw = serialized_json_bytes(item) - .map(|len| i64::try_from(len).unwrap_or(i64::MAX)) - .unwrap_or_default(); - let (image_payload_bytes, image_replacement_bytes) = - image_data_url_estimate_adjustment(item); - let (audio_payload_bytes, audio_replacement_bytes) = - audio_data_url_estimate_adjustment(item); - let (encrypted_payload_bytes, encrypted_replacement_bytes) = - encrypted_function_output_estimate_adjustment(item); - // Replace raw base64 payload bytes with per-modality estimates. - // We intentionally preserve the data URL prefix and JSON - // wrapper bytes already included in `raw`. - let raw = raw - .saturating_sub(image_payload_bytes) - .saturating_add(image_replacement_bytes) - .saturating_sub(audio_payload_bytes) - .saturating_add(audio_replacement_bytes); - raw.saturating_sub(encrypted_payload_bytes) - .saturating_add(encrypted_replacement_bytes) + ResponseItem::FunctionCall { + name, + namespace, + arguments: input, + .. } + | ResponseItem::CustomToolCall { + name, + namespace, + input, + .. + } => text_bytes(name) + .saturating_add(text_bytes( + namespace.as_deref().unwrap_or(DEFAULT_FUNCTION_NAMESPACE), + )) + .saturating_add(text_bytes(input)), + ResponseItem::FunctionCallOutput { + call_id, + name, + namespace, + output, + .. + } => estimate_function_output_bytes(&output.body) + .saturating_add(text_bytes(call_id.as_deref().unwrap_or_default())) + .saturating_add(text_bytes(name.as_deref().unwrap_or_default())) + .saturating_add(text_bytes(namespace.as_deref().unwrap_or_default())), + ResponseItem::CustomToolCallOutput { + call_id, + name, + output, + .. + } => estimate_function_output_bytes(&output.body) + .saturating_add(text_bytes(call_id)) + .saturating_add(text_bytes(name.as_deref().unwrap_or_default())), + // These payloads are themselves JSON arguments, rather than transport envelopes + // around text. Keep their JSON syntax in the estimate. + ResponseItem::AdditionalTools { tools, .. } => json_content_bytes(tools), + ResponseItem::ToolSearchCall { arguments, .. } => json_content_bytes(arguments), + ResponseItem::ToolSearchOutput { tools, .. } => json_content_bytes(tools), + ResponseItem::LocalShellCall { action, .. } => json_content_bytes(action), + ResponseItem::WebSearchCall { action, .. } => { + action.as_ref().map(json_content_bytes).unwrap_or_default() + } + ResponseItem::ImageGenerationCall { + revised_prompt, + result, + .. + } => text_bytes(revised_prompt.as_deref().unwrap_or_default()).saturating_add( + if result.is_empty() { + 0 + } else { + RESIZED_IMAGE_BYTES_ESTIMATE + }, + ), + ResponseItem::ContextCompaction { + encrypted_content: None, + .. + } => 0, + // Plaintext reasoning is excluded from replay accounting. + ResponseItem::Reasoning { + encrypted_content: None, + .. + } => 0, + ResponseItem::ConfigurationUpdate { .. } + | ResponseItem::CompactionTrigger { .. } + | ResponseItem::Other => 0, } } -/// Returns the base64 payload byte length for inline image data URLs that are -/// eligible for token-estimation discounting. -/// -/// We only discount payloads for `data:image/...;base64,...` URLs (case -/// insensitive markers) and leave everything else at raw serialized size. +fn text_bytes(text: &str) -> i64 { + i64::try_from(text.len()).unwrap_or(i64::MAX) +} + +fn json_content_bytes(value: &(impl serde::Serialize + ?Sized)) -> i64 { + serialized_json_bytes(value) + .map(|len| i64::try_from(len).unwrap_or(i64::MAX)) + .unwrap_or_default() +} + +/// Extracts inline image bytes for the original-detail dimension estimate. fn parse_base64_image_data_url(url: &str) -> Option<&str> { - parse_base64_data_url(url, "image/") -} - -/// Returns the base64 payload for inline audio data URLs that are eligible for -/// token-estimation discounting. -fn parse_base64_audio_data_url(url: &str) -> Option<&str> { - parse_base64_data_url(url, "audio/") -} - -fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&'a str> { if !url .get(.."data:".len()) .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) @@ -908,8 +980,8 @@ fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&' let mime_type = metadata_parts.next().unwrap_or_default(); let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64")); if !mime_type - .get(..media_type_prefix.len()) - .is_some_and(|prefix| prefix.eq_ignore_ascii_case(media_type_prefix)) + .get(.."image/".len()) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/")) { return None; } @@ -955,7 +1027,7 @@ fn estimate_original_image_bytes(image_url: &str) -> Option { }) } -/// Shared image estimate, excluding the data URL prefix and message framing. +/// Shared image estimate, excluding message framing. pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option) -> i64 { match detail { Some(ImageDetail::Original) => { @@ -965,130 +1037,35 @@ pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option) } } -/// Scans one response item for discount-eligible inline image data URLs and -/// returns: -/// - total base64 payload bytes to subtract from raw serialized size -/// - total replacement byte estimate for those images -fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { - let mut payload_bytes = 0i64; - let mut replacement_bytes = 0i64; - - let mut accumulate = |image_url: &str, detail: Option| { - if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) { - payload_bytes = - payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX)); - replacement_bytes = - replacement_bytes.saturating_add(estimate_image_bytes(image_url, detail)); - } - }; - - match item { - ResponseItem::Message { content, .. } => { - for content_item in content { - if let ContentItem::InputImage { image_url, detail } = content_item { - accumulate(image_url, *detail); - } - } - } - ResponseItem::FunctionCallOutput { output, .. } - | ResponseItem::CustomToolCallOutput { output, .. } => { - if let FunctionCallOutputBody::ContentItems(items) = &output.body { - for content_item in items { - if let FunctionCallOutputContentItem::InputImage { image_url, detail } = - content_item - { - accumulate(image_url, *detail); - } - } - } - } - _ => {} - } - - (payload_bytes, replacement_bytes) +fn estimate_audio_bytes(audio_url: &str) -> i64 { + i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count( + audio_url, + ))) + .unwrap_or(i64::MAX) } -/// Scans one response item for inline base64 audio data URLs and returns: -/// - total base64 payload bytes to subtract from raw serialized size -/// - total replacement byte estimate for those audio inputs -fn audio_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { - let mut payload_bytes = 0i64; - let mut replacement_bytes = 0i64; - - let mut accumulate = |audio_url: &str| { - if let Some(payload_len) = parse_base64_audio_data_url(audio_url).map(str::len) { - payload_bytes = - payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX)); - replacement_bytes = replacement_bytes.saturating_add( - i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count( - audio_url, - ))) - .unwrap_or(i64::MAX), - ); - } - }; - - match item { - ResponseItem::Message { content, .. } => { - for content_item in content { - if let ContentItem::InputAudio { audio_url } = content_item { - accumulate(audio_url); +fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 { + match output { + FunctionCallOutputBody::Text(text) => text_bytes(text), + FunctionCallOutputBody::ContentItems(items) => items + .iter() + .map(|part| match part { + FunctionCallOutputContentItem::InputText { text } => text_bytes(text), + FunctionCallOutputContentItem::InputImage { image_url, detail } => { + estimate_image_bytes(image_url, *detail) } - } - } - ResponseItem::FunctionCallOutput { output, .. } - | ResponseItem::CustomToolCallOutput { output, .. } => { - if let FunctionCallOutputBody::ContentItems(items) = &output.body { - for content_item in items { - if let FunctionCallOutputContentItem::InputAudio { audio_url } = content_item { - accumulate(audio_url); - } + FunctionCallOutputContentItem::InputAudio { audio_url } => { + estimate_audio_bytes(audio_url) } - } - } - _ => {} + FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => { + i64::try_from(estimate_encrypted_function_output_length( + encrypted_content.len(), + )) + .unwrap_or(i64::MAX) + } + }) + .fold(0i64, i64::saturating_add), } - - (payload_bytes, replacement_bytes) -} - -fn encrypted_function_output_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { - let mut payload_bytes = 0i64; - let mut replacement_bytes = 0i64; - let mut accumulate = |encrypted_content: &str| { - payload_bytes = payload_bytes - .saturating_add(i64::try_from(encrypted_content.len()).unwrap_or(i64::MAX)); - replacement_bytes = replacement_bytes.saturating_add( - i64::try_from(estimate_encrypted_function_output_length( - encrypted_content.len(), - )) - .unwrap_or(i64::MAX), - ); - }; - - match item { - ResponseItem::FunctionCallOutput { output, .. } => { - if let FunctionCallOutputBody::ContentItems(items) = &output.body { - for item in items { - if let FunctionCallOutputContentItem::EncryptedContent { encrypted_content } = - item - { - accumulate(encrypted_content); - } - } - } - } - ResponseItem::AgentMessage { content, .. } => { - for item in content { - if let AgentMessageInputContent::EncryptedContent { encrypted_content } = item { - accumulate(encrypted_content); - } - } - } - _ => {} - } - - (payload_bytes, replacement_bytes) } fn is_model_generated_item(item: &ResponseItem) -> bool { diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index e9a2198f48..651ff3a8d7 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -2433,7 +2433,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() { let raw_len = serde_json::to_string(&image_item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&image_item); - let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE; + let expected = "Here is the screenshot".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE; let text_only_estimated = estimate_response_item_model_visible_bytes(&text_only_item); assert_eq!(estimated, expected); @@ -2464,7 +2464,8 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() { let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE; + let expected = + "call-abc".len() as i64 + "Screenshot captured".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE; assert_eq!(estimated, expected); assert!(estimated < raw_len); @@ -2492,7 +2493,9 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() { let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE; + let expected = "call-js-repl".len() as i64 + + "Screenshot captured".len() as i64 + + RESIZED_IMAGE_BYTES_ESTIMATE; assert_eq!(estimated, expected); assert!(estimated < raw_len); @@ -2500,7 +2503,7 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() { #[test] fn audio_data_url_payload_does_not_dominate_message_estimate() { - let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 801); + let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 801); let item = ResponseItem::Message { id: None, role: "user".to_string(), @@ -2511,7 +2514,7 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() { let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 2) as i64; + let expected = approx_bytes_for_tokens(/*tokens*/ 2) as i64; assert_eq!(estimated, expected); assert!(estimated < raw_len); @@ -2519,7 +2522,7 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() { #[test] fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() { - let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 800); + let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 800); let item = ResponseItem::FunctionCallOutput { id: None, call_id: Some("call-audio".to_string()), @@ -2533,7 +2536,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() { let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64; + let expected = "call-audio".len() as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64; assert_eq!(estimated, expected); assert!(estimated < raw_len); @@ -2541,7 +2544,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() { #[test] fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() { - let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 80_000); + let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 80_000); let item = ResponseItem::CustomToolCallOutput { id: None, call_id: "call-custom-audio".to_string(), @@ -2554,7 +2557,8 @@ fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() { let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64; + let expected = + "call-custom-audio".len() as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64; assert_eq!(estimated, expected); assert!(estimated < raw_len); @@ -2573,10 +2577,9 @@ fn malformed_audio_data_url_falls_back_to_whole_url_size_cost() { internal_chat_message_metadata_passthrough: None, }; - let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - assert_eq!(estimated, raw_len - payload.len() as i64 + fallback_bytes); + assert_eq!(estimated, fallback_bytes); } #[test] @@ -2620,7 +2623,7 @@ fn record_items_omits_audio_that_exceeds_the_output_budget() { } #[test] -fn non_base64_image_urls_are_unchanged() { +fn non_base64_image_urls_use_image_estimates() { let message_item = ResponseItem::Message { id: None, role: "user".to_string(), @@ -2647,11 +2650,11 @@ fn non_base64_image_urls_are_unchanged() { assert_eq!( estimate_response_item_model_visible_bytes(&message_item), - serde_json::to_string(&message_item).unwrap().len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE ); assert_eq!( estimate_response_item_model_visible_bytes(&function_output_item), - serde_json::to_string(&function_output_item).unwrap().len() as i64 + "call-1".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE ); } @@ -2671,9 +2674,8 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() { internal_chat_message_metadata_passthrough: None, }; - let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - encrypted_content.len() as i64 + let expected = "call-encrypted".len() as i64 + estimate_encrypted_function_output_length(encrypted_content.len()) as i64; assert_eq!(estimated, expected); @@ -2686,8 +2688,9 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() { /*trigger_turn*/ true, ) .to_model_input_item(); - let agent_raw_len = serde_json::to_string(&agent_message).unwrap().len() as i64; - let expected_agent = agent_raw_len - encrypted_content.len() as i64 + let expected_agent = "/root".len() as i64 + + "/root/worker".len() as i64 + + "Message Type: NEW_TASK\nTask name: /root/worker\nSender: /root\nPayload:\n".len() as i64 + estimate_encrypted_function_output_length(encrypted_content.len()) as i64; assert_eq!( @@ -2697,7 +2700,7 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() { } #[test] -fn data_url_without_base64_marker_is_unchanged() { +fn data_url_without_base64_marker_uses_image_estimate() { let item = ResponseItem::Message { id: None, role: "user".to_string(), @@ -2711,14 +2714,15 @@ fn data_url_without_base64_marker_is_unchanged() { assert_eq!( estimate_response_item_model_visible_bytes(&item), - serde_json::to_string(&item).unwrap().len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE ); } #[test] -fn non_image_base64_data_url_is_unchanged() { +fn non_image_base64_data_url_uses_image_estimate() { let payload = "C".repeat(4_096); let image_url = format!("data:application/octet-stream;base64,{payload}"); + let expected = "call-octet".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE; let item = ResponseItem::FunctionCallOutput { id: None, call_id: Some("call-octet".to_string()), @@ -2733,10 +2737,9 @@ fn non_image_base64_data_url_is_unchanged() { internal_chat_message_metadata_passthrough: None, }; - let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - assert_eq!(estimated, raw_len); + assert_eq!(estimated, expected); } #[test] @@ -2754,9 +2757,8 @@ fn mixed_case_data_url_markers_are_adjusted() { internal_chat_message_metadata_passthrough: None, }; - let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE; + let expected = RESIZED_IMAGE_BYTES_ESTIMATE; assert_eq!(estimated, expected); } @@ -2787,10 +2789,8 @@ fn multiple_inline_images_apply_multiple_fixed_costs() { internal_chat_message_metadata_passthrough: None, }; - let raw_len = serde_json::to_string(&item).unwrap().len() as i64; - let payload_sum = (payload_one.len() + payload_two.len()) as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - payload_sum + (2 * RESIZED_IMAGE_BYTES_ESTIMATE); + let expected = "images".len() as i64 + (2 * RESIZED_IMAGE_BYTES_ESTIMATE); assert_eq!(estimated, expected); } @@ -2824,9 +2824,8 @@ fn original_detail_images_scale_with_dimensions() { internal_chat_message_metadata_passthrough: None, }; - let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES; + let expected = "call-original".len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES; assert_eq!(estimated, expected); } @@ -2858,11 +2857,10 @@ fn original_detail_images_are_capped_at_max_patch_count() { internal_chat_message_metadata_passthrough: None, }; - let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); let capped_original_detail_image_bytes = i64::try_from(approx_bytes_for_tokens(ORIGINAL_IMAGE_MAX_PATCHES)).unwrap(); - let expected = raw_len - payload.len() as i64 + capped_original_detail_image_bytes; + let expected = "call-original-capped".len() as i64 + capped_original_detail_image_bytes; assert_eq!(estimated, expected); } @@ -2895,15 +2893,14 @@ fn original_detail_webp_images_scale_with_dimensions() { internal_chat_message_metadata_passthrough: None, }; - let raw_len = serde_json::to_string(&item).unwrap().len() as i64; let estimated = estimate_response_item_model_visible_bytes(&item); - let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES; + let expected = "call-original-webp".len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES; assert_eq!(estimated, expected); } #[test] -fn text_only_items_unchanged() { +fn text_only_items_count_decoded_content() { let item = ResponseItem::Message { id: None, role: "assistant".to_string(), @@ -2915,7 +2912,6 @@ fn text_only_items_unchanged() { }; let estimated = estimate_response_item_model_visible_bytes(&item); - let raw_len = serde_json::to_string(&item).unwrap().len() as i64; - assert_eq!(estimated, raw_len); + assert_eq!(estimated, "Hello, \"world\"!\nこんにちは".len() as i64); } diff --git a/codex-rs/core/tests/suite/compact_remote_trimming.rs b/codex-rs/core/tests/suite/compact_remote_trimming.rs index 04ed0f552a..912bf9f6a1 100644 --- a/codex-rs/core/tests/suite/compact_remote_trimming.rs +++ b/codex-rs/core/tests/suite/compact_remote_trimming.rs @@ -14,6 +14,78 @@ fn compact_response() -> String { ]) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remote_compact_v2_token_estimate_ignores_message_bookkeeping_and_json_escaping() +-> Result<()> { + skip_if_no_network!(Ok(())); + + let harness = TestCodexHarness::with_auto_env_builder( + test_codex().with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()), + ) + .await?; + let escaped_text = "a \"quoted\" line\nand\\path"; + let plain_text = "x".repeat(escaped_text.len()); + let mut estimates = Vec::new(); + for (id, text, metadata) in [ + ( + "msg_1", + plain_text.as_str(), + json!({ + "turn_id": "turn-1", + "create_time": 1, + "content_item_kinds": ["unknown"], + }), + ), + ( + "msg_019f0000-0000-7000-8000-000000000001", + escaped_text, + json!({ + "turn_id": "019f0000-0000-7000-8000-000000000002", + "create_time": 1_700_000_000.123, + "content_item_kinds": ["user.text"], + }), + ), + ] { + let codex = harness + .test() + .thread_manager + .start_thread(StartThreadOptions { + environments: Some(vec![ + harness.test().executor_environment().selection().clone(), + ]), + ..StartThreadOptions::new(harness.test().config.clone()) + }) + .await? + .thread; + let message = json!({ + "type": "message", + "id": id, + "role": "user", + "content": [{"type": "input_text", "text": text}], + "internal_chat_message_metadata_passthrough": metadata, + }); + codex + .inject_response_items(vec![serde_json::from_value(message.clone())?]) + .await?; + let mock = mount_sse_once(harness.server(), compact_response()).await; + codex.submit(Op::Compact).await?; + wait_for_turn_complete(&codex).await; + + estimates.push(codex.token_usage_info().await.context("token usage")?); + assert_eq!( + mock.single_request() + .input() + .into_iter() + .find(|item| item["id"] == id), + Some(message), + ); + codex.shutdown_and_wait().await?; + } + + assert_eq!(estimates[0], estimates[1]); + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_compact_v2_trims_tool_search_output_to_empty_tools_array() -> Result<()> { skip_if_no_network!(Ok(()));