openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-09-17 12:23:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Estimate history tokens from content instead of serialized envelopes (#45094)

Browse Source 
## Why

Serialized response items include message IDs, metadata, and JSON escaping that inflate token estimates without adding model-visible content.

## What changed

- Estimate each response item from its content, retaining JSON syntax for structured tool payloads.
- Apply image estimates to all image inputs, including non-base64 URLs, and count audio and encrypted content through their modality-specific estimates.
- Exclude plaintext reasoning and bookkeeping-only items from replay accounting.

## Testing

Update unit expectations for text, images, audio, and encrypted content. Add a remote compaction regression test showing that equal-length text produces identical token usage estimates despite different message IDs, metadata, and JSON escaping, while preserving the submitted messages.

GitOrigin-RevId: 1c43d5abbcc1668b4ab413901a2b4b6511ca5892
This commit is contained in:
Won Park
2026-09-12 18:30:01 +00:00
committed by copyberry
parent 8d3c6cc13d
commit b04a2c2645
3 changed files with 241 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

295
codex-rs/core/src/context_manager/history.rs
View File

@@ -1,6 +1,7 @@
//! Parent model history and bounded host-owned context facts.
//! Compaction replaces only the model window. Snapshots include retained facts atomically;
//! checkpoint replay and source-call rollback share their live lifecycle.
//! Token estimates charge item content rather than transport metadata.
//! Oversized instructions keep an incomplete excerpt for bounded root review, including
//! sources recovered from legacy Guardian checkpoints before their raw history is dropped.
@@ -34,6 +35,7 @@ use codex_history::ResponseItemEnvelope;
use codex_history::RetainedContext;
use codex_history::RetainedContextEvent;
use codex_history::RetainedInputSource;
use codex_protocol::DEFAULT_FUNCTION_NAMESPACE;
use codex_protocol::items::TurnItem;
use codex_protocol::models::AgentMessageInputContent;
use codex_protocol::models::BaseInstructions;
@@ -808,8 +810,7 @@ fn estimate_encrypted_function_output_length(encoded_len: usize) -> usize {
/// Returns the same coarse, model-visible token estimate used for full history estimates.
///
/// Ordinary items are JSON-serialized, so callers estimating many items should reuse these
/// results instead of repeatedly estimating the full history.
/// Counts content directly, excluding transport IDs, metadata, and outer JSON escaping.
pub(crate) fn estimate_item_token_count(item: &ResponseItem) -> i64 {
let model_visible_bytes = estimate_response_item_model_visible_bytes(item);
approx_tokens_from_byte_count_i64(model_visible_bytes)
@@ -839,6 +840,36 @@ static ORIGINAL_IMAGE_ESTIMATE_CACHE: LazyLock<BlockingLruCache<[u8; 20], Option
fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
match item {
ResponseItem::Message { content, .. } => content
.iter()
.map(|part| match part {
ContentItem::InputText { text } | ContentItem::OutputText { text } => {
text_bytes(text)
}
ContentItem::InputImage { image_url, detail } => {
estimate_image_bytes(image_url, *detail)
}
ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),
})
.fold(0i64, i64::saturating_add),
ResponseItem::AgentMessage {
author,
recipient,
content,
..
} => content
.iter()
.map(|part| match part {
AgentMessageInputContent::InputText { text } => text_bytes(text),
AgentMessageInputContent::EncryptedContent { encrypted_content } => i64::try_from(
estimate_encrypted_function_output_length(encrypted_content.len()),
)
.unwrap_or(i64::MAX),
})
.fold(
text_bytes(author).saturating_add(text_bytes(recipient)),
i64::saturating_add,
),
ResponseItem::Reasoning {
encrypted_content: Some(content),
..
@@ -851,46 +882,87 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
encrypted_content: Some(content),
..
} => i64::try_from(estimate_reasoning_length(content.len())).unwrap_or(i64::MAX),
item => {
let raw = serialized_json_bytes(item)
.map(|len| i64::try_from(len).unwrap_or(i64::MAX))
.unwrap_or_default();
let (image_payload_bytes, image_replacement_bytes) =
image_data_url_estimate_adjustment(item);
let (audio_payload_bytes, audio_replacement_bytes) =
audio_data_url_estimate_adjustment(item);
let (encrypted_payload_bytes, encrypted_replacement_bytes) =
encrypted_function_output_estimate_adjustment(item);
// Replace raw base64 payload bytes with per-modality estimates.
// We intentionally preserve the data URL prefix and JSON
// wrapper bytes already included in `raw`.
let raw = raw
.saturating_sub(image_payload_bytes)
.saturating_add(image_replacement_bytes)
.saturating_sub(audio_payload_bytes)
.saturating_add(audio_replacement_bytes);
raw.saturating_sub(encrypted_payload_bytes)
.saturating_add(encrypted_replacement_bytes)
ResponseItem::FunctionCall {
name,
namespace,
arguments: input,
..
}
| ResponseItem::CustomToolCall {
name,
namespace,
input,
..
} => text_bytes(name)
.saturating_add(text_bytes(
namespace.as_deref().unwrap_or(DEFAULT_FUNCTION_NAMESPACE),
))
.saturating_add(text_bytes(input)),
ResponseItem::FunctionCallOutput {
call_id,
name,
namespace,
output,
..
} => estimate_function_output_bytes(&output.body)
.saturating_add(text_bytes(call_id.as_deref().unwrap_or_default()))
.saturating_add(text_bytes(name.as_deref().unwrap_or_default()))
.saturating_add(text_bytes(namespace.as_deref().unwrap_or_default())),
ResponseItem::CustomToolCallOutput {
call_id,
name,
output,
..
} => estimate_function_output_bytes(&output.body)
.saturating_add(text_bytes(call_id))
.saturating_add(text_bytes(name.as_deref().unwrap_or_default())),
// These payloads are themselves JSON arguments, rather than transport envelopes
// around text. Keep their JSON syntax in the estimate.
ResponseItem::AdditionalTools { tools, .. } => json_content_bytes(tools),
ResponseItem::ToolSearchCall { arguments, .. } => json_content_bytes(arguments),
ResponseItem::ToolSearchOutput { tools, .. } => json_content_bytes(tools),
ResponseItem::LocalShellCall { action, .. } => json_content_bytes(action),
ResponseItem::WebSearchCall { action, .. } => {
action.as_ref().map(json_content_bytes).unwrap_or_default()
}
ResponseItem::ImageGenerationCall {
revised_prompt,
result,
..
} => text_bytes(revised_prompt.as_deref().unwrap_or_default()).saturating_add(
if result.is_empty() {
0
} else {
RESIZED_IMAGE_BYTES_ESTIMATE
},
),
ResponseItem::ContextCompaction {
encrypted_content: None,
..
} => 0,
// Plaintext reasoning is excluded from replay accounting.
ResponseItem::Reasoning {
encrypted_content: None,
..
} => 0,
ResponseItem::ConfigurationUpdate { .. }
| ResponseItem::CompactionTrigger { .. }
| ResponseItem::Other => 0,
}
}
/// Returns the base64 payload byte length for inline image data URLs that are
/// eligible for token-estimation discounting.
///
/// We only discount payloads for `data:image/...;base64,...` URLs (case
/// insensitive markers) and leave everything else at raw serialized size.
fn text_bytes(text: &str) -> i64 {
i64::try_from(text.len()).unwrap_or(i64::MAX)
}
fn json_content_bytes(value: &(impl serde::Serialize + ?Sized)) -> i64 {
serialized_json_bytes(value)
.map(|len| i64::try_from(len).unwrap_or(i64::MAX))
.unwrap_or_default()
}
/// Extracts inline image bytes for the original-detail dimension estimate.
fn parse_base64_image_data_url(url: &str) -> Option<&str> {
parse_base64_data_url(url, "image/")
}
/// Returns the base64 payload for inline audio data URLs that are eligible for
/// token-estimation discounting.
fn parse_base64_audio_data_url(url: &str) -> Option<&str> {
parse_base64_data_url(url, "audio/")
}
fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&'a str> {
if !url
.get(.."data:".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
@@ -908,8 +980,8 @@ fn parse_base64_data_url<'a>(url: &'a str, media_type_prefix: &str) -> Option<&'
let mime_type = metadata_parts.next().unwrap_or_default();
let has_base64_marker = metadata_parts.any(|part| part.eq_ignore_ascii_case("base64"));
if !mime_type
.get(..media_type_prefix.len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case(media_type_prefix))
.get(.."image/".len())
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
{
return None;
}
@@ -955,7 +1027,7 @@ fn estimate_original_image_bytes(image_url: &str) -> Option<i64> {
})
}
/// Shared image estimate, excluding the data URL prefix and message framing.
/// Shared image estimate, excluding message framing.
pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>) -> i64 {
match detail {
Some(ImageDetail::Original) => {
@@ -965,130 +1037,35 @@ pub(crate) fn estimate_image_bytes(image_url: &str, detail: Option<ImageDetail>)
}
}
/// Scans one response item for discount-eligible inline image data URLs and
/// returns:
/// - total base64 payload bytes to subtract from raw serialized size
/// - total replacement byte estimate for those images
fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
let mut payload_bytes = 0i64;
let mut replacement_bytes = 0i64;
let mut accumulate = |image_url: &str, detail: Option<ImageDetail>| {
if let Some(payload_len) = parse_base64_image_data_url(image_url).map(str::len) {
payload_bytes =
payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
replacement_bytes =
replacement_bytes.saturating_add(estimate_image_bytes(image_url, detail));
}
};
match item {
ResponseItem::Message { content, .. } => {
for content_item in content {
if let ContentItem::InputImage { image_url, detail } = content_item {
accumulate(image_url, *detail);
}
}
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
for content_item in items {
if let FunctionCallOutputContentItem::InputImage { image_url, detail } =
content_item
{
accumulate(image_url, *detail);
}
}
}
}
_ => {}
}
(payload_bytes, replacement_bytes)
fn estimate_audio_bytes(audio_url: &str) -> i64 {
i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(
audio_url,
)))
.unwrap_or(i64::MAX)
}
/// Scans one response item for inline base64 audio data URLs and returns:
/// - total base64 payload bytes to subtract from raw serialized size
/// - total replacement byte estimate for those audio inputs
fn audio_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
let mut payload_bytes = 0i64;
let mut replacement_bytes = 0i64;
let mut accumulate = |audio_url: &str| {
if let Some(payload_len) = parse_base64_audio_data_url(audio_url).map(str::len) {
payload_bytes =
payload_bytes.saturating_add(i64::try_from(payload_len).unwrap_or(i64::MAX));
replacement_bytes = replacement_bytes.saturating_add(
i64::try_from(approx_bytes_for_tokens(estimate_audio_token_count(
audio_url,
)))
.unwrap_or(i64::MAX),
);
}
};
match item {
ResponseItem::Message { content, .. } => {
for content_item in content {
if let ContentItem::InputAudio { audio_url } = content_item {
accumulate(audio_url);
fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
match output {
FunctionCallOutputBody::Text(text) => text_bytes(text),
FunctionCallOutputBody::ContentItems(items) => items
.iter()
.map(|part| match part {
FunctionCallOutputContentItem::InputText { text } => text_bytes(text),
FunctionCallOutputContentItem::InputImage { image_url, detail } => {
estimate_image_bytes(image_url, *detail)
}
}
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
for content_item in items {
if let FunctionCallOutputContentItem::InputAudio { audio_url } = content_item {
accumulate(audio_url);
}
FunctionCallOutputContentItem::InputAudio { audio_url } => {
estimate_audio_bytes(audio_url)
}
}
}
_ => {}
FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => {
i64::try_from(estimate_encrypted_function_output_length(
encrypted_content.len(),
))
.unwrap_or(i64::MAX)
}
})
.fold(0i64, i64::saturating_add),
}
(payload_bytes, replacement_bytes)
}
fn encrypted_function_output_estimate_adjustment(item: &ResponseItem) -> (i64, i64) {
let mut payload_bytes = 0i64;
let mut replacement_bytes = 0i64;
let mut accumulate = |encrypted_content: &str| {
payload_bytes = payload_bytes
.saturating_add(i64::try_from(encrypted_content.len()).unwrap_or(i64::MAX));
replacement_bytes = replacement_bytes.saturating_add(
i64::try_from(estimate_encrypted_function_output_length(
encrypted_content.len(),
))
.unwrap_or(i64::MAX),
);
};
match item {
ResponseItem::FunctionCallOutput { output, .. } => {
if let FunctionCallOutputBody::ContentItems(items) = &output.body {
for item in items {
if let FunctionCallOutputContentItem::EncryptedContent { encrypted_content } =
item
{
accumulate(encrypted_content);
}
}
}
}
ResponseItem::AgentMessage { content, .. } => {
for item in content {
if let AgentMessageInputContent::EncryptedContent { encrypted_content } = item {
accumulate(encrypted_content);
}
}
}
_ => {}
}
(payload_bytes, replacement_bytes)
}
fn is_model_generated_item(item: &ResponseItem) -> bool {

70
codex-rs/core/src/context_manager/history_tests.rs
View File

@@ -2433,7 +2433,7 @@ fn image_data_url_payload_does_not_dominate_message_estimate() {
let raw_len = serde_json::to_string(&image_item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&image_item);
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let expected = "Here is the screenshot".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let text_only_estimated = estimate_response_item_model_visible_bytes(&text_only_item);
assert_eq!(estimated, expected);
@@ -2464,7 +2464,8 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let expected =
"call-abc".len() as i64 + "Screenshot captured".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2492,7 +2493,9 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let expected = "call-js-repl".len() as i64
+ "Screenshot captured".len() as i64
+ RESIZED_IMAGE_BYTES_ESTIMATE;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2500,7 +2503,7 @@ fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
#[test]
fn audio_data_url_payload_does_not_dominate_message_estimate() {
let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 801);
let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 801);
let item = ResponseItem::Message {
id: None,
role: "user".to_string(),
@@ -2511,7 +2514,7 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 2) as i64;
let expected = approx_bytes_for_tokens(/*tokens*/ 2) as i64;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2519,7 +2522,7 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() {
#[test]
fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {
let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 800);
let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 800);
let item = ResponseItem::FunctionCallOutput {
id: None,
call_id: Some("call-audio".to_string()),
@@ -2533,7 +2536,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64;
let expected = "call-audio".len() as i64 + approx_bytes_for_tokens(/*tokens*/ 1) as i64;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2541,7 +2544,7 @@ fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {
#[test]
fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
let (audio_url, payload_len) = pcm_wav_data_url(/*sample_count*/ 80_000);
let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 80_000);
let item = ResponseItem::CustomToolCallOutput {
id: None,
call_id: "call-custom-audio".to_string(),
@@ -2554,7 +2557,8 @@ fn audio_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() {
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload_len as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64;
let expected =
"call-custom-audio".len() as i64 + approx_bytes_for_tokens(/*tokens*/ 100) as i64;
assert_eq!(estimated, expected);
assert!(estimated < raw_len);
@@ -2573,10 +2577,9 @@ fn malformed_audio_data_url_falls_back_to_whole_url_size_cost() {
internal_chat_message_metadata_passthrough: None,
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
assert_eq!(estimated, raw_len - payload.len() as i64 + fallback_bytes);
assert_eq!(estimated, fallback_bytes);
}
#[test]
@@ -2620,7 +2623,7 @@ fn record_items_omits_audio_that_exceeds_the_output_budget() {
}
#[test]
fn non_base64_image_urls_are_unchanged() {
fn non_base64_image_urls_use_image_estimates() {
let message_item = ResponseItem::Message {
id: None,
role: "user".to_string(),
@@ -2647,11 +2650,11 @@ fn non_base64_image_urls_are_unchanged() {
assert_eq!(
estimate_response_item_model_visible_bytes(&message_item),
serde_json::to_string(&message_item).unwrap().len() as i64
RESIZED_IMAGE_BYTES_ESTIMATE
);
assert_eq!(
estimate_response_item_model_visible_bytes(&function_output_item),
serde_json::to_string(&function_output_item).unwrap().len() as i64
"call-1".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE
);
}
@@ -2671,9 +2674,8 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() {
internal_chat_message_metadata_passthrough: None,
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - encrypted_content.len() as i64
let expected = "call-encrypted".len() as i64
+ estimate_encrypted_function_output_length(encrypted_content.len()) as i64;
assert_eq!(estimated, expected);
@@ -2686,8 +2688,9 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() {
/*trigger_turn*/ true,
)
.to_model_input_item();
let agent_raw_len = serde_json::to_string(&agent_message).unwrap().len() as i64;
let expected_agent = agent_raw_len - encrypted_content.len() as i64
let expected_agent = "/root".len() as i64
+ "/root/worker".len() as i64
+ "Message Type: NEW_TASK\nTask name: /root/worker\nSender: /root\nPayload:\n".len() as i64
+ estimate_encrypted_function_output_length(encrypted_content.len()) as i64;
assert_eq!(
@@ -2697,7 +2700,7 @@ fn encrypted_function_output_uses_plaintext_byte_estimate() {
}
#[test]
fn data_url_without_base64_marker_is_unchanged() {
fn data_url_without_base64_marker_uses_image_estimate() {
let item = ResponseItem::Message {
id: None,
role: "user".to_string(),
@@ -2711,14 +2714,15 @@ fn data_url_without_base64_marker_is_unchanged() {
assert_eq!(
estimate_response_item_model_visible_bytes(&item),
serde_json::to_string(&item).unwrap().len() as i64
RESIZED_IMAGE_BYTES_ESTIMATE
);
}
#[test]
fn non_image_base64_data_url_is_unchanged() {
fn non_image_base64_data_url_uses_image_estimate() {
let payload = "C".repeat(4_096);
let image_url = format!("data:application/octet-stream;base64,{payload}");
let expected = "call-octet".len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let item = ResponseItem::FunctionCallOutput {
id: None,
call_id: Some("call-octet".to_string()),
@@ -2733,10 +2737,9 @@ fn non_image_base64_data_url_is_unchanged() {
internal_chat_message_metadata_passthrough: None,
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
assert_eq!(estimated, raw_len);
assert_eq!(estimated, expected);
}
#[test]
@@ -2754,9 +2757,8 @@ fn mixed_case_data_url_markers_are_adjusted() {
internal_chat_message_metadata_passthrough: None,
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + RESIZED_IMAGE_BYTES_ESTIMATE;
let expected = RESIZED_IMAGE_BYTES_ESTIMATE;
assert_eq!(estimated, expected);
}
@@ -2787,10 +2789,8 @@ fn multiple_inline_images_apply_multiple_fixed_costs() {
internal_chat_message_metadata_passthrough: None,
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let payload_sum = (payload_one.len() + payload_two.len()) as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload_sum + (2 * RESIZED_IMAGE_BYTES_ESTIMATE);
let expected = "images".len() as i64 + (2 * RESIZED_IMAGE_BYTES_ESTIMATE);
assert_eq!(estimated, expected);
}
@@ -2824,9 +2824,8 @@ fn original_detail_images_scale_with_dimensions() {
internal_chat_message_metadata_passthrough: None,
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
let expected = "call-original".len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
assert_eq!(estimated, expected);
}
@@ -2858,11 +2857,10 @@ fn original_detail_images_are_capped_at_max_patch_count() {
internal_chat_message_metadata_passthrough: None,
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let capped_original_detail_image_bytes =
i64::try_from(approx_bytes_for_tokens(ORIGINAL_IMAGE_MAX_PATCHES)).unwrap();
let expected = raw_len - payload.len() as i64 + capped_original_detail_image_bytes;
let expected = "call-original-capped".len() as i64 + capped_original_detail_image_bytes;
assert_eq!(estimated, expected);
}
@@ -2895,15 +2893,14 @@ fn original_detail_webp_images_scale_with_dimensions() {
internal_chat_message_metadata_passthrough: None,
};
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
let estimated = estimate_response_item_model_visible_bytes(&item);
let expected = raw_len - payload.len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
let expected = "call-original-webp".len() as i64 + EXPECTED_ORIGINAL_DETAIL_IMAGE_BYTES;
assert_eq!(estimated, expected);
}
#[test]
fn text_only_items_unchanged() {
fn text_only_items_count_decoded_content() {
let item = ResponseItem::Message {
id: None,
role: "assistant".to_string(),
@@ -2915,7 +2912,6 @@ fn text_only_items_unchanged() {
};
let estimated = estimate_response_item_model_visible_bytes(&item);
let raw_len = serde_json::to_string(&item).unwrap().len() as i64;
assert_eq!(estimated, raw_len);
assert_eq!(estimated, "Hello, \"world\"!\nこんにちは".len() as i64);
}

72
codex-rs/core/tests/suite/compact_remote_trimming.rs
View File

@@ -14,6 +14,78 @@ fn compact_response() -> String {
])
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn remote_compact_v2_token_estimate_ignores_message_bookkeeping_and_json_escaping()
-> Result<()> {
skip_if_no_network!(Ok(()));
let harness = TestCodexHarness::with_auto_env_builder(
test_codex().with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()),
)
.await?;
let escaped_text = "a \"quoted\" line\nand\\path";
let plain_text = "x".repeat(escaped_text.len());
let mut estimates = Vec::new();
for (id, text, metadata) in [
(
"msg_1",
plain_text.as_str(),
json!({
"turn_id": "turn-1",
"create_time": 1,
"content_item_kinds": ["unknown"],
}),
),
(
"msg_019f0000-0000-7000-8000-000000000001",
escaped_text,
json!({
"turn_id": "019f0000-0000-7000-8000-000000000002",
"create_time": 1_700_000_000.123,
"content_item_kinds": ["user.text"],
}),
),
] {
let codex = harness
.test()
.thread_manager
.start_thread(StartThreadOptions {
environments: Some(vec![
harness.test().executor_environment().selection().clone(),
]),
..StartThreadOptions::new(harness.test().config.clone())
})
.await?
.thread;
let message = json!({
"type": "message",
"id": id,
"role": "user",
"content": [{"type": "input_text", "text": text}],
"internal_chat_message_metadata_passthrough": metadata,
});
codex
.inject_response_items(vec![serde_json::from_value(message.clone())?])
.await?;
let mock = mount_sse_once(harness.server(), compact_response()).await;
codex.submit(Op::Compact).await?;
wait_for_turn_complete(&codex).await;
estimates.push(codex.token_usage_info().await.context("token usage")?);
assert_eq!(
mock.single_request()
.input()
.into_iter()
.find(|item| item["id"] == id),
Some(message),
);
codex.shutdown_and_wait().await?;
}
assert_eq!(estimates[0], estimates[1]);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn remote_compact_v2_trims_tool_search_output_to_empty_tools_array() -> Result<()> {
skip_if_no_network!(Ok(()));