Track content provenance through prompt assembly and compaction (#53831)

## Why

Message roles and content classifications do not identify who supplied each context part. Source attribution needs to survive prompt assembly and history truncation, including messages that combine text from different producers.

## What changed

- Add per-part `ContentItemMetadata` for user input, developer instructions, `AGENTS.md`, skills, harness context, command hooks, and tool context with optional callable namespaces.
- Carry attribution alongside content and classifications through contextual fragments, extension prompts, hook output, and agent messages. Keep tool-provided context-window hints separate from harness text.
- Preserve the source of retained text slices during compaction and attribute generated truncation notices to the harness. Default missing historical attribution to unknown.
- Attribute host MCP calls using cached tool namespaces only when catalog revisions remain stable, without starting servers or refreshing catalogs for attribution.
- Send the new metadata to Responses Lite and strip it from requests to endpoints that do not support it.

## Testing

Extend compaction tests to cover mixed sources, missing or extra metadata entries, and discarded media. Update Responses Lite request assertions to verify endpoint-specific attribution handling.

GitOrigin-RevId: b0ebc929b7c60f3ec49d1affbf897312d63edcc8

codex-rs/utils/string/src/lib.rs

@@ -3,9 +3,11 @@ mod truncate;
 
 pub use json::to_ascii_json_string;
 pub use json::to_json_string_bounded;
+pub use truncate::TextTruncation;
 pub use truncate::approx_bytes_for_tokens;
 pub use truncate::approx_token_count;
 pub use truncate::approx_tokens_from_byte_count;
+pub use truncate::plan_token_truncation;
 pub use truncate::truncate_middle_chars;
 pub use truncate::truncate_middle_with_marker;
 pub use truncate::truncate_middle_with_token_budget;

codex-rs/utils/string/src/truncate.rs

@@ -1,15 +1,45 @@
 //! Utilities for truncating large chunks of output while preserving a prefix
 //! and suffix on UTF-8 boundaries.
 
+use std::ops::Range;
+
 const APPROX_BYTES_PER_TOKEN: usize = 4;
 
+/// Replaces one UTF-8-aligned range with a notice. Callers can apply the same
+/// edit to annotated text without losing the sources of the retained slices.
+pub struct TextTruncation {
+    pub removed: Range<usize>,
+    pub marker: String,
+}
+
+/// Plans the existing head/tail token truncation. The marker is outside the
+/// retained-text budget, just as it is in [`truncate_middle_with_token_budget`].
+pub fn plan_token_truncation(s: &str, max_tokens: usize) -> Option<TextTruncation> {
+    let max_bytes = approx_bytes_for_tokens(max_tokens);
+    if s.len() <= max_bytes {
+        return None;
+    }
+    let (left_budget, right_budget) = split_budget(max_bytes);
+    let removed = s.floor_char_boundary(left_budget)..s.ceil_char_boundary(s.len() - right_budget);
+    let removed_tokens = approx_tokens_from_byte_count(s.len() - max_bytes);
+    Some(TextTruncation {
+        removed,
+        marker: format!("…{removed_tokens} tokens truncated…"),
+    })
+}
+
 /// Retain at most `max_bytes` from a string's ends and add a character-count
 /// marker outside that budget.
 ///
 /// Use [`truncate_middle_with_marker`] when the complete result must fit a hard
 /// byte limit.
 pub fn truncate_middle_chars(s: &str, max_bytes: usize) -> String {
-    truncate_with_byte_estimate(s, max_bytes, /*use_tokens*/ false)
+    if s.len() <= max_bytes {
+        return s.to_string();
+    }
+    let (left_budget, right_budget) = split_budget(max_bytes);
+    let (removed_chars, left, right) = split_string(s, left_budget, right_budget);
+    assemble_truncated_output(left, right, &format!("…{removed_chars} chars truncated…"))
 }
 
 /// Truncate the middle of a UTF-8 string so the result, including `marker`, is
@@ -37,18 +67,13 @@ pub fn truncate_middle_with_marker(s: &str, max_bytes: usize, marker: &str) -> S
 /// truncated string and `Some(original_token_count)` if truncation occurred;
 /// otherwise returns the original string and `None`.
 pub fn truncate_middle_with_token_budget(s: &str, max_tokens: usize) -> (String, Option<u64>) {
-    if s.is_empty() {
-        return (String::new(), None);
-    }
-
-    if max_tokens > 0 && s.len() <= approx_bytes_for_tokens(max_tokens) {
+    let Some(truncation) = plan_token_truncation(s, max_tokens) else {
         return (s.to_string(), None);
-    }
-
-    let truncated = truncate_with_byte_estimate(
-        s,
-        approx_bytes_for_tokens(max_tokens),
-        /*use_tokens*/ true,
+    };
+    let truncated = assemble_truncated_output(
+        &s[..truncation.removed.start],
+        &s[truncation.removed.end..],
+        &truncation.marker,
     );
     let total_tokens = u64::try_from(approx_token_count(s)).unwrap_or(u64::MAX);
 
@@ -59,45 +84,6 @@ pub fn truncate_middle_with_token_budget(s: &str, max_tokens: usize) -> (String,
     }
 }
 
-fn truncate_with_byte_estimate(s: &str, max_bytes: usize, use_tokens: bool) -> String {
-    if s.is_empty() {
-        return String::new();
-    }
-
-    if max_bytes == 0 {
-        let total_chars = if use_tokens { 0 } else { s.chars().count() };
-        return format_truncation_marker(
-            use_tokens,
-            removed_units(use_tokens, s.len(), total_chars),
-        );
-    }
-
-    if s.len() <= max_bytes {
-        return s.to_string();
-    }
-
-    let total_bytes = s.len();
-    let (left_budget, right_budget) = split_budget(max_bytes);
-    let (removed_chars, left, right) = if use_tokens {
-        // Token markers depend only on byte counts; avoid scanning the discarded middle.
-        let prefix_end = s.floor_char_boundary(left_budget);
-        let suffix_start = s.ceil_char_boundary(total_bytes - right_budget);
-        (0, &s[..prefix_end], &s[suffix_start..])
-    } else {
-        split_string(s, left_budget, right_budget)
-    };
-    let marker = format_truncation_marker(
-        use_tokens,
-        removed_units(
-            use_tokens,
-            total_bytes.saturating_sub(max_bytes),
-            removed_chars,
-        ),
-    );
-
-    assemble_truncated_output(left, right, &marker)
-}
-
 pub fn approx_token_count(text: &str) -> usize {
     let len = text.len();
     len.saturating_add(APPROX_BYTES_PER_TOKEN.saturating_sub(1)) / APPROX_BYTES_PER_TOKEN
@@ -158,22 +144,6 @@ fn split_budget(budget: usize) -> (usize, usize) {
     (left, budget - left)
 }
 
-fn format_truncation_marker(use_tokens: bool, removed_count: u64) -> String {
-    if use_tokens {
-        format!("…{removed_count} tokens truncated…")
-    } else {
-        format!("…{removed_count} chars truncated…")
-    }
-}
-
-fn removed_units(use_tokens: bool, removed_bytes: usize, removed_chars: usize) -> u64 {
-    if use_tokens {
-        approx_tokens_from_byte_count(removed_bytes)
-    } else {
-        u64::try_from(removed_chars).unwrap_or(u64::MAX)
-    }
-}
-
 fn assemble_truncated_output(prefix: &str, suffix: &str, marker: &str) -> String {
     let mut out = String::with_capacity(prefix.len() + marker.len() + suffix.len() + 1);
     out.push_str(prefix);