mirror of
https://github.com/openai/codex.git
synced 2026-11-14 21:40:39 +00:00
Move JSON byte counting into codex-utils-string (#61465)
Export `serialized_json_bytes` from `codex-utils-string` and update core callers to use the shared helper. Preserve byte counting without retaining the serialized output. GitOrigin-RevId: 9c75b05592d3171b9a0d54517f8f3a4281fde987
This commit is contained in:
committed by
copyberry
parent
ecf54824c6
commit
336b19ab26
@@ -2,9 +2,9 @@
|
||||
//! Only the wire copy changes; model-visible input and WebSocket continuation history stay intact.
|
||||
|
||||
use crate::tools::metadata_metrics;
|
||||
use crate::utils::json::serialized_json_bytes;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::models::bound_executed_tool_calls_for_message;
|
||||
use codex_utils_string::serialized_json_bytes;
|
||||
use serde::Serialize;
|
||||
|
||||
const MAX_RESPONSE_MESSAGE_BYTES: usize = 15 * 1024 * 1024;
|
||||
|
||||
@@ -33,7 +33,6 @@ use crate::guardian::GUARDIAN_MAX_ROOT_MESSAGE_TOKENS;
|
||||
use crate::guardian::guardian_truncate_text;
|
||||
use crate::session::FileAudioDurations;
|
||||
use crate::session::turn_context::TurnContext;
|
||||
use crate::utils::json::serialized_json_bytes;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_attachment_store::AttachmentStore;
|
||||
@@ -79,6 +78,7 @@ use codex_utils_output_truncation::approx_token_count;
|
||||
use codex_utils_output_truncation::approx_tokens_from_byte_count_i64;
|
||||
use codex_utils_output_truncation::truncate_function_output_payload;
|
||||
use codex_utils_output_truncation::with_serialization_allowance;
|
||||
use codex_utils_string::serialized_json_bytes;
|
||||
use image::ImageReader;
|
||||
use std::collections::BTreeSet;
|
||||
use std::io::Cursor;
|
||||
|
||||
@@ -39,7 +39,7 @@ use crate::tools::context::ToolOutput;
|
||||
use crate::tools::context::ToolPayload;
|
||||
use crate::tools::metadata_metrics;
|
||||
use crate::tools::router::ToolCall;
|
||||
use crate::utils::json::serialized_json_bytes;
|
||||
use codex_utils_string::serialized_json_bytes;
|
||||
|
||||
mod seen_ids;
|
||||
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
#[derive(Default)]
|
||||
struct JsonByteCounter(usize);
|
||||
|
||||
impl std::io::Write for JsonByteCounter {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.0 = self.0.saturating_add(bytes.len());
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts serialized JSON bytes without retaining the serialized output.
|
||||
pub(crate) fn serialized_json_bytes<T: serde::Serialize + ?Sized>(
|
||||
value: &T,
|
||||
) -> serde_json::Result<usize> {
|
||||
let mut counter = JsonByteCounter::default();
|
||||
serde_json::to_writer(&mut counter, value)?;
|
||||
Ok(counter.0)
|
||||
}
|
||||
@@ -1,2 +1 @@
|
||||
pub(crate) mod json;
|
||||
pub mod path_utils;
|
||||
|
||||
@@ -1,9 +1,30 @@
|
||||
//! JSON serialization helpers for ASCII-only transports and bounded output.
|
||||
//! JSON serialization helpers for counting bytes, bounding output, and ASCII-only transports.
|
||||
|
||||
use std::io;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Default)]
|
||||
struct JsonByteCounter(usize);
|
||||
|
||||
impl io::Write for JsonByteCounter {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.0 = self.0.saturating_add(bytes.len());
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts serialized JSON bytes without retaining the serialized output.
|
||||
pub fn serialized_json_bytes<T: Serialize + ?Sized>(value: &T) -> serde_json::Result<usize> {
|
||||
let mut counter = JsonByteCounter::default();
|
||||
serde_json::to_writer(&mut counter, value)?;
|
||||
Ok(counter.0)
|
||||
}
|
||||
|
||||
struct AsciiJsonFormatter;
|
||||
|
||||
impl serde_json::ser::Formatter for AsciiJsonFormatter {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
mod json;
|
||||
mod truncate;
|
||||
|
||||
pub use json::serialized_json_bytes;
|
||||
pub use json::to_ascii_json_string;
|
||||
pub use json::to_json_string_bounded;
|
||||
pub use truncate::TextTruncation;
|
||||
|
||||
Reference in New Issue
Block a user