diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 8bdb36c9d2..96316db088 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3003,6 +3003,7 @@ dependencies = [ "codex-utils-cache", "codex-utils-cargo-bin", "codex-utils-git-discovery", + "codex-utils-head-tail-buffer", "codex-utils-home-dir", "codex-utils-image", "codex-utils-output-truncation", @@ -5184,6 +5185,13 @@ dependencies = [ "tracing", ] +[[package]] +name = "codex-utils-head-tail-buffer" +version = "0.0.0" +dependencies = [ + "pretty_assertions", +] + [[package]] name = "codex-utils-home-dir" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 110914fa16..d7bd76b11c 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -130,6 +130,7 @@ members = [ "utils/cache", "utils/critical-path", "utils/git-discovery", + "utils/head-tail-buffer", "utils/image", "utils/json-to-toml", "utils/home-dir", @@ -305,6 +306,7 @@ codex-utils-critical-path = { path = "utils/critical-path" } codex-utils-elapsed = { path = "utils/elapsed" } codex-utils-fuzzy-match = { path = "utils/fuzzy-match" } codex-utils-git-discovery = { path = "utils/git-discovery" } +codex-utils-head-tail-buffer = { path = "utils/head-tail-buffer" } codex-utils-home-dir = { path = "utils/home-dir" } codex-utils-image = { path = "utils/image" } codex-utils-json-to-toml = { path = "utils/json-to-toml" } diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 245f417b18..8fc0efd30d 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -78,6 +78,7 @@ codex-utils-absolute-path = { workspace = true } codex-utils-audio = { workspace = true } codex-utils-cache = { workspace = true } codex-utils-git-discovery = { workspace = true } +codex-utils-head-tail-buffer = { workspace = true } codex-utils-image = { workspace = true } codex-utils-home-dir = { workspace = true } codex-utils-output-truncation = { workspace = true } diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs index c80a89ddd1..6a58a9c373 100644 --- a/codex-rs/core/src/tools/context.rs +++ b/codex-rs/core/src/tools/context.rs @@ -4,7 +4,6 @@ use crate::session::step_context::StepContext; use crate::session::turn_context::TurnContext; use crate::session::turn_context::TurnEnvironment; use crate::turn_diff_tracker::TurnDiffTracker; -use crate::unified_exec::format_output_omission_marker; use crate::unified_exec::resolve_max_tokens; use codex_protocol::ResponseItemId; use codex_protocol::mcp::CallToolResult; @@ -17,6 +16,7 @@ use codex_protocol::models::function_call_output_content_items_to_text; use codex_tools::LoadableToolSpec; use codex_tools::ToolName; use codex_utils_audio::estimate_audio_token_count; +use codex_utils_head_tail_buffer::format_output_omission_marker; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::approx_token_count; use codex_utils_output_truncation::formatted_truncate_text; diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index 4c47abb2de..a7a8899c5e 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -48,7 +48,6 @@ use codex_core_plugins::PluginMetricsSidecar; mod async_watcher; mod errors; -mod head_tail_buffer; mod oneshot; mod process; mod process_manager; @@ -232,10 +231,6 @@ pub(crate) fn resolve_max_tokens(max_tokens: Option) -> usize { max_tokens.unwrap_or(DEFAULT_MAX_OUTPUT_TOKENS) } -pub(crate) fn format_output_omission_marker(omitted_bytes: usize) -> String { - format!("... {omitted_bytes} bytes omitted ...") -} - pub(crate) fn generate_chunk_id() -> String { let mut rng = rng(); (0..6) diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs index b3ddb02952..ed223ac7b4 100644 --- a/codex-rs/core/src/unified_exec/process.rs +++ b/codex-rs/core/src/unified_exec/process.rs @@ -24,6 +24,7 @@ use codex_protocol::protocol::TruncationPolicy; use codex_sandboxing::SandboxType; use codex_sandboxing::is_likely_sandbox_denied; use codex_sandboxing::record_filesystem_sandbox_violation; +use codex_utils_head_tail_buffer::HeadTailBuffer; use codex_utils_output_truncation::formatted_truncate_text; use codex_utils_pty::ExecCommandSession; use codex_utils_pty::ProcessSignal as PtyProcessSignal; @@ -32,7 +33,6 @@ use codex_utils_pty::SpawnedPty; use super::UNIFIED_EXEC_OUTPUT_MAX_BYTES; use super::UNIFIED_EXEC_OUTPUT_MAX_TOKENS; use super::UnifiedExecError; -use super::head_tail_buffer::HeadTailBuffer; use super::process_state::ProcessState; use crate::shell_snapshot::ShellSnapshotFile; diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index 0ba92cb486..b07b410be6 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -64,7 +64,6 @@ use crate::unified_exec::async_watcher::spawn_exit_watcher; use crate::unified_exec::async_watcher::start_streaming_output; use crate::unified_exec::clamp_yield_time; use crate::unified_exec::generate_chunk_id; -use crate::unified_exec::head_tail_buffer::HeadTailBuffer; use crate::unified_exec::process::OutputBuffers; use crate::unified_exec::process::OutputHandles; use crate::unified_exec::process::UnifiedExecProcess; @@ -89,6 +88,7 @@ use codex_protocol::shell_environment::is_non_inheritable_env_var; use codex_sandboxing::SandboxCommand; use codex_shell_command::is_dangerous_command::DangerousCommandPlatform; use codex_tools::ToolName; +use codex_utils_head_tail_buffer::HeadTailBuffer; use codex_utils_output_truncation::approx_tokens_from_byte_count; use codex_utils_path_uri::PathUri; diff --git a/codex-rs/utils/head-tail-buffer/BUILD.bazel b/codex-rs/utils/head-tail-buffer/BUILD.bazel new file mode 100644 index 0000000000..0be34d31dc --- /dev/null +++ b/codex-rs/utils/head-tail-buffer/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "head-tail-buffer", + crate_name = "codex_utils_head_tail_buffer", +) diff --git a/codex-rs/utils/head-tail-buffer/Cargo.toml b/codex-rs/utils/head-tail-buffer/Cargo.toml new file mode 100644 index 0000000000..85304d4dea --- /dev/null +++ b/codex-rs/utils/head-tail-buffer/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "codex-utils-head-tail-buffer" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lints] +workspace = true + +[dev-dependencies] +pretty_assertions = { workspace = true } + +[lib] +doctest = false diff --git a/codex-rs/core/src/unified_exec/head_tail_buffer_tests.rs b/codex-rs/utils/head-tail-buffer/src/head_tail_buffer_tests.rs similarity index 100% rename from codex-rs/core/src/unified_exec/head_tail_buffer_tests.rs rename to codex-rs/utils/head-tail-buffer/src/head_tail_buffer_tests.rs diff --git a/codex-rs/core/src/unified_exec/head_tail_buffer.rs b/codex-rs/utils/head-tail-buffer/src/lib.rs similarity index 87% rename from codex-rs/core/src/unified_exec/head_tail_buffer.rs rename to codex-rs/utils/head-tail-buffer/src/lib.rs index 0c9df7c1be..1b4ee78078 100644 --- a/codex-rs/core/src/unified_exec/head_tail_buffer.rs +++ b/codex-rs/utils/head-tail-buffer/src/lib.rs @@ -1,14 +1,14 @@ -use crate::unified_exec::UNIFIED_EXEC_OUTPUT_MAX_BYTES; -use crate::unified_exec::format_output_omission_marker; +//! A capped byte buffer that preserves a stable prefix and the most recent suffix. +//! Retained bytes never exceed the caller's const-generic budget. + use std::collections::VecDeque; /// A capped buffer that preserves a stable prefix ("head") and suffix ("tail"), /// dropping the middle once it exceeds the configured maximum. The buffer is /// symmetric meaning 50% of the capacity is allocated to the head and 50% is /// allocated to the tail. -#[derive(Debug, Default)] -#[cfg_attr(test, derive(Eq, PartialEq))] -pub(crate) struct HeadTailBuffer { +#[derive(Debug, Default, Eq, PartialEq)] +pub struct HeadTailBuffer { head: Vec, tail: VecDeque, omitted_bytes: usize, @@ -18,22 +18,18 @@ impl HeadTailBuffer { const HEAD_BUDGET: usize = MAX_BYTES / 2; const TAIL_BUDGET: usize = MAX_BYTES.saturating_sub(Self::HEAD_BUDGET); - // Used for tests. - #[allow(dead_code)] /// Total bytes currently retained by the buffer (head + tail). - pub(crate) fn retained_bytes(&self) -> usize { + pub fn retained_bytes(&self) -> usize { self.head.len().saturating_add(self.tail.len()) } - // Used for tests. - #[allow(dead_code)] /// Total bytes that were dropped from the middle due to the size cap. - pub(crate) fn omitted_bytes(&self) -> usize { + pub fn omitted_bytes(&self) -> usize { self.omitted_bytes } /// Total bytes observed by the buffer, including bytes omitted by the cap. - pub(crate) fn total_bytes(&self) -> usize { + pub fn total_bytes(&self) -> usize { self.retained_bytes().saturating_add(self.omitted_bytes) } @@ -42,7 +38,7 @@ impl HeadTailBuffer { /// Bytes are first added to the head until the head budget is full; any /// remaining bytes are added to the tail, with older tail bytes being /// dropped to preserve the tail budget. - pub(crate) fn push_chunk(&mut self, chunk: &[u8]) { + pub fn push_chunk(&mut self, chunk: &[u8]) { let chunk = self.fill_head(chunk); self.push_tail(chunk); } @@ -94,7 +90,7 @@ impl HeadTailBuffer { /// /// The output is formed by concatenating head chunks, then tail chunks. /// Omitted bytes are not represented in the returned value. - pub(crate) fn to_bytes(&self) -> Vec { + pub fn to_bytes(&self) -> Vec { let mut out = Vec::with_capacity(self.retained_bytes()); out.extend_from_slice(&self.head); out.extend(self.tail.iter().copied()); @@ -103,7 +99,7 @@ impl HeadTailBuffer { /// Return the retained output with an explicit marker between the head and /// tail when bytes were omitted. - pub(crate) fn to_bytes_with_omission_marker(&self) -> Vec { + pub fn to_bytes_with_omission_marker(&self) -> Vec { if self.omitted_bytes == 0 { return self.to_bytes(); } @@ -125,7 +121,7 @@ impl HeadTailBuffer { /// Append a later buffer with the same budget. This preserves the summary /// of the original concatenated output, including its omission count. - pub(crate) fn push_buffer(&mut self, buffer: Self) { + pub fn push_buffer(&mut self, buffer: Self) { let Self { head, tail, @@ -164,6 +160,11 @@ impl HeadTailBuffer { } } +/// Render the byte omission notice used by callers of the buffer. +pub fn format_output_omission_marker(omitted_bytes: usize) -> String { + format!("... {omitted_bytes} bytes omitted ...") +} + #[cfg(test)] #[path = "head_tail_buffer_tests.rs"] mod tests;