diff --git a/codex-rs/core/src/conversation_history.rs b/codex-rs/core/src/conversation_history.rs index 4ddb51957d..6a31d638d0 100644 --- a/codex-rs/core/src/conversation_history.rs +++ b/codex-rs/core/src/conversation_history.rs @@ -1,11 +1,8 @@ -use std::sync::OnceLock; - use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TokenUsageInfo; use codex_utils_tokenizer::Tokenizer; -use codex_utils_tokenizer::TokenizerError; use tracing::error; use crate::error::CodexErr; @@ -16,6 +13,7 @@ pub(crate) struct ConversationHistory { /// The oldest items are at the beginning of the vector. items: Vec, token_info: Option, + tokenizer: Option, } impl ConversationHistory { @@ -25,20 +23,20 @@ impl ConversationHistory { last_token_usage: TokenUsage::default(), model_context_window: Some(context_window), }); + let tokenizer = match Tokenizer::try_default() { + Ok(tokenizer) => Some(tokenizer), + Err(error) => { + error!("failed to create tokenizer: {error}"); + None + } + }; Self { items: Vec::new(), token_info, + tokenizer, } } - fn tokenizer() -> Result<&'static Tokenizer, CodexErr> { - static TOKENIZER: OnceLock> = OnceLock::new(); - let tokenizer = TOKENIZER.get_or_init(Tokenizer::try_default); - tokenizer - .as_ref() - .map_err(|e| CodexErr::InvalidInput(format!("tokenizer error: {e}"))) - } - pub(crate) fn token_info(&self) -> Option { self.token_info.clone() } @@ -128,8 +126,9 @@ impl ConversationHistory { let Some(context_window) = info.model_context_window else { return Ok(()); }; - - let tokenizer = Self::tokenizer()?; + let Some(tokenizer) = self.tokenizer.as_ref() else { + return Ok(()); + }; let mut input_tokens: i64 = 0; for item in content { diff --git a/codex-rs/utils/tokenizer/src/lib.rs b/codex-rs/utils/tokenizer/src/lib.rs index 6cda6e635f..bbdcb65bed 100644 --- a/codex-rs/utils/tokenizer/src/lib.rs +++ b/codex-rs/utils/tokenizer/src/lib.rs @@ -107,6 +107,12 @@ impl Tokenizer { } } +impl fmt::Debug for Tokenizer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Tokenizer").finish() + } +} + #[cfg(test)] mod tests { use super::*;