move

codex-rs/Cargo.lock

@@ -1552,6 +1552,7 @@ dependencies = [
  "pretty_assertions",
  "thiserror 2.0.17",
  "tiktoken-rs",
+ "tokio",
 ]
 
 [[package]]

codex-rs/core/src/codex.rs

@@ -133,7 +133,6 @@ use codex_protocol::protocol::InitialHistory;
 use codex_protocol::user_input::UserInput;
 use codex_utils_readiness::Readiness;
 use codex_utils_readiness::ReadinessFlag;
-use codex_utils_tokenizer::shared_default_tokenizer;
 
 /// The high-level interface to the Codex system.
 /// It operates as a queue pair where you send submissions and receive events.
@@ -248,10 +247,6 @@ impl Codex {
     }
 }
 
-async fn warm_up_tokenizer() {
-    let _ = tokio::task::spawn_blocking(shared_default_tokenizer).await;
-}
-
 /// Context for an initialized model agent
 ///
 /// A session has at most 1 running task at a time, and can be interrupted by user input.
@@ -502,7 +497,6 @@ impl Session {
             config.mcp_servers.iter(),
             config.mcp_oauth_credentials_store_mode,
         );
-        let tokenizer_warmup_fut = warm_up_tokenizer();
 
         // Join all independent futures.
         let (
@@ -517,7 +511,6 @@ impl Session {
             default_shell_fut,
             history_meta_fut,
             auth_statuses_fut,
-            tokenizer_warmup_fut
         );
 
         let rollout_recorder = rollout_recorder.map_err(|e| {

codex-rs/core/src/conversation_manager.rs

@@ -17,9 +17,13 @@ use codex_protocol::models::ResponseItem;
 use codex_protocol::protocol::InitialHistory;
 use codex_protocol::protocol::RolloutItem;
 use codex_protocol::protocol::SessionSource;
+use codex_utils_tokenizer::shared_default_tokenizer;
+use codex_utils_tokenizer::warm_up_default_tokenizer;
+use codex_utils_tokenizer::warm_up_tokenizer;
 use std::collections::HashMap;
 use std::path::PathBuf;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::sync::RwLock;
 
 /// Represents a newly created Codex conversation, including the first event
@@ -40,6 +44,7 @@ pub struct ConversationManager {
 
 impl ConversationManager {
     pub fn new(auth_manager: Arc<AuthManager>, session_source: SessionSource) -> Self {
+        warm_up_default_tokenizer();
         Self {
             conversations: Arc::new(RwLock::new(HashMap::new())),
             auth_manager,

codex-rs/utils/tokenizer/Cargo.toml

@@ -10,6 +10,7 @@ workspace = true
 anyhow = { workspace = true }
 thiserror = { workspace = true }
 tiktoken-rs = "0.7"
+tokio.workspace = true
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }

codex-rs/utils/tokenizer/src/lib.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 use std::sync::Arc;
 use std::sync::OnceLock;
+use std::time::Duration;
 
 use anyhow::Context;
 use anyhow::Error as AnyhowError;
@@ -111,6 +112,12 @@ impl Tokenizer {
 
 static DEFAULT_TOKENIZER: OnceLock<Result<Arc<Tokenizer>, TokenizerError>> = OnceLock::new();
 
+pub fn warm_up_default_tokenizer() {
+    tokio::spawn(tokio::time::timeout(Duration::from_secs(5), async {
+        let _ = shared_default_tokenizer();
+    }));
+}
+
 /// Return a shared default tokenizer (`O200kBase`), loading it once per process.
 /// Returns `None` if initialization fails.
 #[must_use]