mirror of
https://github.com/openai/codex.git
synced 2026-10-26 18:38:47 +00:00
add MongoDB thread store and migration
This commit is contained in:
Generated
+36
File diff suppressed because one or more lines are too long
Generated
+475
-53
File diff suppressed because it is too large
Load Diff
@@ -312,6 +312,7 @@ env_logger = "0.11.9"
|
||||
eventsource-stream = "0.2.3"
|
||||
flate2 = "1.1.8"
|
||||
futures = { version = "0.3", default-features = false }
|
||||
mongodb = { version = "2.8", default-features = false, features = ["tokio-runtime"] }
|
||||
gethostname = "1.1.0"
|
||||
gix = { version = "0.81.0", default-features = false, features = ["sha1"] }
|
||||
glob = "0.3"
|
||||
|
||||
@@ -2785,12 +2785,6 @@ impl ThreadRequestProcessor {
|
||||
}
|
||||
let instruction_sources = codex_thread.legacy_instruction_sources().await;
|
||||
let SessionConfiguredEvent { rollout_path, .. } = session_configured;
|
||||
let Some(rollout_path) = rollout_path else {
|
||||
let error =
|
||||
internal_error(format!("rollout path missing for thread {thread_id}"));
|
||||
self.outgoing.send_error(request_id, error).await;
|
||||
return Ok(());
|
||||
};
|
||||
// Auto-attach a thread listener when resuming a thread.
|
||||
log_listener_attach_result(
|
||||
self.ensure_conversation_listener(
|
||||
@@ -2809,7 +2803,7 @@ impl ThreadRequestProcessor {
|
||||
thread_id,
|
||||
codex_thread.as_ref(),
|
||||
&response_history,
|
||||
rollout_path.as_path(),
|
||||
rollout_path.as_deref(),
|
||||
resume_source_thread,
|
||||
include_turns,
|
||||
)
|
||||
@@ -3266,7 +3260,7 @@ impl ThreadRequestProcessor {
|
||||
thread_id: ThreadId,
|
||||
thread: &CodexThread,
|
||||
thread_history: &InitialHistory,
|
||||
rollout_path: &Path,
|
||||
rollout_path: Option<&Path>,
|
||||
resume_source_thread: Option<StoredThread>,
|
||||
include_turns: bool,
|
||||
) -> std::result::Result<Thread, String> {
|
||||
@@ -3335,7 +3329,7 @@ impl ThreadRequestProcessor {
|
||||
thread_id,
|
||||
session_id.clone(),
|
||||
&config_snapshot,
|
||||
Some(rollout_path.into()),
|
||||
rollout_path.map(Path::to_path_buf),
|
||||
);
|
||||
thread.preview = preview_from_rollout_items(items);
|
||||
Ok(thread)
|
||||
@@ -3347,7 +3341,7 @@ impl ThreadRequestProcessor {
|
||||
let mut thread = thread?;
|
||||
thread.id = thread_id.to_string();
|
||||
thread.session_id = session_id;
|
||||
thread.path = Some(rollout_path.to_path_buf());
|
||||
thread.path = rollout_path.map(Path::to_path_buf);
|
||||
if include_turns {
|
||||
let history_items = thread_history.get_rollout_items();
|
||||
populate_thread_turns_from_history(
|
||||
|
||||
@@ -56,6 +56,7 @@ codex-sandboxing = { workspace = true }
|
||||
codex-state = { workspace = true }
|
||||
codex-stdio-to-uds = { workspace = true }
|
||||
codex-terminal-detection = { workspace = true }
|
||||
codex-thread-store = { workspace = true }
|
||||
codex-tui = { workspace = true }
|
||||
codex-utils-absolute-path = { workspace = true }
|
||||
codex-utils-path = { workspace = true }
|
||||
|
||||
@@ -40,6 +40,7 @@ use owo_colors::OwoColorize;
|
||||
use std::collections::HashSet;
|
||||
use std::io::IsTerminal;
|
||||
use std::io::Write;
|
||||
use std::num::NonZeroUsize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use supports_color::Stream;
|
||||
@@ -192,6 +193,9 @@ enum Subcommand {
|
||||
/// Fork a previous interactive session (picker by default; use --last to fork the most recent).
|
||||
Fork(ForkCommand),
|
||||
|
||||
/// Manage persisted Codex sessions.
|
||||
Sessions(SessionsCommand),
|
||||
|
||||
/// [EXPERIMENTAL] Browse tasks from Codex Cloud and apply changes locally.
|
||||
#[clap(name = "cloud", alias = "cloud-tasks")]
|
||||
Cloud(CloudTasksCli),
|
||||
@@ -347,6 +351,55 @@ struct SessionArchiveCommand {
|
||||
config_overrides: SessionArchiveConfigOverrides,
|
||||
}
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
struct SessionsCommand {
|
||||
#[clap(subcommand)]
|
||||
subcommand: SessionsSubcommand,
|
||||
}
|
||||
|
||||
#[derive(Debug, clap::Subcommand)]
|
||||
enum SessionsSubcommand {
|
||||
/// Import local JSONL rollouts into the strict MongoDB thread store.
|
||||
#[clap(name = "migrate-to-mongo")]
|
||||
MigrateToMongo(MigrateToMongoCommand),
|
||||
}
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
struct MigrateToMongoCommand {
|
||||
/// Mongo database name.
|
||||
#[arg(long = "database", default_value = "codex")]
|
||||
database: String,
|
||||
|
||||
/// Environment variable containing the MongoDB URI. If unset, the local
|
||||
/// default mongodb://127.0.0.1:27017 is used.
|
||||
#[arg(long = "uri-env", default_value = "CODEX_MONGODB_URI")]
|
||||
uri_env: String,
|
||||
|
||||
/// Codex home to import. Defaults to the resolved CODEX_HOME.
|
||||
#[arg(long = "codex-home", value_name = "PATH")]
|
||||
codex_home: Option<PathBuf>,
|
||||
|
||||
/// Validate and count rollouts without writing to MongoDB.
|
||||
#[arg(long = "dry-run", default_value_t = false)]
|
||||
dry_run: bool,
|
||||
|
||||
/// Read each imported history back from MongoDB and verify its item count.
|
||||
#[arg(long = "verify", default_value_t = false)]
|
||||
verify: bool,
|
||||
|
||||
/// Number of rollout files to migrate concurrently.
|
||||
#[arg(long = "jobs", default_value = "2")]
|
||||
jobs: NonZeroUsize,
|
||||
|
||||
/// Import only active sessions.
|
||||
#[arg(long = "active-only", default_value_t = false)]
|
||||
active_only: bool,
|
||||
|
||||
/// Import only archived sessions.
|
||||
#[arg(long = "archived-only", default_value_t = false)]
|
||||
archived_only: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args, Clone, Default)]
|
||||
struct SessionArchiveConfigOverrides {
|
||||
#[clap(flatten)]
|
||||
@@ -1309,6 +1362,103 @@ async fn cli_main(
|
||||
.await?;
|
||||
println!("{output}");
|
||||
}
|
||||
Some(Subcommand::Sessions(SessionsCommand {
|
||||
subcommand:
|
||||
SessionsSubcommand::MigrateToMongo(MigrateToMongoCommand {
|
||||
database,
|
||||
uri_env,
|
||||
codex_home,
|
||||
dry_run,
|
||||
verify,
|
||||
jobs,
|
||||
active_only,
|
||||
archived_only,
|
||||
}),
|
||||
})) => {
|
||||
if active_only && archived_only {
|
||||
anyhow::bail!("--active-only and --archived-only cannot be used together");
|
||||
}
|
||||
let codex_home = match codex_home {
|
||||
Some(path) => path,
|
||||
None => find_codex_home()?.to_path_buf(),
|
||||
};
|
||||
let store = codex_thread_store::MongoThreadStore::new(
|
||||
codex_thread_store::MongoThreadStoreConfig {
|
||||
codex_home,
|
||||
database,
|
||||
uri_env,
|
||||
},
|
||||
)
|
||||
.map_err(|err| anyhow::anyhow!("{err}"))?;
|
||||
if store
|
||||
.namespace_has_data()
|
||||
.await
|
||||
.map_err(|err| anyhow::anyhow!("{err}"))?
|
||||
{
|
||||
if dry_run {
|
||||
eprintln!(
|
||||
"MongoDB already contains data for this Codex-home namespace; --dry-run leaves it unchanged."
|
||||
);
|
||||
} else {
|
||||
let clear_existing = confirm(
|
||||
"MongoDB already contains data for this Codex-home namespace. Clear it before migration? [y/N]",
|
||||
)?;
|
||||
if !clear_existing {
|
||||
anyhow::bail!(
|
||||
"migration cancelled because existing MongoDB namespace data was not cleared"
|
||||
);
|
||||
}
|
||||
eprintln!("Clearing existing MongoDB namespace data...");
|
||||
store
|
||||
.clear_namespace_data()
|
||||
.await
|
||||
.map_err(|err| anyhow::anyhow!("{err}"))?;
|
||||
}
|
||||
}
|
||||
let show_progress = std::io::stderr().is_terminal();
|
||||
let progress_verb = if dry_run { "Validating" } else { "Migrating" };
|
||||
let migration_result = store
|
||||
.migrate_codex_home_with_progress_and_jobs(
|
||||
codex_thread_store::MongoMigrationOptions {
|
||||
include_active: !archived_only,
|
||||
include_archived: !active_only,
|
||||
dry_run,
|
||||
verify,
|
||||
jobs: jobs.get(),
|
||||
},
|
||||
|progress| {
|
||||
if show_progress {
|
||||
eprint!(
|
||||
"\r{}",
|
||||
mongo_migration_progress_line(progress_verb, progress)
|
||||
);
|
||||
let _ = std::io::stderr().flush();
|
||||
}
|
||||
},
|
||||
|warning| {
|
||||
if show_progress {
|
||||
eprintln!("\r{warning}");
|
||||
} else {
|
||||
eprintln!("{warning}");
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if show_progress {
|
||||
eprintln!();
|
||||
}
|
||||
let report = migration_result.map_err(|err| anyhow::anyhow!("{err}"))?;
|
||||
let action = if dry_run { "validated" } else { "migrated" };
|
||||
let completion = if dry_run {
|
||||
"Validation complete"
|
||||
} else {
|
||||
"Migration complete"
|
||||
};
|
||||
println!(
|
||||
"{completion}: {} total rollouts, {} {action}, {} skipped, {} rollout items",
|
||||
report.scanned, report.imported, report.skipped, report.items
|
||||
);
|
||||
}
|
||||
Some(Subcommand::Fork(ForkCommand {
|
||||
session_id,
|
||||
last,
|
||||
@@ -2113,6 +2263,7 @@ fn unsupported_subcommand_name_for_strict_config(
|
||||
| Some(Subcommand::Delete(_))
|
||||
| Some(Subcommand::Unarchive(_))
|
||||
| Some(Subcommand::Fork(_))
|
||||
| Some(Subcommand::Sessions(_))
|
||||
| Some(Subcommand::Doctor(_)) => None,
|
||||
Some(Subcommand::AppServer(app_server)) if app_server.subcommand.is_none() => None,
|
||||
Some(Subcommand::AppServer(app_server)) => {
|
||||
@@ -2358,6 +2509,32 @@ fn confirm(prompt: &str) -> std::io::Result<bool> {
|
||||
Ok(answer.eq_ignore_ascii_case("y") || answer.eq_ignore_ascii_case("yes"))
|
||||
}
|
||||
|
||||
fn mongo_migration_progress_line(
|
||||
verb: &str,
|
||||
progress: codex_thread_store::MongoMigrationProgress,
|
||||
) -> String {
|
||||
const BAR_WIDTH: usize = 30;
|
||||
let filled = progress
|
||||
.completed
|
||||
.saturating_mul(BAR_WIDTH)
|
||||
.checked_div(progress.total)
|
||||
.unwrap_or(BAR_WIDTH);
|
||||
let percent = progress
|
||||
.completed
|
||||
.saturating_mul(100)
|
||||
.checked_div(progress.total)
|
||||
.unwrap_or(100);
|
||||
let bar = format!(
|
||||
"{}{}",
|
||||
"=".repeat(filled),
|
||||
"-".repeat(BAR_WIDTH.saturating_sub(filled))
|
||||
);
|
||||
format!(
|
||||
"{verb} sessions [{bar}] {}/{} ({percent:>3}%)",
|
||||
progress.completed, progress.total
|
||||
)
|
||||
}
|
||||
|
||||
/// Build the final `TuiCli` for a `codex resume` invocation.
|
||||
fn finalize_resume_interactive(
|
||||
mut interactive: TuiCli,
|
||||
@@ -4071,6 +4248,20 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mongo_migration_progress_line_renders_rollout_completion() {
|
||||
assert_eq!(
|
||||
mongo_migration_progress_line(
|
||||
"Migrating",
|
||||
codex_thread_store::MongoMigrationProgress {
|
||||
completed: 1,
|
||||
total: 3,
|
||||
}
|
||||
),
|
||||
"Migrating sessions [==========--------------------] 1/3 ( 33%)"
|
||||
);
|
||||
}
|
||||
|
||||
fn strict_config_feature_toggle_error(args: &[&str]) -> anyhow::Error {
|
||||
let cli_args = std::iter::once("codex")
|
||||
.chain(std::iter::once("--strict-config"))
|
||||
|
||||
@@ -554,6 +554,14 @@ pub struct DebugConfigLockToml {
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum ThreadStoreToml {
|
||||
Local {},
|
||||
/// Store all thread metadata and replay history in MongoDB.
|
||||
Mongodb {
|
||||
/// Mongo database name. Defaults to `codex`.
|
||||
database: Option<String>,
|
||||
/// Environment variable containing the Mongo URI. When unset, the
|
||||
/// backend uses mongodb://127.0.0.1:27017.
|
||||
uri_env: Option<String>,
|
||||
},
|
||||
#[schemars(skip)]
|
||||
InMemory {
|
||||
id: String,
|
||||
|
||||
@@ -2906,6 +2906,29 @@
|
||||
"type"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"description": "Store all thread metadata and replay history in MongoDB.",
|
||||
"properties": {
|
||||
"database": {
|
||||
"description": "Mongo database name. Defaults to `codex`.",
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"mongodb"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"uri_env": {
|
||||
"description": "Environment variable containing the Mongo URI. When unset, the backend uses mongodb://127.0.0.1:27017.",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type"
|
||||
],
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -600,6 +600,8 @@ pub enum ThreadStoreConfig {
|
||||
/// Persist threads locally using rollout JSONL files and sqlite metadata.
|
||||
#[default]
|
||||
Local,
|
||||
/// Persist all thread metadata and replay history in MongoDB.
|
||||
Mongodb { database: String, uri_env: String },
|
||||
/// In-memory thread store for test and debug configurations.
|
||||
InMemory { id: String },
|
||||
}
|
||||
@@ -2232,6 +2234,10 @@ fn resolve_tool_suggest_config_from_config(
|
||||
fn thread_store_config(thread_store: Option<ThreadStoreToml>) -> ThreadStoreConfig {
|
||||
match thread_store {
|
||||
Some(ThreadStoreToml::Local {}) => ThreadStoreConfig::Local,
|
||||
Some(ThreadStoreToml::Mongodb { database, uri_env }) => ThreadStoreConfig::Mongodb {
|
||||
database: database.unwrap_or_else(|| "codex".to_string()),
|
||||
uri_env: uri_env.unwrap_or_else(|| "CODEX_MONGODB_URI".to_string()),
|
||||
},
|
||||
Some(ThreadStoreToml::InMemory { id }) => ThreadStoreConfig::InMemory { id },
|
||||
None => ThreadStoreConfig::Local,
|
||||
}
|
||||
|
||||
@@ -66,6 +66,8 @@ use codex_rollout::state_db::StateDbHandle;
|
||||
use codex_thread_store::InMemoryThreadStore;
|
||||
use codex_thread_store::LocalThreadStore;
|
||||
use codex_thread_store::LocalThreadStoreConfig;
|
||||
use codex_thread_store::MongoThreadStore;
|
||||
use codex_thread_store::MongoThreadStoreConfig;
|
||||
use codex_thread_store::ReadThreadByRolloutPathParams;
|
||||
use codex_thread_store::ReadThreadParams;
|
||||
use codex_thread_store::StoredThread;
|
||||
@@ -287,6 +289,15 @@ pub fn thread_store_from_config(
|
||||
state_db,
|
||||
))
|
||||
}
|
||||
ThreadStoreConfig::Mongodb { database, uri_env } => {
|
||||
let store = MongoThreadStore::new(MongoThreadStoreConfig {
|
||||
codex_home: config.codex_home.to_path_buf(),
|
||||
database: database.clone(),
|
||||
uri_env: uri_env.clone(),
|
||||
})
|
||||
.unwrap_or_else(|err| panic!("failed to configure strict Mongo thread store: {err}"));
|
||||
Arc::new(store)
|
||||
}
|
||||
ThreadStoreConfig::InMemory { id } => InMemoryThreadStore::for_id(id),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,11 +21,15 @@ codex-protocol = { workspace = true }
|
||||
codex-rollout = { workspace = true }
|
||||
codex-state = { workspace = true }
|
||||
codex-utils-path = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
mongodb = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
url = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions = { workspace = true }
|
||||
|
||||
@@ -8,6 +8,8 @@ mod error;
|
||||
mod in_memory;
|
||||
mod live_thread;
|
||||
mod local;
|
||||
mod mongodb;
|
||||
mod mongodb_blob;
|
||||
mod store;
|
||||
mod thread_metadata_sync;
|
||||
mod types;
|
||||
@@ -20,6 +22,11 @@ pub use live_thread::LiveThread;
|
||||
pub use live_thread::LiveThreadInitGuard;
|
||||
pub use local::LocalThreadStore;
|
||||
pub use local::LocalThreadStoreConfig;
|
||||
pub use mongodb::MongoMigrationOptions;
|
||||
pub use mongodb::MongoMigrationProgress;
|
||||
pub use mongodb::MongoMigrationReport;
|
||||
pub use mongodb::MongoThreadStore;
|
||||
pub use mongodb::MongoThreadStoreConfig;
|
||||
pub use store::ThreadStore;
|
||||
pub use store::ThreadStoreFuture;
|
||||
pub use types::AppendThreadItemsParams;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,219 @@
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use sha2::Digest;
|
||||
use sha2::Sha256;
|
||||
|
||||
use crate::ThreadStoreError;
|
||||
use crate::ThreadStoreResult;
|
||||
|
||||
pub(super) const EXTERNAL_OUTPUT_THRESHOLD_BYTES: usize = 4 * 1024 * 1024;
|
||||
const BLOB_SUBDIR: &str = "mongodb_blobs";
|
||||
const OUTPUT_JSON_PATH: &str = "/payload/output";
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub(super) struct ExternalItemField {
|
||||
pub(super) json_path: String,
|
||||
pub(super) file_name: String,
|
||||
pub(super) byte_len: usize,
|
||||
pub(super) sha256: String,
|
||||
}
|
||||
|
||||
pub(super) fn externalize_rollout_item(
|
||||
codex_home: &Path,
|
||||
item: &mut RolloutItem,
|
||||
) -> ThreadStoreResult<Vec<ExternalItemField>> {
|
||||
let Some(output) = text_output_mut(item) else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
if output.len() <= EXTERNAL_OUTPUT_THRESHOLD_BYTES {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let raw_output = std::mem::take(output);
|
||||
let sha256 = sha256_hex(raw_output.as_bytes());
|
||||
let file_name = format!("sha256-{sha256}.raw");
|
||||
write_blob(codex_home, &file_name, raw_output.as_bytes())?;
|
||||
Ok(vec![ExternalItemField {
|
||||
json_path: OUTPUT_JSON_PATH.to_string(),
|
||||
file_name,
|
||||
byte_len: raw_output.len(),
|
||||
sha256,
|
||||
}])
|
||||
}
|
||||
|
||||
pub(super) fn hydrate_rollout_item(
|
||||
codex_home: &Path,
|
||||
item: &mut RolloutItem,
|
||||
external_fields: &[ExternalItemField],
|
||||
) -> ThreadStoreResult<()> {
|
||||
for field in external_fields {
|
||||
if field.json_path != OUTPUT_JSON_PATH {
|
||||
return Err(invalid_blob(format!(
|
||||
"unsupported external rollout item path {}",
|
||||
field.json_path
|
||||
)));
|
||||
}
|
||||
let output = text_output_mut(item).ok_or_else(|| {
|
||||
invalid_blob(format!(
|
||||
"external field {} does not refer to a text tool output",
|
||||
field.json_path
|
||||
))
|
||||
})?;
|
||||
let bytes = read_blob(codex_home, field)?;
|
||||
*output = String::from_utf8(bytes)
|
||||
.map_err(|err| invalid_blob(format!("blob {} is not UTF-8: {err}", field.file_name)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn remove_blob(codex_home: &Path, file_name: &str) -> ThreadStoreResult<()> {
|
||||
validate_file_name(file_name)?;
|
||||
let path = blob_dir(codex_home).join(file_name);
|
||||
match std::fs::remove_file(&path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(blob_io_error(&path, err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn clear_blob_dir(codex_home: &Path) -> ThreadStoreResult<()> {
|
||||
let path = blob_dir(codex_home);
|
||||
match std::fs::remove_dir_all(&path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(blob_io_error(&path, err)),
|
||||
}
|
||||
}
|
||||
|
||||
fn text_output_mut(item: &mut RolloutItem) -> Option<&mut String> {
|
||||
let RolloutItem::ResponseItem(
|
||||
ResponseItem::FunctionCallOutput { output, .. }
|
||||
| ResponseItem::CustomToolCallOutput { output, .. },
|
||||
) = item
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
let FunctionCallOutputBody::Text(output) = &mut output.body else {
|
||||
return None;
|
||||
};
|
||||
Some(output)
|
||||
}
|
||||
|
||||
fn write_blob(codex_home: &Path, file_name: &str, bytes: &[u8]) -> ThreadStoreResult<()> {
|
||||
validate_file_name(file_name)?;
|
||||
let dir = blob_dir(codex_home);
|
||||
std::fs::create_dir_all(&dir).map_err(|err| blob_io_error(&dir, err))?;
|
||||
let final_path = dir.join(file_name);
|
||||
if final_path
|
||||
.metadata()
|
||||
.is_ok_and(|metadata| metadata.len() == bytes.len() as u64)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for attempt in 0..100 {
|
||||
let temp_path = dir.join(format!(
|
||||
".{file_name}.{}.{}.tmp",
|
||||
std::process::id(),
|
||||
attempt
|
||||
));
|
||||
let mut file = match OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&temp_path)
|
||||
{
|
||||
Ok(file) => file,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||
Err(err) => return Err(blob_io_error(&temp_path, err)),
|
||||
};
|
||||
file.write_all(bytes)
|
||||
.map_err(|err| blob_io_error(&temp_path, err))?;
|
||||
file.sync_all()
|
||||
.map_err(|err| blob_io_error(&temp_path, err))?;
|
||||
drop(file);
|
||||
match std::fs::rename(&temp_path, &final_path) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) if final_path.exists() => {
|
||||
std::fs::remove_file(&temp_path)
|
||||
.map_err(|remove_err| blob_io_error(&temp_path, remove_err))?;
|
||||
if final_path
|
||||
.metadata()
|
||||
.is_ok_and(|metadata| metadata.len() == bytes.len() as u64)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
return Err(blob_io_error(&final_path, err));
|
||||
}
|
||||
Err(err) => return Err(blob_io_error(&final_path, err)),
|
||||
}
|
||||
}
|
||||
Err(invalid_blob(format!(
|
||||
"could not allocate a temporary file for {file_name}"
|
||||
)))
|
||||
}
|
||||
|
||||
fn read_blob(codex_home: &Path, field: &ExternalItemField) -> ThreadStoreResult<Vec<u8>> {
|
||||
validate_file_name(&field.file_name)?;
|
||||
let path = blob_dir(codex_home).join(&field.file_name);
|
||||
let bytes = std::fs::read(&path).map_err(|err| blob_io_error(&path, err))?;
|
||||
if bytes.len() != field.byte_len {
|
||||
return Err(invalid_blob(format!(
|
||||
"blob {} has {} bytes, expected {}",
|
||||
field.file_name,
|
||||
bytes.len(),
|
||||
field.byte_len
|
||||
)));
|
||||
}
|
||||
let actual_sha256 = sha256_hex(&bytes);
|
||||
if actual_sha256 != field.sha256 {
|
||||
return Err(invalid_blob(format!(
|
||||
"blob {} has SHA-256 {actual_sha256}, expected {}",
|
||||
field.file_name, field.sha256
|
||||
)));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn validate_file_name(file_name: &str) -> ThreadStoreResult<()> {
|
||||
let path = Path::new(file_name);
|
||||
if file_name.is_empty()
|
||||
|| path.components().count() != 1
|
||||
|| path.file_name().and_then(|name| name.to_str()) != Some(file_name)
|
||||
{
|
||||
return Err(invalid_blob(format!(
|
||||
"invalid blob file name {file_name:?}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn blob_dir(codex_home: &Path) -> PathBuf {
|
||||
codex_home.join(BLOB_SUBDIR)
|
||||
}
|
||||
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
format!("{:x}", Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
fn blob_io_error(path: &Path, err: std::io::Error) -> ThreadStoreError {
|
||||
ThreadStoreError::Internal {
|
||||
message: format!(
|
||||
"Mongo thread store blob failure at {}: {err}",
|
||||
path.display()
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_blob(message: String) -> ThreadStoreError {
|
||||
ThreadStoreError::Internal {
|
||||
message: format!("Mongo thread store blob failure: {message}"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use super::DEFAULT_MONGODB_URI;
|
||||
use super::MongoThreadStoreConfig;
|
||||
use super::reconstruct_rollout_line;
|
||||
use crate::mongodb_blob::EXTERNAL_OUTPUT_THRESHOLD_BYTES;
|
||||
use crate::mongodb_blob::externalize_rollout_item;
|
||||
use crate::mongodb_blob::hydrate_rollout_item;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::RolloutItem;
|
||||
|
||||
#[test]
|
||||
fn missing_uri_env_uses_local_default() {
|
||||
let config = MongoThreadStoreConfig {
|
||||
codex_home: PathBuf::from("/tmp/codex-mongo-test"),
|
||||
database: "codex".to_string(),
|
||||
uri_env: "CODEX_MONGODB_URI_TEST_UNSET".to_string(),
|
||||
};
|
||||
assert_eq!(
|
||||
config.resolved_uri().expect("default URI"),
|
||||
DEFAULT_MONGODB_URI
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconstructs_line_with_missing_timestamp() {
|
||||
let line = json!({
|
||||
"type": "event_msg",
|
||||
"payload": {
|
||||
"type": "turn_aborted",
|
||||
"reason": "interrupted"
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
let reconstructed = reconstruct_rollout_line(&line).expect("reconstructed rollout line");
|
||||
assert_eq!(reconstructed.timestamp, "1970-01-01T00:00:00.000Z");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconstructs_legacy_absolute_cwd_as_path_uri() {
|
||||
let line = json!({
|
||||
"timestamp": "2026-04-29T15:29:36.000Z",
|
||||
"type": "event_msg",
|
||||
"payload": {
|
||||
"type": "exec_command_end",
|
||||
"call_id": "call-1",
|
||||
"turn_id": "turn-1",
|
||||
"command": ["pwd"],
|
||||
"cwd": "/home/dev-user/code/codex",
|
||||
"parsed_cmd": [],
|
||||
"stdout": "",
|
||||
"stderr": "",
|
||||
"aggregated_output": "{",
|
||||
"exit_code": 0,
|
||||
"duration": {
|
||||
"secs": 0,
|
||||
"nanos": 0
|
||||
},
|
||||
"formatted_output": "",
|
||||
"status": "completed"
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
reconstruct_rollout_line(&line).expect("legacy cwd should reconstruct");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconstructs_first_json_object_when_line_has_trailing_content() {
|
||||
let line = format!(
|
||||
"{} trailing rollout noise",
|
||||
json!({
|
||||
"type": "event_msg",
|
||||
"payload": {
|
||||
"type": "turn_aborted",
|
||||
"reason": "interrupted"
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
reconstruct_rollout_line(&line).expect("rollout envelope should reconstruct");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_line_missing_core_rollout_fields() {
|
||||
let error = match reconstruct_rollout_line(r#"{"timestamp":"2026-01-01T00:00:00.000Z"}"#) {
|
||||
Ok(_) => panic!("missing core fields should not reconstruct"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error, "missing required type or payload fields");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn externalizes_and_hydrates_large_text_tool_outputs() {
|
||||
let codex_home = tempdir().expect("temporary codex home");
|
||||
let output = "x".repeat(EXTERNAL_OUTPUT_THRESHOLD_BYTES + 1);
|
||||
let mut item = RolloutItem::ResponseItem(ResponseItem::CustomToolCallOutput {
|
||||
id: None,
|
||||
call_id: "call-1".to_string(),
|
||||
name: None,
|
||||
output: FunctionCallOutputPayload::from_text(output.clone()),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
});
|
||||
|
||||
let fields =
|
||||
externalize_rollout_item(codex_home.path(), &mut item).expect("externalized output");
|
||||
assert_eq!(fields.len(), 1);
|
||||
assert!(!fields[0].file_name.contains('/'));
|
||||
let RolloutItem::ResponseItem(ResponseItem::CustomToolCallOutput {
|
||||
output: externalized_output,
|
||||
..
|
||||
}) = &item
|
||||
else {
|
||||
panic!("expected custom tool output");
|
||||
};
|
||||
assert_eq!(externalized_output.text_content(), Some(""));
|
||||
|
||||
hydrate_rollout_item(codex_home.path(), &mut item, &fields).expect("hydrated output");
|
||||
let RolloutItem::ResponseItem(ResponseItem::CustomToolCallOutput {
|
||||
output: hydrated_output,
|
||||
..
|
||||
}) = &item
|
||||
else {
|
||||
panic!("expected custom tool output");
|
||||
};
|
||||
assert_eq!(hydrated_output.text_content(), Some(output.as_str()));
|
||||
}
|
||||
@@ -330,7 +330,7 @@ fn strip_user_message_prefix(text: &str) -> &str {
|
||||
}
|
||||
}
|
||||
|
||||
fn user_message_preview(user: &UserMessageEvent) -> Option<String> {
|
||||
pub(crate) fn user_message_preview(user: &UserMessageEvent) -> Option<String> {
|
||||
let message = strip_user_message_prefix(user.message.as_str());
|
||||
if !message.is_empty() {
|
||||
return Some(message.to_string());
|
||||
|
||||
Reference in New Issue
Block a user