openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-02 19:43:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

P4

Browse Source
This commit is contained in:
jimmyfraiture
2025-09-29 11:06:44 +01:00
parent 2efe961ac1
commit c9f6b5dffc
14 changed files with 364 additions and 414 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/Cargo.lock
Generated
+3
View File
@@ -580,6 +580,7 @@ dependencies = [
"anyhow",
"async-trait",
"base64",
"codex-file-search",
"codex-protocol",
"core_test_support",
"libc",
@@ -591,7 +592,9 @@ dependencies = [
"shlex",
"similar",
"thiserror 2.0.16",
"time",
"tokio",
"tracing",
"uuid",
"which",
"wildmatch",
codex-rs/codex-agent/Cargo.toml
+3
View File
@@ -21,6 +21,9 @@ tokio = { workspace = true, features = ["macros", "process", "rt-multi-thread",
uuid = { workspace = true, features = ["serde", "v4"] }
which = { workspace = true }
wildmatch = { workspace = true }
codex-file-search = { workspace = true }
time = { workspace = true, features = ["formatting", "parsing", "local-offset", "macros"] }
tracing = { workspace = true }
[dev-dependencies]
core_test_support = { workspace = true }
codex-rs/codex-agent/src/conversation_history.rs
+117
View File
@@ -0,0 +1,117 @@
use codex_protocol::models::ResponseItem;
/// Transcript of conversation history shared across agent hosts.
#[derive(Debug, Clone, Default)]
pub struct ConversationHistory {
/// Oldest items appear at the start of the vector.
items: Vec<ResponseItem>,
}
impl ConversationHistory {
pub fn new() -> Self {
Self { items: Vec::new() }
}
/// Returns a clone of the stored transcript.
pub fn contents(&self) -> Vec<ResponseItem> {
self.items.clone()
}
/// Records additional response items, filtering out non-API messages.
pub fn record_items<I>(&mut self, items: I)
where
I: IntoIterator,
I::Item: std::ops::Deref<Target = ResponseItem>,
{
for item in items {
if !is_api_message(&item) {
continue;
}
self.items.push(item.clone());
}
}
pub fn replace(&mut self, items: Vec<ResponseItem>) {
self.items = items;
}
}
/// Detects whether the given message should be persisted to history.
fn is_api_message(message: &ResponseItem) -> bool {
match message {
ResponseItem::Message { role, .. } => role.as_str() != "system",
ResponseItem::FunctionCallOutput { .. }
| ResponseItem::FunctionCall { .. }
| ResponseItem::CustomToolCall { .. }
| ResponseItem::CustomToolCallOutput { .. }
| ResponseItem::LocalShellCall { .. }
| ResponseItem::Reasoning { .. }
| ResponseItem::WebSearchCall { .. } => true,
ResponseItem::Other => false,
}
}
#[cfg(test)]
mod tests {
use super::*;
use codex_protocol::models::ContentItem;
fn assistant_msg(text: &str) -> ResponseItem {
ResponseItem::Message {
id: None,
role: "assistant".to_string(),
content: vec![ContentItem::OutputText {
text: text.to_string(),
}],
}
}
fn user_msg(text: &str) -> ResponseItem {
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::OutputText {
text: text.to_string(),
}],
}
}
#[test]
fn filters_non_api_messages() {
let mut h = ConversationHistory::default();
let system = ResponseItem::Message {
id: None,
role: "system".to_string(),
content: vec![ContentItem::OutputText {
text: "ignored".to_string(),
}],
};
h.record_items([&system, &ResponseItem::Other]);
let u = user_msg("hi");
let a = assistant_msg("hello");
h.record_items([&u, &a]);
let items = h.contents();
assert_eq!(
items,
vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::OutputText {
text: "hi".to_string()
}]
},
ResponseItem::Message {
id: None,
role: "assistant".to_string(),
content: vec![ContentItem::OutputText {
text: "hello".to_string()
}]
}
]
);
}
}
codex-rs/codex-agent/src/lib.rs
+4
View File
@@ -1,9 +1,11 @@
pub mod config_types;
pub mod conversation_history;
pub mod exec_command;
pub mod function_tool;
pub mod model_family;
pub mod model_provider;
pub mod notifications;
pub mod rollout;
pub mod runtime;
pub mod runtime_config;
pub mod services;
@@ -15,10 +17,12 @@ pub mod turn_diff_tracker;
pub mod unified_exec;
pub use config_types::*;
pub use conversation_history::*;
pub use function_tool::*;
pub use model_family::*;
pub use model_provider::*;
pub use notifications::*;
pub use rollout::*;
pub use runtime::*;
pub use runtime_config::*;
pub use services::*;
codex-rs/core/src/rollout/list.rs → codex-rs/codex-agent/src/rollout/list.rs
+66 -122
View File
@@ -1,9 +1,13 @@
use std::cmp::Reverse;
use std::io::{self};
use std::io;
use std::path::Path;
use std::path::PathBuf;
use codex_file_search as file_search;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use serde_json::Value;
use std::num::NonZero;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
@@ -11,40 +15,29 @@ use time::OffsetDateTime;
use time::PrimitiveDateTime;
use time::format_description::FormatItem;
use time::macros::format_description;
use tokio::fs;
use tokio::io::AsyncBufReadExt;
use uuid::Uuid;
use super::SESSIONS_SUBDIR;
use crate::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
/// Returned page of conversation summaries.
#[derive(Debug, Default, PartialEq)]
pub struct ConversationsPage {
/// Conversation summaries ordered newest first.
pub items: Vec<ConversationItem>,
/// Opaque pagination token to resume after the last item, or `None` if end.
pub next_cursor: Option<Cursor>,
/// Total number of files touched while scanning this request.
pub num_scanned_files: usize,
/// True if a hard scan cap was hit; consider resuming with `next_cursor`.
pub reached_scan_cap: bool,
}
/// Summary information for a conversation rollout file.
#[derive(Debug, PartialEq)]
pub struct ConversationItem {
/// Absolute path to the rollout file.
pub path: PathBuf,
/// First up to 5 JSONL records parsed as JSON (includes meta line).
pub head: Vec<serde_json::Value>,
pub head: Vec<Value>,
}
/// Hard cap to bound worst‑case work per request.
const MAX_SCAN_FILES: usize = 100;
const HEAD_RECORD_LIMIT: usize = 10;
/// Pagination cursor identifying a file by timestamp and UUID.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Cursor {
ts: OffsetDateTime,
@@ -82,10 +75,7 @@ impl<'de> serde::Deserialize<'de> for Cursor {
}
}
/// Retrieve recorded conversation file paths with token pagination. The returned `next_cursor`
/// can be supplied on the next call to resume after the last returned item, resilient to
/// concurrent new sessions being appended. Ordering is stable by timestamp desc, then UUID desc.
pub(crate) async fn get_conversations(
pub async fn get_conversations(
codex_home: &Path,
page_size: usize,
cursor: Option<&Cursor>,
@@ -94,31 +84,56 @@ pub(crate) async fn get_conversations(
root.push(SESSIONS_SUBDIR);
if !root.exists() {
return Ok(ConversationsPage {
items: Vec::new(),
next_cursor: None,
num_scanned_files: 0,
reached_scan_cap: false,
});
return Ok(ConversationsPage::default());
}
let anchor = cursor.cloned();
let result = traverse_directories_for_paths(root.clone(), page_size, anchor).await?;
Ok(result)
traverse_directories_for_paths(root, page_size, anchor).await
}
/// Load the full contents of a single conversation session file at `path`.
/// Returns the entire file contents as a String.
#[allow(dead_code)]
pub(crate) async fn get_conversation(path: &Path) -> io::Result<String> {
tokio::fs::read_to_string(path).await
pub async fn get_conversation(path: &Path) -> io::Result<String> {
fs::read_to_string(path).await
}
pub async fn find_conversation_path_by_id_str(
codex_home: &Path,
id_str: &str,
) -> io::Result<Option<PathBuf>> {
if Uuid::parse_str(id_str).is_err() {
return Ok(None);
}
let mut root = codex_home.to_path_buf();
root.push(SESSIONS_SUBDIR);
if !root.exists() {
return Ok(None);
}
let limit = NonZero::new(1).unwrap();
let threads = NonZero::new(2).unwrap();
let cancel = Arc::new(AtomicBool::new(false));
let exclude: Vec<String> = Vec::new();
let compute_indices = false;
let results = file_search::run(
id_str,
limit,
&root,
exclude,
threads,
cancel,
compute_indices,
)
.map_err(|e| io::Error::other(format!("file search failed: {e}")))?;
Ok(results
.matches
.into_iter()
.next()
.map(|m| root.join(m.path)))
}
/// Load conversation file paths from disk using directory traversal.
///
/// Directory layout: `~/.codex/sessions/YYYY/MM/DD/rollout-YYYY-MM-DDThh-mm-ss-<uuid>.jsonl`
/// Returned newest (latest) first.
async fn traverse_directories_for_paths(
root: PathBuf,
page_size: usize,
@@ -157,8 +172,7 @@ async fn traverse_directories_for_paths(
.map(|(ts, id)| (ts, id, name_str.to_string(), path.to_path_buf()))
})
.await?;
// Stable ordering within the same second: (timestamp desc, uuid desc)
day_files.sort_by_key(|(ts, sid, _name_str, _path)| (Reverse(*ts), Reverse(*sid)));
day_files.sort_by_key(|(ts, sid, _, _)| (Reverse(*ts), Reverse(*sid)));
for (ts, sid, _name_str, path) in day_files.into_iter() {
scanned_files += 1;
if scanned_files >= MAX_SCAN_FILES && items.len() >= page_size {
@@ -174,13 +188,10 @@ async fn traverse_directories_for_paths(
if items.len() == page_size {
break 'outer;
}
// Read head and simultaneously detect message events within the same
// first N JSONL records to avoid a second file read.
let (head, saw_session_meta, saw_user_event) =
read_head_and_flags(&path, HEAD_RECORD_LIMIT)
.await
.unwrap_or((Vec::new(), false, false));
// Apply filters: must have session meta and at least one user message event
if saw_session_meta && saw_user_event {
items.push(ConversationItem { path, head });
}
@@ -198,23 +209,6 @@ async fn traverse_directories_for_paths(
})
}
/// Pagination cursor token format: "<file_ts>|<uuid>" where `file_ts` matches the
/// filename timestamp portion (YYYY-MM-DDThh-mm-ss) used in rollout filenames.
/// The cursor orders files by timestamp desc, then UUID desc.
fn parse_cursor(token: &str) -> Option<Cursor> {
let (file_ts, uuid_str) = token.split_once('|')?;
let Ok(uuid) = Uuid::parse_str(uuid_str) else {
return None;
};
let format: &[FormatItem] =
format_description!("[year]-[month]-[day]T[hour]-[minute]-[second]");
let ts = PrimitiveDateTime::parse(file_ts, format).ok()?.assume_utc();
Some(Cursor::new(ts, uuid))
}
fn build_next_cursor(items: &[ConversationItem]) -> Option<Cursor> {
let last = items.last()?;
let file_name = last.path.file_name()?.to_string_lossy();
@@ -222,14 +216,12 @@ fn build_next_cursor(items: &[ConversationItem]) -> Option<Cursor> {
Some(Cursor::new(ts, id))
}
/// Collects immediate subdirectories of `parent`, parses their (string) names with `parse`,
/// and returns them sorted descending by the parsed key.
async fn collect_dirs_desc<T, F>(parent: &Path, parse: F) -> io::Result<Vec<(T, PathBuf)>>
where
T: Ord + Copy,
F: Fn(&str) -> Option<T>,
{
let mut dir = tokio::fs::read_dir(parent).await?;
let mut dir = fs::read_dir(parent).await?;
let mut vec: Vec<(T, PathBuf)> = Vec::new();
while let Some(entry) = dir.next_entry().await? {
if entry
@@ -247,12 +239,11 @@ where
Ok(vec)
}
/// Collects files in a directory and parses them with `parse`.
async fn collect_files<T, F>(parent: &Path, parse: F) -> io::Result<Vec<T>>
where
F: Fn(&str, &Path) -> Option<T>,
{
let mut dir = tokio::fs::read_dir(parent).await?;
let mut dir = fs::read_dir(parent).await?;
let mut collected: Vec<T> = Vec::new();
while let Some(entry) = dir.next_entry().await? {
if entry
@@ -270,15 +261,11 @@ where
}
fn parse_timestamp_uuid_from_filename(name: &str) -> Option<(OffsetDateTime, Uuid)> {
// Expected: rollout-YYYY-MM-DDThh-mm-ss-<uuid>.jsonl
let core = name.strip_prefix("rollout-")?.strip_suffix(".jsonl")?;
// Scan from the right for a '-' such that the suffix parses as a UUID.
let (sep_idx, uuid) = core
.match_indices('-')
.rev()
.find_map(|(i, _)| Uuid::parse_str(&core[i + 1..]).ok().map(|u| (i, u)))?;
let ts_str = &core[..sep_idx];
let format: &[FormatItem] =
format_description!("[year]-[month]-[day]T[hour]-[minute]-[second]");
@@ -286,16 +273,23 @@ fn parse_timestamp_uuid_from_filename(name: &str) -> Option<(OffsetDateTime, Uui
Some((ts, uuid))
}
fn parse_cursor(token: &str) -> Option<Cursor> {
let (file_ts, uuid_str) = token.split_once('|')?;
let uuid = Uuid::parse_str(uuid_str).ok()?;
let format: &[FormatItem] =
format_description!("[year]-[month]-[day]T[hour]-[minute]-[second]");
let ts = PrimitiveDateTime::parse(file_ts, format).ok()?.assume_utc();
Some(Cursor::new(ts, uuid))
}
async fn read_head_and_flags(
path: &Path,
max_records: usize,
) -> io::Result<(Vec<serde_json::Value>, bool, bool)> {
use tokio::io::AsyncBufReadExt;
) -> io::Result<(Vec<Value>, bool, bool)> {
let file = tokio::fs::File::open(path).await?;
let reader = tokio::io::BufReader::new(file);
let mut lines = reader.lines();
let mut head: Vec<serde_json::Value> = Vec::new();
let mut head: Vec<Value> = Vec::new();
let mut saw_session_meta = false;
let mut saw_user_event = false;
@@ -322,12 +316,7 @@ async fn read_head_and_flags(
head.push(val);
}
}
RolloutItem::TurnContext(_) => {
// Not included in `head`; skip.
}
RolloutItem::Compacted(_) => {
// Not included in `head`; skip.
}
RolloutItem::TurnContext(_) | RolloutItem::Compacted(_) => {}
RolloutItem::EventMsg(ev) => {
if matches!(ev, EventMsg::UserMessage(_)) {
saw_user_event = true;
@@ -338,48 +327,3 @@ async fn read_head_and_flags(
Ok((head, saw_session_meta, saw_user_event))
}
/// Locate a recorded conversation rollout file by its UUID string using the existing
/// paginated listing implementation. Returns `Ok(Some(path))` if found, `Ok(None)` if not present
/// or the id is invalid.
pub async fn find_conversation_path_by_id_str(
codex_home: &Path,
id_str: &str,
) -> io::Result<Option<PathBuf>> {
// Validate UUID format early.
if Uuid::parse_str(id_str).is_err() {
return Ok(None);
}
let mut root = codex_home.to_path_buf();
root.push(SESSIONS_SUBDIR);
if !root.exists() {
return Ok(None);
}
// This is safe because we know the values are valid.
#[allow(clippy::unwrap_used)]
let limit = NonZero::new(1).unwrap();
// This is safe because we know the values are valid.
#[allow(clippy::unwrap_used)]
let threads = NonZero::new(2).unwrap();
let cancel = Arc::new(AtomicBool::new(false));
let exclude: Vec<String> = Vec::new();
let compute_indices = false;
let results = file_search::run(
id_str,
limit,
&root,
exclude,
threads,
cancel,
compute_indices,
)
.map_err(|e| io::Error::other(format!("file search failed: {e}")))?;
Ok(results
.matches
.into_iter()
.next()
.map(|m| root.join(m.path)))
}
codex-rs/codex-agent/src/rollout/mod.rs
+11
View File
@@ -0,0 +1,11 @@
pub const SESSIONS_SUBDIR: &str = "sessions";
pub const ARCHIVED_SESSIONS_SUBDIR: &str = "archived_sessions";
pub mod list;
pub mod policy;
pub mod recorder;
pub use recorder::GitInfoCollector;
pub use recorder::RolloutConfig;
pub use recorder::RolloutRecorder;
pub use recorder::RolloutRecorderParams;
codex-rs/core/src/rollout/policy.rs → codex-rs/codex-agent/src/rollout/policy.rs
+5 -6
View File
@@ -1,14 +1,13 @@
use crate::protocol::EventMsg;
use crate::protocol::RolloutItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
/// Whether a rollout `item` should be persisted in rollout files.
#[inline]
pub(crate) fn is_persisted_response_item(item: &RolloutItem) -> bool {
pub fn is_persisted_response_item(item: &RolloutItem) -> bool {
match item {
RolloutItem::ResponseItem(item) => should_persist_response_item(item),
RolloutItem::EventMsg(ev) => should_persist_event_msg(ev),
// Persist Codex executive markers so we can analyze flows (e.g., compaction, API turns).
RolloutItem::Compacted(_) | RolloutItem::TurnContext(_) | RolloutItem::SessionMeta(_) => {
true
}
@@ -17,7 +16,7 @@ pub(crate) fn is_persisted_response_item(item: &RolloutItem) -> bool {
/// Whether a `ResponseItem` should be persisted in rollout files.
#[inline]
pub(crate) fn should_persist_response_item(item: &ResponseItem) -> bool {
pub fn should_persist_response_item(item: &ResponseItem) -> bool {
match item {
ResponseItem::Message { .. }
| ResponseItem::Reasoning { .. }
@@ -33,7 +32,7 @@ pub(crate) fn should_persist_response_item(item: &ResponseItem) -> bool {
/// Whether an `EventMsg` should be persisted in rollout files.
#[inline]
pub(crate) fn should_persist_event_msg(ev: &EventMsg) -> bool {
pub fn should_persist_event_msg(ev: &EventMsg) -> bool {
match ev {
EventMsg::UserMessage(_)
| EventMsg::AgentMessage(_)
codex-rs/core/src/rollout/recorder.rs → codex-rs/codex-agent/src/rollout/recorder.rs
+95 -128
View File
@@ -1,19 +1,26 @@
//! Persist Codex session rollouts (.jsonl) so sessions can be replayed or inspected later.
use std::fs;
use std::fs::File;
use std::fs::{self};
use std::io::Error as IoError;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use codex_protocol::mcp_protocol::ConversationId;
use codex_protocol::protocol::GitInfo;
use codex_protocol::protocol::InitialHistory;
use codex_protocol::protocol::ResumedHistory;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
use serde_json::Value;
use time::OffsetDateTime;
use time::format_description::FormatItem;
use time::macros::format_description;
use tokio::io::AsyncWriteExt;
use tokio::sync::mpsc;
use tokio::sync::mpsc::Sender;
use tokio::sync::mpsc::{self};
use tokio::sync::oneshot;
use tracing::info;
use tracing::warn;
@@ -23,35 +30,31 @@ use super::list::ConversationsPage;
use super::list::Cursor;
use super::list::get_conversations;
use super::policy::is_persisted_response_item;
use crate::agent_config::AgentConfig;
use crate::default_client::ORIGINATOR;
use crate::git_info::collect_git_info;
use codex_protocol::protocol::InitialHistory;
use codex_protocol::protocol::ResumedHistory;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
/// Records all [`ResponseItem`]s for a session and flushes them to disk after
/// every update.
///
/// Rollouts are recorded as JSONL and can be inspected with tools such as:
///
/// ```ignore
/// $ jq -C . ~/.codex/sessions/rollout-2025-05-07T17-24-21-5973b6c0-94b8-487b-a530-2aeb6098ae0e.jsonl
/// $ fx ~/.codex/sessions/rollout-2025-05-07T17-24-21-5973b6c0-94b8-487b-a530-2aeb6098ae0e.jsonl
/// ```
#[async_trait]
pub trait GitInfoCollector: Send + Sync {
async fn collect(&self, cwd: &Path) -> Option<GitInfo>;
}
#[derive(Clone)]
pub struct RolloutConfig {
pub codex_home: PathBuf,
pub originator: String,
pub cli_version: String,
pub git_info_collector: Option<Arc<dyn GitInfoCollector>>,
}
#[derive(Clone)]
pub struct RolloutRecorder {
tx: Sender<RolloutCmd>,
pub(crate) rollout_path: PathBuf,
rollout_path: PathBuf,
}
#[derive(Clone)]
pub enum RolloutRecorderParams {
Create {
conversation_id: ConversationId,
cwd: PathBuf,
instructions: Option<String>,
},
Resume {
@@ -61,19 +64,19 @@ pub enum RolloutRecorderParams {
enum RolloutCmd {
AddItems(Vec<RolloutItem>),
/// Ensure all prior writes are processed; respond when flushed.
Flush {
ack: oneshot::Sender<()>,
},
Shutdown {
ack: oneshot::Sender<()>,
},
Flush { ack: oneshot::Sender<()> },
Shutdown { ack: oneshot::Sender<()> },
}
impl RolloutRecorderParams {
pub fn new(conversation_id: ConversationId, instructions: Option<String>) -> Self {
pub fn new(
conversation_id: ConversationId,
cwd: PathBuf,
instructions: Option<String>,
) -> Self {
Self::Create {
conversation_id,
cwd,
instructions,
}
}
@@ -84,7 +87,6 @@ impl RolloutRecorderParams {
}
impl RolloutRecorder {
/// List conversations (rollout files) under the provided Codex home directory.
pub async fn list_conversations(
codex_home: &Path,
page_size: usize,
@@ -93,13 +95,14 @@ impl RolloutRecorder {
get_conversations(codex_home, page_size, cursor).await
}
/// Attempt to create a new [`RolloutRecorder`]. If the sessions directory
/// cannot be created or the rollout file cannot be opened we return the
/// error so the caller can decide whether to disable persistence.
pub async fn new(config: &AgentConfig, params: RolloutRecorderParams) -> std::io::Result<Self> {
let (file, rollout_path, meta) = match params {
pub async fn new(
config: &RolloutConfig,
params: RolloutRecorderParams,
) -> std::io::Result<Self> {
let (file, rollout_path, meta, cwd) = match params {
RolloutRecorderParams::Create {
conversation_id,
cwd,
instructions,
} => {
let LogFileInfo {
@@ -107,7 +110,7 @@ impl RolloutRecorder {
path,
conversation_id: session_id,
timestamp,
} = create_log_file(config, conversation_id)?;
} = create_log_file(&config.codex_home, conversation_id)?;
let timestamp_format: &[FormatItem] = format_description!(
"[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
@@ -117,18 +120,16 @@ impl RolloutRecorder {
.format(timestamp_format)
.map_err(|e| IoError::other(format!("failed to format timestamp: {e}")))?;
(
tokio::fs::File::from_std(file),
path,
Some(SessionMeta {
id: session_id,
timestamp,
cwd: config.cwd.clone(),
originator: ORIGINATOR.value.clone(),
cli_version: env!("CARGO_PKG_VERSION").to_string(),
instructions,
}),
)
let meta = SessionMeta {
id: session_id,
timestamp,
cwd: cwd.clone(),
originator: config.originator.clone(),
cli_version: config.cli_version.clone(),
instructions,
};
(tokio::fs::File::from_std(file), path, Some(meta), Some(cwd))
}
RolloutRecorderParams::Resume { path } => (
tokio::fs::OpenOptions::new()
@@ -137,31 +138,21 @@ impl RolloutRecorder {
.await?,
path,
None,
None,
),
};
// Clone the cwd for the spawned task to collect git info asynchronously
let cwd = config.cwd.clone();
// A reasonably-sized bounded channel. If the buffer fills up the send
// future will yield, which is fine – we only need to ensure we do not
// perform *blocking* I/O on the caller's thread.
let (tx, rx) = mpsc::channel::<RolloutCmd>(256);
let collector = config.git_info_collector.clone();
// Spawn a Tokio task that owns the file handle and performs async
// writes. Using `tokio::fs::File` keeps everything on the async I/O
// driver instead of blocking the runtime.
tokio::task::spawn(rollout_writer(file, rx, meta, cwd));
tokio::task::spawn(rollout_writer(file, rx, meta, cwd, collector));
Ok(Self { tx, rollout_path })
}
pub(crate) async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> {
pub async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> {
let mut filtered = Vec::new();
for item in items {
// Note that function calls may look a bit strange if they are
// "fully qualified MCP tool calls," so we could consider
// reformatting them in that case.
if is_persisted_response_item(item) {
filtered.push(item.clone());
}
@@ -175,7 +166,6 @@ impl RolloutRecorder {
.map_err(|e| IoError::other(format!("failed to queue rollout items: {e}")))
}
/// Flush all queued writes and wait until they are committed by the writer task.
pub async fn flush(&self) -> std::io::Result<()> {
let (tx, rx) = oneshot::channel();
self.tx
@@ -186,7 +176,26 @@ impl RolloutRecorder {
.map_err(|e| IoError::other(format!("failed waiting for rollout flush: {e}")))
}
pub(crate) async fn get_rollout_history(path: &Path) -> std::io::Result<InitialHistory> {
pub async fn shutdown(&self) -> std::io::Result<()> {
let (tx_done, rx_done) = oneshot::channel();
match self.tx.send(RolloutCmd::Shutdown { ack: tx_done }).await {
Ok(_) => rx_done
.await
.map_err(|e| IoError::other(format!("failed waiting for rollout shutdown: {e}"))),
Err(e) => {
warn!("failed to send rollout shutdown command: {e}");
Err(IoError::other(format!(
"failed to send rollout shutdown command: {e}"
)))
}
}
}
pub fn get_rollout_path(&self) -> PathBuf {
self.rollout_path.clone()
}
pub async fn get_rollout_history(path: &Path) -> std::io::Result<InitialHistory> {
info!("Resuming rollout from {path:?}");
let text = tokio::fs::read_to_string(path).await?;
if text.trim().is_empty() {
@@ -207,33 +216,17 @@ impl RolloutRecorder {
}
};
// Parse the rollout line structure
match serde_json::from_value::<RolloutLine>(v.clone()) {
Ok(rollout_line) => match rollout_line.item {
RolloutItem::SessionMeta(session_meta_line) => {
// Use the FIRST SessionMeta encountered in the file as the canonical
// conversation id and main session information. Keep all items intact.
if conversation_id.is_none() {
conversation_id = Some(session_meta_line.meta.id);
}
items.push(RolloutItem::SessionMeta(session_meta_line));
}
RolloutItem::ResponseItem(item) => {
items.push(RolloutItem::ResponseItem(item));
}
RolloutItem::Compacted(item) => {
items.push(RolloutItem::Compacted(item));
}
RolloutItem::TurnContext(item) => {
items.push(RolloutItem::TurnContext(item));
}
RolloutItem::EventMsg(_ev) => {
items.push(RolloutItem::EventMsg(_ev));
}
other => items.push(other),
},
Err(e) => {
warn!("failed to parse rollout line: {v:?}, error: {e}");
}
Err(e) => warn!("failed to parse rollout line: {v:?}, error: {e}"),
}
}
@@ -256,57 +249,28 @@ impl RolloutRecorder {
rollout_path: path.to_path_buf(),
}))
}
pub(crate) fn get_rollout_path(&self) -> PathBuf {
self.rollout_path.clone()
}
pub async fn shutdown(&self) -> std::io::Result<()> {
let (tx_done, rx_done) = oneshot::channel();
match self.tx.send(RolloutCmd::Shutdown { ack: tx_done }).await {
Ok(_) => rx_done
.await
.map_err(|e| IoError::other(format!("failed waiting for rollout shutdown: {e}"))),
Err(e) => {
warn!("failed to send rollout shutdown command: {e}");
Err(IoError::other(format!(
"failed to send rollout shutdown command: {e}"
)))
}
}
}
}
struct LogFileInfo {
/// Opened file handle to the rollout file.
file: File,
/// Full path to the rollout file.
path: PathBuf,
/// Session ID (also embedded in filename).
conversation_id: ConversationId,
/// Timestamp for the start of the session.
timestamp: OffsetDateTime,
}
fn create_log_file(
config: &AgentConfig,
codex_home: &Path,
conversation_id: ConversationId,
) -> std::io::Result<LogFileInfo> {
// Resolve ~/.codex/sessions/YYYY/MM/DD and create it if missing.
let timestamp = OffsetDateTime::now_local()
.map_err(|e| IoError::other(format!("failed to get local time: {e}")))?;
let mut dir = config.codex_home.clone();
let mut dir = codex_home.to_path_buf();
dir.push(SESSIONS_SUBDIR);
dir.push(timestamp.year().to_string());
dir.push(format!("{:02}", u8::from(timestamp.month())));
dir.push(format!("{:02}", timestamp.day()));
fs::create_dir_all(&dir)?;
// Custom format for YYYY-MM-DDThh-mm-ss. Use `-` instead of `:` for
// compatibility with filesystems that do not allow colons in filenames.
let format: &[FormatItem] =
format_description!("[year]-[month]-[day]T[hour]-[minute]-[second]");
let date_str = timestamp
@@ -314,7 +278,6 @@ fn create_log_file(
.map_err(|e| IoError::other(format!("failed to format timestamp: {e}")))?;
let filename = format!("rollout-{date_str}-{conversation_id}.jsonl");
let path = dir.join(filename);
let file = std::fs::OpenOptions::new()
.append(true)
@@ -333,25 +296,27 @@ async fn rollout_writer(
file: tokio::fs::File,
mut rx: mpsc::Receiver<RolloutCmd>,
mut meta: Option<SessionMeta>,
cwd: std::path::PathBuf,
cwd: Option<PathBuf>,
git_info_collector: Option<Arc<dyn GitInfoCollector>>,
) -> std::io::Result<()> {
let mut writer = JsonlWriter { file };
// If we have a meta, collect git info asynchronously and write meta first
if let Some(session_meta) = meta.take() {
let git_info = collect_git_info(&cwd).await;
let git_info =
if let (Some(provider), Some(cwd)) = (git_info_collector.as_ref(), cwd.as_ref()) {
provider.collect(cwd.as_path()).await
} else {
None
};
let session_meta_line = SessionMetaLine {
meta: session_meta,
git: git_info,
};
// Write the SessionMeta as the first item in the file, wrapped in a rollout line
writer
.write_rollout_item(RolloutItem::SessionMeta(session_meta_line))
.await?;
}
// Process rollout commands
while let Some(cmd) = rx.recv().await {
match cmd {
RolloutCmd::AddItems(items) => {
@@ -362,7 +327,6 @@ async fn rollout_writer(
}
}
RolloutCmd::Flush { ack } => {
// Ensure underlying file is flushed and then ack.
if let Err(e) = writer.file.flush().await {
let _ = ack.send(());
return Err(e);
@@ -397,11 +361,14 @@ impl JsonlWriter {
};
self.write_line(&line).await
}
async fn write_line(&mut self, item: &impl serde::Serialize) -> std::io::Result<()> {
let mut json = serde_json::to_string(item)?;
json.push('\n');
self.file.write_all(json.as_bytes()).await?;
self.file.flush().await?;
Ok(())
let mut buf = serde_json::to_vec(item)
.map_err(|e| IoError::other(format!("failed to serialise rollout line: {e}")))?;
buf.push(b'\n');
self.file
.write_all(&buf)
.await
.map_err(|e| IoError::other(format!("failed to write rollout line: {e}")))
}
}
codex-rs/codex-agent/src/services.rs
+20
View File
@@ -11,6 +11,7 @@ use crate::exec_command::ExecCommandOutput;
use crate::exec_command::ExecCommandParams;
use crate::exec_command::WriteStdinParams;
use crate::notifications::UserNotification;
use crate::rollout::RolloutRecorder;
use crate::token_data::PlanType;
use crate::unified_exec::UnifiedExecError;
use crate::unified_exec::UnifiedExecRequest;
@@ -68,6 +69,25 @@ pub trait RolloutSink: Send + Sync {
fn get_rollout_path(&self) -> PathBuf;
}
#[async_trait]
impl RolloutSink for RolloutRecorder {
async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> {
RolloutRecorder::record_items(self, items).await
}
async fn flush(&self) -> std::io::Result<()> {
RolloutRecorder::flush(self).await
}
async fn shutdown(&self) -> std::io::Result<()> {
RolloutRecorder::shutdown(self).await
}
fn get_rollout_path(&self) -> PathBuf {
RolloutRecorder::get_rollout_path(self)
}
}
/// Handles sandboxed exec orchestration, including long-running sessions.
#[async_trait]
pub trait SandboxManager: Send + Sync {
codex-rs/core/src/agent_services.rs
-24
View File
@@ -9,7 +9,6 @@ use codex_agent::services::CredentialsProvider;
use codex_agent::services::McpInterface;
use codex_agent::services::Notifier;
use codex_agent::services::ProviderAuth;
use codex_agent::services::RolloutSink;
use codex_agent::services::SandboxManager;
use codex_agent::token_data::PlanType;
use codex_protocol::mcp_protocol::AuthMode;
@@ -24,7 +23,6 @@ use crate::exec_command::ExecCommandParams;
use crate::exec_command::ExecSessionManager;
use crate::exec_command::WriteStdinParams;
use crate::mcp_connection_manager::McpConnectionManager;
use crate::rollout::RolloutRecorder;
use crate::unified_exec::UnifiedExecError;
use crate::unified_exec::UnifiedExecRequest;
use crate::unified_exec::UnifiedExecResult;
@@ -87,28 +85,6 @@ impl McpInterface for McpConnectionManager {
}
}
#[async_trait]
impl RolloutSink for RolloutRecorder {
async fn record_items(
&self,
items: &[codex_protocol::protocol::RolloutItem],
) -> std::io::Result<()> {
RolloutRecorder::record_items(self, items).await
}
async fn flush(&self) -> std::io::Result<()> {
RolloutRecorder::flush(self).await
}
async fn shutdown(&self) -> std::io::Result<()> {
RolloutRecorder::shutdown(self).await
}
fn get_rollout_path(&self) -> PathBuf {
RolloutRecorder::get_rollout_path(self)
}
}
/// Default [`SandboxManager`] used by the CLI runtime. Wraps the existing exec
/// session managers and exposes their functionality via the trait-based
/// interface so other hosts can substitute different implementations.
codex-rs/core/src/codex.rs
+14 -2
View File
@@ -115,6 +115,8 @@ use crate::protocol::TokenCountEvent;
use crate::protocol::TokenUsage;
use crate::protocol::TurnDiffEvent;
use crate::protocol::WebSearchBeginEvent;
use crate::rollout::CoreGitInfoCollector;
use crate::rollout::RolloutConfig;
use crate::rollout::RolloutRecorder;
use crate::rollout::RolloutRecorderParams;
use crate::sandbox::BackendRegistry;
@@ -385,7 +387,11 @@ impl Session {
let conversation_id = ConversationId::default();
(
conversation_id,
RolloutRecorderParams::new(conversation_id, user_instructions.clone()),
RolloutRecorderParams::new(
conversation_id,
agent_config.cwd.clone(),
user_instructions.clone(),
),
)
}
InitialHistory::Resumed(resumed_history) => (
@@ -403,7 +409,13 @@ impl Session {
// - spin up MCP connection manager
// - perform default shell discovery
// - load history metadata
let rollout_fut = RolloutRecorder::new(&agent_config, rollout_params);
let rollout_config = RolloutConfig {
codex_home: agent_config.codex_home.clone(),
originator: crate::default_client::ORIGINATOR.value.clone(),
cli_version: env!("CARGO_PKG_VERSION").to_string(),
git_info_collector: Some(Arc::new(CoreGitInfoCollector)),
};
let rollout_fut = RolloutRecorder::new(&rollout_config, rollout_params);
let mcp_fut = McpConnectionManager::new(agent_config.mcp_servers.clone());
let default_shell_fut = shell::default_user_shell();
codex-rs/core/src/conversation_history.rs
+1 -120
View File
@@ -1,120 +1 @@
use codex_protocol::models::ResponseItem;
/// Transcript of conversation history
#[derive(Debug, Clone, Default)]
pub(crate) struct ConversationHistory {
/// The oldest items are at the beginning of the vector.
items: Vec<ResponseItem>,
}
impl ConversationHistory {
pub(crate) fn new() -> Self {
Self { items: Vec::new() }
}
/// Returns a clone of the contents in the transcript.
pub(crate) fn contents(&self) -> Vec<ResponseItem> {
self.items.clone()
}
/// `items` is ordered from oldest to newest.
pub(crate) fn record_items<I>(&mut self, items: I)
where
I: IntoIterator,
I::Item: std::ops::Deref<Target = ResponseItem>,
{
for item in items {
if !is_api_message(&item) {
continue;
}
self.items.push(item.clone());
}
}
pub(crate) fn replace(&mut self, items: Vec<ResponseItem>) {
self.items = items;
}
}
/// Anything that is not a system message or "reasoning" message is considered
/// an API message.
fn is_api_message(message: &ResponseItem) -> bool {
match message {
ResponseItem::Message { role, .. } => role.as_str() != "system",
ResponseItem::FunctionCallOutput { .. }
| ResponseItem::FunctionCall { .. }
| ResponseItem::CustomToolCall { .. }
| ResponseItem::CustomToolCallOutput { .. }
| ResponseItem::LocalShellCall { .. }
| ResponseItem::Reasoning { .. }
| ResponseItem::WebSearchCall { .. } => true,
ResponseItem::Other => false,
}
}
#[cfg(test)]
mod tests {
use super::*;
use codex_protocol::models::ContentItem;
fn assistant_msg(text: &str) -> ResponseItem {
ResponseItem::Message {
id: None,
role: "assistant".to_string(),
content: vec![ContentItem::OutputText {
text: text.to_string(),
}],
}
}
fn user_msg(text: &str) -> ResponseItem {
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::OutputText {
text: text.to_string(),
}],
}
}
#[test]
fn filters_non_api_messages() {
let mut h = ConversationHistory::default();
// System message is not an API message; Other is ignored.
let system = ResponseItem::Message {
id: None,
role: "system".to_string(),
content: vec![ContentItem::OutputText {
text: "ignored".to_string(),
}],
};
h.record_items([&system, &ResponseItem::Other]);
// User and assistant should be retained.
let u = user_msg("hi");
let a = assistant_msg("hello");
h.record_items([&u, &a]);
let items = h.contents();
assert_eq!(
items,
vec![
ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![ContentItem::OutputText {
text: "hi".to_string()
}]
},
ResponseItem::Message {
id: None,
role: "assistant".to_string(),
content: vec![ContentItem::OutputText {
text: "hello".to_string()
}]
}
]
);
}
}
pub use codex_agent::ConversationHistory;
codex-rs/core/src/lib.rs
+2 -2
View File
@@ -68,14 +68,14 @@ pub mod spawn;
pub mod terminal;
mod tool_apply_patch;
pub mod turn_diff_tracker;
pub use codex_protocol::protocol::SessionMeta;
pub use rollout::ARCHIVED_SESSIONS_SUBDIR;
pub use rollout::RolloutRecorder;
pub use rollout::SESSIONS_SUBDIR;
pub use rollout::SessionMeta;
pub use rollout::find_conversation_path_by_id_str;
pub use rollout::list::ConversationItem;
pub use rollout::list::ConversationsPage;
pub use rollout::list::Cursor;
pub use rollout::list::find_conversation_path_by_id_str;
mod function_tool;
mod state;
mod tasks;
codex-rs/core/src/rollout/mod.rs
+23 -10
View File
@@ -1,16 +1,29 @@
//! Rollout module: persistence and discovery of session rollout files.
use std::path::Path;
pub const SESSIONS_SUBDIR: &str = "sessions";
pub const ARCHIVED_SESSIONS_SUBDIR: &str = "archived_sessions";
use async_trait::async_trait;
use codex_agent::rollout::GitInfoCollector as SharedGitInfoCollector;
use codex_protocol::protocol::GitInfo;
pub mod list;
pub(crate) mod policy;
pub mod recorder;
pub use codex_agent::rollout::ARCHIVED_SESSIONS_SUBDIR;
pub use codex_agent::rollout::RolloutConfig;
pub use codex_agent::rollout::RolloutRecorder;
pub use codex_agent::rollout::RolloutRecorderParams;
pub use codex_agent::rollout::SESSIONS_SUBDIR;
pub use codex_protocol::protocol::SessionMeta;
pub use list::find_conversation_path_by_id_str;
pub use recorder::RolloutRecorder;
pub use recorder::RolloutRecorderParams;
pub mod list {
pub use codex_agent::rollout::list::*;
}
#[cfg(test)]
pub mod tests;
use crate::git_info::collect_git_info;
pub struct CoreGitInfoCollector;
#[async_trait]
impl SharedGitInfoCollector for CoreGitInfoCollector {
async fn collect(&self, cwd: &Path) -> Option<GitInfo> {
collect_git_info(cwd).await
}
}