feat: record messages from user in ~/.codex/history.jsonl

This commit is contained in:
Michael Bolin
2025-05-15 12:22:46 -07:00
parent 5fc9fc3e3e
commit 970e2cba4d
12 changed files with 535 additions and 8 deletions

11
codex-rs/Cargo.lock generated
View File

@@ -523,6 +523,7 @@ dependencies = [
"env-flags",
"eventsource-stream",
"fs-err",
"fs2",
"futures",
"landlock",
"libc",
@@ -1244,6 +1245,16 @@ dependencies = [
"autocfg",
]
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "futures"
version = "0.3.31"

View File

@@ -23,7 +23,9 @@ This folder is the root of a Cargo workspace. It contains quite a bit of experim
## Config
The CLI can be configured via `~/.codex/config.toml`. It supports the following options:
The CLI can be configured via a file named `config.toml`. By default, configuration is read from `~/.codex/config.toml`, though the `CODEX_HOME` environment variable can be used to specify a directory other than `~/.codex`.
The `config.toml` file supports the following options:
### model
@@ -297,6 +299,17 @@ To have Codex use this script for notifications, you would configure it via `not
notify = ["python3", "/Users/mbolin/.codex/notify.py"]
```
### history
By default, Codex CLI records messages sent to the model in `$CODEX_HOME/history.jsonl`. Note that on UNIX, the file permissions are set to `o600`, so it should only be readable and writable by the owner.
To disable this behavior, configure `[history]` as follows:
```toml
[history]
persistence = "none" # "save-all" is the default value
```
### project_doc_max_bytes
Maximum number of bytes to read from an `AGENTS.md` file to include in the instructions sent with the first turn of a session. Defaults to 32 KiB.

View File

@@ -20,6 +20,7 @@ codex-mcp-client = { path = "../mcp-client" }
dirs = "6"
env-flags = "0.1.1"
eventsource-stream = "0.2.3"
fs2 = "0.4.3"
fs-err = "3.1.0"
futures = "0.3"
mcp-types = { path = "../mcp-types" }

View File

@@ -48,6 +48,7 @@ use crate::flags::OPENAI_STREAM_MAX_RETRIES;
use crate::mcp_connection_manager::McpConnectionManager;
use crate::mcp_connection_manager::try_parse_fully_qualified_tool_name;
use crate::mcp_tool_call::handle_mcp_tool_call;
use crate::message_history;
use crate::models::ContentItem;
use crate::models::FunctionCallOutputPayload;
use crate::models::ReasoningItemReasoningSummary;
@@ -110,6 +111,7 @@ impl Codex {
cwd: config.cwd.clone(),
};
let config = Arc::new(config);
tokio::spawn(submission_loop(config, rx_sub, tx_event, ctrl_c));
let codex = Codex {
next_id: AtomicU64::new(0),
@@ -483,11 +485,14 @@ impl AgentTask {
}
async fn submission_loop(
config: Config,
config: Arc<Config>,
rx_sub: Receiver<Submission>,
tx_event: Sender<Event>,
ctrl_c: Arc<Notify>,
) {
// Generate a unique ID for the lifetime of this Codex session.
let session_id = Uuid::new_v4();
let mut sess: Option<Arc<Session>> = None;
// shorthand - send an event when there is no active session
let send_no_session_event = |sub_id: String| async {
@@ -608,7 +613,9 @@ async fn submission_loop(
// Attempt to create a RolloutRecorder *before* moving the
// `instructions` value into the Session struct.
let session_id = Uuid::new_v4();
// TODO: if ConfigureSession is sent twice, we will create an
// overlapping rollout file. Consider passing RolloutRecorder
// from above.
let rollout_recorder =
match RolloutRecorder::new(&config, session_id, instructions.clone()).await {
Ok(r) => Some(r),
@@ -633,10 +640,41 @@ async fn submission_loop(
rollout: Mutex::new(rollout_recorder),
}));
// Gather history metadata for SessionConfiguredEvent.
let config_clone = config.clone();
let (history_log_id, history_entry_count) =
tokio::task::spawn_blocking(move || {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let mut path = config_clone.codex_home.clone();
path.push("history.jsonl");
let log_id = std::fs::metadata(&path).map(|m| m.ino()).unwrap_or(0);
let count = crate::message_history::read_history(&config_clone)
.map(|v| v.len())
.unwrap_or(0);
(log_id, count)
}
#[cfg(not(unix))]
{
let count = crate::message_history::read_history(&config_clone)
.map(|v| v.len())
.unwrap_or(0);
(0, count)
}
})
.await
.unwrap_or((0, 0));
// ack
let events = std::iter::once(Event {
id: sub.id.clone(),
msg: EventMsg::SessionConfigured(SessionConfiguredEvent { session_id, model }),
msg: EventMsg::SessionConfigured(SessionConfiguredEvent {
session_id,
model,
history_log_id,
history_entry_count,
}),
})
.chain(mcp_connection_errors.into_iter());
for event in events {
@@ -691,6 +729,47 @@ async fn submission_loop(
other => sess.notify_approval(&id, other),
}
}
Op::AddToHistory { text } => {
// Perform blocking I/O inside a blocking task so we do not
// stall the async runtime.
let id = session_id;
let config = config.clone();
tokio::task::spawn_blocking(move || {
if let Err(e) = message_history::append_entry(&text, &id, &config) {
tracing::warn!("failed to append to message history: {e}");
}
});
}
Op::GetHistoryEntryRequest { offset, log_id } => {
let config = config.clone();
let tx_event = tx_event.clone();
let sub_id = sub.id.clone();
tokio::spawn(async move {
// Run lookup in blocking thread because it does file IO + locking.
let entry_opt = tokio::task::spawn_blocking(move || {
crate::message_history::lookup(log_id, offset, &config)
})
.await
.unwrap_or(None);
let event = Event {
id: sub_id,
msg: EventMsg::GetHistoryEntryResponse(
crate::protocol::GetHistoryEntryResponseEvent {
offset,
log_id,
entry: entry_opt,
},
),
};
if let Err(e) = tx_event.send(event).await {
tracing::warn!("failed to send GetHistoryEntryResponse event: {e}");
}
});
}
}
}
debug!("Agent loop exited");

View File

@@ -81,6 +81,30 @@ pub struct Config {
/// Directory containing all Codex state (defaults to `~/.codex` but can be
/// overridden by the `CODEX_HOME` environment variable).
pub codex_home: PathBuf,
/// Settings that govern if and what will be written to `~/.codex/history.jsonl`.
pub history: History,
}
/// Settings that govern if and what will be written to `~/.codex/history.jsonl`.
#[derive(Deserialize, Debug, Clone, PartialEq, Default)]
pub struct History {
/// If true, history entries will not be written to disk.
pub persistence: HistoryPersistence,
/// If set, the maximum size of the history file in bytes.
/// TODO(mbolin): Not currently honored.
pub max_bytes: Option<usize>,
}
#[derive(Deserialize, Debug, Clone, PartialEq, Default)]
#[serde(rename_all = "kebab-case")]
pub enum HistoryPersistence {
/// Save all history entries to disk.
#[default]
SaveAll,
/// Do not write history to disk.
None,
}
/// Base config deserialized from ~/.codex/config.toml.
@@ -130,6 +154,10 @@ pub struct ConfigToml {
/// Named profiles to facilitate switching between different configurations.
#[serde(default)]
pub profiles: HashMap<String, ConfigProfile>,
/// Settings that govern if and what will be written to `~/.codex/history.jsonl`.
#[serde(default)]
pub history: Option<History>,
}
impl ConfigToml {
@@ -297,6 +325,8 @@ impl Config {
}
};
let history = cfg.history.unwrap_or_default();
let config = Self {
model: model
.or(config_profile.model)
@@ -320,6 +350,7 @@ impl Config {
model_providers,
project_doc_max_bytes: cfg.project_doc_max_bytes.unwrap_or(PROJECT_DOC_MAX_BYTES),
codex_home,
history,
};
Ok(config)
}
@@ -468,6 +499,40 @@ mod tests {
);
}
#[test]
fn test_toml_parsing() {
let history_with_persistence = r#"
[history]
persistence = "save-all"
"#;
let history_with_persistence_cfg: ConfigToml =
toml::from_str::<ConfigToml>(history_with_persistence)
.expect("TOML deserialization should succeed");
assert_eq!(
Some(History {
persistence: HistoryPersistence::SaveAll,
max_bytes: None,
}),
history_with_persistence_cfg.history
);
let history_no_persistence = r#"
[history]
persistence = "none"
"#;
let history_no_persistence_cfg: ConfigToml =
toml::from_str::<ConfigToml>(history_no_persistence)
.expect("TOML deserialization should succeed");
assert_eq!(
Some(History {
persistence: HistoryPersistence::None,
max_bytes: None,
}),
history_no_persistence_cfg.history
);
}
/// Deserializing a TOML string containing an *invalid* permission should
/// fail with a helpful error rather than silently defaulting or
/// succeeding.
@@ -620,6 +685,7 @@ disable_response_storage = true
model_providers: fixture.model_provider_map.clone(),
project_doc_max_bytes: PROJECT_DOC_MAX_BYTES,
codex_home: fixture.codex_home(),
history: History::default(),
},
o3_profile_config
);
@@ -654,6 +720,7 @@ disable_response_storage = true
model_providers: fixture.model_provider_map.clone(),
project_doc_max_bytes: PROJECT_DOC_MAX_BYTES,
codex_home: fixture.codex_home(),
history: History::default(),
};
assert_eq!(expected_gpt3_profile_config, gpt3_profile_config);
@@ -703,6 +770,7 @@ disable_response_storage = true
model_providers: fixture.model_provider_map.clone(),
project_doc_max_bytes: PROJECT_DOC_MAX_BYTES,
codex_home: fixture.codex_home(),
history: History::default(),
};
assert_eq!(expected_zdr_profile_config, zdr_profile_config);

View File

@@ -24,6 +24,7 @@ pub mod landlock;
mod mcp_connection_manager;
pub mod mcp_server_config;
mod mcp_tool_call;
mod message_history;
mod model_provider_info;
pub use model_provider_info::ModelProviderInfo;
pub use model_provider_info::WireApi;

View File

@@ -0,0 +1,308 @@
//! Persistence layer for the global, append-only *message history* file.
//!
//! The history is stored at `~/.codex/history.jsonl` with **one JSON object per
//! line** so that it can be efficiently appended to and parsed with standard
//! JSON-Lines tooling. Each record has the following schema:
//!
//! ````text
//! {"session_id":"<uuid>","ts":<unix_seconds>,"text":"<message>"}
//! ````
//!
//! To minimise the chance of interleaved writes when multiple processes are
//! appending concurrently, callers should *prepare the full line* (record +
//! trailing `\n`) and write it with a **single `write(2)` system call** while
//! the file descriptor is opened with the `O_APPEND` flag. POSIX guarantees
//! that writes up to `PIPE_BUF` bytes are atomic in that case.
use std::fs::OpenOptions;
use std::io::BufRead;
use std::io::BufReader;
use std::io::Write;
use serde::Deserialize;
use serde::Serialize;
use std::time::Duration;
use uuid::Uuid;
use crate::config::Config;
use crate::config::HistoryPersistence;
#[cfg(unix)]
use std::os::unix::fs::OpenOptionsExt;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
/// Filename that stores the message history inside `~/.codex`.
const HISTORY_FILENAME: &str = "history.jsonl";
const MAX_RETRIES: usize = 10;
const RETRY_SLEEP: Duration = Duration::from_millis(100);
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct HistoryEntry {
pub session_id: String,
pub ts: u64,
pub text: String,
}
/// Append a `text` entry associated with `session_id` to the history file.
///
/// This uses a *single* `write(2)` on a file opened with the `O_APPEND` flag.
/// POSIX guarantees that such writes up to `PIPE_BUF` bytes are atomic no
/// other process can interleave its own data within the same call. Because
/// each history record is tiny (≪ `PIPE_BUF`) we can rely on this property to
/// avoid additional synchronisation primitives or file locking.
///
/// Owing to the blocking nature of the syscall the function itself is kept
/// **synchronous**; callers running in an async context should wrap it in
/// `tokio::task::spawn_blocking` so the write does not obstruct the async
/// scheduler.
pub(crate) fn append_entry(text: &str, session_id: &Uuid, config: &Config) -> std::io::Result<()> {
match config.history.persistence {
HistoryPersistence::SaveAll => {
// Save everything: proceed.
}
HistoryPersistence::None => {
// No history persistence requested.
return Ok(());
}
}
// TODO: check `text` for sensitive patterns
// Resolve `~/.codex/history.jsonl` and ensure the parent directory exists.
let codex_home = config.codex_home.clone();
std::fs::create_dir_all(&codex_home)?;
let mut history_file = codex_home;
history_file.push(HISTORY_FILENAME);
// Compute timestamp (seconds since the Unix epoch).
let ts = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::Other,
format!("system clock before Unix epoch: {e}"),
)
})?
.as_secs();
// Construct the JSON line first so we can write it in a single syscall.
let entry = HistoryEntry {
session_id: session_id.to_string(),
ts,
text: text.to_string(),
};
let mut line = serde_json::to_string(&entry).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::Other,
format!("failed to serialise history entry: {e}"),
)
})?;
line.push('\n');
// Open in append-only mode so concurrent writers do not overwrite each
// other. Using O_APPEND ensures that the kernel appends each write atomically.
// We also open the file for reading so that `fs2` locking works on all
// platforms.
let mut options = OpenOptions::new();
options.append(true).read(true).create(true);
#[cfg(unix)]
{
// Ensure file is created with permissions 0o600.
options.mode(0o600);
}
let mut file = options.open(&history_file)?;
// For files that already existed, adjust permissions if necessary.
ensure_owner_only_permissions(&history_file)?;
// Acquire an exclusive advisory lock with a bounded retry loop so that we
// do not block indefinitely if another process keeps the file locked.
acquire_exclusive_lock_with_retry(&file)?;
// TODO: honor `config.history.max_size` and truncate the file if necessary.
// Apparently Bash only does this check on startup, so over the course of
// execution, it can exceed max_size. This seems like a good tradeoff, as
// it keeps the amend logic simple.
file.write_all(line.as_bytes())?;
file.flush()?;
// The lock is automatically released when `file` goes out of scope.
Ok(())
}
/// Attempt to acquire an exclusive advisory lock on `file`, retrying up to 10
/// times (100 ms apart) if the lock is currently held by another process. This
/// prevents a potential indefinite wait while still giving other writers some
/// time to finish their operation.
fn acquire_exclusive_lock_with_retry(file: &std::fs::File) -> std::io::Result<()> {
for _ in 0..MAX_RETRIES {
match fs2::FileExt::try_lock_exclusive(file) {
Ok(()) => return Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
std::thread::sleep(RETRY_SLEEP);
}
Err(e) => return Err(e),
}
}
Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"could not acquire exclusive lock on history file after multiple attempts",
))
}
/// Read the full contents of the history file and return a vector containing
/// every line (entry) as a `String`. If the history file does not exist yet,
/// an empty vector is returned.
///
/// The function acquires a shared advisory lock to avoid reading while another
/// process is writing, using the same bounded retry strategy as
/// `append_entry`.
pub(crate) fn read_history(config: &Config) -> std::io::Result<Vec<String>> {
match config.history.persistence {
HistoryPersistence::SaveAll => { /* proceed */ }
HistoryPersistence::None => return Ok(Vec::new()),
}
let mut path = config.codex_home.clone();
path.push(HISTORY_FILENAME);
let file = match OpenOptions::new().read(true).open(&path) {
Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// History file does not exist yet.
return Ok(Vec::new());
}
Err(e) => return Err(e),
};
// Ensure the file has the correct permissions before reading.
ensure_owner_only_permissions(&path)?;
// Acquire a shared lock so that writers (who take an exclusive lock) are
// blocked, ensuring we do not read partially-written data.
acquire_shared_lock_with_retry(&file)?;
let reader = BufReader::new(&file);
let mut lines = Vec::new();
for line_res in reader.lines() {
lines.push(line_res?);
}
Ok(lines)
}
// ---------------------------------------------------------------------------
// Random access helper
// ---------------------------------------------------------------------------
/// Given a `log_id` (on Unix this is the file's inode number) and a zero-based
/// `offset`, return the corresponding `HistoryEntry` if the identifier matches
/// the current history file **and** the requested offset exists. Any I/O or
/// parsing errors are logged and result in `None`.
#[cfg(unix)]
pub(crate) fn lookup(log_id: u64, offset: usize, config: &Config) -> Option<HistoryEntry> {
use std::os::unix::fs::MetadataExt;
let mut path = config.codex_home.clone();
path.push(HISTORY_FILENAME);
let metadata = match std::fs::metadata(&path) {
Ok(m) => m,
Err(e) => {
tracing::warn!(error = %e, "failed to stat history file");
return None;
}
};
if metadata.ino() != log_id {
return None;
}
// Open & lock file for reading.
if let Err(e) = ensure_owner_only_permissions(&path) {
tracing::warn!(error = %e, "failed to set history file permissions");
return None;
}
let file = match OpenOptions::new().read(true).open(&path) {
Ok(f) => f,
Err(e) => {
tracing::warn!(error = %e, "failed to open history file");
return None;
}
};
if let Err(e) = acquire_shared_lock_with_retry(&file) {
tracing::warn!(error = %e, "failed to acquire shared lock on history file");
return None;
}
let reader = BufReader::new(&file);
for (idx, line_res) in reader.lines().enumerate() {
let line = match line_res {
Ok(l) => l,
Err(e) => {
tracing::warn!(error = %e, "failed to read line from history file");
return None;
}
};
if idx == offset {
match serde_json::from_str::<HistoryEntry>(&line) {
Ok(entry) => return Some(entry),
Err(e) => {
tracing::warn!(error = %e, "failed to parse history entry");
return None;
}
}
}
}
None
}
/// Fallback stub for non-Unix systems: currently always returns `None`.
#[cfg(not(unix))]
pub(crate) fn lookup(log_id: u64, offset: usize, config: &Config) -> Option<HistoryEntry> {
let _ = (log_id, offset, config);
None
}
fn acquire_shared_lock_with_retry(file: &std::fs::File) -> std::io::Result<()> {
for _ in 0..MAX_RETRIES {
match fs2::FileExt::try_lock_shared(file) {
Ok(()) => return Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
std::thread::sleep(RETRY_SLEEP);
}
Err(e) => return Err(e),
}
}
Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"could not acquire shared lock on history file after multiple attempts",
))
}
/// On Unix systems ensure the file permissions are `0o600` (rw-------). On
/// non-Unix platforms this function is a no-op. If the permissions cannot be
/// changed the error is propagated to the caller.
fn ensure_owner_only_permissions<P: AsRef<std::path::Path>>(path: P) -> std::io::Result<()> {
#[cfg(unix)]
{
use std::fs;
let metadata = fs::metadata(&path)?;
let current_mode = metadata.permissions().mode() & 0o777;
if current_mode != 0o600 {
let mut perms = metadata.permissions();
perms.set_mode(0o600);
fs::set_permissions(&path, perms)?;
}
}
// On non-Unix simply succeed.
Ok(())
}

View File

@@ -12,6 +12,7 @@ use serde::Deserialize;
use serde::Serialize;
use uuid::Uuid;
use crate::message_history::HistoryEntry;
use crate::model_provider_info::ModelProviderInfo;
/// Submission Queue Entry - requests from user
@@ -88,6 +89,18 @@ pub enum Op {
/// The user's decision in response to the request.
decision: ReviewDecision,
},
/// Append an entry to the persistent cross-session message history.
///
/// Note the entry is not guaranteed to be logged if the user has
/// history disabled, it matches the list of "sensitive" patterns, etc.
AddToHistory {
/// The message text to be stored.
text: String,
},
/// Request a single history entry identified by `log_id` + `offset`.
GetHistoryEntryRequest { offset: usize, log_id: u64 },
}
/// Determines how liberally commands are autoapproved by the system.
@@ -340,6 +353,9 @@ pub enum EventMsg {
/// Notification that a patch application has finished.
PatchApplyEnd(PatchApplyEndEvent),
/// Response to GetHistoryEntryRequest.
GetHistoryEntryResponse(GetHistoryEntryResponseEvent),
}
// Individual event payload types matching each `EventMsg` variant.
@@ -452,6 +468,15 @@ pub struct PatchApplyEndEvent {
pub success: bool,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct GetHistoryEntryResponseEvent {
pub offset: usize,
pub log_id: u64,
/// The entry at the requested offset, if available and parseable.
#[serde(skip_serializing_if = "Option::is_none")]
pub entry: Option<HistoryEntry>,
}
#[derive(Debug, Default, Clone, Deserialize, Serialize)]
pub struct SessionConfiguredEvent {
/// Unique id for this session.
@@ -459,6 +484,12 @@ pub struct SessionConfiguredEvent {
/// Tell the client what model is being queried.
pub model: String,
/// Identifier of the history log file (inode on Unix, 0 otherwise).
pub history_log_id: u64,
/// Current number of entries in the history log.
pub history_entry_count: usize,
}
/// User's decision in response to an ExecApprovalRequest.
@@ -519,12 +550,14 @@ mod tests {
msg: EventMsg::SessionConfigured(SessionConfiguredEvent {
session_id,
model: "o4-mini".to_string(),
history_log_id: 0,
history_entry_count: 0,
}),
};
let serialized = serde_json::to_string(&event).unwrap();
assert_eq!(
serialized,
r#"{"id":"1234","msg":{"type":"session_configured","session_id":"67e55044-10b1-426f-9247-bb680e5fe0c8","model":"o4-mini"}}"#
r#"{"id":"1234","msg":{"type":"session_configured","session_id":"67e55044-10b1-426f-9247-bb680e5fe0c8","model":"o4-mini","history_log_id":0,"history_entry_count":0}}"#
);
}
}

View File

@@ -375,9 +375,12 @@ impl EventProcessor {
println!("thinking: {}", agent_reasoning_event.text);
}
EventMsg::SessionConfigured(session_configured_event) => {
let SessionConfiguredEvent { session_id, model } = session_configured_event;
let SessionConfiguredEvent { session_id, model, .. } = session_configured_event;
println!("session {session_id} with model {model}");
}
EventMsg::GetHistoryEntryResponse(_) => {
// Currently ignored in exec output.
}
}
}
}

View File

@@ -166,7 +166,8 @@ pub async fn run_codex_tool_session(
| EventMsg::ExecCommandEnd(_)
| EventMsg::BackgroundEvent(_)
| EventMsg::PatchApplyBegin(_)
| EventMsg::PatchApplyEnd(_) => {
| EventMsg::PatchApplyEnd(_)
| EventMsg::GetHistoryEntryResponse(_) => {
// For now, we do not do anything extra for these
// events. Note that
// send(codex_event_to_notification(&event)) above has

View File

@@ -195,6 +195,15 @@ impl ChatWidget<'_> {
tracing::error!("failed to send message: {e}");
});
// Persist the text to cross-session message history.
if !text.is_empty() {
self.codex_op_tx
.send(Op::AddToHistory { text: text.clone() })
.unwrap_or_else(|e| {
tracing::error!("failed to send AddHistory op: {e}");
});
}
// Only show text portion in conversation history for now.
if !text.is_empty() {
self.conversation_history.add_user_message(text);

View File

@@ -100,7 +100,7 @@ impl HistoryCell {
event: SessionConfiguredEvent,
is_first_event: bool,
) -> Self {
let SessionConfiguredEvent { model, session_id } = event;
let SessionConfiguredEvent { model, session_id, .. } = event;
if is_first_event {
let mut lines: Vec<Line<'static>> = vec![
Line::from(vec![