Merge 0a53d55d9d into sapling-pr-archive-bolinfest

This commit is contained in:
Michael Bolin
2026-05-04 12:50:21 -07:00
committed by GitHub
173 changed files with 4891 additions and 1183 deletions
+7
View File
@@ -29,6 +29,7 @@ pub use codex_app_server::in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY;
pub use codex_app_server::in_process::InProcessServerEvent;
use codex_app_server::in_process::InProcessStartArgs;
use codex_app_server::in_process::LogDbLayer;
pub use codex_app_server::in_process::StateDbHandle;
use codex_app_server_protocol::ClientInfo;
use codex_app_server_protocol::ClientNotification;
use codex_app_server_protocol::ClientRequest;
@@ -345,6 +346,8 @@ pub struct InProcessClientStartArgs {
pub feedback: CodexFeedback,
/// SQLite tracing layer used to flush recently emitted logs before feedback upload.
pub log_db: Option<LogDbLayer>,
/// Process-wide SQLite state handle shared with the embedded app-server.
pub state_db: Option<StateDbHandle>,
/// Environment manager used by core execution and filesystem operations.
pub environment_manager: Arc<EnvironmentManager>,
/// Startup warnings emitted after initialize succeeds.
@@ -407,6 +410,7 @@ impl InProcessClientStartArgs {
thread_config_loader,
feedback: self.feedback,
log_db: self.log_db,
state_db: self.state_db,
environment_manager: self.environment_manager,
config_warnings: self.config_warnings,
session_source: self.session_source,
@@ -987,6 +991,7 @@ mod tests {
cloud_requirements: CloudRequirementsLoader::default(),
feedback: CodexFeedback::new(),
log_db: None,
state_db: None,
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source,
@@ -2062,6 +2067,7 @@ mod tests {
cloud_requirements: CloudRequirementsLoader::default(),
feedback: CodexFeedback::new(),
log_db: None,
state_db: None,
environment_manager: environment_manager.clone(),
config_warnings: Vec::new(),
session_source: SessionSource::Exec,
@@ -2102,6 +2108,7 @@ mod tests {
cloud_requirements: CloudRequirementsLoader::default(),
feedback: CodexFeedback::new(),
log_db: None,
state_db: None,
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source: SessionSource::Exec,
@@ -3631,8 +3631,9 @@ pub struct ThreadStartParams {
#[experimental("thread/start.experimentalRawEvents")]
#[serde(default)]
pub experimental_raw_events: bool,
/// If true, persist additional rollout EventMsg variants required to
/// reconstruct a richer thread history on resume/fork/read.
/// If true, persist additional EventMsg variants to the rollout file.
/// However, `thread/read`, `thread/resume`, and `thread/fork` still only
/// return the limited form of thread history for scalability reasons.
#[experimental("thread/start.persistFullHistory")]
#[serde(default)]
pub persist_extended_history: bool,
@@ -3762,8 +3763,9 @@ pub struct ThreadResumeParams {
#[experimental("thread/resume.excludeTurns")]
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub exclude_turns: bool,
/// If true, persist additional rollout EventMsg variants required to
/// reconstruct a richer thread history on subsequent resume/fork/read.
/// If true, persist additional EventMsg variants to the rollout file.
/// However, `thread/read`, `thread/resume`, and `thread/fork` still only
/// return the limited form of thread history for scalability reasons.
#[experimental("thread/resume.persistFullHistory")]
#[serde(default)]
pub persist_extended_history: bool,
@@ -3867,8 +3869,9 @@ pub struct ThreadForkParams {
#[experimental("thread/fork.excludeTurns")]
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub exclude_turns: bool,
/// If true, persist additional rollout EventMsg variants required to
/// reconstruct a richer thread history on subsequent resume/fork/read.
/// If true, persist additional EventMsg variants to the rollout file.
/// However, `thread/read`, `thread/resume`, and `thread/fork` still only
/// return the limited form of thread history for scalability reasons.
#[experimental("thread/fork.persistFullHistory")]
#[serde(default)]
pub persist_extended_history: bool,
+1 -3
View File
@@ -307,8 +307,6 @@ To branch from a stored session, call `thread/fork` with the `thread.id`. This c
Like `thread/resume`, experimental clients can pass `excludeTurns: true` to `thread/fork` to return only thread metadata in `thread.turns` and page history with `thread/turns/list`. In that mode the server skips replaying restored `thread/tokenUsage/updated`, which keeps the fork path from rebuilding turns just to attribute historical usage.
Experimental API: `thread/start`, `thread/resume`, and `thread/fork` accept `persistExtendedHistory: true` to persist a richer subset of ThreadItems for non-lossy history when calling `thread/read`, `thread/resume`, and `thread/fork` later. This does not backfill events that were not persisted previously.
### Example: List threads (with pagination & filters)
`thread/list` lets you render a history UI. Results default to `createdAt` (newest first) descending. Pass any combination of:
@@ -403,7 +401,7 @@ Later, after the idle unload timeout:
### Example: Read a thread
Use `thread/read` to fetch a stored thread by id without resuming it. Pass `includeTurns` when you want the full rollout history loaded into `thread.turns`. The returned thread includes `agentNickname` and `agentRole` for AgentControl-spawned thread sub-agents when available.
Use `thread/read` to fetch a stored thread by id without resuming it. Pass `includeTurns` when you want thread history loaded into `thread.turns`. The returned thread includes `agentNickname` and `agentRole` for AgentControl-spawned thread sub-agents when available.
```json
{ "method": "thread/read", "id": 22, "params": { "threadId": "thr_123" } }
@@ -2,6 +2,7 @@ use crate::error_code::internal_error;
use crate::error_code::invalid_request;
use crate::outgoing_message::ClientRequestResult;
use crate::outgoing_message::ThreadScopedOutgoingMessageSender;
use crate::request_processors::build_api_turns_from_rollout_items;
use crate::request_processors::read_rollout_items_from_rollout;
use crate::request_processors::read_summary_from_rollout;
use crate::request_processors::summary_to_thread;
@@ -81,7 +82,6 @@ use codex_app_server_protocol::TurnStartedNotification;
use codex_app_server_protocol::TurnStatus;
use codex_app_server_protocol::WarningNotification;
use codex_app_server_protocol::build_item_from_guardian_event;
use codex_app_server_protocol::build_turns_from_rollout_items;
use codex_app_server_protocol::guardian_auto_approval_review_notification;
use codex_app_server_protocol::item_event_to_server_notification;
use codex_core::CodexThread;
@@ -1179,7 +1179,7 @@ pub(crate) async fn apply_bespoke_event_handling(
let mut thread = summary_to_thread(summary, &fallback_cwd);
match read_rollout_items_from_rollout(rollout_path.as_path()).await {
Ok(items) => {
thread.turns = build_turns_from_rollout_items(&items);
thread.turns = build_api_turns_from_rollout_items(&items);
thread.status = thread_watch_manager
.loaded_status_for_thread(&thread.id)
.await;
+32 -7
View File
@@ -86,6 +86,7 @@ use codex_exec_server::EnvironmentManager;
use codex_feedback::CodexFeedback;
use codex_login::AuthManager;
use codex_protocol::protocol::SessionSource;
pub use codex_rollout::StateDbHandle;
pub use codex_state::log_db::LogDbLayer;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
@@ -128,6 +129,8 @@ pub struct InProcessStartArgs {
pub feedback: CodexFeedback,
/// SQLite tracing layer used to flush recently emitted logs before feedback upload.
pub log_db: Option<LogDbLayer>,
/// Process-wide SQLite state handle shared with embedded app-server consumers.
pub state_db: Option<StateDbHandle>,
/// Environment manager used by core execution and filesystem operations.
pub environment_manager: Arc<EnvironmentManager>,
/// Startup warnings emitted after initialize succeeds.
@@ -253,6 +256,8 @@ pub struct InProcessClientHandle {
client: InProcessClientSender,
event_rx: mpsc::Receiver<InProcessServerEvent>,
runtime_handle: tokio::task::JoinHandle<()>,
#[cfg(test)]
_test_codex_home: Option<tempfile::TempDir>,
}
impl InProcessClientHandle {
@@ -421,6 +426,7 @@ fn start_uninitialized(args: InProcessStartArgs) -> InProcessClientHandle {
environment_manager: args.environment_manager,
feedback: args.feedback,
log_db: args.log_db,
state_db: args.state_db,
config_warnings: args.config_warnings,
session_source: args.session_source,
auth_manager,
@@ -720,6 +726,8 @@ fn start_uninitialized(args: InProcessStartArgs) -> InProcessClientHandle {
client: InProcessClientSender { client_tx },
event_rx,
runtime_handle,
#[cfg(test)]
_test_codex_home: None,
}
}
@@ -741,13 +749,22 @@ mod tests {
use codex_app_server_protocol::TurnStatus;
use codex_core::config::ConfigBuilder;
use pretty_assertions::assert_eq;
use std::path::Path;
use tempfile::TempDir;
async fn build_test_config() -> Config {
match ConfigBuilder::default().build().await {
async fn build_test_config(codex_home: &Path) -> Config {
match ConfigBuilder::default()
.codex_home(codex_home.to_path_buf())
.build()
.await
{
Ok(config) => config,
Err(_) => Config::load_default_with_cli_overrides(Vec::new())
.await
.expect("default config should load"),
Err(_) => Config::load_default_with_cli_overrides_for_codex_home(
codex_home.to_path_buf(),
Vec::new(),
)
.await
.expect("default config should load"),
}
}
@@ -755,9 +772,14 @@ mod tests {
session_source: SessionSource,
channel_capacity: usize,
) -> InProcessClientHandle {
let codex_home = TempDir::new().expect("temp dir");
let config = Arc::new(build_test_config(codex_home.path()).await);
let state_db = codex_rollout::state_db::try_init(config.as_ref())
.await
.expect("state db should initialize for in-process test");
let args = InProcessStartArgs {
arg0_paths: Arg0DispatchPaths::default(),
config: Arc::new(build_test_config().await),
config,
cli_overrides: Vec::new(),
loader_overrides: LoaderOverrides::default(),
strict_config: false,
@@ -765,6 +787,7 @@ mod tests {
thread_config_loader: Arc::new(codex_config::NoopThreadConfigLoader),
feedback: CodexFeedback::new(),
log_db: None,
state_db: Some(state_db),
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source,
@@ -779,7 +802,9 @@ mod tests {
},
channel_capacity,
};
start(args).await.expect("in-process runtime should start")
let mut client = start(args).await.expect("in-process runtime should start");
client._test_codex_home = Some(codex_home);
client
}
async fn start_test_client(session_source: SessionSource) -> InProcessClientHandle {
+57 -32
View File
@@ -54,6 +54,7 @@ use codex_exec_server::EnvironmentManager;
use codex_exec_server::ExecServerRuntimePaths;
use codex_feedback::CodexFeedback;
use codex_protocol::protocol::SessionSource;
use codex_rollout::state_db as rollout_state_db;
use codex_state::log_db;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
@@ -455,23 +456,6 @@ pub async fn run_main_with_transport_options(
.await
{
Ok(config) => {
let effective_toml = config.config_layer_stack.effective_config();
match effective_toml.try_into() {
Ok(config_toml) => {
if let Err(err) = codex_core::personality_migration::maybe_migrate_personality(
&config.codex_home,
&config_toml,
)
.await
{
warn!(error = %err, "Failed to run personality migration");
}
}
Err(err) => {
warn!(error = %err, "Failed to deserialize config for personality migration");
}
}
let discovered_thread_config_loader = configured_thread_config_loader(&config);
config_manager
.replace_thread_config_loader(Arc::clone(&discovered_thread_config_loader));
@@ -485,23 +469,70 @@ pub async fn run_main_with_transport_options(
}
};
let mut config_warnings = Vec::new();
let config = match config_manager
let (mut config, should_run_personality_migration) = match config_manager
.load_latest_config(/*fallback_cwd*/ None)
.await
{
Ok(config) => config,
Ok(config) => (config, true),
Err(err) => {
let message = config_warning_from_error("Invalid configuration; using defaults.", &err);
config_warnings.push(message);
config_manager.load_default_config().await.map_err(|e| {
std::io::Error::new(
ErrorKind::InvalidData,
format!("error loading default config after config error: {e}"),
)
})?
(
config_manager.load_default_config().await.map_err(|e| {
std::io::Error::new(
ErrorKind::InvalidData,
format!("error loading default config after config error: {e}"),
)
})?,
false,
)
}
};
let state_db_result = rollout_state_db::try_init(&config).await;
let state_db_init_error = state_db_result.as_ref().err().map(ToString::to_string);
let state_db = state_db_result.ok();
if should_run_personality_migration {
let effective_toml = config.config_layer_stack.effective_config();
match effective_toml.try_into() {
Ok(config_toml) => {
match codex_core::personality_migration::maybe_migrate_personality(
&config.codex_home,
&config_toml,
state_db.clone(),
)
.await
{
Ok(codex_core::personality_migration::PersonalityMigrationStatus::Applied) => {
config = config_manager
.load_latest_config(/*fallback_cwd*/ None)
.await
.map_err(|err| {
std::io::Error::new(
ErrorKind::InvalidData,
format!(
"error reloading config after personality migration: {err}"
),
)
})?;
}
Ok(
codex_core::personality_migration::PersonalityMigrationStatus::SkippedMarker
| codex_core::personality_migration::PersonalityMigrationStatus::SkippedExplicitPersonality
| codex_core::personality_migration::PersonalityMigrationStatus::SkippedNoSessions,
) => {}
Err(err) => {
warn!(error = %err, "Failed to run personality migration");
}
}
}
Err(err) => {
warn!(error = %err, "Failed to deserialize config for personality migration");
}
}
}
if let Ok(Some(err)) = check_execpolicy_for_warnings(&config.config_layer_stack).await {
let (path, range) = exec_policy_warning_location(&err);
let message = ConfigWarningNotification {
@@ -569,13 +600,6 @@ pub async fn run_main_with_transport_options(
let feedback_layer = feedback.logger_layer();
let feedback_metadata_layer = feedback.metadata_layer();
let state_db_result = codex_state::StateRuntime::init(
config.sqlite_home.clone(),
config.model_provider_id.clone(),
)
.await;
let state_db_init_error = state_db_result.as_ref().err().map(ToString::to_string);
let state_db = state_db_result.ok();
let log_db = state_db.clone().map(log_db::start);
let log_db_layer = log_db
.clone()
@@ -747,6 +771,7 @@ pub async fn run_main_with_transport_options(
environment_manager,
feedback: feedback.clone(),
log_db,
state_db: state_db.clone(),
config_warnings,
session_source,
auth_manager,
+10 -6
View File
@@ -72,6 +72,7 @@ use codex_login::auth::ExternalAuthTokens;
use codex_protocol::ThreadId;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::W3cTraceContext;
use codex_rollout::StateDbHandle;
use codex_state::log_db::LogDbLayer;
use tokio::sync::Mutex;
use tokio::sync::Semaphore;
@@ -251,6 +252,7 @@ pub(crate) struct MessageProcessorArgs {
pub(crate) environment_manager: Arc<EnvironmentManager>,
pub(crate) feedback: CodexFeedback,
pub(crate) log_db: Option<LogDbLayer>,
pub(crate) state_db: Option<StateDbHandle>,
pub(crate) config_warnings: Vec<ConfigWarningNotification>,
pub(crate) session_source: SessionSource,
pub(crate) auth_manager: Arc<AuthManager>,
@@ -272,6 +274,7 @@ impl MessageProcessor {
environment_manager,
feedback,
log_db,
state_db,
config_warnings,
session_source,
auth_manager,
@@ -285,7 +288,7 @@ impl MessageProcessor {
// The thread store is intentionally process-scoped. Config reloads can
// affect per-thread behavior, but they must not move newly started,
// resumed, or forked threads to a different persistence backend/root.
let thread_store = thread_store_from_config(config.as_ref());
let thread_store = thread_store_from_config(config.as_ref(), state_db.clone());
let thread_manager = Arc::new(ThreadManager::new(
config.as_ref(),
auth_manager.clone(),
@@ -293,6 +296,7 @@ impl MessageProcessor {
environment_manager,
Some(analytics_events_client.clone()),
Arc::clone(&thread_store),
state_db.clone(),
));
thread_manager
.plugins_manager()
@@ -337,6 +341,7 @@ impl MessageProcessor {
Arc::clone(&config),
feedback,
log_db,
state_db.clone(),
);
let git_processor = GitRequestProcessor::new();
let initialize_processor = InitializeRequestProcessor::new(
@@ -371,6 +376,7 @@ impl MessageProcessor {
outgoing.clone(),
Arc::clone(&config),
thread_state_manager.clone(),
state_db.clone(),
);
let thread_processor = ThreadRequestProcessor::new(
auth_manager.clone(),
@@ -386,6 +392,7 @@ impl MessageProcessor {
thread_watch_manager.clone(),
Arc::clone(&thread_list_state_permit),
thread_goal_processor.clone(),
state_db.clone(),
);
let turn_processor = TurnRequestProcessor::new(
auth_manager.clone(),
@@ -399,6 +406,7 @@ impl MessageProcessor {
thread_state_manager,
thread_watch_manager,
thread_list_state_permit,
state_db.clone(),
);
if matches!(plugin_startup_tasks, crate::PluginStartupTasks::Start) {
// Keep plugin startup warmups aligned at app-server startup.
@@ -429,11 +437,7 @@ impl MessageProcessor {
arg0_paths,
config.codex_home.to_path_buf(),
);
let device_key_processor = DeviceKeyRequestProcessor::new(
outgoing.clone(),
config.sqlite_home.clone(),
config.model_provider_id.clone(),
);
let device_key_processor = DeviceKeyRequestProcessor::new(outgoing.clone(), state_db);
let fs_processor = FsRequestProcessor::new(
thread_manager
.environment_manager()
@@ -291,6 +291,7 @@ async fn build_test_processor(
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
feedback: CodexFeedback::new(),
log_db: None,
state_db: None,
config_warnings: Vec::new(),
session_source: SessionSource::VSCode,
auth_manager,
+13 -2
View File
@@ -169,6 +169,7 @@ use codex_app_server_protocol::ThreadGoalSetParams;
use codex_app_server_protocol::ThreadGoalSetResponse;
use codex_app_server_protocol::ThreadGoalStatus;
use codex_app_server_protocol::ThreadGoalUpdatedNotification;
use codex_app_server_protocol::ThreadHistoryBuilder;
use codex_app_server_protocol::ThreadIncrementElicitationParams;
use codex_app_server_protocol::ThreadIncrementElicitationResponse;
use codex_app_server_protocol::ThreadInjectItemsParams;
@@ -233,7 +234,6 @@ use codex_app_server_protocol::WindowsSandboxSetupCompletedNotification;
use codex_app_server_protocol::WindowsSandboxSetupMode;
use codex_app_server_protocol::WindowsSandboxSetupStartParams;
use codex_app_server_protocol::WindowsSandboxSetupStartResponse;
use codex_app_server_protocol::build_turns_from_rollout_items;
use codex_arg0::Arg0DispatchPaths;
use codex_backend_client::AddCreditsNudgeCreditType as BackendAddCreditsNudgeCreditType;
use codex_backend_client::Client as BackendClient;
@@ -366,8 +366,9 @@ use codex_protocol::protocol::W3cTraceContext;
use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS;
use codex_protocol::user_input::UserInput as CoreInputItem;
use codex_rmcp_client::perform_oauth_login_return_url;
use codex_rollout::EventPersistenceMode;
use codex_rollout::is_persisted_rollout_item;
use codex_rollout::state_db::StateDbHandle;
use codex_rollout::state_db::get_state_db;
use codex_rollout::state_db::reconcile_rollout;
use codex_state::StateRuntime;
use codex_state::ThreadMetadata;
@@ -479,3 +480,13 @@ use self::thread_summary::*;
pub(crate) use self::thread_summary::read_rollout_items_from_rollout;
pub(crate) use self::thread_summary::read_summary_from_rollout;
pub(crate) use self::thread_summary::summary_to_thread;
pub(crate) fn build_api_turns_from_rollout_items(items: &[RolloutItem]) -> Vec<Turn> {
let mut builder = ThreadHistoryBuilder::new();
for item in items {
if is_persisted_rollout_item(item, EventPersistenceMode::Limited) {
builder.handle_rollout_item(item);
}
}
builder.finish()
}
@@ -1,6 +1,5 @@
use std::fmt;
use std::future::Future;
use std::path::PathBuf;
use std::sync::Arc;
use crate::error_code::internal_error;
@@ -36,7 +35,6 @@ use codex_device_key::RemoteControlClientEnrollmentAudience;
use codex_device_key::RemoteControlClientEnrollmentSignPayload;
use codex_state::DeviceKeyBindingRecord;
use codex_state::StateRuntime;
use tokio::sync::OnceCell;
#[derive(Clone)]
pub(crate) struct DeviceKeyRequestProcessor {
@@ -47,15 +45,11 @@ pub(crate) struct DeviceKeyRequestProcessor {
impl DeviceKeyRequestProcessor {
pub(crate) fn new(
outgoing: Arc<OutgoingMessageSender>,
sqlite_home: PathBuf,
default_provider: String,
state_db: Option<Arc<StateRuntime>>,
) -> Self {
Self {
outgoing,
store: DeviceKeyStore::new(Arc::new(StateDeviceKeyBindingStore::new(
sqlite_home,
default_provider,
))),
store: DeviceKeyStore::new(Arc::new(StateDeviceKeyBindingStore::new(state_db))),
}
}
@@ -176,39 +170,25 @@ async fn sign_device_key(
}
struct StateDeviceKeyBindingStore {
sqlite_home: PathBuf,
default_provider: String,
state_db: OnceCell<Arc<StateRuntime>>,
state_db: Option<Arc<StateRuntime>>,
}
impl StateDeviceKeyBindingStore {
fn new(sqlite_home: PathBuf, default_provider: String) -> Self {
Self {
sqlite_home,
default_provider,
state_db: OnceCell::new(),
}
fn new(state_db: Option<Arc<StateRuntime>>) -> Self {
Self { state_db }
}
async fn state_db(&self) -> Result<Arc<StateRuntime>, DeviceKeyError> {
let sqlite_home = self.sqlite_home.clone();
let default_provider = self.default_provider.clone();
self.state_db
.get_or_try_init(|| async move {
StateRuntime::init(sqlite_home, default_provider)
.await
.map_err(|err| DeviceKeyError::Platform(err.to_string()))
})
.await
.cloned()
.clone()
.ok_or_else(|| DeviceKeyError::Platform("sqlite state db unavailable".to_string()))
}
}
impl fmt::Debug for StateDeviceKeyBindingStore {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("StateDeviceKeyBindingStore")
.field("sqlite_home", &self.sqlite_home)
.field("default_provider", &self.default_provider)
.field("has_state_db", &self.state_db.is_some())
.finish_non_exhaustive()
}
}
@@ -7,6 +7,7 @@ pub(crate) struct FeedbackRequestProcessor {
config: Arc<Config>,
feedback: CodexFeedback,
log_db: Option<LogDbLayer>,
state_db: Option<StateDbHandle>,
}
impl FeedbackRequestProcessor {
@@ -16,6 +17,7 @@ impl FeedbackRequestProcessor {
config: Arc<Config>,
feedback: CodexFeedback,
log_db: Option<LogDbLayer>,
state_db: Option<StateDbHandle>,
) -> Self {
Self {
auth_manager,
@@ -23,6 +25,7 @@ impl FeedbackRequestProcessor {
config,
feedback,
log_db,
state_db,
}
}
@@ -75,7 +78,7 @@ impl FeedbackRequestProcessor {
if let Some(log_db) = self.log_db.as_ref() {
log_db.flush().await;
}
let state_db_ctx = get_state_db(&self.config).await;
let state_db_ctx = self.state_db.clone();
let feedback_thread_ids = match conversation_id {
Some(conversation_id) => match self
.thread_manager
@@ -7,6 +7,7 @@ pub(crate) struct ThreadGoalRequestProcessor {
outgoing: Arc<OutgoingMessageSender>,
config: Arc<Config>,
thread_state_manager: ThreadStateManager,
state_db: Option<StateDbHandle>,
}
impl ThreadGoalRequestProcessor {
@@ -15,12 +16,14 @@ impl ThreadGoalRequestProcessor {
outgoing: Arc<OutgoingMessageSender>,
config: Arc<Config>,
thread_state_manager: ThreadStateManager,
state_db: Option<StateDbHandle>,
) -> Self {
Self {
thread_manager,
outgoing,
config,
thread_state_manager,
state_db,
}
}
@@ -78,7 +81,7 @@ impl ThreadGoalRequestProcessor {
if let Some(state_db) = thread.state_db() {
Some(state_db)
} else {
open_state_db_for_direct_thread_lookup(&self.config).await
self.state_db.clone()
}
} else {
None
@@ -104,12 +107,16 @@ impl ThreadGoalRequestProcessor {
"ephemeral thread does not support goals: {thread_id}"
))
})?,
None => find_thread_path_by_id_str(&self.config.codex_home, &thread_id.to_string())
.await
.map_err(|err| {
internal_error(format!("failed to locate thread id {thread_id}: {err}"))
})?
.ok_or_else(|| invalid_request(format!("thread not found: {thread_id}")))?,
None => find_thread_path_by_id_str(
&self.config.codex_home,
&thread_id.to_string(),
self.state_db.as_deref(),
)
.await
.map_err(|err| {
internal_error(format!("failed to locate thread id {thread_id}: {err}"))
})?
.ok_or_else(|| invalid_request(format!("thread not found: {thread_id}")))?,
};
reconcile_rollout(
Some(&state_db),
@@ -247,12 +254,16 @@ impl ThreadGoalRequestProcessor {
"ephemeral thread does not support goals: {thread_id}"
))
})?,
None => find_thread_path_by_id_str(&self.config.codex_home, &thread_id.to_string())
.await
.map_err(|err| {
internal_error(format!("failed to locate thread id {thread_id}: {err}"))
})?
.ok_or_else(|| invalid_request(format!("thread not found: {thread_id}")))?,
None => find_thread_path_by_id_str(
&self.config.codex_home,
&thread_id.to_string(),
self.state_db.as_deref(),
)
.await
.map_err(|err| {
internal_error(format!("failed to locate thread id {thread_id}: {err}"))
})?
.ok_or_else(|| invalid_request(format!("thread not found: {thread_id}")))?,
};
reconcile_rollout(
Some(&state_db),
@@ -307,16 +318,20 @@ impl ThreadGoalRequestProcessor {
return Ok(state_db);
}
} else {
find_thread_path_by_id_str(&self.config.codex_home, &thread_id.to_string())
.await
.map_err(|err| {
internal_error(format!("failed to locate thread id {thread_id}: {err}"))
})?
.ok_or_else(|| invalid_request(format!("thread not found: {thread_id}")))?;
find_thread_path_by_id_str(
&self.config.codex_home,
&thread_id.to_string(),
self.state_db.as_deref(),
)
.await
.map_err(|err| {
internal_error(format!("failed to locate thread id {thread_id}: {err}"))
})?
.ok_or_else(|| invalid_request(format!("thread not found: {thread_id}")))?;
}
open_state_db_for_direct_thread_lookup(&self.config)
.await
self.state_db
.clone()
.ok_or_else(|| internal_error("sqlite state db unavailable for thread goals"))
}
@@ -542,17 +542,12 @@ pub(super) async fn handle_pending_thread_resume_request(
let request_id = pending.request_id;
let connection_id = request_id.connection_id;
let mut thread = pending.thread_summary;
if pending.include_turns
&& let Err(message) = populate_thread_turns_from_history(
if pending.include_turns {
populate_thread_turns_from_history(
&mut thread,
&pending.history_items,
active_turn.as_ref(),
)
{
outgoing
.send_error(request_id, internal_error(message))
.await;
return;
);
}
let thread_status = thread_watch_manager
@@ -711,13 +706,12 @@ pub(super) fn populate_thread_turns_from_history(
thread: &mut Thread,
items: &[RolloutItem],
active_turn: Option<&Turn>,
) -> std::result::Result<(), String> {
let mut turns = build_turns_from_rollout_items(items);
) {
let mut turns = build_api_turns_from_rollout_items(items);
if let Some(active_turn) = active_turn {
merge_turn_history_with_active_turn(&mut turns, active_turn.clone());
}
thread.turns = turns;
Ok(())
}
pub(super) async fn resolve_pending_server_request(
@@ -259,6 +259,7 @@ pub(crate) struct ThreadRequestProcessor {
pub(super) thread_watch_manager: ThreadWatchManager,
pub(super) thread_list_state_permit: Arc<Semaphore>,
pub(super) thread_goal_processor: ThreadGoalRequestProcessor,
pub(super) state_db: Option<StateDbHandle>,
pub(super) background_tasks: TaskTracker,
}
@@ -278,6 +279,7 @@ impl ThreadRequestProcessor {
thread_watch_manager: ThreadWatchManager,
thread_list_state_permit: Arc<Semaphore>,
thread_goal_processor: ThreadGoalRequestProcessor,
state_db: Option<StateDbHandle>,
) -> Self {
Self {
auth_manager,
@@ -293,6 +295,7 @@ impl ThreadRequestProcessor {
thread_watch_manager,
thread_list_state_permit,
thread_goal_processor,
state_db,
background_tasks: TaskTracker::new(),
}
}
@@ -1170,7 +1173,7 @@ impl ThreadRequestProcessor {
.map_err(|err| invalid_request(format!("invalid thread id: {err}")))?;
let mut thread_ids = vec![thread_id];
if let Some(state_db_ctx) = get_state_db(&self.config).await {
if let Some(state_db_ctx) = self.state_db.as_ref() {
let descendants = state_db_ctx
.list_thread_spawn_descendants(thread_id)
.await
@@ -1391,14 +1394,10 @@ impl ThreadRequestProcessor {
}
async fn memory_reset_response_inner(&self) -> Result<MemoryResetResponse, JSONRPCErrorError> {
let state_db = StateRuntime::init(
self.config.sqlite_home.clone(),
self.config.model_provider_id.clone(),
)
.await
.map_err(|err| {
internal_error(format!("failed to open state db for memory reset: {err}"))
})?;
let state_db = self
.state_db
.clone()
.ok_or_else(|| internal_error("sqlite state db unavailable for memory reset"))?;
state_db.clear_memory_data().await.map_err(|err| {
internal_error(format!("failed to clear memory rows in state db: {err}"))
@@ -1445,7 +1444,7 @@ impl ThreadRequestProcessor {
let loaded_thread = self.thread_manager.get_thread(thread_uuid).await.ok();
let mut state_db_ctx = loaded_thread.as_ref().and_then(|thread| thread.state_db());
if state_db_ctx.is_none() {
state_db_ctx = get_state_db(&self.config).await;
state_db_ctx = self.state_db.clone();
}
let Some(state_db_ctx) = state_db_ctx else {
return Err(internal_error(format!(
@@ -1583,33 +1582,37 @@ impl ThreadRequestProcessor {
return Ok(());
}
let rollout_path =
match find_thread_path_by_id_str(&self.config.codex_home, &thread_uuid.to_string())
.await
let rollout_path = match find_thread_path_by_id_str(
&self.config.codex_home,
&thread_uuid.to_string(),
self.state_db.as_deref(),
)
.await
{
Ok(Some(path)) => path,
Ok(None) => match find_archived_thread_path_by_id_str(
&self.config.codex_home,
&thread_uuid.to_string(),
self.state_db.as_deref(),
)
.await
{
Ok(Some(path)) => path,
Ok(None) => match find_archived_thread_path_by_id_str(
&self.config.codex_home,
&thread_uuid.to_string(),
)
.await
{
Ok(Some(path)) => path,
Ok(None) => {
return Err(invalid_request(format!("thread not found: {thread_uuid}")));
}
Err(err) => {
return Err(internal_error(format!(
"failed to locate archived thread id {thread_uuid}: {err}"
)));
}
},
Ok(None) => {
return Err(invalid_request(format!("thread not found: {thread_uuid}")));
}
Err(err) => {
return Err(internal_error(format!(
"failed to locate thread id {thread_uuid}: {err}"
"failed to locate archived thread id {thread_uuid}: {err}"
)));
}
};
},
Err(err) => {
return Err(internal_error(format!(
"failed to locate thread id {thread_uuid}: {err}"
)));
}
};
reconcile_rollout(
Some(state_db_ctx),
@@ -2048,7 +2051,7 @@ impl ThreadRequestProcessor {
let (mut thread, history) =
thread_from_stored_thread(stored_thread, fallback_provider, &self.config.cwd);
if include_turns && let Some(history) = history {
thread.turns = build_turns_from_rollout_items(&history.items);
thread.turns = build_api_turns_from_rollout_items(&history.items);
}
Ok(Some(thread))
}
@@ -2113,7 +2116,7 @@ impl ThreadRequestProcessor {
.load_history(/*include_archived*/ true)
.await
.map_err(|err| thread_read_history_load_error(thread_id, err))?;
thread.turns = build_turns_from_rollout_items(&history.items);
thread.turns = build_api_turns_from_rollout_items(&history.items);
}
Ok(())
@@ -2555,7 +2558,7 @@ impl ThreadRequestProcessor {
let InitialHistory::Resumed(resumed_history) = thread_history else {
return None;
};
let state_db_ctx = get_state_db(&self.config).await?;
let state_db_ctx = self.state_db.clone()?;
let persisted_metadata = state_db_ctx
.get_thread(resumed_history.conversation_id)
.await
@@ -2662,17 +2665,11 @@ impl ThreadRequestProcessor {
}
let mut summary_source_thread = source_thread;
summary_source_thread.history = None;
let thread_summary = match self
.stored_thread_to_api_thread(
summary_source_thread,
config_snapshot.model_provider_id.as_str(),
/*include_turns*/ false,
)
.await
{
Ok(thread) => thread,
Err(message) => return Err(internal_error(message)),
};
let thread_summary = self.stored_thread_to_api_thread(
summary_source_thread,
config_snapshot.model_provider_id.as_str(),
/*include_turns*/ false,
);
let mut config_for_instruction_sources = self.config.as_ref().clone();
config_for_instruction_sources.cwd = config_snapshot.cwd.clone();
let instruction_sources =
@@ -2798,12 +2795,12 @@ impl ThreadRequestProcessor {
}))
}
async fn stored_thread_to_api_thread(
fn stored_thread_to_api_thread(
&self,
stored_thread: StoredThread,
fallback_provider: &str,
include_turns: bool,
) -> std::result::Result<Thread, String> {
) -> Thread {
let (mut thread, history) =
thread_from_stored_thread(stored_thread, fallback_provider, &self.config.cwd);
if include_turns && let Some(history) = history {
@@ -2811,9 +2808,9 @@ impl ThreadRequestProcessor {
&mut thread,
&history.items,
/*active_turn*/ None,
)?;
);
}
Ok(thread)
thread
}
async fn read_stored_thread_for_new_fork(
@@ -2921,14 +2918,16 @@ impl ThreadRequestProcessor {
&mut thread,
&history_items,
/*active_turn*/ None,
)?;
);
}
self.attach_thread_name(thread_id, &mut thread).await;
Ok(thread)
}
async fn attach_thread_name(&self, thread_id: ThreadId, thread: &mut Thread) {
if let Some(title) = title_from_state_db(&self.config, thread_id).await {
if let Some(title) =
title_from_state_db(&self.config, self.state_db.as_ref(), thread_id).await
{
set_thread_name_from_title(thread, title);
}
}
@@ -3066,7 +3065,7 @@ impl ThreadRequestProcessor {
// Persistent forks materialize their own rollout immediately. Ephemeral forks stay
// pathless, so they rebuild their visible history from the copied source history instead.
let mut thread = if let Some(fork_rollout_path) = session_configured.rollout_path.as_ref() {
let mut thread = if session_configured.rollout_path.is_some() {
let stored_thread = self
.read_stored_thread_for_new_fork(thread_id, include_turns)
.await?;
@@ -3075,13 +3074,6 @@ impl ThreadRequestProcessor {
fallback_model_provider.as_str(),
include_turns,
)
.await
.map_err(|message| {
internal_error(format!(
"failed to load rollout `{}` for thread {thread_id}: {message}",
fork_rollout_path.display()
))
})?
} else {
let config_snapshot = forked_thread.config_snapshot().await;
// forked thread names do not inherit the source thread name
@@ -3094,8 +3086,7 @@ impl ThreadRequestProcessor {
&mut thread,
&history_items,
/*active_turn*/ None,
)
.map_err(internal_error)?;
);
}
thread
};
@@ -3466,16 +3457,6 @@ fn parse_thread_turns_cursor(cursor: &str) -> Result<ThreadTurnsCursor, JSONRPCE
})
}
fn reconstruct_thread_turns_from_rollout_items(
items: &[RolloutItem],
loaded_status: ThreadStatus,
has_live_in_progress_turn: bool,
) -> Vec<Turn> {
let mut turns = build_turns_from_rollout_items(items);
normalize_thread_turns_status(&mut turns, loaded_status, has_live_in_progress_turn);
turns
}
fn reconstruct_thread_turns_for_turns_list(
items: &[RolloutItem],
loaded_status: ThreadStatus,
@@ -3486,11 +3467,8 @@ fn reconstruct_thread_turns_for_turns_list(
|| active_turn
.as_ref()
.is_some_and(|turn| matches!(turn.status, TurnStatus::InProgress));
let mut turns = reconstruct_thread_turns_from_rollout_items(
items,
loaded_status,
has_live_in_progress_turn,
);
let mut turns = build_api_turns_from_rollout_items(items);
normalize_thread_turns_status(&mut turns, loaded_status, has_live_in_progress_turn);
if let Some(active_turn) = active_turn {
merge_turn_history_with_active_turn(&mut turns, active_turn);
}
@@ -3710,8 +3688,12 @@ async fn read_summary_from_state_db_context_by_thread_id(
Some(summary_from_thread_metadata(&metadata))
}
async fn title_from_state_db(config: &Config, thread_id: ThreadId) -> Option<String> {
if let Some(state_db_ctx) = open_state_db_for_direct_thread_lookup(config).await
async fn title_from_state_db(
config: &Config,
state_db_ctx: Option<&StateDbHandle>,
thread_id: ThreadId,
) -> Option<String> {
if let Some(state_db_ctx) = state_db_ctx
&& let Some(metadata) = state_db_ctx.get_thread(thread_id).await.ok().flatten()
&& let Some(title) = distinct_title(&metadata)
{
@@ -1,13 +1,5 @@
use super::*;
pub(super) async fn open_state_db_for_direct_thread_lookup(
config: &Config,
) -> Option<StateDbHandle> {
StateRuntime::init(config.sqlite_home.clone(), config.model_provider_id.clone())
.await
.ok()
}
pub(crate) async fn read_summary_from_rollout(
path: &Path,
fallback_provider: &str,
@@ -13,6 +13,7 @@ pub(crate) struct TurnRequestProcessor {
thread_state_manager: ThreadStateManager,
thread_watch_manager: ThreadWatchManager,
thread_list_state_permit: Arc<Semaphore>,
state_db: Option<StateDbHandle>,
}
impl TurnRequestProcessor {
@@ -29,6 +30,7 @@ impl TurnRequestProcessor {
thread_state_manager: ThreadStateManager,
thread_watch_manager: ThreadWatchManager,
thread_list_state_permit: Arc<Semaphore>,
state_db: Option<StateDbHandle>,
) -> Self {
Self {
auth_manager,
@@ -42,6 +44,7 @@ impl TurnRequestProcessor {
thread_state_manager,
thread_watch_manager,
thread_list_state_permit,
state_db,
}
}
@@ -891,16 +894,20 @@ impl TurnRequestProcessor {
let rollout_path = if let Some(path) = parent_thread.rollout_path() {
path
} else {
find_thread_path_by_id_str(&self.config.codex_home, &parent_thread_id.to_string())
.await
.map_err(|err| {
internal_error(format!(
"failed to locate thread id {parent_thread_id}: {err}"
))
})?
.ok_or_else(|| {
invalid_request(format!("no rollout found for thread id {parent_thread_id}"))
})?
find_thread_path_by_id_str(
&self.config.codex_home,
&parent_thread_id.to_string(),
self.state_db.as_deref(),
)
.await
.map_err(|err| {
internal_error(format!(
"failed to locate thread id {parent_thread_id}: {err}"
))
})?
.ok_or_else(|| {
invalid_request(format!("no rollout found for thread id {parent_thread_id}"))
})?
};
let mut config = self.config.as_ref().clone();
@@ -205,6 +205,7 @@ async fn mcp_resource_read_returns_error_for_unknown_thread() -> Result<()> {
thread_config_loader: Arc::new(codex_config::NoopThreadConfigLoader),
feedback: CodexFeedback::new(),
log_db: None,
state_db: None,
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source: SessionSource::Cli,
@@ -81,6 +81,7 @@ async fn thread_start_with_non_local_thread_store_does_not_create_local_persiste
thread_config_loader: Arc::new(NoopThreadConfigLoader),
feedback: CodexFeedback::new(),
log_db: None,
state_db: None,
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source: SessionSource::Cli,
@@ -63,7 +63,7 @@ async fn thread_archive_requires_materialized_rollout() -> Result<()> {
rollout_path.display()
);
assert!(
find_thread_path_by_id_str(codex_home.path(), &thread.id)
find_thread_path_by_id_str(codex_home.path(), &thread.id, /*state_db_ctx*/ None)
.await?
.is_none(),
"thread id should not be discoverable before rollout materialization"
@@ -118,9 +118,10 @@ async fn thread_archive_requires_materialized_rollout() -> Result<()> {
rollout_path.display()
);
let discovered_path = find_thread_path_by_id_str(codex_home.path(), &thread.id)
.await?
.expect("expected rollout path for thread id to exist after materialization");
let discovered_path =
find_thread_path_by_id_str(codex_home.path(), &thread.id, /*state_db_ctx*/ None)
.await?
.expect("expected rollout path for thread id to exist after materialization");
assert_paths_match_on_disk(&discovered_path, &rollout_path)?;
let archive_id = mcp
@@ -252,15 +253,23 @@ async fn thread_archive_archives_spawned_descendants() -> Result<()> {
for thread_id in [parent_thread_id, child_thread_id, grandchild_thread_id] {
assert!(
find_thread_path_by_id_str(codex_home.path(), &thread_id.to_string())
.await?
.is_none(),
find_thread_path_by_id_str(
codex_home.path(),
&thread_id.to_string(),
/*state_db_ctx*/ None,
)
.await?
.is_none(),
"expected active rollout for {thread_id} to be archived"
);
assert!(
find_archived_thread_path_by_id_str(codex_home.path(), &thread_id.to_string())
.await?
.is_some(),
find_archived_thread_path_by_id_str(
codex_home.path(),
&thread_id.to_string(),
/*state_db_ctx*/ None,
)
.await?
.is_some(),
"expected archived rollout for {thread_id} to exist"
);
}
@@ -322,9 +331,10 @@ async fn thread_archive_succeeds_when_descendant_archive_fails() -> Result<()> {
)
.await?;
let child_rollout_path = find_thread_path_by_id_str(codex_home.path(), &child_id)
.await?
.expect("child rollout path");
let child_rollout_path =
find_thread_path_by_id_str(codex_home.path(), &child_id, /*state_db_ctx*/ None)
.await?
.expect("child rollout path");
let archived_child_path = codex_home
.path()
.join(ARCHIVED_SESSIONS_SUBDIR)
@@ -381,15 +391,23 @@ async fn thread_archive_succeeds_when_descendant_archive_fails() -> Result<()> {
);
for thread_id in [parent_thread_id, grandchild_thread_id] {
assert!(
find_thread_path_by_id_str(codex_home.path(), &thread_id.to_string())
.await?
.is_none(),
find_thread_path_by_id_str(
codex_home.path(),
&thread_id.to_string(),
/*state_db_ctx*/ None,
)
.await?
.is_none(),
"expected active rollout for {thread_id} to be archived"
);
assert!(
find_archived_thread_path_by_id_str(codex_home.path(), &thread_id.to_string())
.await?
.is_some(),
find_archived_thread_path_by_id_str(
codex_home.path(),
&thread_id.to_string(),
/*state_db_ctx*/ None,
)
.await?
.is_some(),
"expected archived rollout for {thread_id} to exist"
);
}
@@ -455,15 +473,19 @@ async fn thread_archive_succeeds_when_spawned_descendant_is_missing() -> Result<
assert_eq!(archived_notification.thread_id, parent_id);
assert!(
find_thread_path_by_id_str(codex_home.path(), &parent_id)
find_thread_path_by_id_str(codex_home.path(), &parent_id, /*state_db_ctx*/ None)
.await?
.is_none(),
"parent should be archived even when a descendant is missing"
);
assert!(
find_archived_thread_path_by_id_str(codex_home.path(), &parent_id)
.await?
.is_some(),
find_archived_thread_path_by_id_str(
codex_home.path(),
&parent_id,
/*state_db_ctx*/ None,
)
.await?
.is_some(),
"parent should be moved into archived sessions"
);
@@ -614,6 +614,7 @@ sqlite = true
generate_memories: false,
};
let repaired_page = codex_core::RolloutRecorder::list_threads(
Some(state_db.clone()),
&rollout_config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -211,9 +211,10 @@ async fn thread_name_update_rollout_count(
codex_home: &Path,
conversation_id: &str,
) -> Result<usize> {
let rollout_path = find_thread_path_by_id_str(codex_home, conversation_id)
.await?
.context("rollout path")?;
let rollout_path =
find_thread_path_by_id_str(codex_home, conversation_id, /*state_db_ctx*/ None)
.await?
.context("rollout path")?;
let contents = tokio::fs::read_to_string(rollout_path).await?;
Ok(contents
.lines()
@@ -301,6 +301,7 @@ async fn thread_turns_list_reads_store_history_without_rollout_path() -> Result<
thread_config_loader: Arc::new(codex_config::NoopThreadConfigLoader),
feedback: CodexFeedback::new(),
log_db: None,
state_db: None,
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source: SessionSource::Cli.into(),
@@ -365,6 +366,7 @@ async fn thread_read_loaded_include_turns_reads_store_history_without_rollout_pa
thread_config_loader: Arc::new(codex_config::NoopThreadConfigLoader),
feedback: CodexFeedback::new(),
log_db: None,
state_db: None,
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source: SessionSource::Cli.into(),
@@ -450,6 +452,7 @@ async fn thread_list_includes_store_thread_without_rollout_path() -> Result<()>
thread_config_loader: Arc::new(codex_config::NoopThreadConfigLoader),
feedback: CodexFeedback::new(),
log_db: None,
state_db: None,
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
config_warnings: Vec::new(),
session_source: SessionSource::Cli.into(),
@@ -15,6 +15,9 @@ use codex_app_server_protocol::ItemStartedNotification;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerRequest;
use codex_app_server_protocol::SortDirection;
use codex_app_server_protocol::ThreadForkParams;
use codex_app_server_protocol::ThreadForkResponse;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadReadParams;
use codex_app_server_protocol::ThreadReadResponse;
@@ -22,6 +25,8 @@ use codex_app_server_protocol::ThreadShellCommandParams;
use codex_app_server_protocol::ThreadShellCommandResponse;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::ThreadStartResponse;
use codex_app_server_protocol::ThreadTurnsListParams;
use codex_app_server_protocol::ThreadTurnsListResponse;
use codex_app_server_protocol::TurnCompletedNotification;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartResponse;
@@ -38,7 +43,8 @@ use tokio::time::timeout;
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
#[tokio::test]
async fn thread_shell_command_runs_as_standalone_turn_and_persists_history() -> Result<()> {
async fn thread_shell_command_history_responses_exclude_persisted_command_executions() -> Result<()>
{
let tmp = TempDir::new()?;
let codex_home = tmp.path().join("codex_home");
std::fs::create_dir(&codex_home)?;
@@ -126,7 +132,7 @@ async fn thread_shell_command_runs_as_standalone_turn_and_persists_history() ->
let read_id = mcp
.send_thread_read_request(ThreadReadParams {
thread_id: thread.id,
thread_id: thread.id.clone(),
include_turns: true,
})
.await?;
@@ -137,22 +143,40 @@ async fn thread_shell_command_runs_as_standalone_turn_and_persists_history() ->
.await??;
let ThreadReadResponse { thread, .. } = to_response::<ThreadReadResponse>(read_resp)?;
assert_eq!(thread.turns.len(), 1);
let ThreadItem::CommandExecution {
source,
status,
aggregated_output,
..
} = thread.turns[0]
.items
.iter()
.find(|item| matches!(item, ThreadItem::CommandExecution { .. }))
.expect("expected persisted command execution item")
else {
unreachable!("matched command execution item");
};
assert_eq!(source, &CommandExecutionSource::UserShell);
assert_eq!(status, &CommandExecutionStatus::Completed);
assert_eq!(aggregated_output.as_deref(), Some(expected_output.as_str()));
assert_no_command_executions(&thread.turns[0].items, "thread/read");
let turns_list_id = mcp
.send_thread_turns_list_request(ThreadTurnsListParams {
thread_id: thread.id.clone(),
cursor: None,
limit: None,
sort_direction: Some(SortDirection::Asc),
})
.await?;
let turns_list_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turns_list_id)),
)
.await??;
let ThreadTurnsListResponse { data, .. } =
to_response::<ThreadTurnsListResponse>(turns_list_resp)?;
assert_eq!(data.len(), 1);
assert_no_command_executions(&data[0].items, "thread/turns/list");
let fork_id = mcp
.send_thread_fork_request(ThreadForkParams {
thread_id: thread.id,
..Default::default()
})
.await?;
let fork_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(fork_id)),
)
.await??;
let ThreadForkResponse { thread, .. } = to_response::<ThreadForkResponse>(fork_resp)?;
assert_eq!(thread.turns.len(), 1);
assert_no_command_executions(&thread.turns[0].items, "thread/fork");
Ok(())
}
@@ -307,23 +331,20 @@ async fn thread_shell_command_uses_existing_active_turn() -> Result<()> {
.await??;
let ThreadReadResponse { thread, .. } = to_response::<ThreadReadResponse>(read_resp)?;
assert_eq!(thread.turns.len(), 1);
assert!(
thread.turns[0].items.iter().any(|item| {
matches!(
item,
ThreadItem::CommandExecution {
source: CommandExecutionSource::UserShell,
aggregated_output,
..
} if aggregated_output.as_deref() == Some(expected_output.as_str())
)
}),
"expected active-turn shell command to be persisted on the existing turn"
);
assert_no_command_executions(&thread.turns[0].items, "thread/read");
Ok(())
}
fn assert_no_command_executions(items: &[ThreadItem], context: &str) {
assert!(
items
.iter()
.all(|item| !matches!(item, ThreadItem::CommandExecution { .. })),
"{context} should always exclude command executions from returned turns"
);
}
fn current_shell_output_command(text: &str) -> Result<(String, String)> {
let command_and_output = match default_user_shell().name() {
"powershell" => {
@@ -75,9 +75,10 @@ async fn thread_unarchive_moves_rollout_back_into_sessions_directory() -> Result
)
.await??;
let found_rollout_path = find_thread_path_by_id_str(codex_home.path(), &thread.id)
.await?
.expect("expected rollout path for thread id to exist");
let found_rollout_path =
find_thread_path_by_id_str(codex_home.path(), &thread.id, /*state_db_ctx*/ None)
.await?
.expect("expected rollout path for thread id to exist");
assert_paths_match_on_disk(&found_rollout_path, &rollout_path)?;
let archive_id = mcp
@@ -92,9 +93,13 @@ async fn thread_unarchive_moves_rollout_back_into_sessions_directory() -> Result
.await??;
let _: ThreadArchiveResponse = to_response::<ThreadArchiveResponse>(archive_resp)?;
let archived_path = find_archived_thread_path_by_id_str(codex_home.path(), &thread.id)
.await?
.expect("expected archived rollout path for thread id to exist");
let archived_path = find_archived_thread_path_by_id_str(
codex_home.path(),
&thread.id,
/*state_db_ctx*/ None,
)
.await?
.expect("expected archived rollout path for thread id to exist");
let archived_path_display = archived_path.display();
assert!(
archived_path.exists(),
+2 -2
View File
@@ -446,7 +446,7 @@ struct AppServerCommand {
#[derive(Debug, Parser)]
struct ExecServerCommand {
/// Transport endpoint URL. Supported values: `ws://IP:PORT` (default).
/// Transport endpoint URL. Supported values: `ws://IP:PORT` (default), `stdio`, `stdio://`.
#[arg(
long = "listen",
value_name = "URL",
@@ -1393,7 +1393,7 @@ async fn run_debug_prompt_input_command(
});
}
let prompt_input = codex_core::build_prompt_input(config, input).await?;
let prompt_input = codex_core::build_prompt_input(config, input, /*state_db*/ None).await?;
println!("{}", serde_json::to_string_pretty(&prompt_input)?);
Ok(())
@@ -40,7 +40,7 @@ fn create_test_tool(server_name: &str, tool_name: &str) -> ToolInfo {
server_name: server_name.to_string(),
callable_name: tool_name.to_string(),
callable_namespace: tool_namespace,
server_instructions: None,
namespace_description: None,
tool: Tool {
name: tool_name.to_string().into(),
title: None,
@@ -55,7 +55,6 @@ fn create_test_tool(server_name: &str, tool_name: &str) -> ToolInfo {
connector_id: None,
connector_name: None,
plugin_display_names: Vec::new(),
connector_description: None,
}
}
+1 -6
View File
@@ -71,12 +71,7 @@ pub fn mcp_permission_prompt_is_auto_approved(
permission_profile: &PermissionProfile,
context: McpPermissionPromptAutoApproveContext,
) -> bool {
if matches!(
approval_policy,
AskForApproval::OnRequest | AskForApproval::Granular(_)
) && context.approvals_reviewer == Some(ApprovalsReviewer::AutoReview)
&& context.tool_approval_mode == Some(AppToolApproval::Approve)
{
if context.tool_approval_mode == Some(AppToolApproval::Approve) {
return true;
}
+24 -27
View File
@@ -74,16 +74,11 @@ fn mcp_prompt_auto_approval_honors_unrestricted_managed_profiles() {
}
#[test]
fn mcp_prompt_auto_approval_honors_auto_review_approved_tools() {
assert!(mcp_permission_prompt_is_auto_approved(
fn mcp_prompt_auto_approval_honors_approved_tools_in_all_permission_modes() {
for approval_policy in [
AskForApproval::UnlessTrusted,
AskForApproval::OnFailure,
AskForApproval::OnRequest,
&PermissionProfile::read_only(),
McpPermissionPromptAutoApproveContext {
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
tool_approval_mode: Some(AppToolApproval::Approve),
},
));
assert!(mcp_permission_prompt_is_auto_approved(
AskForApproval::Granular(GranularApprovalConfig {
sandbox_approval: true,
rules: true,
@@ -91,34 +86,36 @@ fn mcp_prompt_auto_approval_honors_auto_review_approved_tools() {
request_permissions: true,
mcp_elicitations: true,
}),
AskForApproval::Never,
] {
assert!(mcp_permission_prompt_is_auto_approved(
approval_policy,
&PermissionProfile::read_only(),
McpPermissionPromptAutoApproveContext {
approvals_reviewer: Some(ApprovalsReviewer::User),
tool_approval_mode: Some(AppToolApproval::Approve),
},
));
}
assert!(!mcp_permission_prompt_is_auto_approved(
AskForApproval::OnRequest,
&PermissionProfile::read_only(),
McpPermissionPromptAutoApproveContext {
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
tool_approval_mode: Some(AppToolApproval::Approve),
tool_approval_mode: Some(AppToolApproval::Auto),
},
));
}
#[test]
fn mcp_prompt_auto_approval_rejects_auto_mode_in_default_permission_mode() {
assert!(!mcp_permission_prompt_is_auto_approved(
AskForApproval::OnRequest,
&PermissionProfile::read_only(),
McpPermissionPromptAutoApproveContext {
approvals_reviewer: Some(ApprovalsReviewer::User),
tool_approval_mode: Some(AppToolApproval::Approve),
},
));
assert!(!mcp_permission_prompt_is_auto_approved(
AskForApproval::OnFailure,
&PermissionProfile::read_only(),
McpPermissionPromptAutoApproveContext {
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
tool_approval_mode: Some(AppToolApproval::Approve),
},
));
assert!(!mcp_permission_prompt_is_auto_approved(
AskForApproval::UnlessTrusted,
&PermissionProfile::read_only(),
McpPermissionPromptAutoApproveContext {
approvals_reviewer: Some(ApprovalsReviewer::AutoReview),
tool_approval_mode: Some(AppToolApproval::Approve),
tool_approval_mode: Some(AppToolApproval::Auto),
},
));
}
+9 -2
View File
@@ -362,16 +362,23 @@ pub(crate) async fn list_tools_for_client_uncached(
tool_def.title = Some(normalized_title);
}
}
let has_connector_metadata = connector_id.is_some()
|| connector_name.is_some()
|| connector_description.is_some();
let namespace_description = if has_connector_metadata {
connector_description
} else {
server_instructions.map(str::to_string)
};
ToolInfo {
server_name: server_name.to_owned(),
callable_name,
callable_namespace,
server_instructions: server_instructions.map(str::to_string),
namespace_description,
tool: tool_def,
connector_id,
connector_name,
plugin_display_names: Vec::new(),
connector_description,
}
})
.collect();
+4 -4
View File
@@ -35,16 +35,16 @@ pub struct ToolInfo {
/// Model-visible namespace used for deferred tool loading.
#[serde(rename = "tool_namespace", alias = "callable_namespace")]
pub callable_namespace: String,
/// Instructions from the MCP server initialize result.
#[serde(default)]
pub server_instructions: Option<String>,
/// Model-visible namespace description.
// Keep the old serialized field name readable for cached ToolInfo values.
#[serde(default, alias = "connector_description")]
pub namespace_description: Option<String>,
/// Raw MCP tool definition; `tool.name` is sent back to the MCP server.
pub tool: Tool,
pub connector_id: Option<String>,
pub connector_name: Option<String>,
#[serde(default)]
pub plugin_display_names: Vec<String>,
pub connector_description: Option<String>,
}
impl ToolInfo {
+4
View File
@@ -104,6 +104,8 @@ pub struct TuiGlobalKeymap {
pub toggle_shortcuts: Option<KeybindingsSpec>,
/// Toggle Vim mode for the composer input.
pub toggle_vim_mode: Option<KeybindingsSpec>,
/// Toggle Fast mode.
pub toggle_fast_mode: Option<KeybindingsSpec>,
}
/// Chat context keybindings.
@@ -169,6 +171,8 @@ pub struct TuiEditorKeymap {
pub delete_forward_word: Option<KeybindingsSpec>,
/// Kill text from cursor to line start.
pub kill_line_start: Option<KeybindingsSpec>,
/// Kill the current line.
pub kill_whole_line: Option<KeybindingsSpec>,
/// Kill text from cursor to line end.
pub kill_line_end: Option<KeybindingsSpec>,
/// Yank the kill buffer.
+2
View File
@@ -27,6 +27,7 @@ pub use codex_core::ForkSnapshot;
pub use codex_core::McpManager;
pub use codex_core::NewThread;
pub use codex_core::StartThreadOptions;
pub use codex_core::StateDbHandle;
pub use codex_core::ThreadManager;
pub use codex_core::ThreadShutdownReport;
pub use codex_core::config::Config;
@@ -37,6 +38,7 @@ pub use codex_core::config::Permissions;
pub use codex_core::config::TerminalResizeReflowConfig;
pub use codex_core::config::ThreadStoreConfig;
pub use codex_core::config::find_codex_home;
pub use codex_core::init_state_db;
pub use codex_core::skills::SkillsManager;
pub use codex_core::thread_store_from_config;
pub use codex_exec_server::EnvironmentManager;
+20
View File
@@ -2441,6 +2441,7 @@
"insert_newline": null,
"kill_line_end": null,
"kill_line_start": null,
"kill_whole_line": null,
"move_down": null,
"move_left": null,
"move_line_end": null,
@@ -2458,6 +2459,7 @@
"open_transcript": null,
"queue": null,
"submit": null,
"toggle_fast_mode": null,
"toggle_shortcuts": null,
"toggle_vim_mode": null
},
@@ -2811,6 +2813,14 @@
],
"description": "Kill text from cursor to line start."
},
"kill_whole_line": {
"allOf": [
{
"$ref": "#/definitions/KeybindingsSpec"
}
],
"description": "Kill the current line."
},
"move_down": {
"allOf": [
{
@@ -2938,6 +2948,14 @@
],
"description": "Submit the current composer draft."
},
"toggle_fast_mode": {
"allOf": [
{
"$ref": "#/definitions/KeybindingsSpec"
}
],
"description": "Toggle Fast mode."
},
"toggle_shortcuts": {
"allOf": [
{
@@ -3018,6 +3036,7 @@
"insert_newline": null,
"kill_line_end": null,
"kill_line_start": null,
"kill_whole_line": null,
"move_down": null,
"move_left": null,
"move_line_end": null,
@@ -3042,6 +3061,7 @@
"open_transcript": null,
"queue": null,
"submit": null,
"toggle_fast_mode": null,
"toggle_shortcuts": null,
"toggle_vim_mode": null
}
+2 -2
View File
@@ -29,7 +29,6 @@ use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::TurnEnvironmentSelection;
use codex_protocol::user_input::UserInput;
use codex_rollout::state_db;
use codex_state::DirectionalThreadSpawnEdgeStatus;
use codex_thread_store::ReadThreadParams;
use serde::Serialize;
@@ -526,6 +525,7 @@ impl AgentControl {
let _ = config.features.disable(Feature::Collab);
}
let state = self.upgrade()?;
let state_db_ctx = state.state_db();
let mut reservation = self.state.reserve_spawn_slot(config.agent_max_threads)?;
let (session_source, agent_metadata) = match session_source {
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
@@ -536,7 +536,7 @@ impl AgentControl {
agent_nickname: _,
}) => {
let (resumed_agent_nickname, resumed_agent_role) =
if let Some(state_db_ctx) = state_db::get_state_db(&config).await {
if let Some(state_db_ctx) = state_db_ctx.as_ref() {
match state_db_ctx.get_thread(thread_id).await {
Ok(Some(metadata)) => (metadata.agent_nickname, metadata.agent_role),
Ok(None) | Err(_) => (None, None),
+15 -3
View File
@@ -1,5 +1,6 @@
use super::*;
use crate::CodexThread;
use crate::StateDbHandle;
use crate::ThreadManager;
use crate::agent::agent_status_from_event;
use crate::config::AgentRoleConfig;
@@ -7,6 +8,7 @@ use crate::config::Config;
use crate::config::ConfigBuilder;
use crate::context::ContextualUserFragment;
use crate::context::SubagentNotification;
use crate::init_state_db;
use assert_matches::assert_matches;
use codex_features::Feature;
use codex_login::CodexAuth;
@@ -84,6 +86,7 @@ fn spawn_agent_call(call_id: &str) -> ResponseItem {
struct AgentControlHarness {
_home: TempDir,
config: Config,
state_db: Option<StateDbHandle>,
manager: ThreadManager,
control: AgentControl,
}
@@ -91,16 +94,19 @@ struct AgentControlHarness {
impl AgentControlHarness {
async fn new() -> Self {
let (home, config) = test_config().await;
let manager = ThreadManager::with_models_provider_and_home_for_tests(
let state_db = init_state_db(&config).await;
let manager = ThreadManager::with_models_provider_home_and_state_for_tests(
CodexAuth::from_api_key("dummy"),
config.model_provider.clone(),
config.codex_home.to_path_buf(),
std::sync::Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
state_db.clone(),
);
let control = manager.agent_control();
Self {
_home: home,
config,
state_db,
manager,
control,
}
@@ -1537,16 +1543,19 @@ async fn resume_thread_subagent_restores_stored_nickname_and_role() {
.features
.enable(Feature::Sqlite)
.expect("test config should allow sqlite");
let manager = ThreadManager::with_models_provider_and_home_for_tests(
let state_db = init_state_db(&config).await;
let manager = ThreadManager::with_models_provider_home_and_state_for_tests(
CodexAuth::from_api_key("dummy"),
config.model_provider.clone(),
config.codex_home.to_path_buf(),
std::sync::Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
state_db.clone(),
);
let control = manager.agent_control();
let harness = AgentControlHarness {
_home: home,
config,
state_db,
manager,
control,
};
@@ -1695,7 +1704,10 @@ async fn resume_agent_from_rollout_reads_archived_rollout_path() {
.shutdown_live_agent(child_thread_id)
.await
.expect("child shutdown should succeed");
let store = LocalThreadStore::new(LocalThreadStoreConfig::from_config(&harness.config));
let store = LocalThreadStore::new(
LocalThreadStoreConfig::from_config(&harness.config),
harness.state_db.clone(),
);
store
.archive_thread(ArchiveThreadParams {
thread_id: child_thread_id,
+91 -101
View File
@@ -107,6 +107,7 @@ use tracing::warn;
use crate::client_common::Prompt;
use crate::client_common::ResponseEvent;
use crate::client_common::ResponseStream;
use crate::feedback_tags;
use crate::flags::CODEX_RS_SSE_FIXTURE;
use crate::util::emit_feedback_auth_recovery_tags;
use codex_api::map_api_error;
@@ -433,36 +434,33 @@ impl ModelClient {
RequestRouteTelemetry::for_endpoint(RESPONSES_COMPACT_ENDPOINT),
self.state.auth_env_telemetry.clone(),
);
let request = self.build_responses_request(
&client_setup.api_provider,
prompt,
model_info,
effort,
summary,
/*service_tier*/ None,
)?;
let ResponsesApiRequest {
model,
instructions,
input,
tools,
parallel_tool_calls,
reasoning,
text,
..
} = request;
let client =
ApiCompactClient::new(transport, client_setup.api_provider, client_setup.api_auth)
.with_telemetry(Some(request_telemetry));
let instructions = prompt.base_instructions.text.clone();
let input = prompt.get_formatted_input();
let tools = create_tools_json_for_responses_api(&prompt.tools)?;
let reasoning = Self::build_reasoning(model_info, effort, summary);
let verbosity = if model_info.support_verbosity {
self.state.model_verbosity.or(model_info.default_verbosity)
} else {
if self.state.model_verbosity.is_some() {
warn!(
"model_verbosity is set but ignored as the model does not support verbosity: {}",
model_info.slug
);
}
None
};
let text = create_text_param_for_request(
verbosity,
&prompt.output_schema,
prompt.output_schema_strict,
);
let payload = ApiCompactionInput {
model: &model_info.slug,
model: &model,
input: &input,
instructions: &instructions,
tools,
parallel_tool_calls: prompt.parallel_tool_calls,
parallel_tool_calls,
reasoning,
text,
};
@@ -664,6 +662,67 @@ impl ModelClient {
}
}
fn build_responses_request(
&self,
provider: &codex_api::Provider,
prompt: &Prompt,
model_info: &ModelInfo,
effort: Option<ReasoningEffortConfig>,
summary: ReasoningSummaryConfig,
service_tier: Option<ServiceTier>,
) -> Result<ResponsesApiRequest> {
let instructions = &prompt.base_instructions.text;
let input = prompt.get_formatted_input();
let tools = create_tools_json_for_responses_api(&prompt.tools)?;
let reasoning = Self::build_reasoning(model_info, effort, summary);
let include = if reasoning.is_some() {
vec!["reasoning.encrypted_content".to_string()]
} else {
Vec::new()
};
let verbosity = if model_info.support_verbosity {
self.state.model_verbosity.or(model_info.default_verbosity)
} else {
if self.state.model_verbosity.is_some() {
warn!(
"model_verbosity is set but ignored as the model does not support verbosity: {}",
model_info.slug
);
}
None
};
let text = create_text_param_for_request(
verbosity,
&prompt.output_schema,
prompt.output_schema_strict,
);
let prompt_cache_key = Some(self.state.conversation_id.to_string());
let request = ResponsesApiRequest {
model: model_info.slug.clone(),
instructions: instructions.clone(),
input,
tools,
tool_choice: "auto".to_string(),
parallel_tool_calls: prompt.parallel_tool_calls,
reasoning,
store: provider.is_azure_responses_endpoint(),
stream: true,
include,
service_tier: match service_tier {
Some(ServiceTier::Fast) => Some("priority".to_string()),
Some(service_tier) => Some(service_tier.to_string()),
None => None,
},
prompt_cache_key,
text,
client_metadata: Some(HashMap::from([(
X_CODEX_INSTALLATION_ID_HEADER.to_string(),
self.state.installation_id.clone(),
)])),
};
Ok(request)
}
/// Returns whether the Responses-over-WebSocket transport is active for this session.
///
/// WebSocket use is controlled by provider capability and session-scoped fallback state.
@@ -833,82 +892,6 @@ impl ModelClientSession {
.set_connection_reused(/*connection_reused*/ false);
}
fn build_responses_request(
&self,
provider: &codex_api::Provider,
prompt: &Prompt,
model_info: &ModelInfo,
effort: Option<ReasoningEffortConfig>,
summary: ReasoningSummaryConfig,
service_tier: Option<ServiceTier>,
) -> Result<ResponsesApiRequest> {
let instructions = &prompt.base_instructions.text;
let input = prompt.get_formatted_input();
let tools = create_tools_json_for_responses_api(&prompt.tools)?;
let default_reasoning_effort = model_info.default_reasoning_level;
let reasoning = if model_info.supports_reasoning_summaries {
Some(Reasoning {
effort: effort.or(default_reasoning_effort),
summary: if summary == ReasoningSummaryConfig::None {
None
} else {
Some(summary)
},
})
} else {
None
};
let include = if reasoning.is_some() {
vec!["reasoning.encrypted_content".to_string()]
} else {
Vec::new()
};
let verbosity = if model_info.support_verbosity {
self.client
.state
.model_verbosity
.or(model_info.default_verbosity)
} else {
if self.client.state.model_verbosity.is_some() {
warn!(
"model_verbosity is set but ignored as the model does not support verbosity: {}",
model_info.slug
);
}
None
};
let text = create_text_param_for_request(
verbosity,
&prompt.output_schema,
prompt.output_schema_strict,
);
let prompt_cache_key = Some(self.client.state.conversation_id.to_string());
let request = ResponsesApiRequest {
model: model_info.slug.clone(),
instructions: instructions.clone(),
input,
tools,
tool_choice: "auto".to_string(),
parallel_tool_calls: prompt.parallel_tool_calls,
reasoning,
store: provider.is_azure_responses_endpoint(),
stream: true,
include,
service_tier: match service_tier {
Some(ServiceTier::Fast) => Some("priority".to_string()),
Some(service_tier) => Some(service_tier.to_string()),
None => None,
},
prompt_cache_key,
text,
client_metadata: Some(HashMap::from([(
X_CODEX_INSTALLATION_ID_HEADER.to_string(),
self.client.state.installation_id.clone(),
)])),
};
Ok(request)
}
#[allow(clippy::too_many_arguments)]
/// Builds shared Responses API transport options and request-body options.
///
@@ -1209,7 +1192,7 @@ impl ModelClientSession {
let compression = self.responses_request_compression(client_setup.auth.as_ref());
let options = self.build_responses_options(turn_metadata_header, compression);
let request = self.build_responses_request(
let request = self.client.build_responses_request(
&client_setup.api_provider,
prompt,
model_info,
@@ -1315,7 +1298,7 @@ impl ModelClientSession {
let compression = self.responses_request_compression(client_setup.auth.as_ref());
let options = self.build_responses_options(turn_metadata_header, compression);
let request = self.build_responses_request(
let request = self.client.build_responses_request(
&client_setup.api_provider,
prompt,
model_info,
@@ -1703,6 +1686,9 @@ where
let mut items_added: Vec<ResponseItem> = Vec::new();
let mut api_stream = api_stream;
let upstream_request_id = upstream_request_id.as_deref();
if let Some(upstream_request_id) = upstream_request_id {
feedback_tags!(last_model_request_id = upstream_request_id);
}
loop {
let event = tokio::select! {
_ = consumer_dropped.cancelled() => {
@@ -1739,6 +1725,7 @@ where
token_usage,
end_turn,
}) => {
feedback_tags!(last_model_response_id = &response_id);
if let Some(usage) = &token_usage {
session_telemetry.sse_event_completed(
usage.input_tokens,
@@ -1787,6 +1774,9 @@ where
extract_response_debug_context_from_api_error(&err);
let upstream_request_id =
upstream_request_id.or(response_debug_context.request_id.as_deref());
if let Some(upstream_request_id) = upstream_request_id {
feedback_tags!(last_model_request_id = upstream_request_id);
}
let mapped = map_api_error(err);
inference_trace_attempt.record_failed(
&mapped,
+81
View File
@@ -31,14 +31,24 @@ use codex_rollout_trace::replay_bundle;
use futures::StreamExt;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::collections::BTreeMap;
use std::collections::VecDeque;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::Mutex;
use std::task::Context;
use std::task::Poll;
use std::time::Duration;
use tempfile::TempDir;
use tokio::sync::Notify;
use tracing::Event;
use tracing::Subscriber;
use tracing::field::Visit;
use tracing_subscriber::Layer;
use tracing_subscriber::layer::Context as LayerContext;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::util::SubscriberInitExt;
fn test_model_client(session_source: SessionSource) -> ModelClient {
let provider = create_oss_provider_with_base_url("https://example.com/v1", WireApi::Responses);
@@ -100,6 +110,42 @@ fn test_session_telemetry() -> SessionTelemetry {
)
}
#[derive(Default)]
struct TagCollectorVisitor {
tags: BTreeMap<String, String>,
}
impl Visit for TagCollectorVisitor {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.tags
.insert(field.name().to_string(), value.to_string());
}
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.tags
.insert(field.name().to_string(), format!("{value:?}"));
}
}
#[derive(Clone)]
struct TagCollectorLayer {
tags: Arc<Mutex<BTreeMap<String, String>>>,
}
impl<S> Layer<S> for TagCollectorLayer
where
S: Subscriber + for<'a> LookupSpan<'a>,
{
fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) {
if event.metadata().target() != "feedback_tags" {
return;
}
let mut visitor = TagCollectorVisitor::default();
event.record(&mut visitor);
self.tags.lock().unwrap().extend(visitor.tags);
}
}
fn started_inference_attempt(temp: &TempDir) -> anyhow::Result<InferenceTraceAttempt> {
let writer = Arc::new(TraceWriter::create(
temp.path(),
@@ -316,6 +362,41 @@ async fn dropped_response_stream_traces_cancelled_partial_output() -> anyhow::Re
Ok(())
}
#[tokio::test]
async fn response_stream_records_last_model_feedback_ids() {
let tags = Arc::new(Mutex::new(BTreeMap::new()));
let _guard = tracing_subscriber::registry()
.with(TagCollectorLayer { tags: tags.clone() })
.set_default();
let api_stream = futures::stream::iter([
Ok(ResponseEvent::Created),
Ok(ResponseEvent::Completed {
response_id: "resp-123".to_string(),
token_usage: None,
end_turn: Some(true),
}),
]);
let (mut stream, _) = super::map_response_events(
Some("req-123".to_string()),
api_stream,
test_session_telemetry(),
InferenceTraceAttempt::disabled(),
);
while stream.next().await.is_some() {}
let tags = tags.lock().unwrap().clone();
assert_eq!(
tags.get("last_model_request_id").map(String::as_str),
Some("\"req-123\"")
);
assert_eq!(
tags.get("last_model_response_id").map(String::as_str),
Some("\"resp-123\"")
);
}
#[tokio::test]
async fn dropped_backpressured_response_stream_traces_cancelled_partial_output()
-> anyhow::Result<()> {
+1 -4
View File
@@ -30,7 +30,6 @@ use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use crate::config::Config;
use crate::environment_selection::ResolvedTurnEnvironments;
use crate::guardian::GuardianApprovalRequest;
use crate::guardian::new_guardian_review_id;
use crate::guardian::routes_approval_to_guardian;
@@ -94,9 +93,7 @@ pub(crate) async fn run_codex_thread_interactive(
inherited_exec_policy: Some(Arc::clone(&parent_session.services.exec_policy)),
parent_rollout_thread_trace: codex_rollout_trace::ThreadTraceContext::disabled(),
parent_trace: None,
environment_selections: ResolvedTurnEnvironments {
turn_environments: parent_ctx.environments.clone(),
},
environment_selections: parent_ctx.environments.clone(),
analytics_events_client: Some(parent_session.services.analytics_events_client.clone()),
thread_store: Arc::clone(&parent_session.services.thread_store),
}))
+1 -1
View File
@@ -525,7 +525,7 @@ pub(crate) fn accessible_connectors_from_mcp_tools(
Some(codex_connectors::accessible::AccessibleConnectorTool {
connector_id: connector_id.to_string(),
connector_name: tool.connector_name.clone(),
connector_description: tool.connector_description.clone(),
connector_description: tool.namespace_description.clone(),
plugin_display_names: tool.plugin_display_names.clone(),
})
});
+3 -6
View File
@@ -119,11 +119,10 @@ fn codex_app_tool(
server_name: CODEX_APPS_MCP_SERVER_NAME.to_string(),
callable_name: tool_name.to_string(),
callable_namespace: tool_namespace,
server_instructions: None,
namespace_description: None,
tool: test_tool_definition(tool_name),
connector_id: Some(connector_id.to_string()),
connector_name: connector_name.map(ToOwned::to_owned),
connector_description: None,
plugin_display_names: plugin_names(plugin_display_names),
}
}
@@ -198,11 +197,10 @@ fn accessible_connectors_from_mcp_tools_carries_plugin_display_names() {
server_name: "sample".to_string(),
callable_name: "echo".to_string(),
callable_namespace: "sample".to_string(),
server_instructions: None,
namespace_description: None,
tool: test_tool_definition("echo"),
connector_id: None,
connector_name: None,
connector_description: None,
plugin_display_names: plugin_names(&["ignored"]),
},
),
@@ -323,7 +321,7 @@ fn accessible_connectors_from_mcp_tools_preserves_description() {
server_name: CODEX_APPS_MCP_SERVER_NAME.to_string(),
callable_name: "calendar_create_event".to_string(),
callable_namespace: "mcp__codex_apps__calendar".to_string(),
server_instructions: None,
namespace_description: Some("Plan events".to_string()),
tool: Tool {
name: "calendar_create_event".to_string().into(),
title: None,
@@ -337,7 +335,6 @@ fn accessible_connectors_from_mcp_tools_preserves_description() {
},
connector_id: Some("calendar".to_string()),
connector_name: Some("Calendar".to_string()),
connector_description: Some("Plan events".to_string()),
plugin_display_names: Vec::new(),
},
)]);
@@ -176,7 +176,9 @@ impl EnvironmentContext {
pub(crate) fn from_turn_context(turn_context: &TurnContext) -> Self {
Self::new(
EnvironmentContextEnvironment::from_turn_environments(&turn_context.environments),
EnvironmentContextEnvironment::from_turn_environments(
&turn_context.environments.turn_environments,
),
turn_context.current_date.clone(),
turn_context.timezone.clone(),
Self::network_from_turn_context(turn_context),
+5 -5
View File
@@ -24,7 +24,7 @@ pub(crate) fn default_thread_environment_selections(
.collect()
}
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Default)]
pub(crate) struct ResolvedTurnEnvironments {
pub(crate) turn_environments: Vec<TurnEnvironment>,
}
@@ -37,17 +37,17 @@ impl ResolvedTurnEnvironments {
.collect()
}
pub(crate) fn primary_turn_environment(&self) -> Option<&TurnEnvironment> {
pub(crate) fn primary(&self) -> Option<&TurnEnvironment> {
self.turn_environments.first()
}
pub(crate) fn primary_environment(&self) -> Option<Arc<codex_exec_server::Environment>> {
self.primary_turn_environment()
self.primary()
.map(|environment| Arc::clone(&environment.environment))
}
pub(crate) fn primary_filesystem(&self) -> Option<Arc<dyn ExecutorFileSystem>> {
self.primary_turn_environment()
self.primary()
.map(|environment| environment.environment.get_filesystem())
}
}
@@ -171,7 +171,7 @@ mod tests {
assert_eq!(
resolved
.primary_turn_environment()
.primary()
.expect("primary environment")
.environment_id,
"local"
+1 -1
View File
@@ -145,7 +145,7 @@ pub(crate) mod shell_snapshot;
pub mod spawn;
pub(crate) mod state_db_bridge;
pub use state_db_bridge::StateDbHandle;
pub use state_db_bridge::get_state_db;
pub use state_db_bridge::init_state_db;
mod thread_rollout_truncation;
mod tools;
pub(crate) mod turn_diff_tracker;
+63 -63
View File
@@ -18,6 +18,7 @@ use codex_hooks::HooksConfig;
use codex_model_provider::create_model_provider;
use codex_protocol::models::PermissionProfile;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::GranularApprovalConfig;
use core_test_support::PathExt;
use core_test_support::responses::ev_assistant_message;
use core_test_support::responses::ev_completed;
@@ -2328,7 +2329,7 @@ async fn prompt_mode_waits_for_approval_when_annotations_do_not_require_approval
}
#[tokio::test]
async fn approve_mode_blocks_when_arc_returns_interrupt_for_model() {
async fn approve_mode_skips_arc_interrupt_for_model() {
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
@@ -2349,7 +2350,7 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_for_model() {
"why": "high-risk action",
}],
})))
.expect(1)
.expect(0)
.mount(&server)
.await;
@@ -2391,16 +2392,11 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_for_model() {
)
.await;
assert_eq!(
decision,
Some(McpToolApprovalDecision::BlockedBySafetyMonitor(
"Tool call was cancelled because of safety risks: high-risk action".to_string(),
))
);
assert_eq!(decision, None);
}
#[tokio::test]
async fn custom_approve_mode_blocks_when_arc_returns_interrupt_for_model() {
async fn custom_approve_mode_skips_arc_interrupt_for_model() {
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
@@ -2421,7 +2417,7 @@ async fn custom_approve_mode_blocks_when_arc_returns_interrupt_for_model() {
"why": "high-risk action",
}],
})))
.expect(1)
.expect(0)
.mount(&server)
.await;
@@ -2463,16 +2459,11 @@ async fn custom_approve_mode_blocks_when_arc_returns_interrupt_for_model() {
)
.await;
assert_eq!(
decision,
Some(McpToolApprovalDecision::BlockedBySafetyMonitor(
"Tool call was cancelled because of safety risks: high-risk action".to_string(),
))
);
assert_eq!(decision, None);
}
#[tokio::test]
async fn approve_mode_blocks_when_arc_returns_interrupt_without_annotations() {
async fn approve_mode_skips_arc_interrupt_without_annotations() {
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
@@ -2493,7 +2484,7 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_without_annotations() {
"why": "high-risk action",
}],
})))
.expect(1)
.expect(0)
.mount(&server)
.await;
@@ -2535,12 +2526,7 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_without_annotations() {
)
.await;
assert_eq!(
decision,
Some(McpToolApprovalDecision::BlockedBySafetyMonitor(
"Tool call was cancelled because of safety risks: high-risk action".to_string(),
))
);
assert_eq!(decision, None);
}
#[tokio::test]
@@ -2622,7 +2608,7 @@ async fn full_access_mode_skips_arc_monitor_for_all_approval_modes() {
}
#[tokio::test]
async fn approve_mode_skips_arc_and_guardian_when_guardian_reviewer_is_enabled() {
async fn approve_mode_skips_arc_and_guardian_in_every_permission_mode() {
use wiremock::Mock;
use wiremock::ResponseTemplate;
use wiremock::matchers::method;
@@ -2652,33 +2638,6 @@ async fn approve_mode_skips_arc_and_guardian_when_guardian_reviewer_is_enabled()
.mount(&server)
.await;
let (mut session, mut turn_context) = make_session_and_context().await;
turn_context.auth_manager = Some(crate::test_support::auth_manager_from_auth(
codex_login::CodexAuth::create_dummy_chatgpt_auth_for_testing(),
));
turn_context
.approval_policy
.set(AskForApproval::OnRequest)
.expect("test setup should allow updating approval policy");
let mut config = (*turn_context.config).clone();
config.chatgpt_base_url = server.uri();
config.model_provider.base_url = Some(format!("{}/v1", server.uri()));
config.approvals_reviewer = ApprovalsReviewer::AutoReview;
let config = Arc::new(config);
let models_manager = models_manager_with_provider(
config.codex_home.to_path_buf(),
Arc::clone(&session.services.auth_manager),
config.model_provider.clone(),
);
session.services.models_manager = models_manager;
turn_context.config = Arc::clone(&config);
turn_context.provider = create_model_provider(
config.model_provider.clone(),
turn_context.auth_manager.clone(),
);
let session = Arc::new(session);
let turn_context = Arc::new(turn_context);
let invocation = McpInvocation {
server: CODEX_APPS_MCP_SERVER_NAME.to_string(),
tool: "dangerous_tool".to_string(),
@@ -2696,16 +2655,57 @@ async fn approve_mode_skips_arc_and_guardian_when_guardian_reviewer_is_enabled()
openai_file_input_params: None,
};
let decision = maybe_request_mcp_tool_approval(
&session,
&turn_context,
"call-3",
&invocation,
"mcp__test__tool",
Some(&metadata),
AppToolApproval::Approve,
)
.await;
for approval_policy in [
AskForApproval::UnlessTrusted,
AskForApproval::OnFailure,
AskForApproval::OnRequest,
AskForApproval::Granular(GranularApprovalConfig {
sandbox_approval: true,
rules: true,
skill_approval: true,
request_permissions: true,
mcp_elicitations: true,
}),
AskForApproval::Never,
] {
let (mut session, mut turn_context) = make_session_and_context().await;
turn_context.auth_manager = Some(crate::test_support::auth_manager_from_auth(
codex_login::CodexAuth::create_dummy_chatgpt_auth_for_testing(),
));
turn_context
.approval_policy
.set(approval_policy)
.expect("test setup should allow updating approval policy");
let mut config = (*turn_context.config).clone();
config.chatgpt_base_url = server.uri();
config.model_provider.base_url = Some(format!("{}/v1", server.uri()));
config.approvals_reviewer = ApprovalsReviewer::User;
let config = Arc::new(config);
let models_manager = models_manager_with_provider(
config.codex_home.to_path_buf(),
Arc::clone(&session.services.auth_manager),
config.model_provider.clone(),
);
session.services.models_manager = models_manager;
turn_context.config = Arc::clone(&config);
turn_context.provider = create_model_provider(
config.model_provider.clone(),
turn_context.auth_manager.clone(),
);
assert_eq!(decision, None);
let session = Arc::new(session);
let turn_context = Arc::new(turn_context);
let decision = maybe_request_mcp_tool_approval(
&session,
&turn_context,
"call-3",
&invocation,
"mcp__test__tool",
Some(&metadata),
AppToolApproval::Approve,
)
.await;
assert_eq!(decision, None);
}
}
+1 -2
View File
@@ -58,7 +58,7 @@ fn make_mcp_tool(
server_name: server_name.to_string(),
callable_name: tool_name.to_string(),
callable_namespace: tool_namespace,
server_instructions: None,
namespace_description: None,
tool: Tool {
name: tool_name.to_string().into(),
title: None,
@@ -73,7 +73,6 @@ fn make_mcp_tool(
connector_id: connector_id.map(str::to_string),
connector_name: connector_name.map(str::to_string),
plugin_display_names: Vec::new(),
connector_description: None,
}
}
+16 -7
View File
@@ -1,6 +1,7 @@
use crate::config::edit::ConfigEditsBuilder;
use codex_config::config_toml::ConfigToml;
use codex_protocol::config_types::Personality;
use codex_rollout::state_db::StateDbHandle;
use codex_thread_store::ListThreadsParams;
use codex_thread_store::LocalThreadStore;
use codex_thread_store::LocalThreadStoreConfig;
@@ -24,6 +25,7 @@ pub enum PersonalityMigrationStatus {
pub async fn maybe_migrate_personality(
codex_home: &Path,
config_toml: &ConfigToml,
state_db: Option<StateDbHandle>,
) -> io::Result<PersonalityMigrationStatus> {
let marker_path = codex_home.join(PERSONALITY_MIGRATION_FILENAME);
if tokio::fs::try_exists(&marker_path).await? {
@@ -43,7 +45,7 @@ pub async fn maybe_migrate_personality(
.or_else(|| config_toml.model_provider.clone())
.unwrap_or_else(|| "openai".to_string());
if !has_recorded_sessions(codex_home, model_provider_id.as_str()).await? {
if !has_recorded_sessions(codex_home, model_provider_id.as_str(), state_db).await? {
create_marker(&marker_path).await?;
return Ok(PersonalityMigrationStatus::SkippedNoSessions);
}
@@ -60,12 +62,19 @@ pub async fn maybe_migrate_personality(
Ok(PersonalityMigrationStatus::Applied)
}
async fn has_recorded_sessions(codex_home: &Path, default_provider: &str) -> io::Result<bool> {
let store = LocalThreadStore::new(LocalThreadStoreConfig {
codex_home: codex_home.to_path_buf(),
sqlite_home: codex_home.to_path_buf(),
default_model_provider_id: default_provider.to_string(),
});
async fn has_recorded_sessions(
codex_home: &Path,
default_provider: &str,
state_db: Option<StateDbHandle>,
) -> io::Result<bool> {
let store = LocalThreadStore::new(
LocalThreadStoreConfig {
codex_home: codex_home.to_path_buf(),
sqlite_home: codex_home.to_path_buf(),
default_model_provider_id: default_provider.to_string(),
},
state_db,
);
if has_threads(&store, /*archived*/ false).await? {
return Ok(true);
}
@@ -87,7 +87,7 @@ async fn applies_when_sessions_exist_and_no_personality() -> io::Result<()> {
write_session_with_user_event(temp.path()).await?;
let config_toml = ConfigToml::default();
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::Applied);
assert!(temp.path().join(PERSONALITY_MIGRATION_FILENAME).exists());
@@ -103,7 +103,7 @@ async fn applies_when_only_archived_sessions_exist_and_no_personality() -> io::R
write_archived_session_with_user_event(temp.path()).await?;
let config_toml = ConfigToml::default();
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::Applied);
assert!(temp.path().join(PERSONALITY_MIGRATION_FILENAME).exists());
@@ -119,7 +119,7 @@ async fn skips_when_marker_exists() -> io::Result<()> {
create_marker(&temp.path().join(PERSONALITY_MIGRATION_FILENAME)).await?;
let config_toml = ConfigToml::default();
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::SkippedMarker);
assert!(!temp.path().join("config.toml").exists());
@@ -136,7 +136,7 @@ async fn skips_when_personality_explicit() -> io::Result<()> {
.map_err(|err| io::Error::other(format!("failed to write config: {err}")))?;
let config_toml = read_config_toml(temp.path()).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None).await?;
assert_eq!(
status,
@@ -153,7 +153,7 @@ async fn skips_when_personality_explicit() -> io::Result<()> {
async fn skips_when_no_sessions() -> io::Result<()> {
let temp = TempDir::new()?;
let config_toml = ConfigToml::default();
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::SkippedNoSessions);
assert!(temp.path().join(PERSONALITY_MIGRATION_FILENAME).exists());
+5 -1
View File
@@ -16,6 +16,7 @@ use crate::config::Config;
use crate::session::session::Session;
use crate::session::turn::build_prompt;
use crate::session::turn::built_tools;
use crate::state_db_bridge::StateDbHandle;
use crate::thread_manager::ThreadManager;
use crate::thread_manager::thread_store_from_config;
@@ -24,6 +25,7 @@ use crate::thread_manager::thread_store_from_config;
pub async fn build_prompt_input(
mut config: Config,
input: Vec<UserInput>,
state_db: Option<StateDbHandle>,
) -> CodexResult<Vec<ResponseItem>> {
config.ephemeral = true;
@@ -35,13 +37,15 @@ pub async fn build_prompt_input(
config.codex_linux_sandbox_exe.clone(),
)?;
let thread_store = thread_store_from_config(&config, state_db.clone());
let thread_manager = ThreadManager::new(
&config,
Arc::clone(&auth_manager),
SessionSource::Exec,
Arc::new(EnvironmentManager::new(EnvironmentManagerArgs::new(local_runtime_paths)).await),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store,
state_db.clone(),
);
let thread = thread_manager.start_thread(config).await?;
+1 -1
View File
@@ -221,7 +221,7 @@ impl Session {
let mcp_servers = with_codex_apps_mcp(mcp_servers, auth.as_ref(), &mcp_config);
let auth_statuses =
compute_auth_statuses(mcp_servers.iter(), store_mode, auth.as_ref()).await;
let mcp_runtime_environment = match turn_context.primary_environment() {
let mcp_runtime_environment = match turn_context.environments.primary() {
Some(turn_environment) => McpRuntimeEnvironment::new(
Arc::clone(&turn_environment.environment),
turn_environment.cwd.to_path_buf(),
+11 -1
View File
@@ -355,6 +355,7 @@ use codex_protocol::protocol::TokenUsage;
use codex_protocol::protocol::TokenUsageInfo;
use codex_protocol::protocol::WarningEvent;
use codex_protocol::user_input::UserInput;
use codex_tools::ToolEnvironmentMode;
use codex_tools::ToolsConfig;
use codex_tools::ToolsConfigParams;
use codex_utils_absolute_path::AbsolutePathBuf;
@@ -555,7 +556,15 @@ impl Codex {
};
match thread_id {
Some(thread_id) => {
let state_db_ctx = state_db::get_state_db(&config).await;
let state_db_ctx = if config.ephemeral {
None
} else if let Some(local_store) =
thread_store.as_any().downcast_ref::<LocalThreadStore>()
{
local_store.state_db().await
} else {
None
};
state_db::get_dynamic_tools(state_db_ctx.as_deref(), thread_id, "codex_spawn")
.await
}
@@ -1297,6 +1306,7 @@ impl Session {
self.services.user_shell.as_ref().clone(),
self.services.shell_snapshot_tx.clone(),
self.services.session_telemetry.clone(),
self.services.state_db.clone(),
);
}
+1 -1
View File
@@ -52,7 +52,7 @@ pub(super) async fn spawn_review_thread(
)
.with_web_search_config(/*web_search_config*/ None)
.with_allow_login_shell(config.permissions.allow_login_shell)
.with_has_environment(parent_turn_context.tools_config.has_environment)
.with_environment_mode(parent_turn_context.tools_config.environment_mode)
.with_spawn_agent_usage_hint(config.multi_agent_v2.usage_hint_enabled)
.with_spawn_agent_usage_hint_text(config.multi_agent_v2.usage_hint_text.clone())
.with_hide_spawn_agent_metadata(config.multi_agent_v2.hide_spawn_agent_metadata)
+2 -1
View File
@@ -726,6 +726,7 @@ impl Session {
session_configuration.cwd.clone(),
&mut default_shell,
session_telemetry.clone(),
state_db_ctx.clone(),
)
}
} else {
@@ -968,7 +969,7 @@ impl Session {
"unknown stored MCP environment id",
))
})?
.primary_turn_environment()
.primary()
.cloned();
let mcp_runtime_environment = match turn_environment {
Some(turn_environment) => McpRuntimeEnvironment::new(
+101 -45
View File
@@ -2944,13 +2944,15 @@ pub(crate) async fn make_session_configuration_for_tests() -> SessionConfigurati
fn turn_environments_for_tests(
environment: &Arc<codex_exec_server::Environment>,
cwd: &codex_utils_absolute_path::AbsolutePathBuf,
) -> Vec<TurnEnvironment> {
vec![TurnEnvironment {
environment_id: codex_exec_server::LOCAL_ENVIRONMENT_ID.to_string(),
environment: Arc::clone(environment),
cwd: cwd.clone(),
shell: "bash".to_string(),
}]
) -> crate::environment_selection::ResolvedTurnEnvironments {
crate::environment_selection::ResolvedTurnEnvironments {
turn_environments: vec![TurnEnvironment {
environment_id: codex_exec_server::LOCAL_ENVIRONMENT_ID.to_string(),
environment: Arc::clone(environment),
cwd: cwd.clone(),
shell: "bash".to_string(),
}],
}
}
#[tokio::test]
@@ -3399,7 +3401,7 @@ async fn absolute_cwd_update_with_turn_environment_is_allowed() {
assert_eq!(turn_context.cwd, absolute_cwd);
assert_eq!(turn_context.config.cwd, absolute_cwd);
assert_eq!(turn_context.environments.len(), 1);
assert_eq!(turn_context.environments.turn_environments.len(), 1);
}
#[tokio::test]
@@ -3490,6 +3492,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_zsh_path() {
/*analytics_events_client*/ None,
Arc::new(codex_thread_store::LocalThreadStore::new(
codex_thread_store::LocalThreadStoreConfig::from_config(config.as_ref()),
/*state_db*/ None,
)),
codex_rollout_trace::ThreadTraceContext::disabled(),
)
@@ -3637,6 +3640,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
live_thread: None,
thread_store: Arc::new(codex_thread_store::LocalThreadStore::new(
codex_thread_store::LocalThreadStoreConfig::from_config(config.as_ref()),
/*state_db*/ None,
)),
model_client: ModelClient::new(
Some(auth_manager.clone()),
@@ -3808,6 +3812,7 @@ async fn make_session_with_config_and_rx(
/*analytics_events_client*/ None,
Arc::new(codex_thread_store::LocalThreadStore::new(
codex_thread_store::LocalThreadStoreConfig::from_config(config.as_ref()),
/*state_db*/ None,
)),
codex_rollout_trace::ThreadTraceContext::disabled(),
)
@@ -4418,15 +4423,16 @@ async fn turn_environments_set_primary_environment() {
.expect("turn should start");
let turn_environments = &turn_context.environments;
assert_eq!(turn_environments.len(), 1);
assert_eq!(turn_environments.turn_environments.len(), 1);
let turn_environment = turn_context
.primary_environment()
.environments
.primary()
.expect("primary environment should be set");
assert!(std::sync::Arc::ptr_eq(
&turn_environment.environment,
&turn_environments[0].environment
&turn_environments.turn_environments[0].environment
));
assert!(!turn_context.environments.is_empty());
assert!(!turn_context.environments.turn_environments.is_empty());
assert_eq!(turn_context.cwd.as_path(), selected_cwd.as_path());
assert_eq!(turn_context.config.cwd.as_path(), selected_cwd.as_path());
}
@@ -4449,13 +4455,14 @@ async fn default_turn_overlays_session_cwd_onto_stored_thread_environments() {
let turn_context = session.new_default_turn().await;
let turn_environments = &turn_context.environments;
assert_eq!(turn_environments.len(), 1);
assert_eq!(turn_environments.turn_environments.len(), 1);
let turn_environment = turn_context
.primary_environment()
.environments
.primary()
.expect("primary environment should be set");
assert!(std::sync::Arc::ptr_eq(
&turn_environment.environment,
&turn_environments[0].environment
&turn_environments.turn_environments[0].environment
));
assert_eq!(turn_context.cwd, session_cwd);
assert_eq!(turn_context.config.cwd, session_cwd);
@@ -4473,28 +4480,32 @@ async fn default_turn_honors_empty_stored_thread_environments() {
let turn_context = session.new_default_turn().await;
assert!(turn_context.primary_environment().is_none());
assert!(turn_context.environments.is_empty());
assert!(turn_context.environments.primary().is_none());
assert!(turn_context.environments.turn_environments.is_empty());
assert_eq!(turn_context.cwd, session_cwd);
assert_eq!(turn_context.config.cwd, session_cwd);
assert_eq!(turn_context.environments.len(), 0);
assert_eq!(turn_context.environments.turn_environments.len(), 0);
}
#[tokio::test]
async fn primary_environment_uses_first_turn_environment() {
let (_session, mut turn_context) = make_session_and_context().await;
let first_environment = turn_context.environments[0].clone();
let first_environment = turn_context.environments.turn_environments[0].clone();
let second_cwd = turn_context.cwd.join("second");
turn_context.environments.push(TurnEnvironment {
environment_id: "second".to_string(),
environment: Arc::clone(&first_environment.environment),
cwd: second_cwd.clone(),
shell: first_environment.shell.clone(),
});
turn_context
.environments
.turn_environments
.push(TurnEnvironment {
environment_id: "second".to_string(),
environment: Arc::clone(&first_environment.environment),
cwd: second_cwd.clone(),
shell: first_environment.shell.clone(),
});
assert_eq!(
turn_context
.primary_environment()
.environments
.primary()
.expect("primary environment")
.environment_id,
first_environment.environment_id
@@ -4502,14 +4513,18 @@ async fn primary_environment_uses_first_turn_environment() {
assert_eq!(
turn_context
.environments
.turn_environments
.iter()
.find(|environment| environment.environment_id == "second")
.expect("second environment")
.cwd,
second_cwd
);
assert_eq!(turn_context.environments.len(), 2);
assert_eq!(turn_context.environments[1].cwd, second_cwd);
assert_eq!(turn_context.environments.turn_environments.len(), 2);
assert_eq!(
turn_context.environments.turn_environments[1].cwd,
second_cwd
);
}
#[tokio::test]
@@ -4527,8 +4542,8 @@ async fn empty_turn_environments_clear_primary_environment() {
.await
.expect("turn should start");
assert!(turn_context.primary_environment().is_none());
assert!(turn_context.environments.is_empty());
assert!(turn_context.environments.primary().is_none());
assert!(turn_context.environments.turn_environments.is_empty());
assert_eq!(turn_context.cwd, session.get_config().await.cwd);
assert_eq!(turn_context.config.cwd, session.get_config().await.cwd);
}
@@ -4986,10 +5001,44 @@ async fn make_session_and_context_with_auth_and_config_and_rx<F>(
where
F: FnOnce(&mut Config),
{
let (tx_event, rx_event) = async_channel::unbounded();
let codex_home = tempfile::tempdir().expect("create temp dir");
let mut config = build_test_config(codex_home.path()).await;
make_session_and_context_with_auth_config_home_and_rx(
auth,
dynamic_tools,
codex_home.path(),
configure_config,
)
.await
}
async fn make_session_and_context_with_auth_config_home_and_rx<F>(
auth: CodexAuth,
dynamic_tools: Vec<DynamicToolSpec>,
codex_home: &Path,
configure_config: F,
) -> (
Arc<Session>,
Arc<TurnContext>,
async_channel::Receiver<Event>,
)
where
F: FnOnce(&mut Config),
{
let (tx_event, rx_event) = async_channel::unbounded();
let mut config = build_test_config(codex_home).await;
configure_config(&mut config);
let state_db = if config.features.enabled(Feature::Goals) {
Some(
codex_state::StateRuntime::init(
config.sqlite_home.clone(),
config.model_provider_id.clone(),
)
.await
.expect("goal tests should initialize sqlite state db"),
)
} else {
None
};
let config = Arc::new(config);
let conversation_id = ThreadId::default();
let auth_manager = AuthManager::from_auth_for_testing(auth);
@@ -5115,10 +5164,11 @@ where
agent_control,
network_proxy: None,
network_approval: Arc::clone(&network_approval),
state_db: None,
state_db: state_db.clone(),
live_thread: None,
thread_store: Arc::new(codex_thread_store::LocalThreadStore::new(
codex_thread_store::LocalThreadStoreConfig::from_config(config.as_ref()),
state_db,
)),
model_client: ModelClient::new(
Some(Arc::clone(&auth_manager)),
@@ -5213,10 +5263,13 @@ async fn make_goal_session_and_context_with_rx() -> (
Arc<Session>,
Arc<TurnContext>,
async_channel::Receiver<Event>,
tempfile::TempDir,
) {
let (session, turn_context, rx) = make_session_and_context_with_auth_and_config_and_rx(
let codex_home = tempfile::tempdir().expect("create temp dir");
let (session, turn_context, rx) = make_session_and_context_with_auth_config_home_and_rx(
CodexAuth::from_api_key("Test API Key"),
Vec::new(),
codex_home.path(),
|config| {
config
.features
@@ -5226,14 +5279,14 @@ async fn make_goal_session_and_context_with_rx() -> (
)
.await;
upsert_goal_test_thread(session.as_ref()).await;
(session, turn_context, rx)
(session, turn_context, rx, codex_home)
}
async fn upsert_goal_test_thread(session: &Session) {
let config = session.get_config().await;
let state_db = goal_test_state_db(session)
.await
.expect("goal test state db should initialize");
let state_db = session
.state_db()
.expect("goal test session should have a state db");
let mut builder = codex_state::ThreadMetadataBuilder::new(
session.conversation_id,
config
@@ -7050,7 +7103,7 @@ async fn abort_empty_active_turn_preserves_pending_input() {
#[tokio::test]
async fn interrupt_accounts_active_goal_before_pausing() -> anyhow::Result<()> {
let (sess, tc, _rx) = make_goal_session_and_context_with_rx().await;
let (sess, tc, _rx, _codex_home) = make_goal_session_and_context_with_rx().await;
sess.set_thread_goal(
tc.as_ref(),
SetGoalRequest {
@@ -7304,6 +7357,9 @@ fn post_goal_token_usage() -> TokenUsage {
}
async fn goal_test_state_db(sess: &Session) -> anyhow::Result<crate::StateDbHandle> {
if let Some(state_db) = sess.state_db() {
return Ok(state_db);
}
let config = sess.get_config().await;
codex_state::StateRuntime::init(config.sqlite_home.clone(), config.model_provider_id.clone())
.await
@@ -7311,7 +7367,7 @@ async fn goal_test_state_db(sess: &Session) -> anyhow::Result<crate::StateDbHand
#[tokio::test]
async fn budget_limited_accounting_steers_active_turn_without_aborting() -> anyhow::Result<()> {
let (sess, tc, rx) = make_goal_session_and_context_with_rx().await;
let (sess, tc, rx, _codex_home) = make_goal_session_and_context_with_rx().await;
sess.set_thread_goal(
tc.as_ref(),
SetGoalRequest {
@@ -7411,7 +7467,7 @@ async fn budget_limited_accounting_steers_active_turn_without_aborting() -> anyh
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn external_goal_mutation_accounts_active_turn_before_status_change() -> anyhow::Result<()> {
let (sess, tc, _rx) = make_goal_session_and_context_with_rx().await;
let (sess, tc, _rx, _codex_home) = make_goal_session_and_context_with_rx().await;
sess.set_thread_goal(
tc.as_ref(),
SetGoalRequest {
@@ -7473,7 +7529,7 @@ async fn external_goal_mutation_accounts_active_turn_before_status_change() -> a
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn external_active_goal_set_marks_current_turn_for_accounting() -> anyhow::Result<()> {
let (sess, tc, _rx) = make_goal_session_and_context_with_rx().await;
let (sess, tc, _rx, _codex_home) = make_goal_session_and_context_with_rx().await;
sess.spawn_task(
Arc::clone(&tc),
Vec::new(),
@@ -8137,7 +8193,7 @@ async fn sample_rollout(
#[tokio::test]
async fn create_goal_tool_rejects_existing_goal() {
let (session, turn_context, _rx) = make_goal_session_and_context_with_rx().await;
let (session, turn_context, _rx, _codex_home) = make_goal_session_and_context_with_rx().await;
let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new()));
let handler = GoalHandler;
@@ -8199,7 +8255,7 @@ async fn create_goal_tool_rejects_existing_goal() {
#[tokio::test]
async fn update_goal_tool_rejects_pausing_goal() {
let (session, turn_context, _rx) = make_goal_session_and_context_with_rx().await;
let (session, turn_context, _rx, _codex_home) = make_goal_session_and_context_with_rx().await;
let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new()));
let handler = GoalHandler;
@@ -8259,7 +8315,7 @@ async fn update_goal_tool_rejects_pausing_goal() {
#[tokio::test]
async fn update_goal_tool_marks_goal_complete() {
let (session, turn_context, _rx) = make_goal_session_and_context_with_rx().await;
let (session, turn_context, _rx, _codex_home) = make_goal_session_and_context_with_rx().await;
let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new()));
let handler = GoalHandler;
@@ -731,6 +731,7 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() {
let skills_watcher = Arc::new(SkillsWatcher::noop());
let thread_store = Arc::new(codex_thread_store::LocalThreadStore::new(
codex_thread_store::LocalThreadStoreConfig::from_config(&config),
/*state_db*/ None,
));
let CodexSpawnOk { codex, .. } = Codex::spawn(CodexSpawnArgs {
+11 -13
View File
@@ -1,5 +1,6 @@
use super::*;
use crate::config::GhostSnapshotConfig;
use crate::environment_selection::ResolvedTurnEnvironments;
use codex_model_provider::SharedModelProvider;
use codex_model_provider::create_model_provider;
use codex_protocol::models::AdditionalPermissionProfile;
@@ -60,7 +61,7 @@ pub(crate) struct TurnContext {
pub(crate) reasoning_effort: Option<ReasoningEffortConfig>,
pub(crate) reasoning_summary: ReasoningSummaryConfig,
pub(crate) session_source: SessionSource,
pub(crate) environments: Vec<TurnEnvironment>,
pub(crate) environments: ResolvedTurnEnvironments,
/// The session's absolute working directory. All relative paths provided
/// by the model as well as sandbox policies are resolved against this path
/// instead of `std::env::current_dir()`.
@@ -106,10 +107,6 @@ impl TurnContext {
self.permission_profile.network_sandbox_policy()
}
pub(crate) fn primary_environment(&self) -> Option<&TurnEnvironment> {
self.environments.first()
}
pub(crate) fn sandbox_policy(&self) -> SandboxPolicy {
let file_system_sandbox_policy = self.file_system_sandbox_policy();
let network_sandbox_policy = self.network_sandbox_policy();
@@ -198,7 +195,7 @@ impl TurnContext {
.with_unified_exec_shell_mode(self.tools_config.unified_exec_shell_mode.clone())
.with_web_search_config(self.tools_config.web_search_config.clone())
.with_allow_login_shell(self.tools_config.allow_login_shell)
.with_has_environment(self.tools_config.has_environment)
.with_environment_mode(self.tools_config.environment_mode)
.with_spawn_agent_usage_hint(config.multi_agent_v2.usage_hint_enabled)
.with_spawn_agent_usage_hint_text(config.multi_agent_v2.usage_hint_text.clone())
.with_hide_spawn_agent_metadata(config.multi_agent_v2.hide_spawn_agent_metadata)
@@ -435,7 +432,7 @@ impl Session {
model_info: ModelInfo,
models_manager: &SharedModelsManager,
network: Option<NetworkProxy>,
environments: Vec<TurnEnvironment>,
environments: ResolvedTurnEnvironments,
cwd: AbsolutePathBuf,
sub_id: String,
skills_outcome: Arc<SkillLoadOutcome>,
@@ -476,7 +473,9 @@ impl Session {
)
.with_web_search_config(per_turn_config.web_search_config.clone())
.with_allow_login_shell(per_turn_config.permissions.allow_login_shell)
.with_has_environment(!environments.is_empty())
.with_environment_mode(ToolEnvironmentMode::from_count(
environments.turn_environments.len(),
))
.with_spawn_agent_usage_hint(per_turn_config.multi_agent_v2.usage_hint_enabled)
.with_spawn_agent_usage_hint_text(per_turn_config.multi_agent_v2.usage_hint_text.clone())
.with_hide_spawn_agent_metadata(per_turn_config.multi_agent_v2.hide_spawn_agent_metadata)
@@ -647,12 +646,11 @@ impl Session {
fn resolve_turn_environments(
&self,
environments: &[TurnEnvironmentSelection],
) -> CodexResult<Vec<TurnEnvironment>> {
) -> CodexResult<ResolvedTurnEnvironments> {
crate::environment_selection::resolve_environment_selections(
self.services.environment_manager.as_ref(),
environments,
)
.map(|resolved| resolved.turn_environments)
}
async fn new_turn_from_configuration(
@@ -660,9 +658,9 @@ impl Session {
sub_id: String,
session_configuration: SessionConfiguration,
final_output_json_schema: Option<Option<Value>>,
turn_environments: Vec<TurnEnvironment>,
turn_environments: ResolvedTurnEnvironments,
) -> Arc<TurnContext> {
let primary_turn_environment = turn_environments.first();
let primary_turn_environment = turn_environments.primary();
let cwd = primary_turn_environment
.map(|turn_environment| turn_environment.cwd.clone())
.unwrap_or_else(|| session_configuration.cwd.clone());
@@ -769,7 +767,7 @@ impl Session {
Ok(turn_environments) => turn_environments,
Err(err) => {
warn!("failed to resolve stored session environments: {err}");
Vec::new()
ResolvedTurnEnvironments::default()
}
};
+22 -6
View File
@@ -5,6 +5,7 @@ use std::sync::Arc;
use std::time::Duration;
use std::time::SystemTime;
use crate::StateDbHandle;
use crate::rollout::list::find_thread_path_by_id_str;
use crate::shell::Shell;
use crate::shell::ShellType;
@@ -41,6 +42,7 @@ impl ShellSnapshot {
session_cwd: AbsolutePathBuf,
shell: &mut Shell,
session_telemetry: SessionTelemetry,
state_db: Option<StateDbHandle>,
) -> watch::Sender<Option<Arc<ShellSnapshot>>> {
let (shell_snapshot_tx, shell_snapshot_rx) = watch::channel(None);
shell.shell_snapshot = shell_snapshot_rx;
@@ -52,6 +54,7 @@ impl ShellSnapshot {
shell.clone(),
shell_snapshot_tx.clone(),
session_telemetry,
state_db,
);
shell_snapshot_tx
@@ -64,6 +67,7 @@ impl ShellSnapshot {
shell: Shell,
shell_snapshot_tx: watch::Sender<Option<Arc<ShellSnapshot>>>,
session_telemetry: SessionTelemetry,
state_db: Option<StateDbHandle>,
) {
Self::spawn_snapshot_task(
codex_home,
@@ -72,6 +76,7 @@ impl ShellSnapshot {
shell,
shell_snapshot_tx,
session_telemetry,
state_db,
);
}
@@ -82,15 +87,21 @@ impl ShellSnapshot {
snapshot_shell: Shell,
shell_snapshot_tx: watch::Sender<Option<Arc<ShellSnapshot>>>,
session_telemetry: SessionTelemetry,
state_db: Option<StateDbHandle>,
) {
let snapshot_span = info_span!("shell_snapshot", thread_id = %session_id);
tokio::spawn(
async move {
let timer = session_telemetry.start_timer("codex.shell_snapshot.duration_ms", &[]);
let snapshot =
ShellSnapshot::try_new(&codex_home, session_id, &session_cwd, &snapshot_shell)
.await
.map(Arc::new);
let snapshot = ShellSnapshot::try_new(
&codex_home,
session_id,
&session_cwd,
&snapshot_shell,
state_db,
)
.await
.map(Arc::new);
let success = snapshot.is_ok();
let success_tag = if success { "true" } else { "false" };
let _ = timer.map(|timer| timer.record(&[("success", success_tag)]));
@@ -110,6 +121,7 @@ impl ShellSnapshot {
session_id: ThreadId,
session_cwd: &AbsolutePathBuf,
shell: &Shell,
state_db: Option<StateDbHandle>,
) -> std::result::Result<Self, &'static str> {
// File to store the snapshot
let extension = match shell.shell_type {
@@ -131,7 +143,9 @@ impl ShellSnapshot {
let codex_home = codex_home.clone();
let cleanup_session_id = session_id;
tokio::spawn(async move {
if let Err(err) = cleanup_stale_snapshots(&codex_home, cleanup_session_id).await {
if let Err(err) =
cleanup_stale_snapshots(&codex_home, cleanup_session_id, state_db).await
{
tracing::warn!("Failed to clean up shell snapshots: {err:?}");
}
});
@@ -486,6 +500,7 @@ $envVars | ForEach-Object {
pub async fn cleanup_stale_snapshots(
codex_home: &AbsolutePathBuf,
active_session_id: ThreadId,
state_db: Option<StateDbHandle>,
) -> Result<()> {
let snapshot_dir = codex_home.join(SNAPSHOT_DIR);
@@ -515,7 +530,8 @@ pub async fn cleanup_stale_snapshots(
continue;
}
let rollout_path = find_thread_path_by_id_str(codex_home, session_id).await?;
let rollout_path =
find_thread_path_by_id_str(codex_home, session_id, state_db.as_deref()).await?;
let Some(rollout_path) = rollout_path else {
remove_snapshot_file(&path).await;
continue;
+22 -11
View File
@@ -202,6 +202,7 @@ async fn try_new_creates_and_deletes_snapshot_file() -> Result<()> {
ThreadId::new(),
&dir.path().abs(),
&shell,
/*state_db*/ None,
)
.await
.expect("snapshot should be created");
@@ -227,14 +228,24 @@ async fn try_new_uses_distinct_generation_paths() -> Result<()> {
shell_snapshot: crate::shell::empty_shell_snapshot_receiver(),
};
let initial_snapshot =
ShellSnapshot::try_new(&dir.path().abs(), session_id, &dir.path().abs(), &shell)
.await
.expect("initial snapshot should be created");
let refreshed_snapshot =
ShellSnapshot::try_new(&dir.path().abs(), session_id, &dir.path().abs(), &shell)
.await
.expect("refreshed snapshot should be created");
let initial_snapshot = ShellSnapshot::try_new(
&dir.path().abs(),
session_id,
&dir.path().abs(),
&shell,
/*state_db*/ None,
)
.await
.expect("initial snapshot should be created");
let refreshed_snapshot = ShellSnapshot::try_new(
&dir.path().abs(),
session_id,
&dir.path().abs(),
&shell,
/*state_db*/ None,
)
.await
.expect("refreshed snapshot should be created");
let initial_path = initial_snapshot.path.clone();
let refreshed_path = refreshed_snapshot.path.clone();
@@ -428,7 +439,7 @@ async fn cleanup_stale_snapshots_removes_orphans_and_keeps_live() -> Result<()>
fs::write(&orphan_snapshot, "orphan").await?;
fs::write(&invalid_snapshot, "invalid").await?;
cleanup_stale_snapshots(&codex_home, ThreadId::new()).await?;
cleanup_stale_snapshots(&codex_home, ThreadId::new(), /*state_db*/ None).await?;
assert_eq!(live_snapshot.exists(), true);
assert_eq!(orphan_snapshot.exists(), false);
@@ -451,7 +462,7 @@ async fn cleanup_stale_snapshots_removes_stale_rollouts() -> Result<()> {
set_file_mtime(&rollout_path, SNAPSHOT_RETENTION + Duration::from_secs(60))?;
cleanup_stale_snapshots(&codex_home, ThreadId::new()).await?;
cleanup_stale_snapshots(&codex_home, ThreadId::new(), /*state_db*/ None).await?;
assert_eq!(stale_snapshot.exists(), false);
Ok(())
@@ -472,7 +483,7 @@ async fn cleanup_stale_snapshots_skips_active_session() -> Result<()> {
set_file_mtime(&rollout_path, SNAPSHOT_RETENTION + Duration::from_secs(60))?;
cleanup_stale_snapshots(&codex_home, active_session).await?;
cleanup_stale_snapshots(&codex_home, active_session, /*state_db*/ None).await?;
assert_eq!(active_snapshot.exists(), true);
Ok(())
+2 -2
View File
@@ -3,6 +3,6 @@ pub use codex_rollout::state_db::StateDbHandle;
use crate::config::Config;
pub async fn get_state_db(config: &Config) -> Option<StateDbHandle> {
rollout_state_db::get_state_db(config).await
pub async fn init_state_db(config: &Config) -> Option<StateDbHandle> {
rollout_state_db::init(config).await
}
+7 -4
View File
@@ -138,8 +138,11 @@ pub(crate) async fn record_completed_response_item(
.await;
}
mark_thread_memory_mode_polluted_if_external_context(sess, turn_context, item).await;
let has_memory_citation =
record_stage1_output_usage_and_detect_memory_citation(turn_context, item).await;
let has_memory_citation = record_stage1_output_usage_and_detect_memory_citation(
sess.services.state_db.as_ref(),
item,
)
.await;
if has_memory_citation {
sess.record_memory_citation_for_turn(&turn_context.sub_id)
.await;
@@ -174,7 +177,7 @@ pub(crate) async fn mark_thread_memory_mode_polluted_if_external_context(
}
async fn record_stage1_output_usage_and_detect_memory_citation(
turn_context: &TurnContext,
state_db_ctx: Option<&state_db::StateDbHandle>,
item: &ResponseItem,
) -> bool {
let Some(raw_text) = raw_assistant_output_text_from_item(item) else {
@@ -190,7 +193,7 @@ async fn record_stage1_output_usage_and_detect_memory_citation(
return true;
}
if let Some(db) = state_db::get_state_db(turn_context.config.as_ref()).await {
if let Some(db) = state_db_ctx {
let _ = db.record_stage1_output_usage(&thread_ids).await;
}
true
+16
View File
@@ -73,6 +73,22 @@ pub fn thread_manager_with_models_provider_and_home(
)
}
pub fn thread_manager_with_models_provider_home_and_state(
auth: CodexAuth,
provider: ModelProviderInfo,
codex_home: PathBuf,
environment_manager: Arc<EnvironmentManager>,
state_db: Option<crate::StateDbHandle>,
) -> ThreadManager {
ThreadManager::with_models_provider_home_and_state_for_tests(
auth,
provider,
codex_home,
environment_manager,
state_db,
)
}
pub async fn start_thread_with_user_shell_override(
thread_manager: &ThreadManager,
config: Config,
+36 -5
View File
@@ -50,6 +50,7 @@ use codex_protocol::protocol::TurnAbortReason;
use codex_protocol::protocol::TurnAbortedEvent;
use codex_protocol::protocol::TurnEnvironmentSelection;
use codex_protocol::protocol::W3cTraceContext;
use codex_rollout::state_db::StateDbHandle;
use codex_state::DirectionalThreadSpawnEdgeStatus;
use codex_thread_store::InMemoryThreadStore;
use codex_thread_store::LocalThreadStore;
@@ -248,6 +249,7 @@ pub(crate) struct ThreadManagerState {
thread_store: Arc<dyn ThreadStore>,
session_source: SessionSource,
analytics_events_client: Option<AnalyticsEventsClient>,
state_db: Option<StateDbHandle>,
// Captures submitted ops for testing purpose when test mode is enabled.
ops_log: Option<SharedCapturedOps>,
}
@@ -263,10 +265,14 @@ pub fn build_models_manager(
)
}
pub fn thread_store_from_config(config: &Config) -> Arc<dyn ThreadStore> {
pub fn thread_store_from_config(
config: &Config,
state_db: Option<StateDbHandle>,
) -> Arc<dyn ThreadStore> {
match &config.experimental_thread_store {
ThreadStoreConfig::Local => Arc::new(LocalThreadStore::new(
LocalThreadStoreConfig::from_config(config),
state_db,
)),
ThreadStoreConfig::Remote { endpoint } => Arc::new(RemoteThreadStore::new(endpoint)),
ThreadStoreConfig::InMemory { id } => InMemoryThreadStore::for_id(id),
@@ -281,6 +287,7 @@ impl ThreadManager {
environment_manager: Arc<EnvironmentManager>,
analytics_events_client: Option<AnalyticsEventsClient>,
thread_store: Arc<dyn ThreadStore>,
state_db: Option<StateDbHandle>,
) -> Self {
let codex_home = config.codex_home.clone();
let restriction_product = session_source.restriction_product();
@@ -310,6 +317,7 @@ impl ThreadManager {
auth_manager,
session_source,
analytics_events_client,
state_db,
ops_log: should_use_test_thread_manager_behavior()
.then(|| Arc::new(std::sync::Mutex::new(Vec::new()))),
}),
@@ -347,6 +355,22 @@ impl ThreadManager {
provider: ModelProviderInfo,
codex_home: PathBuf,
environment_manager: Arc<EnvironmentManager>,
) -> Self {
Self::with_models_provider_home_and_state_for_tests(
auth,
provider,
codex_home,
environment_manager,
/*state_db*/ None,
)
}
pub(crate) fn with_models_provider_home_and_state_for_tests(
auth: CodexAuth,
provider: ModelProviderInfo,
codex_home: PathBuf,
environment_manager: Arc<EnvironmentManager>,
state_db: Option<StateDbHandle>,
) -> Self {
set_thread_manager_test_mode_for_tests(/*enabled*/ true);
let auth_manager = AuthManager::from_auth_for_testing(auth);
@@ -369,12 +393,14 @@ impl ThreadManager {
let skills_watcher = build_skills_watcher(Arc::clone(&skills_manager));
// This test constructor has no Config input. Tests that need a non-local
// process store should construct ThreadManager::new with an explicit store.
let thread_store: Arc<dyn ThreadStore> =
Arc::new(LocalThreadStore::new(LocalThreadStoreConfig {
let thread_store: Arc<dyn ThreadStore> = Arc::new(LocalThreadStore::new(
LocalThreadStoreConfig {
codex_home: codex_home.clone(),
sqlite_home: codex_home.clone(),
default_model_provider_id: OPENAI_PROVIDER_ID.to_string(),
}));
},
state_db.clone(),
));
Self {
state: Arc::new(ThreadManagerState {
threads: Arc::new(RwLock::new(HashMap::new())),
@@ -390,6 +416,7 @@ impl ThreadManager {
auth_manager,
session_source: SessionSource::Exec,
analytics_events_client: None,
state_db,
ops_log: should_use_test_thread_manager_behavior()
.then(|| Arc::new(std::sync::Mutex::new(Vec::new()))),
}),
@@ -837,6 +864,10 @@ impl ThreadManager {
}
impl ThreadManagerState {
pub(crate) fn state_db(&self) -> Option<StateDbHandle> {
self.state_db.clone()
}
pub(crate) async fn list_thread_ids(&self) -> Vec<ThreadId> {
self.threads
.read()
@@ -1100,7 +1131,7 @@ impl ThreadManagerState {
}
let environment_selections =
resolve_environment_selections(self.environment_manager.as_ref(), &environments)?;
let watch_registration = match environment_selections.primary_turn_environment() {
let watch_registration = match environment_selections.primary() {
Some(turn_environment) if !turn_environment.environment.is_remote() => {
self.skills_watcher
.register_config(
+39 -14
View File
@@ -1,5 +1,6 @@
use super::*;
use crate::config::test_config;
use crate::init_state_db;
use crate::rollout::RolloutRecorder;
use crate::session::session::SessionSettingsUpdate;
use crate::session::tests::make_session_and_context;
@@ -389,7 +390,8 @@ async fn resume_and_fork_do_not_restore_thread_environments_from_rollout() {
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, /*state_db*/ None),
/*state_db*/ None,
);
let selected_cwd =
AbsolutePathBuf::try_from(config.cwd.as_path().join("selected")).expect("absolute path");
@@ -444,9 +446,15 @@ async fn resume_and_fork_do_not_restore_thread_environments_from_rollout() {
.new_turn_with_sub_id("resume-turn".to_string(), SessionSettingsUpdate::default())
.await
.expect("build resumed turn context");
assert_eq!(resumed_turn.environments.len(), 1);
assert_eq!(resumed_turn.environments[0].cwd, default_cwd);
assert_ne!(resumed_turn.environments[0].cwd, selected_cwd);
assert_eq!(resumed_turn.environments.turn_environments.len(), 1);
assert_eq!(
resumed_turn.environments.turn_environments[0].cwd,
default_cwd
);
assert_ne!(
resumed_turn.environments.turn_environments[0].cwd,
selected_cwd
);
let forked = manager
.fork_thread(
@@ -465,9 +473,15 @@ async fn resume_and_fork_do_not_restore_thread_environments_from_rollout() {
.new_turn_with_sub_id("fork-turn".to_string(), SessionSettingsUpdate::default())
.await
.expect("build forked turn context");
assert_eq!(forked_turn.environments.len(), 1);
assert_eq!(forked_turn.environments[0].cwd, default_cwd);
assert_ne!(forked_turn.environments[0].cwd, selected_cwd);
assert_eq!(forked_turn.environments.turn_environments.len(), 1);
assert_eq!(
forked_turn.environments.turn_environments[0].cwd,
default_cwd
);
assert_ne!(
forked_turn.environments.turn_environments[0].cwd,
selected_cwd
);
}
#[tokio::test]
@@ -486,7 +500,8 @@ async fn resume_active_thread_from_rollout_returns_running_thread() {
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, /*state_db*/ None),
/*state_db*/ None,
);
let source = manager
@@ -539,7 +554,8 @@ async fn resume_stopped_thread_from_rollout_spawns_new_thread() {
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, /*state_db*/ None),
/*state_db*/ None,
);
let source = manager
@@ -602,7 +618,8 @@ async fn new_uses_active_provider_for_model_refresh() {
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, /*state_db*/ None),
/*state_db*/ None,
);
let _ = manager.list_models(RefreshStrategy::Online).await;
@@ -807,13 +824,15 @@ async fn interrupted_fork_snapshot_does_not_synthesize_turn_id_for_legacy_histor
let auth_manager =
AuthManager::from_auth_for_testing(CodexAuth::create_dummy_chatgpt_auth_for_testing());
let state_db = init_state_db(&config).await;
let manager = ThreadManager::new(
&config,
auth_manager.clone(),
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, state_db.clone()),
state_db.clone(),
);
let source = manager
@@ -909,13 +928,15 @@ async fn interrupted_fork_snapshot_preserves_explicit_turn_id() {
let auth_manager =
AuthManager::from_auth_for_testing(CodexAuth::create_dummy_chatgpt_auth_for_testing());
let state_db = init_state_db(&config).await;
let manager = ThreadManager::new(
&config,
auth_manager.clone(),
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, state_db.clone()),
state_db.clone(),
);
let source = manager
@@ -1000,13 +1021,15 @@ async fn interrupted_fork_snapshot_uses_persisted_mid_turn_history_without_live_
let auth_manager =
AuthManager::from_auth_for_testing(CodexAuth::create_dummy_chatgpt_auth_for_testing());
let state_db = init_state_db(&config).await;
let manager = ThreadManager::new(
&config,
auth_manager.clone(),
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, state_db.clone()),
state_db.clone(),
);
let source = manager
@@ -1136,13 +1159,15 @@ async fn resumed_thread_keeps_paused_goal_paused() -> anyhow::Result<()> {
let auth_manager =
AuthManager::from_auth_for_testing(CodexAuth::create_dummy_chatgpt_auth_for_testing());
let state_db = init_state_db(&config).await;
let manager = ThreadManager::new(
&config,
auth_manager.clone(),
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, state_db.clone()),
state_db.clone(),
);
let source = manager
@@ -6,7 +6,6 @@ use crate::config::Config;
use crate::function_tool::FunctionCallError;
use crate::session::session::Session;
use crate::session::turn_context::TurnContext;
use crate::session::turn_context::TurnEnvironment;
use crate::tools::context::FunctionToolOutput;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolPayload;
@@ -541,12 +540,7 @@ async fn run_agent_job_loop(
"agent_job:{job_id}"
)))),
SpawnAgentOptions {
environments: Some(
turn.environments
.iter()
.map(TurnEnvironment::selection)
.collect(),
),
environments: Some(turn.environments.to_selections()),
..Default::default()
},
)
@@ -363,7 +363,7 @@ impl ToolHandler for ApplyPatchHandler {
// Avoid building temporary ExecParams/command vectors; derive directly from inputs.
let cwd = turn.cwd.clone();
let command = vec!["apply_patch".to_string(), patch_input.clone()];
let Some(turn_environment) = turn.primary_environment() else {
let Some(turn_environment) = turn.environments.primary() else {
return Err(FunctionCallError::RespondToModel(
"apply_patch is unavailable in this session".to_string(),
));
@@ -475,7 +475,8 @@ pub(crate) async fn intercept_apply_patch(
tool_name: &str,
) -> Result<Option<FunctionToolOutput>, FunctionCallError> {
let sandbox = turn
.primary_environment()
.environments
.primary()
.filter(|env| env.environment.is_remote())
.map(|_| turn.file_system_sandbox_context(/*additional_permissions*/ None));
match codex_apply_patch::maybe_parse_apply_patch_verified(command, cwd, fs, sandbox.as_ref())
@@ -6,7 +6,6 @@ use crate::agent::exceeds_thread_spawn_depth_limit;
use crate::agent::next_thread_spawn_depth;
use crate::agent::role::DEFAULT_ROLE_NAME;
use crate::agent::role::apply_role_to_config;
use crate::session::turn_context::TurnEnvironment;
pub(crate) struct Handler;
@@ -83,29 +82,22 @@ impl ToolHandler for Handler {
apply_spawn_agent_runtime_overrides(&mut config, turn.as_ref())?;
apply_spawn_agent_overrides(&mut config, child_depth);
let result = Box::pin(
session.services.agent_control.spawn_agent_with_metadata(
config,
input_items,
Some(thread_spawn_source(
session.conversation_id,
&turn.session_source,
child_depth,
role_name,
/*task_name*/ None,
)?),
SpawnAgentOptions {
fork_parent_spawn_call_id: args.fork_context.then(|| call_id.clone()),
fork_mode: args.fork_context.then_some(SpawnAgentForkMode::FullHistory),
environments: Some(
turn.environments
.iter()
.map(TurnEnvironment::selection)
.collect(),
),
},
),
)
let result = Box::pin(session.services.agent_control.spawn_agent_with_metadata(
config,
input_items,
Some(thread_spawn_source(
session.conversation_id,
&turn.session_source,
child_depth,
role_name,
/*task_name*/ None,
)?),
SpawnAgentOptions {
fork_parent_spawn_call_id: args.fork_context.then(|| call_id.clone()),
fork_mode: args.fork_context.then_some(SpawnAgentForkMode::FullHistory),
environments: Some(turn.environments.to_selections()),
},
))
.await
.map_err(collab_spawn_error);
let (new_thread_id, new_agent_metadata, status) = match &result {
@@ -3,8 +3,10 @@ use crate::ThreadManager;
use crate::config::AgentRoleConfig;
use crate::config::DEFAULT_AGENT_MAX_DEPTH;
use crate::function_tool::FunctionCallError;
use crate::init_state_db;
use crate::session::tests::make_session_and_context;
use crate::session_prefix::format_subagent_notification_message;
use crate::thread_manager::thread_store_from_config;
use crate::tools::context::ToolOutput;
use crate::tools::handlers::multi_agents_v2::CloseAgentHandler as CloseAgentHandlerV2;
use crate::tools::handlers::multi_agents_v2::FollowupTaskHandler as FollowupTaskHandlerV2;
@@ -3149,13 +3151,22 @@ async fn close_agent_submits_shutdown_and_returns_previous_status() {
#[tokio::test]
async fn tool_handlers_cascade_close_and_resume_and_keep_explicitly_closed_subtrees_closed() {
let (_session, turn) = make_session_and_context().await;
let manager = thread_manager();
let mut config = turn.config.as_ref().clone();
config.agent_max_depth = 3;
config
.features
.enable(Feature::Sqlite)
.expect("test config should allow sqlite");
let state_db = init_state_db(&config).await;
let manager = ThreadManager::new(
&config,
AuthManager::from_auth_for_testing(CodexAuth::from_api_key("dummy")),
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config, state_db.clone()),
state_db.clone(),
);
let parent = manager
.start_thread(config.clone())
@@ -5,7 +5,6 @@ use crate::agent::control::render_input_preview;
use crate::agent::next_thread_spawn_depth;
use crate::agent::role::DEFAULT_ROLE_NAME;
use crate::agent::role::apply_role_to_config;
use crate::session::turn_context::TurnEnvironment;
use codex_protocol::AgentPath;
use codex_protocol::protocol::InterAgentCommunication;
use codex_protocol::protocol::Op;
@@ -118,12 +117,7 @@ impl ToolHandler for Handler {
SpawnAgentOptions {
fork_parent_spawn_call_id: fork_mode.as_ref().map(|_| call_id.clone()),
fork_mode,
environments: Some(
turn.environments
.iter()
.map(TurnEnvironment::selection)
.collect(),
),
environments: Some(turn.environments.to_selections()),
},
)
.await
+1 -1
View File
@@ -412,7 +412,7 @@ impl ShellHandler {
} = args;
let mut exec_params = exec_params;
let Some(turn_environment) = turn.primary_environment() else {
let Some(turn_environment) = turn.environments.primary() else {
return Err(FunctionCallError::RespondToModel(
"shell is unavailable in this session".to_string(),
));
@@ -392,7 +392,7 @@ mod tests {
server_name: server_name.to_string(),
callable_name: tool_name.to_string(),
callable_namespace: format!("mcp__{server_name}__"),
server_instructions: None,
namespace_description: None,
tool: Tool {
name: tool_name.to_string().into(),
title: None,
@@ -411,7 +411,6 @@ mod tests {
connector_id: None,
connector_name: None,
plugin_display_names: Vec::new(),
connector_description: None,
}
}
@@ -196,7 +196,7 @@ impl ToolHandler for UnifiedExecHandler {
}
};
let Some(turn_environment) = turn.primary_environment() else {
let Some(turn_environment) = turn.environments.primary() else {
return Err(FunctionCallError::RespondToModel(
"unified exec is unavailable in this session".to_string(),
));
@@ -88,7 +88,7 @@ impl ToolHandler for ViewImageHandler {
};
let abs_path = turn.resolve_path(Some(args.path));
let Some(environment) = turn.primary_environment() else {
let Some(environment) = turn.environments.primary() else {
return Err(FunctionCallError::RespondToModel(
"view_image is unavailable in this session".to_string(),
));
@@ -191,7 +191,7 @@ impl ToolRuntime<ApplyPatchRequest, ExecToolCallOutput> for ApplyPatchRuntime {
attempt: &SandboxAttempt<'_>,
ctx: &ToolCtx,
) -> Result<ExecToolCallOutput, ToolError> {
let turn_environment = ctx.turn.primary_environment().ok_or_else(|| {
let turn_environment = ctx.turn.environments.primary().ok_or_else(|| {
ToolError::Rejected("apply_patch is unavailable in this session".to_string())
})?;
let started_at = Instant::now();
@@ -254,7 +254,8 @@ impl<'a> ToolRuntime<UnifiedExecRequest, UnifiedExecProcess> for UnifiedExecRunt
}
let environment_is_remote = ctx
.turn
.primary_environment()
.environments
.primary()
.is_some_and(|turn_environment| turn_environment.environment.is_remote());
let command = if environment_is_remote {
base_command.to_vec()
@@ -292,7 +293,7 @@ impl<'a> ToolRuntime<UnifiedExecRequest, UnifiedExecProcess> for UnifiedExecRunt
.await?
{
Some(prepared) => {
let Some(turn_environment) = ctx.turn.primary_environment() else {
let Some(turn_environment) = ctx.turn.environments.primary() else {
return Err(ToolError::Rejected(
"exec_command is unavailable in this session".to_string(),
));
@@ -337,7 +338,7 @@ impl<'a> ToolRuntime<UnifiedExecRequest, UnifiedExecProcess> for UnifiedExecRunt
.env_for(command, options, managed_network)
.map_err(|err| ToolError::Codex(err.into()))?;
exec_env.exec_server_env_config = req.exec_server_env_config.clone();
let Some(turn_environment) = ctx.turn.primary_environment() else {
let Some(turn_environment) = ctx.turn.environments.primary() else {
return Err(ToolError::Rejected(
"exec_command is unavailable in this session".to_string(),
));
+2 -5
View File
@@ -57,10 +57,7 @@ fn map_mcp_tools_for_plan(mcp_tools: &HashMap<String, ToolInfo>) -> McpToolPlanI
tool.callable_namespace.clone(),
ToolNamespace {
name: tool.callable_namespace.clone(),
description: tool
.connector_description
.clone()
.or_else(|| tool.server_instructions.clone()),
description: tool.namespace_description.clone(),
},
)
})
@@ -118,7 +115,7 @@ pub(crate) fn build_specs_with_discoverable_tools(
name: tool.canonical_tool_name(),
server_name: tool.server_name.as_str(),
connector_name: tool.connector_name.as_deref(),
connector_description: tool.connector_description.as_deref(),
description: tool.namespace_description.as_deref(),
})
.collect::<Vec<_>>()
});
+6 -12
View File
@@ -62,12 +62,11 @@ fn mcp_tool_info(tool: rmcp::model::Tool) -> ToolInfo {
server_name: "test_server".to_string(),
callable_name: tool.name.to_string(),
callable_namespace: "mcp__test_server__".to_string(),
server_instructions: None,
namespace_description: None,
tool,
connector_id: None,
connector_name: None,
plugin_display_names: Vec::new(),
connector_description: None,
}
}
@@ -81,12 +80,11 @@ fn mcp_tool_info_with_display_name(display_name: &str, tool: rmcp::model::Tool)
server_name: "test_server".to_string(),
callable_name,
callable_namespace,
server_instructions: None,
namespace_description: None,
tool,
connector_id: None,
connector_name: None,
plugin_display_names: Vec::new(),
connector_description: None,
}
}
@@ -898,7 +896,7 @@ async fn search_tool_description_falls_back_to_connector_name_without_descriptio
server_name: CODEX_APPS_MCP_SERVER_NAME.to_string(),
callable_name: "_create_event".to_string(),
callable_namespace: "mcp__codex_apps__calendar".to_string(),
server_instructions: None,
namespace_description: None,
tool: mcp_tool(
"calendar_create_event",
"Create calendar event",
@@ -907,7 +905,6 @@ async fn search_tool_description_falls_back_to_connector_name_without_descriptio
connector_id: Some("calendar".to_string()),
connector_name: Some("Calendar".to_string()),
plugin_display_names: Vec::new(),
connector_description: None,
},
)])),
&[],
@@ -950,7 +947,7 @@ async fn search_tool_registers_namespaced_mcp_tool_aliases() {
server_name: CODEX_APPS_MCP_SERVER_NAME.to_string(),
callable_name: "_create_event".to_string(),
callable_namespace: "mcp__codex_apps__calendar".to_string(),
server_instructions: None,
namespace_description: None,
tool: mcp_tool(
"calendar-create-event",
"Create calendar event",
@@ -958,7 +955,6 @@ async fn search_tool_registers_namespaced_mcp_tool_aliases() {
),
connector_id: Some("calendar".to_string()),
connector_name: Some("Calendar".to_string()),
connector_description: None,
plugin_display_names: Vec::new(),
},
),
@@ -968,7 +964,7 @@ async fn search_tool_registers_namespaced_mcp_tool_aliases() {
server_name: CODEX_APPS_MCP_SERVER_NAME.to_string(),
callable_name: "_list_events".to_string(),
callable_namespace: "mcp__codex_apps__calendar".to_string(),
server_instructions: None,
namespace_description: None,
tool: mcp_tool(
"calendar-list-events",
"List calendar events",
@@ -976,7 +972,6 @@ async fn search_tool_registers_namespaced_mcp_tool_aliases() {
),
connector_id: Some("calendar".to_string()),
connector_name: Some("Calendar".to_string()),
connector_description: None,
plugin_display_names: Vec::new(),
},
),
@@ -986,11 +981,10 @@ async fn search_tool_registers_namespaced_mcp_tool_aliases() {
server_name: "rmcp".to_string(),
callable_name: "echo".to_string(),
callable_namespace: "mcp__rmcp__".to_string(),
server_instructions: None,
namespace_description: None,
tool: mcp_tool("echo", "Echo", serde_json::json!({"type": "object"})),
connector_id: None,
connector_name: None,
connector_description: None,
plugin_display_names: Vec::new(),
},
),
+4 -4
View File
@@ -80,7 +80,7 @@ fn mcp_tool_search_entry(info: &ToolInfo) -> Result<ToolSearchEntry, serde_json:
tool_name: info.callable_name.as_str(),
tool: &info.tool,
connector_name: info.connector_name.as_deref(),
connector_description: info.connector_description.as_deref(),
description: info.namespace_description.as_deref(),
})?,
limit_bucket: Some(info.server_name.clone()),
})
@@ -120,10 +120,10 @@ fn build_mcp_search_text(info: &ToolInfo) -> String {
parts.push(connector_name.to_string());
}
if let Some(connector_description) = info.connector_description.as_deref()
&& !connector_description.trim().is_empty()
if let Some(description) = info.namespace_description.as_deref()
&& !description.trim().is_empty()
{
parts.push(connector_description.to_string());
parts.push(description.to_string());
}
parts.extend(
+6 -3
View File
@@ -96,7 +96,8 @@ async fn exec_command_with_tty(
&request,
tty,
Box::new(NoopSpawnLifecycle),
turn.primary_environment()
turn.environments
.primary()
.expect("turn environment")
.environment
.as_ref(),
@@ -594,7 +595,8 @@ async fn remote_exec_server_rejects_inherited_fd_launches() -> anyhow::Result<()
let remote_test_env = remote_test_env().await?;
let (_, mut turn) = make_session_and_context().await;
turn.environments[0].environment = Arc::new(remote_test_env.environment().clone());
turn.environments.turn_environments[0].environment =
Arc::new(remote_test_env.environment().clone());
let request = test_exec_request(
&turn,
@@ -612,7 +614,8 @@ async fn remote_exec_server_rejects_inherited_fd_launches() -> anyhow::Result<()
Box::new(TestSpawnLifecycle {
inherited_fds: vec![42],
}),
turn.primary_environment()
turn.environments
.primary()
.expect("turn environment")
.environment
.as_ref(),
+5 -2
View File
@@ -423,6 +423,7 @@ impl TestCodexBuilder {
environment_manager: Arc<codex_exec_server::EnvironmentManager>,
) -> anyhow::Result<TestCodex> {
let auth = self.auth.clone();
let state_db = codex_core::init_state_db(&config).await;
let thread_manager = if config.model_catalog.is_some() {
ThreadManager::new(
&config,
@@ -430,14 +431,16 @@ impl TestCodexBuilder {
SessionSource::Exec,
Arc::clone(&environment_manager),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, state_db.clone()),
state_db.clone(),
)
} else {
codex_core::test_support::thread_manager_with_models_provider_and_home(
codex_core::test_support::thread_manager_with_models_provider_home_and_state(
auth.clone(),
config.model_provider.clone(),
config.codex_home.to_path_buf(),
Arc::clone(&environment_manager),
state_db.clone(),
)
};
let thread_manager = Arc::new(thread_manager);
+8 -2
View File
@@ -770,10 +770,15 @@ async fn includes_conversation_id_and_model_headers_in_request() {
let installation_id =
std::fs::read_to_string(test.codex_home_path().join(INSTALLATION_ID_FILENAME))
.expect("read installation id");
let session_id_string = session_id.to_string();
assert_eq!(request_session_id, session_id.to_string());
assert_eq!(request_session_id, session_id_string);
assert_eq!(request_originator, originator().value);
assert_eq!(request_authorization, "Bearer Test API Key");
assert_eq!(
request_body["prompt_cache_key"].as_str(),
Some(session_id_string.as_str())
);
assert_eq!(
request_body["client_metadata"]["x-codex-installation-id"].as_str(),
Some(installation_id.as_str())
@@ -1107,7 +1112,8 @@ async fn prefers_apikey_when_config_prefers_apikey_even_with_chatgpt_tokens() {
SessionSource::Exec,
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, /*state_db*/ None),
/*state_db*/ None,
);
let NewThread { thread: codex, .. } = thread_manager
.start_thread(config.clone())
@@ -141,7 +141,8 @@ async fn migration_marker_exists_no_sessions_no_change() -> io::Result<()> {
let marker_path = temp.path().join(PERSONALITY_MIGRATION_FILENAME);
tokio::fs::write(&marker_path, "v1\n").await?;
let status = maybe_migrate_personality(temp.path(), &ConfigToml::default()).await?;
let status =
maybe_migrate_personality(temp.path(), &ConfigToml::default(), /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::SkippedMarker);
assert_eq!(
@@ -155,7 +156,8 @@ async fn migration_marker_exists_no_sessions_no_change() -> io::Result<()> {
async fn no_marker_no_sessions_no_change() -> io::Result<()> {
let temp = TempDir::new()?;
let status = maybe_migrate_personality(temp.path(), &ConfigToml::default()).await?;
let status =
maybe_migrate_personality(temp.path(), &ConfigToml::default(), /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::SkippedNoSessions);
assert_eq!(
@@ -174,7 +176,8 @@ async fn no_marker_sessions_sets_personality() -> io::Result<()> {
let temp = TempDir::new()?;
write_session_with_user_event(temp.path()).await?;
let status = maybe_migrate_personality(temp.path(), &ConfigToml::default()).await?;
let status =
maybe_migrate_personality(temp.path(), &ConfigToml::default(), /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::Applied);
assert_eq!(
@@ -194,7 +197,7 @@ async fn no_marker_sessions_preserves_existing_config_fields() -> io::Result<()>
tokio::fs::write(temp.path().join("config.toml"), "model = \"gpt-5.4\"\n").await?;
let config_toml = read_config_toml(temp.path()).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::Applied);
let persisted = read_config_toml(temp.path()).await?;
@@ -208,7 +211,8 @@ async fn no_marker_meta_only_rollout_is_treated_as_no_sessions() -> io::Result<(
let temp = TempDir::new()?;
write_session_with_meta_only(temp.path()).await?;
let status = maybe_migrate_personality(temp.path(), &ConfigToml::default()).await?;
let status =
maybe_migrate_personality(temp.path(), &ConfigToml::default(), /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::SkippedNoSessions);
assert_eq!(
@@ -228,7 +232,7 @@ async fn no_marker_explicit_global_personality_skips_migration() -> io::Result<(
write_session_with_user_event(temp.path()).await?;
let config_toml = parse_config_toml("personality = \"friendly\"\n")?;
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None).await?;
assert_eq!(
status,
@@ -258,7 +262,7 @@ personality = "friendly"
"#,
)?;
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None).await?;
assert_eq!(
status,
@@ -281,7 +285,7 @@ async fn marker_short_circuits_invalid_profile_resolution() -> io::Result<()> {
tokio::fs::write(temp.path().join(PERSONALITY_MIGRATION_FILENAME), "v1\n").await?;
let config_toml = parse_config_toml("profile = \"missing\"\n")?;
let status = maybe_migrate_personality(temp.path(), &config_toml).await?;
let status = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::SkippedMarker);
Ok(())
@@ -292,7 +296,7 @@ async fn invalid_selected_profile_returns_error_and_does_not_write_marker() -> i
let temp = TempDir::new()?;
let config_toml = parse_config_toml("profile = \"missing\"\n")?;
let err = maybe_migrate_personality(temp.path(), &config_toml)
let err = maybe_migrate_personality(temp.path(), &config_toml, /*state_db*/ None)
.await
.expect_err("missing profile should fail");
@@ -309,8 +313,10 @@ async fn applied_migration_is_idempotent_on_second_run() -> io::Result<()> {
let temp = TempDir::new()?;
write_session_with_user_event(temp.path()).await?;
let first_status = maybe_migrate_personality(temp.path(), &ConfigToml::default()).await?;
let second_status = maybe_migrate_personality(temp.path(), &ConfigToml::default()).await?;
let first_status =
maybe_migrate_personality(temp.path(), &ConfigToml::default(), /*state_db*/ None).await?;
let second_status =
maybe_migrate_personality(temp.path(), &ConfigToml::default(), /*state_db*/ None).await?;
assert_eq!(first_status, PersonalityMigrationStatus::Applied);
assert_eq!(second_status, PersonalityMigrationStatus::SkippedMarker);
@@ -324,7 +330,8 @@ async fn no_marker_archived_sessions_sets_personality() -> io::Result<()> {
let temp = TempDir::new()?;
write_archived_session_with_user_event(temp.path()).await?;
let status = maybe_migrate_personality(temp.path(), &ConfigToml::default()).await?;
let status =
maybe_migrate_personality(temp.path(), &ConfigToml::default(), /*state_db*/ None).await?;
assert_eq!(status, PersonalityMigrationStatus::Applied);
assert_eq!(
@@ -29,6 +29,7 @@ async fn build_prompt_input_includes_context_and_user_message() -> Result<()> {
text: "hello from debug prompt".to_string(),
text_elements: Vec::new(),
}],
/*state_db*/ None,
)
.await?;
+40 -22
View File
@@ -14,6 +14,7 @@ use codex_core::find_thread_path_by_id_str;
use codex_protocol::ThreadId;
use codex_protocol::models::BaseInstructions;
use codex_protocol::protocol::SessionSource;
use codex_rollout::StateDbHandle;
use codex_state::StateRuntime;
use codex_state::ThreadMetadataBuilder;
use pretty_assertions::assert_eq;
@@ -27,7 +28,13 @@ fn write_minimal_rollout_with_id_in_subdir(codex_home: &Path, subdir: &str, id:
std::fs::create_dir_all(&sessions).unwrap();
let file = sessions.join(format!("rollout-2024-01-01T00-00-00-{id}.jsonl"));
let mut f = std::fs::File::create(&file).unwrap();
write_minimal_rollout_with_id_at_path(&file, id);
file
}
fn write_minimal_rollout_with_id_at_path(file: &Path, id: Uuid) {
let mut f = std::fs::File::create(file).unwrap();
// Minimal first line: session_meta with the id so content search can find it
writeln!(
f,
@@ -46,8 +53,6 @@ fn write_minimal_rollout_with_id_in_subdir(codex_home: &Path, subdir: &str, id:
})
)
.unwrap();
file
}
/// Create sessions/YYYY/MM/DD and write a minimal rollout file containing the
@@ -56,7 +61,11 @@ fn write_minimal_rollout_with_id(codex_home: &Path, id: Uuid) -> PathBuf {
write_minimal_rollout_with_id_in_subdir(codex_home, "sessions", id)
}
async fn upsert_thread_metadata(codex_home: &Path, thread_id: ThreadId, rollout_path: PathBuf) {
async fn upsert_thread_metadata(
codex_home: &Path,
thread_id: ThreadId,
rollout_path: PathBuf,
) -> StateDbHandle {
let runtime = StateRuntime::init(codex_home.to_path_buf(), "test-provider".to_string())
.await
.unwrap();
@@ -73,6 +82,7 @@ async fn upsert_thread_metadata(codex_home: &Path, thread_id: ThreadId, rollout_
builder.cwd = codex_home.to_path_buf();
let metadata = builder.build("test-provider");
runtime.upsert_thread(&metadata).await.unwrap();
runtime
}
#[tokio::test]
@@ -81,9 +91,10 @@ async fn find_locates_rollout_file_by_id() {
let id = Uuid::new_v4();
let expected = write_minimal_rollout_with_id(home.path(), id);
let found = find_thread_path_by_id_str(home.path(), &id.to_string())
.await
.unwrap();
let found =
find_thread_path_by_id_str(home.path(), &id.to_string(), /*state_db_ctx*/ None)
.await
.unwrap();
assert_eq!(found.unwrap(), expected);
}
@@ -97,9 +108,10 @@ async fn find_handles_gitignore_covering_codex_home_directory() {
let id = Uuid::new_v4();
let expected = write_minimal_rollout_with_id(&codex_home, id);
let found = find_thread_path_by_id_str(&codex_home, &id.to_string())
.await
.unwrap();
let found =
find_thread_path_by_id_str(&codex_home, &id.to_string(), /*state_db_ctx*/ None)
.await
.unwrap();
assert_eq!(found, Some(expected));
}
@@ -113,11 +125,11 @@ async fn find_prefers_sqlite_path_by_id() {
"sessions/2030/12/30/rollout-2030-12-30T00-00-00-{id}.jsonl"
));
std::fs::create_dir_all(db_path.parent().unwrap()).unwrap();
std::fs::write(&db_path, "").unwrap();
write_minimal_rollout_with_id_at_path(&db_path, id);
write_minimal_rollout_with_id(home.path(), id);
upsert_thread_metadata(home.path(), thread_id, db_path.clone()).await;
let state_db = upsert_thread_metadata(home.path(), thread_id, db_path.clone()).await;
let found = find_thread_path_by_id_str(home.path(), &id.to_string())
let found = find_thread_path_by_id_str(home.path(), &id.to_string(), Some(&state_db))
.await
.unwrap();
@@ -134,9 +146,9 @@ async fn find_falls_back_to_filesystem_when_sqlite_has_no_match() {
let unrelated_path = home
.path()
.join("sessions/2030/12/30/rollout-2030-12-30T00-00-00-unrelated.jsonl");
upsert_thread_metadata(home.path(), unrelated_thread_id, unrelated_path).await;
let state_db = upsert_thread_metadata(home.path(), unrelated_thread_id, unrelated_path).await;
let found = find_thread_path_by_id_str(home.path(), &id.to_string())
let found = find_thread_path_by_id_str(home.path(), &id.to_string(), Some(&state_db))
.await
.unwrap();
@@ -150,9 +162,10 @@ async fn find_ignores_granular_gitignore_rules() {
let expected = write_minimal_rollout_with_id(home.path(), id);
std::fs::write(home.path().join("sessions/.gitignore"), "*.jsonl\n").unwrap();
let found = find_thread_path_by_id_str(home.path(), &id.to_string())
.await
.unwrap();
let found =
find_thread_path_by_id_str(home.path(), &id.to_string(), /*state_db_ctx*/ None)
.await
.unwrap();
assert_eq!(found, Some(expected));
}
@@ -197,7 +210,8 @@ async fn find_locates_rollout_file_written_by_recorder() -> std::io::Result<()>
),
)?;
let found = find_thread_meta_by_name_str(home.path(), thread_name).await?;
let found =
find_thread_meta_by_name_str(home.path(), thread_name, /*state_db_ctx*/ None).await?;
let (path, session_meta) = found.expect("expected rollout path to be found");
assert_eq!(session_meta.meta.id, thread_id);
@@ -214,9 +228,13 @@ async fn find_archived_locates_rollout_file_by_id() {
let id = Uuid::new_v4();
let expected = write_minimal_rollout_with_id_in_subdir(home.path(), "archived_sessions", id);
let found = find_archived_thread_path_by_id_str(home.path(), &id.to_string())
.await
.unwrap();
let found = find_archived_thread_path_by_id_str(
home.path(),
&id.to_string(),
/*state_db_ctx*/ None,
)
.await
.unwrap();
assert_eq!(found, Some(expected));
}
+89
View File
@@ -1029,3 +1029,92 @@ async fn tool_search_indexes_only_enabled_non_app_mcp_tools() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn tool_search_uses_non_app_mcp_server_instructions_as_namespace_description() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let apps_server = AppsTestServer::mount_searchable(&server).await?;
let search_call_id = "tool-search-echo";
let mock = mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
ev_tool_search_call(
search_call_id,
&json!({
"query": "Echo back the provided message and include environment data.",
"limit": 8,
}),
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_assistant_message("msg-1", "done"),
ev_completed("resp-2"),
]),
],
)
.await;
let rmcp_test_server_bin = stdio_server_bin()?;
let mut builder =
configured_builder(apps_server.chatgpt_base_url.clone()).with_config(move |config| {
let mut servers = config.mcp_servers.get().clone();
servers.insert(
"rmcp".to_string(),
McpServerConfig {
transport: McpServerTransportConfig::Stdio {
command: rmcp_test_server_bin,
args: Vec::new(),
env: None,
env_vars: Vec::new(),
cwd: None,
},
experimental_environment: None,
enabled: true,
required: false,
disabled_reason: None,
startup_timeout_sec: Some(Duration::from_secs(10)),
tool_timeout_sec: None,
default_tools_approval_mode: None,
enabled_tools: Some(vec!["echo".to_string()]),
disabled_tools: None,
scopes: None,
oauth_resource: None,
supports_parallel_tool_calls: false,
tools: HashMap::new(),
},
);
config
.mcp_servers
.set(servers)
.expect("test mcp servers should accept any configuration");
});
let test = builder.build(&server).await?;
test.submit_turn_with_approval_and_permission_profile(
"Find the rmcp echo tool.",
AskForApproval::Never,
PermissionProfile::Disabled,
)
.await?;
let requests = mock.requests();
assert_eq!(requests.len(), 2);
let tools = tool_search_output_tools(&requests[1], search_call_id);
let rmcp_namespace = tools
.iter()
.find(|tool| tool.get("name").and_then(Value::as_str) == Some("mcp__rmcp__"))
.expect("tool_search should return the rmcp namespace");
assert_eq!(
rmcp_namespace.get("description").and_then(Value::as_str),
Some("Use these tools to exercise the rmcp test server.")
);
Ok(())
}
+2 -1
View File
@@ -246,7 +246,8 @@ async fn list_skills_skips_cwd_roots_when_environment_disabled() -> Result<()> {
)?,
)),
/*analytics_events_client*/ None,
thread_store_from_config(&config),
thread_store_from_config(&config, /*state_db*/ None),
/*state_db*/ None,
);
let new_thread = thread_manager.start_thread(config.clone()).await?;
let cwd = config.cwd.to_path_buf();
-6
View File
@@ -8,13 +8,9 @@ use tokio::sync::watch;
use tokio_tungstenite::WebSocketStream;
use tokio_tungstenite::tungstenite::Message;
#[cfg(test)]
use tokio::io::AsyncBufReadExt;
#[cfg(test)]
use tokio::io::AsyncWriteExt;
#[cfg(test)]
use tokio::io::BufReader;
#[cfg(test)]
use tokio::io::BufWriter;
pub(crate) const CHANNEL_CAPACITY: usize = 128;
@@ -34,7 +30,6 @@ pub(crate) struct JsonRpcConnection {
}
impl JsonRpcConnection {
#[cfg(test)]
pub(crate) fn from_stdio<R, W>(reader: R, writer: W, connection_label: String) -> Self
where
R: AsyncRead + Unpin + Send + 'static,
@@ -298,7 +293,6 @@ async fn send_malformed_message(
.await;
}
#[cfg(test)]
async fn write_jsonrpc_line_message<W>(
writer: &mut BufWriter<W>,
message: &JSONRPCMessage,
+54 -7
View File
@@ -1,5 +1,8 @@
use std::io::Write as _;
use std::net::SocketAddr;
use tokio::io;
use tokio::io::AsyncRead;
use tokio::io::AsyncWrite;
use tokio::net::TcpListener;
use tokio_tungstenite::accept_async;
use tracing::warn;
@@ -10,6 +13,12 @@ use crate::server::processor::ConnectionProcessor;
pub const DEFAULT_LISTEN_URL: &str = "ws://127.0.0.1:0";
#[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) enum ExecServerListenTransport {
WebSocket(SocketAddr),
Stdio,
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum ExecServerListenUrlParseError {
UnsupportedListenUrl(String),
@@ -21,7 +30,7 @@ impl std::fmt::Display for ExecServerListenUrlParseError {
match self {
ExecServerListenUrlParseError::UnsupportedListenUrl(listen_url) => write!(
f,
"unsupported --listen URL `{listen_url}`; expected `ws://IP:PORT`"
"unsupported --listen URL `{listen_url}`; expected `ws://IP:PORT` or `stdio`"
),
ExecServerListenUrlParseError::InvalidWebSocketListenUrl(listen_url) => write!(
f,
@@ -35,11 +44,18 @@ impl std::error::Error for ExecServerListenUrlParseError {}
pub(crate) fn parse_listen_url(
listen_url: &str,
) -> Result<SocketAddr, ExecServerListenUrlParseError> {
) -> Result<ExecServerListenTransport, ExecServerListenUrlParseError> {
if matches!(listen_url, "stdio" | "stdio://") {
return Ok(ExecServerListenTransport::Stdio);
}
if let Some(socket_addr) = listen_url.strip_prefix("ws://") {
return socket_addr.parse::<SocketAddr>().map_err(|_| {
ExecServerListenUrlParseError::InvalidWebSocketListenUrl(listen_url.to_string())
});
return socket_addr
.parse::<SocketAddr>()
.map(ExecServerListenTransport::WebSocket)
.map_err(|_| {
ExecServerListenUrlParseError::InvalidWebSocketListenUrl(listen_url.to_string())
});
}
Err(ExecServerListenUrlParseError::UnsupportedListenUrl(
@@ -51,8 +67,39 @@ pub(crate) async fn run_transport(
listen_url: &str,
runtime_paths: ExecServerRuntimePaths,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let bind_address = parse_listen_url(listen_url)?;
run_websocket_listener(bind_address, runtime_paths).await
match parse_listen_url(listen_url)? {
ExecServerListenTransport::WebSocket(bind_address) => {
run_websocket_listener(bind_address, runtime_paths).await
}
ExecServerListenTransport::Stdio => run_stdio_connection(runtime_paths).await,
}
}
async fn run_stdio_connection(
runtime_paths: ExecServerRuntimePaths,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
run_stdio_connection_with_io(io::stdin(), io::stdout(), runtime_paths).await
}
async fn run_stdio_connection_with_io<R, W>(
reader: R,
writer: W,
runtime_paths: ExecServerRuntimePaths,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
where
R: AsyncRead + Unpin + Send + 'static,
W: AsyncWrite + Unpin + Send + 'static,
{
let processor = ConnectionProcessor::new(runtime_paths);
tracing::info!("codex-exec-server listening on stdio");
processor
.run_connection(JsonRpcConnection::from_stdio(
reader,
writer,
"exec-server stdio".to_string(),
))
.await;
Ok(())
}
async fn run_websocket_listener(
@@ -1,31 +1,127 @@
use std::net::SocketAddr;
use std::time::Duration;
use codex_app_server_protocol::JSONRPCMessage;
use codex_app_server_protocol::JSONRPCNotification;
use codex_app_server_protocol::JSONRPCRequest;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::RequestId;
use pretty_assertions::assert_eq;
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncWriteExt;
use tokio::io::BufReader;
use tokio::io::duplex;
use tokio::time::timeout;
use super::DEFAULT_LISTEN_URL;
use super::ExecServerListenTransport;
use super::parse_listen_url;
use super::run_stdio_connection_with_io;
use crate::ExecServerRuntimePaths;
use crate::protocol::INITIALIZE_METHOD;
use crate::protocol::INITIALIZED_METHOD;
use crate::protocol::InitializeParams;
use crate::protocol::InitializeResponse;
#[test]
fn parse_listen_url_accepts_default_websocket_url() {
let bind_address =
parse_listen_url(DEFAULT_LISTEN_URL).expect("default listen URL should parse");
let transport = parse_listen_url(DEFAULT_LISTEN_URL).expect("default listen URL should parse");
assert_eq!(
bind_address,
"127.0.0.1:0"
.parse::<SocketAddr>()
.expect("valid socket address")
transport,
ExecServerListenTransport::WebSocket(
"127.0.0.1:0"
.parse::<SocketAddr>()
.expect("valid socket address")
)
);
}
#[test]
fn parse_listen_url_accepts_stdio() {
let transport = parse_listen_url("stdio").expect("stdio listen URL should parse");
assert_eq!(transport, ExecServerListenTransport::Stdio);
}
#[test]
fn parse_listen_url_accepts_stdio_url() {
let transport = parse_listen_url("stdio://").expect("stdio listen URL should parse");
assert_eq!(transport, ExecServerListenTransport::Stdio);
}
#[tokio::test]
async fn stdio_listen_transport_serves_initialize() {
let transport = parse_listen_url("stdio").expect("stdio listen URL should parse");
let ExecServerListenTransport::Stdio = transport else {
panic!("expected stdio listen transport, got {transport:?}");
};
let (mut client_writer, server_reader) = duplex(1 << 20);
let (server_writer, client_reader) = duplex(1 << 20);
let server_task = tokio::spawn(run_stdio_connection_with_io(
server_reader,
server_writer,
test_runtime_paths(),
));
let mut client_lines = BufReader::new(client_reader).lines();
let initialize = JSONRPCMessage::Request(JSONRPCRequest {
id: RequestId::Integer(1),
method: INITIALIZE_METHOD.to_string(),
params: Some(
serde_json::to_value(InitializeParams {
client_name: "exec-server-transport-test".to_string(),
resume_session_id: None,
})
.expect("initialize params should serialize"),
),
trace: None,
});
write_jsonrpc_line(&mut client_writer, &initialize).await;
let response = timeout(Duration::from_secs(1), client_lines.next_line())
.await
.expect("initialize response should arrive")
.expect("initialize response read should succeed")
.expect("initialize response should be present");
let response: JSONRPCMessage =
serde_json::from_str(&response).expect("initialize response should parse");
let JSONRPCMessage::Response(JSONRPCResponse { id, result }) = response else {
panic!("expected initialize response, got {response:?}");
};
assert_eq!(id, RequestId::Integer(1));
let initialize_response: InitializeResponse =
serde_json::from_value(result).expect("initialize response should decode");
assert!(
!initialize_response.session_id.is_empty(),
"initialize should return a session id"
);
let initialized = JSONRPCMessage::Notification(JSONRPCNotification {
method: INITIALIZED_METHOD.to_string(),
params: Some(serde_json::to_value(()).expect("initialized params should serialize")),
});
write_jsonrpc_line(&mut client_writer, &initialized).await;
drop(client_writer);
drop(client_lines);
timeout(Duration::from_secs(1), server_task)
.await
.expect("stdio transport should finish after client disconnect")
.expect("stdio transport task should join")
.expect("stdio transport should not fail");
}
#[test]
fn parse_listen_url_accepts_websocket_url() {
let bind_address =
let transport =
parse_listen_url("ws://127.0.0.1:1234").expect("websocket listen URL should parse");
assert_eq!(
bind_address,
"127.0.0.1:1234"
.parse::<SocketAddr>()
.expect("valid socket address")
transport,
ExecServerListenTransport::WebSocket(
"127.0.0.1:1234"
.parse::<SocketAddr>()
.expect("valid socket address")
)
);
}
@@ -45,6 +141,26 @@ fn parse_listen_url_rejects_unsupported_url() {
parse_listen_url("http://127.0.0.1:1234").expect_err("unsupported scheme should fail");
assert_eq!(
err.to_string(),
"unsupported --listen URL `http://127.0.0.1:1234`; expected `ws://IP:PORT`"
"unsupported --listen URL `http://127.0.0.1:1234`; expected `ws://IP:PORT` or `stdio`"
);
}
async fn write_jsonrpc_line(writer: &mut tokio::io::DuplexStream, message: &JSONRPCMessage) {
let encoded = serde_json::to_vec(message).expect("JSON-RPC message should serialize");
writer
.write_all(&encoded)
.await
.expect("JSON-RPC message should write");
writer
.write_all(b"\n")
.await
.expect("JSON-RPC newline should write");
}
fn test_runtime_paths() -> ExecServerRuntimePaths {
ExecServerRuntimePaths::new(
std::env::current_exe().expect("current exe"),
/*codex_linux_sandbox_exe*/ None,
)
.expect("runtime paths")
}
+1 -1
View File
@@ -198,7 +198,7 @@ set -euo pipefail
for arg in "$@"; do
if [[ "${arg}" == "--help" ]]; then
echo "Usage: bwrap --argv0"
echo "Usage: bwrap --argv0 --perms"
exit 0
fi
done
+13 -3
View File
@@ -58,6 +58,7 @@ use codex_config::ConfigLoadError;
use codex_config::ConfigLoadOptions;
use codex_config::LoaderOverrides;
use codex_config::format_config_error_with_source;
use codex_core::StateDbHandle;
use codex_core::check_execpolicy_for_warnings;
use codex_core::config::Config;
use codex_core::config::ConfigBuilder;
@@ -195,6 +196,7 @@ impl RequestIdSequencer {
struct ExecRunArgs {
in_process_start_args: InProcessClientStartArgs,
state_db: Option<StateDbHandle>,
command: Option<ExecCommand>,
config: Config,
dangerously_bypass_approvals_and_sandbox: bool,
@@ -509,6 +511,7 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result
arg0_paths.codex_self_exe.clone(),
arg0_paths.codex_linux_sandbox_exe.clone(),
)?;
let state_db = codex_core::init_state_db(&config).await;
let in_process_start_args = InProcessClientStartArgs {
arg0_paths,
config: std::sync::Arc::new(config.clone()),
@@ -518,6 +521,7 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result
cloud_requirements: run_cloud_requirements,
feedback: CodexFeedback::new(),
log_db: None,
state_db: state_db.clone(),
environment_manager: std::sync::Arc::new(
EnvironmentManager::new(EnvironmentManagerArgs::new(local_runtime_paths)).await,
),
@@ -532,6 +536,7 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result
};
run_exec_session(ExecRunArgs {
in_process_start_args,
state_db,
command,
config,
dangerously_bypass_approvals_and_sandbox,
@@ -553,6 +558,7 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result
async fn run_exec_session(args: ExecRunArgs) -> anyhow::Result<()> {
let ExecRunArgs {
in_process_start_args,
state_db,
command,
config,
dangerously_bypass_approvals_and_sandbox,
@@ -679,7 +685,9 @@ async fn run_exec_session(args: ExecRunArgs) -> anyhow::Result<()> {
let (primary_thread_id, fallback_session_configured) = if let Some(ExecCommand::Resume(args)) =
command.as_ref()
{
if let Some(thread_id) = resolve_resume_thread_id(&client, &config, args).await? {
if let Some(thread_id) =
resolve_resume_thread_id(&client, &config, state_db.as_ref(), args).await?
{
let response: ThreadResumeResponse = send_request_with_response(
&client,
ClientRequest::ThreadResume {
@@ -1325,6 +1333,7 @@ fn cwds_match(current_cwd: &Path, session_cwd: &Path) -> bool {
async fn resolve_resume_thread_id(
client: &InProcessAppServerClient,
config: &Config,
state_db: Option<&StateDbHandle>,
args: &crate::cli::ResumeArgs,
) -> anyhow::Result<Option<String>> {
let model_providers = resume_lookup_model_providers(config, args);
@@ -1372,7 +1381,7 @@ async fn resolve_resume_thread_id(
if Uuid::parse_str(session_id).is_ok() {
return Ok(Some(session_id.to_string()));
}
if let Some(state_db) = codex_core::get_state_db(config).await {
if let Some(state_db) = state_db {
let cwd = (!args.all).then_some(config.cwd.as_path());
let resolved = state_db
.find_thread_by_exact_title(
@@ -1387,7 +1396,8 @@ async fn resolve_resume_thread_id(
return Ok(Some(thread.id.to_string()));
}
if let Some((_, session_meta)) =
find_thread_meta_by_name_str(&config.codex_home, session_id).await?
find_thread_meta_by_name_str(&config.codex_home, session_id, Some(state_db.as_ref()))
.await?
&& (args.all || cwds_match(config.cwd.as_path(), &session_meta.meta.cwd))
{
return Ok(Some(session_meta.meta.id.to_string()));
+48 -8
View File
@@ -23,6 +23,12 @@ struct SystemBwrapLauncher {
supports_argv0: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct SystemBwrapCapabilities {
supports_argv0: bool,
supports_perms: bool,
}
pub(crate) fn exec_bwrap(argv: Vec<String>, preserved_files: Vec<File>) -> ! {
match preferred_bwrap_launcher() {
BubblewrapLauncher::System(launcher) => {
@@ -43,18 +49,24 @@ fn preferred_bwrap_launcher() -> BubblewrapLauncher {
}
fn preferred_bwrap_launcher_for_path(system_bwrap_path: &Path) -> BubblewrapLauncher {
preferred_bwrap_launcher_for_path_with_probe(system_bwrap_path, system_bwrap_supports_argv0)
preferred_bwrap_launcher_for_path_with_probe(system_bwrap_path, system_bwrap_capabilities)
}
fn preferred_bwrap_launcher_for_path_with_probe(
system_bwrap_path: &Path,
system_bwrap_supports_argv0: impl FnOnce(&Path) -> bool,
system_bwrap_capabilities: impl FnOnce(&Path) -> Option<SystemBwrapCapabilities>,
) -> BubblewrapLauncher {
if !system_bwrap_path.is_file() {
return BubblewrapLauncher::Vendored;
}
let supports_argv0 = system_bwrap_supports_argv0(system_bwrap_path);
let Some(SystemBwrapCapabilities {
supports_argv0,
supports_perms: true,
}) = system_bwrap_capabilities(system_bwrap_path)
else {
return BubblewrapLauncher::Vendored;
};
let system_bwrap_path = match AbsolutePathBuf::from_absolute_path(system_bwrap_path) {
Ok(path) => path,
Err(err) => panic!(
@@ -75,7 +87,7 @@ pub(crate) fn preferred_bwrap_supports_argv0() -> bool {
}
}
fn system_bwrap_supports_argv0(system_bwrap_path: &Path) -> bool {
fn system_bwrap_capabilities(system_bwrap_path: &Path) -> Option<SystemBwrapCapabilities> {
// bubblewrap added `--argv0` in v0.9.0:
// https://github.com/containers/bubblewrap/releases/tag/v0.9.0
// Older distro packages (for example Ubuntu 20.04/22.04) ship builds that
@@ -83,11 +95,14 @@ fn system_bwrap_supports_argv0(system_bwrap_path: &Path) -> bool {
// in that case.
let output = match Command::new(system_bwrap_path).arg("--help").output() {
Ok(output) => output,
Err(_) => return false,
Err(_) => return None,
};
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
stdout.contains("--argv0") || stderr.contains("--argv0")
Some(SystemBwrapCapabilities {
supports_argv0: stdout.contains("--argv0") || stderr.contains("--argv0"),
supports_perms: stdout.contains("--perms") || stderr.contains("--perms"),
})
}
fn exec_system_bwrap(
@@ -164,7 +179,12 @@ mod tests {
let expected = AbsolutePathBuf::from_absolute_path(fake_bwrap_path).expect("absolute");
assert_eq!(
preferred_bwrap_launcher_for_path_with_probe(fake_bwrap_path, |_| true),
preferred_bwrap_launcher_for_path_with_probe(fake_bwrap_path, |_| {
Some(SystemBwrapCapabilities {
supports_argv0: true,
supports_perms: true,
})
}),
BubblewrapLauncher::System(SystemBwrapLauncher {
program: expected,
supports_argv0: true,
@@ -178,7 +198,12 @@ mod tests {
let fake_bwrap_path = fake_bwrap.path();
assert_eq!(
preferred_bwrap_launcher_for_path_with_probe(fake_bwrap_path, |_| false),
preferred_bwrap_launcher_for_path_with_probe(fake_bwrap_path, |_| {
Some(SystemBwrapCapabilities {
supports_argv0: false,
supports_perms: true,
})
}),
BubblewrapLauncher::System(SystemBwrapLauncher {
program: AbsolutePathBuf::from_absolute_path(fake_bwrap_path).expect("absolute"),
supports_argv0: false,
@@ -186,6 +211,21 @@ mod tests {
);
}
#[test]
fn falls_back_to_vendored_when_system_bwrap_lacks_perms() {
let fake_bwrap = NamedTempFile::new().expect("temp file");
assert_eq!(
preferred_bwrap_launcher_for_path_with_probe(fake_bwrap.path(), |_| {
Some(SystemBwrapCapabilities {
supports_argv0: false,
supports_perms: false,
})
}),
BubblewrapLauncher::Vendored
);
}
#[test]
fn falls_back_to_vendored_when_system_bwrap_is_missing() {
assert_eq!(
+2
View File
@@ -86,6 +86,7 @@ pub async fn run_main(
std::io::Error::new(ErrorKind::InvalidData, format!("error loading config: {e}"))
})?;
set_default_client_residency_requirement(config.enforce_residency.value());
let state_db = codex_core::init_state_db(&config).await;
let otel = codex_core::otel_init::build_provider(
&config,
@@ -147,6 +148,7 @@ pub async fn run_main(
arg0_paths,
Arc::new(config),
environment_manager,
state_db,
)
.await;
async move {
+4 -1
View File
@@ -2,6 +2,7 @@ use std::collections::HashMap;
use std::sync::Arc;
use codex_arg0::Arg0DispatchPaths;
use codex_core::StateDbHandle;
use codex_core::ThreadManager;
use codex_core::config::Config;
use codex_core::thread_store_from_config;
@@ -53,6 +54,7 @@ impl MessageProcessor {
arg0_paths: Arg0DispatchPaths,
config: Arc<Config>,
environment_manager: Arc<EnvironmentManager>,
state_db: Option<StateDbHandle>,
) -> Self {
let outgoing = Arc::new(outgoing);
let auth_manager = AuthManager::shared_from_config(
@@ -66,7 +68,8 @@ impl MessageProcessor {
SessionSource::Mcp,
environment_manager,
/*analytics_events_client*/ None,
thread_store_from_config(config.as_ref()),
thread_store_from_config(config.as_ref(), state_db.clone()),
state_db.clone(),
));
Self {
outgoing,
@@ -401,6 +401,7 @@ impl ServerHandler for TestToolServer {
)]));
ServerInfo {
instructions: Some("Use these tools to exercise the rmcp test server.".to_string()),
capabilities,
..ServerInfo::default()
}
+1
View File
@@ -53,6 +53,7 @@ pub use list::read_thread_item_from_rollout;
pub use list::rollout_date_parts;
pub use metadata::builder_from_items;
pub use policy::EventPersistenceMode;
pub use policy::is_persisted_rollout_item;
pub use policy::should_persist_response_item_for_memories;
pub use recorder::RolloutRecorder;
pub use recorder::RolloutRecorderParams;
+41 -15
View File
@@ -1239,6 +1239,7 @@ async fn find_thread_path_by_id_str_in_subdir(
codex_home: &Path,
subdir: &str,
id_str: &str,
state_db_ctx: Option<&codex_state::StateRuntime>,
) -> io::Result<Option<PathBuf>> {
// Validate UUID format early.
if Uuid::parse_str(id_str).is_err() {
@@ -1253,8 +1254,8 @@ async fn find_thread_path_by_id_str_in_subdir(
_ => None,
};
let thread_id = ThreadId::from_string(id_str).ok();
let state_db_ctx = state_db::open_if_present(codex_home, "").await;
if let Some(state_db_ctx) = state_db_ctx.as_deref()
let mut unverified_db_path = None;
if let Some(state_db_ctx) = state_db_ctx
&& let Some(thread_id) = thread_id
&& let Some(db_path) = state_db::find_rollout_path_by_id(
Some(state_db_ctx),
@@ -1265,21 +1266,43 @@ async fn find_thread_path_by_id_str_in_subdir(
.await
{
if tokio::fs::try_exists(&db_path).await.unwrap_or(false) {
return Ok(Some(db_path));
match read_session_meta_line(&db_path).await {
Ok(meta_line) if meta_line.meta.id == thread_id => {
return Ok(Some(db_path));
}
Ok(meta_line) => {
tracing::error!(
"state db returned rollout path for thread {id_str} but file belongs to thread {}: {}",
meta_line.meta.id,
db_path.display()
);
tracing::warn!(
"state db discrepancy during find_thread_path_by_id_str_in_subdir: mismatched_db_path"
);
}
Err(err) => {
tracing::debug!(
"state db returned rollout path for thread {id_str} that could not be verified: {}: {err}",
db_path.display()
);
unverified_db_path = Some(db_path);
}
}
} else {
tracing::error!(
"state db returned stale rollout path for thread {id_str}: {}",
db_path.display()
);
tracing::warn!(
"state db discrepancy during find_thread_path_by_id_str_in_subdir: stale_db_path"
);
}
tracing::error!(
"state db returned stale rollout path for thread {id_str}: {}",
db_path.display()
);
tracing::warn!(
"state db discrepancy during find_thread_path_by_id_str_in_subdir: stale_db_path"
);
}
let mut root = codex_home.to_path_buf();
root.push(subdir);
if !root.exists() {
return Ok(None);
return Ok(unverified_db_path);
}
// This is safe because we know the values are valid.
#[allow(clippy::unwrap_used)]
@@ -1301,7 +1324,7 @@ async fn find_thread_path_by_id_str_in_subdir(
"state db discrepancy during find_thread_path_by_id_str_in_subdir: falling_back"
);
state_db::read_repair_rollout_path(
state_db_ctx.as_deref(),
state_db_ctx,
thread_id,
archived_only,
found_path.as_path(),
@@ -1309,7 +1332,7 @@ async fn find_thread_path_by_id_str_in_subdir(
.await;
}
Ok(found)
Ok(found.or(unverified_db_path))
}
/// Locate a recorded thread rollout file by its UUID string using the existing
@@ -1318,16 +1341,19 @@ async fn find_thread_path_by_id_str_in_subdir(
pub async fn find_thread_path_by_id_str(
codex_home: &Path,
id_str: &str,
state_db_ctx: Option<&codex_state::StateRuntime>,
) -> io::Result<Option<PathBuf>> {
find_thread_path_by_id_str_in_subdir(codex_home, SESSIONS_SUBDIR, id_str).await
find_thread_path_by_id_str_in_subdir(codex_home, SESSIONS_SUBDIR, id_str, state_db_ctx).await
}
/// Locate an archived thread rollout file by its UUID string.
pub async fn find_archived_thread_path_by_id_str(
codex_home: &Path,
id_str: &str,
state_db_ctx: Option<&codex_state::StateRuntime>,
) -> io::Result<Option<PathBuf>> {
find_thread_path_by_id_str_in_subdir(codex_home, ARCHIVED_SESSIONS_SUBDIR, id_str).await
find_thread_path_by_id_str_in_subdir(codex_home, ARCHIVED_SESSIONS_SUBDIR, id_str, state_db_ctx)
.await
}
/// Extract the `YYYY/MM/DD` directory components from a rollout filename.
+16 -1
View File
@@ -136,6 +136,21 @@ pub(crate) async fn backfill_sessions(
runtime: &codex_state::StateRuntime,
codex_home: &Path,
default_provider: &str,
) {
backfill_sessions_with_lease(
runtime,
codex_home,
default_provider,
BACKFILL_LEASE_SECONDS,
)
.await;
}
pub(crate) async fn backfill_sessions_with_lease(
runtime: &codex_state::StateRuntime,
codex_home: &Path,
default_provider: &str,
backfill_lease_seconds: i64,
) {
let metric_client = codex_otel::global();
let timer = metric_client
@@ -154,7 +169,7 @@ pub(crate) async fn backfill_sessions(
if backfill_state.status == BackfillStatus::Complete {
return;
}
let claimed = match runtime.try_claim_backfill(BACKFILL_LEASE_SECONDS).await {
let claimed = match runtime.try_claim_backfill(backfill_lease_seconds).await {
Ok(claimed) => claimed,
Err(err) => {
warn!(
+3 -3
View File
@@ -11,7 +11,7 @@ pub enum EventPersistenceMode {
/// Whether a rollout `item` should be persisted in rollout files for the
/// provided persistence `mode`.
pub fn is_persisted_response_item(item: &RolloutItem, mode: EventPersistenceMode) -> bool {
pub fn is_persisted_rollout_item(item: &RolloutItem, mode: EventPersistenceMode) -> bool {
match item {
RolloutItem::ResponseItem(item) => should_persist_response_item(item),
RolloutItem::EventMsg(ev) => should_persist_event_msg(ev, mode),
@@ -102,10 +102,12 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option<EventPersistenceMode> {
| EventMsg::ContextCompacted(_)
| EventMsg::EnteredReviewMode(_)
| EventMsg::ExitedReviewMode(_)
| EventMsg::McpToolCallEnd(_)
| EventMsg::ThreadRolledBack(_)
| EventMsg::TurnAborted(_)
| EventMsg::TurnStarted(_)
| EventMsg::TurnComplete(_)
| EventMsg::WebSearchEnd(_)
| EventMsg::ImageGenerationEnd(_) => Some(EventPersistenceMode::Limited),
EventMsg::ItemCompleted(event) => {
// Plan items are derived from streaming tags and are not part of the
@@ -119,9 +121,7 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option<EventPersistenceMode> {
}
EventMsg::Error(_)
| EventMsg::GuardianAssessment(_)
| EventMsg::WebSearchEnd(_)
| EventMsg::ExecCommandEnd(_)
| EventMsg::McpToolCallEnd(_)
| EventMsg::ViewImageToolCall(_)
| EventMsg::CollabAgentSpawnEnd(_)
| EventMsg::CollabAgentInteractionEnd(_)
+12 -10
View File
@@ -45,7 +45,7 @@ use super::list::parse_cursor;
use super::list::parse_timestamp_uuid_from_filename;
use super::metadata;
use super::policy::EventPersistenceMode;
use super::policy::is_persisted_response_item;
use super::policy::is_persisted_rollout_item;
use super::session_index::find_thread_names_by_ids;
use crate::config::RolloutConfigView;
use crate::default_client::originator;
@@ -79,7 +79,6 @@ pub struct RolloutRecorder {
tx: Sender<RolloutCmd>,
writer_task: Arc<RolloutWriterTask>,
pub(crate) rollout_path: PathBuf,
state_db: Option<StateDbHandle>,
event_persistence_mode: EventPersistenceMode,
}
@@ -230,6 +229,7 @@ impl RolloutRecorder {
/// List threads (rollout files) under the provided Codex home directory.
#[allow(clippy::too_many_arguments)]
pub async fn list_threads(
state_db_ctx: Option<StateDbHandle>,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
@@ -242,6 +242,7 @@ impl RolloutRecorder {
search_term: Option<&str>,
) -> std::io::Result<ThreadsPage> {
Self::list_threads_with_db_fallback(
state_db_ctx,
config,
page_size,
cursor,
@@ -260,6 +261,7 @@ impl RolloutRecorder {
#[allow(clippy::too_many_arguments)]
pub async fn list_threads_from_state_db(
state_db_ctx: Option<StateDbHandle>,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
@@ -272,6 +274,7 @@ impl RolloutRecorder {
search_term: Option<&str>,
) -> std::io::Result<ThreadsPage> {
Self::list_threads_with_db_fallback(
state_db_ctx,
config,
page_size,
cursor,
@@ -291,6 +294,7 @@ impl RolloutRecorder {
/// List archived threads (rollout files) under the archived sessions directory.
#[allow(clippy::too_many_arguments)]
pub async fn list_archived_threads(
state_db_ctx: Option<StateDbHandle>,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
@@ -303,6 +307,7 @@ impl RolloutRecorder {
search_term: Option<&str>,
) -> std::io::Result<ThreadsPage> {
Self::list_threads_with_db_fallback(
state_db_ctx,
config,
page_size,
cursor,
@@ -321,6 +326,7 @@ impl RolloutRecorder {
#[allow(clippy::too_many_arguments)]
pub async fn list_archived_threads_from_state_db(
state_db_ctx: Option<StateDbHandle>,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
@@ -333,6 +339,7 @@ impl RolloutRecorder {
search_term: Option<&str>,
) -> std::io::Result<ThreadsPage> {
Self::list_threads_with_db_fallback(
state_db_ctx,
config,
page_size,
cursor,
@@ -351,6 +358,7 @@ impl RolloutRecorder {
#[allow(clippy::too_many_arguments)]
async fn list_threads_with_db_fallback(
state_db_ctx: Option<StateDbHandle>,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
@@ -365,7 +373,6 @@ impl RolloutRecorder {
search_term: Option<&str>,
) -> std::io::Result<ThreadsPage> {
let codex_home = config.codex_home();
let state_db_ctx = state_db::get_state_db(config).await;
let archived = match archive_filter {
ThreadListArchiveFilter::Active => false,
ThreadListArchiveFilter::Archived => true,
@@ -575,6 +582,7 @@ impl RolloutRecorder {
/// Find the newest recorded thread path, optionally filtering to a matching cwd.
#[allow(clippy::too_many_arguments)]
pub async fn find_latest_thread_path(
state_db_ctx: Option<StateDbHandle>,
config: &impl RolloutConfigView,
page_size: usize,
cursor: Option<&Cursor>,
@@ -585,7 +593,6 @@ impl RolloutRecorder {
filter_cwd: Option<&Path>,
) -> std::io::Result<Option<PathBuf>> {
let codex_home = config.codex_home();
let state_db_ctx = state_db::get_state_db(config).await;
let cwd_filter = filter_cwd.map(Path::to_path_buf);
if state_db_ctx.is_some() {
let mut db_cursor = cursor.cloned();
@@ -770,7 +777,6 @@ impl RolloutRecorder {
tx,
writer_task,
rollout_path,
state_db: state_db_ctx,
event_persistence_mode,
})
}
@@ -779,17 +785,13 @@ impl RolloutRecorder {
self.rollout_path.as_path()
}
pub fn state_db(&self) -> Option<StateDbHandle> {
self.state_db.clone()
}
pub async fn record_items(&self, items: &[RolloutItem]) -> std::io::Result<()> {
let mut filtered = Vec::new();
for item in items {
// Note that function calls may look a bit strange if they are
// "fully qualified MCP tool calls," so we could consider
// reformatting them in that case.
if is_persisted_response_item(item, self.event_persistence_mode) {
if is_persisted_rollout_item(item, self.event_persistence_mode) {
filtered.push(sanitize_rollout_item_for_persistence(
item.clone(),
self.event_persistence_mode,
+89
View File
@@ -3,6 +3,7 @@
use super::*;
use crate::config::RolloutConfig;
use chrono::TimeZone;
use codex_protocol::ThreadId;
use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::AgentMessageEvent;
@@ -11,6 +12,9 @@ use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RolloutItem;
use codex_protocol::protocol::RolloutLine;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::TurnContextItem;
use codex_protocol::protocol::UserMessageEvent;
use pretty_assertions::assert_eq;
@@ -65,6 +69,77 @@ fn write_session_file(root: &Path, ts: &str, uuid: Uuid) -> std::io::Result<Path
Ok(path)
}
#[tokio::test]
async fn state_db_init_backfills_before_returning() -> anyhow::Result<()> {
let home = TempDir::new().expect("temp dir");
let uuid = Uuid::new_v4();
let thread_id = ThreadId::from_string(&uuid.to_string())?;
let rollout_path = home.path().join(format!(
"sessions/2026/01/27/rollout-2026-01-27T12-34-56-{uuid}.jsonl"
));
let parent = rollout_path
.parent()
.expect("rollout path should have parent");
fs::create_dir_all(parent)?;
let session_meta_line = SessionMetaLine {
meta: SessionMeta {
id: thread_id,
forked_from_id: None,
timestamp: "2026-01-27T12:34:56Z".to_string(),
cwd: home.path().to_path_buf(),
originator: "test".to_string(),
cli_version: "test".to_string(),
source: SessionSource::Cli,
agent_path: None,
agent_nickname: None,
agent_role: None,
model_provider: None,
base_instructions: None,
dynamic_tools: None,
memory_mode: None,
},
git: None,
};
let lines = [
RolloutLine {
timestamp: "2026-01-27T12:34:56Z".to_string(),
item: RolloutItem::SessionMeta(session_meta_line),
},
RolloutLine {
timestamp: "2026-01-27T12:34:57Z".to_string(),
item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent {
message: "hello from startup backfill".to_string(),
images: None,
local_images: Vec::new(),
text_elements: Vec::new(),
})),
},
];
let jsonl = lines
.iter()
.map(serde_json::to_string)
.collect::<Result<Vec<_>, _>>()?
.join("\n");
fs::write(&rollout_path, format!("{jsonl}\n"))?;
let runtime = crate::state_db::init(&test_config(home.path()))
.await
.expect("state db should initialize");
let metadata = runtime
.get_thread(thread_id)
.await?
.expect("thread should be backfilled before init returns");
assert_eq!(metadata.rollout_path, rollout_path);
assert_eq!(
runtime.get_backfill_state().await?.status,
codex_state::BackfillStatus::Complete
);
Ok(())
}
#[tokio::test]
async fn load_rollout_items_skips_legacy_ghost_snapshot_lines() -> std::io::Result<()> {
let home = TempDir::new().expect("temp dir");
@@ -526,6 +601,7 @@ async fn list_threads_db_disabled_does_not_skip_paginated_items() -> std::io::Re
let default_provider = config.model_provider_id.clone();
let page1 = RolloutRecorder::list_threads(
/*state_db_ctx*/ None,
&config,
/*page_size*/ 1,
/*cursor*/ None,
@@ -543,6 +619,7 @@ async fn list_threads_db_disabled_does_not_skip_paginated_items() -> std::io::Re
let cursor = page1.next_cursor.clone().expect("cursor should be present");
let page2 = RolloutRecorder::list_threads(
/*state_db_ctx*/ None,
&config,
/*page_size*/ 1,
Some(&cursor),
@@ -602,6 +679,7 @@ async fn list_threads_db_enabled_drops_missing_rollout_paths() -> std::io::Resul
let default_provider = config.model_provider_id.clone();
let page = RolloutRecorder::list_threads(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -666,6 +744,7 @@ async fn list_threads_db_enabled_repairs_stale_rollout_paths() -> std::io::Resul
let default_provider = config.model_provider_id.clone();
let page = RolloutRecorder::list_threads(
Some(runtime.clone()),
&config,
/*page_size*/ 1,
/*cursor*/ None,
@@ -738,6 +817,7 @@ async fn list_threads_state_db_only_skips_jsonl_repair_scan() -> std::io::Result
let cwd_filters = [home.path().to_path_buf()];
let state_db_only_page = RolloutRecorder::list_threads_from_state_db(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -753,6 +833,7 @@ async fn list_threads_state_db_only_skips_jsonl_repair_scan() -> std::io::Result
assert_eq!(state_db_only_page.items.len(), 0);
let repaired_page = RolloutRecorder::list_threads(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -768,6 +849,7 @@ async fn list_threads_state_db_only_skips_jsonl_repair_scan() -> std::io::Result
assert_eq!(repaired_page.items.len(), 1);
let repaired_state_db_only_page = RolloutRecorder::list_threads_from_state_db(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -825,6 +907,7 @@ async fn list_threads_default_filter_returns_filesystem_scan_results() -> std::i
let cwd_filters = [stale_cwd];
let state_db_only_page = RolloutRecorder::list_threads_from_state_db(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -840,6 +923,7 @@ async fn list_threads_default_filter_returns_filesystem_scan_results() -> std::i
assert_eq!(state_db_only_page.items.len(), 1);
let scanned_page = RolloutRecorder::list_threads(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -855,6 +939,7 @@ async fn list_threads_default_filter_returns_filesystem_scan_results() -> std::i
assert_eq!(scanned_page.items.len(), 0);
let repaired_state_db_only_page = RolloutRecorder::list_threads_from_state_db(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -913,6 +998,7 @@ async fn list_threads_metadata_filter_overlays_state_db_list_metadata() -> std::
.expect("state db upsert should succeed");
let page = RolloutRecorder::list_threads(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -1039,6 +1125,7 @@ async fn list_threads_search_repairs_stale_state_db_hits_before_returning() -> s
.expect("state db upsert should succeed");
let stale_state_db_only_page = RolloutRecorder::list_threads_from_state_db(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -1054,6 +1141,7 @@ async fn list_threads_search_repairs_stale_state_db_hits_before_returning() -> s
assert_eq!(stale_state_db_only_page.items.len(), 1);
let scanned_page = RolloutRecorder::list_threads(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
@@ -1069,6 +1157,7 @@ async fn list_threads_search_repairs_stale_state_db_hits_before_returning() -> s
assert_eq!(scanned_page.items.len(), 0);
let repaired_state_db_only_page = RolloutRecorder::list_threads_from_state_db(
Some(runtime.clone()),
&config,
/*page_size*/ 10,
/*cursor*/ None,
+7 -2
View File
@@ -117,6 +117,7 @@ pub async fn find_thread_names_by_ids(
pub async fn find_thread_meta_by_name_str(
codex_home: &Path,
name: &str,
state_db_ctx: Option<&codex_state::StateRuntime>,
) -> std::io::Result<Option<(PathBuf, SessionMetaLine)>> {
if name.trim().is_empty() {
return Ok(None);
@@ -135,8 +136,12 @@ pub async fn find_thread_meta_by_name_str(
while let Some(thread_id) = rx.recv().await {
// Keep walking until a matching id resolves to a loadable rollout so an unsaved or partial
// rename cannot shadow an older persisted session with the same name.
if let Some(path) =
super::list::find_thread_path_by_id_str(codex_home, &thread_id.to_string()).await?
if let Some(path) = super::list::find_thread_path_by_id_str(
codex_home,
&thread_id.to_string(),
state_db_ctx,
)
.await?
&& let Ok(session_meta) = super::list::read_session_meta_line(&path).await
{
drop(rx);

Some files were not shown because too many files have changed in this diff Show More