diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 1e5aac36df..268ae574e5 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -1120,7 +1120,7 @@ pub(crate) async fn apply_bespoke_event_handling( { Ok(summary) => { let conversation_modalities = resolve_conversation_modalities( - conversation.state_db(), + conversation.state_db().as_ref(), conversation_id, None, Some(rollout_path.as_path()), diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 70b800fd35..cb2706b53b 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -216,6 +216,7 @@ use codex_protocol::config_types::Personality; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::dynamic_tools::DynamicToolSpec as CoreDynamicToolSpec; use codex_protocol::items::TurnItem; +use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::input_modalities_from_mask; @@ -1911,7 +1912,7 @@ impl CodexMessageProcessor { { Ok(summary) => { let conversation_modalities = resolve_conversation_modalities( - state_db_ctx.as_deref(), + state_db_ctx.as_ref(), thread_id, None, Some(rollout_path.as_path()), @@ -2244,7 +2245,7 @@ impl CodexMessageProcessor { message: format!("failed to update unarchived thread timestamp: {err}"), data: None, })?; - if let Some(ctx) = state_db_ctx { + if let Some(ctx) = state_db_ctx.as_ref() { let _ = ctx .mark_unarchived(thread_id, restored_path.as_path()) .await; @@ -2258,7 +2259,7 @@ impl CodexMessageProcessor { data: None, })?; let conversation_modalities = resolve_conversation_modalities( - state_db_ctx.as_deref(), + state_db_ctx.as_ref(), summary.conversation_id, None, Some(summary.path.as_path()), @@ -2429,7 +2430,7 @@ impl CodexMessageProcessor { let mut data = Vec::with_capacity(summaries.len()); for summary in summaries { let conversation_modalities = resolve_conversation_modalities( - state_db_ctx.as_deref(), + state_db_ctx.as_ref(), summary.conversation_id, None, Some(summary.path.as_path()), @@ -2554,7 +2555,7 @@ impl CodexMessageProcessor { let mut thread = if let Some(summary) = db_summary { let conversation_modalities = resolve_conversation_modalities( - state_db_ctx.as_deref(), + state_db_ctx.as_ref(), thread_uuid, None, rollout_path.as_deref(), @@ -2566,7 +2567,7 @@ impl CodexMessageProcessor { match read_summary_from_rollout(rollout_path, fallback_provider).await { Ok(summary) => { let conversation_modalities = resolve_conversation_modalities( - state_db_ctx.as_deref(), + state_db_ctx.as_ref(), thread_uuid, None, Some(rollout_path.as_path()), @@ -2862,7 +2863,7 @@ impl CodexMessageProcessor { } }; let conversation_modalities = resolve_conversation_modalities( - state_db_ctx.as_deref(), + state_db_ctx.as_ref(), thread_id, None, Some(rollout_path.as_path()), @@ -3077,8 +3078,9 @@ impl CodexMessageProcessor { return; } }; + let state_db_ctx = get_state_db(&self.config, None).await; let conversation_modalities = resolve_conversation_modalities( - state_db_ctx.as_deref(), + state_db_ctx.as_ref(), thread_id, None, Some(rollout_path.as_path()), @@ -4403,7 +4405,7 @@ impl CodexMessageProcessor { if has_image_input { let modalities = conversation_modalities_for_has_image(true); persist_conversation_modalities( - conversation.state_db(), + conversation.state_db().as_ref(), conversation_id, &modalities, ) @@ -4469,7 +4471,7 @@ impl CodexMessageProcessor { if has_image_input { let modalities = conversation_modalities_for_has_image(true); persist_conversation_modalities( - conversation.state_db(), + conversation.state_db().as_ref(), conversation_id, &modalities, ) @@ -5127,7 +5129,7 @@ impl CodexMessageProcessor { Ok(summary) => { let state_db_ctx = get_state_db(&self.config, None).await; let conversation_modalities = resolve_conversation_modalities( - state_db_ctx.as_deref(), + state_db_ctx.as_ref(), summary.conversation_id, None, Some(rollout_path.as_path()), @@ -6067,11 +6069,12 @@ pub(crate) async fn resolve_conversation_modalities( }; match read_rollout_has_image_context(rollout_path).await { - Ok(has_image_context) => { + Ok(Some(has_image_context)) => { let modalities = conversation_modalities_for_has_image(has_image_context); persist_conversation_modalities(state_db_ctx, thread_id, &modalities).await; Some(modalities) } + Ok(None) => None, Err(err) => { warn!( "failed to determine conversation modalities for rollout {}: {err}", @@ -6082,24 +6085,57 @@ pub(crate) async fn resolve_conversation_modalities( } } -pub(crate) async fn read_rollout_has_image_context(path: &Path) -> std::io::Result { +pub(crate) async fn read_rollout_has_image_context( + path: &Path, +) -> std::io::Result> { let items = match RolloutRecorder::get_rollout_history(path).await? { InitialHistory::New => Vec::new(), InitialHistory::Forked(items) => items, InitialHistory::Resumed(resumed) => resumed.history, }; - Ok(items.into_iter().rev().any(|item| match item { - RolloutItem::ResponseItem(response_item) => response_item.has_input_image(), - RolloutItem::EventMsg(EventMsg::UserMessage(user_message)) => { - user_message - .images - .as_ref() - .is_some_and(|images| !images.is_empty()) - || !user_message.local_images.is_empty() + let mut saw_input = false; + for item in items.into_iter().rev() { + match item { + RolloutItem::ResponseItem(response_item) => { + if response_item.has_input_image() { + return Ok(Some(true)); + } + if response_item_has_input_text(&response_item) { + saw_input = true; + } + } + RolloutItem::EventMsg(EventMsg::UserMessage(user_message)) => { + if user_message + .images + .as_ref() + .is_some_and(|images| !images.is_empty()) + || !user_message.local_images.is_empty() + { + return Ok(Some(true)); + } + saw_input = true; + } + _ => {} + } + } + + if saw_input { + Ok(Some(false)) + } else { + Ok(None) + } +} + +fn response_item_has_input_text(item: &ResponseItem) -> bool { + match item { + ResponseItem::Message { content, .. } => { + content + .iter() + .any(|item| matches!(item, ContentItem::InputText { .. } | ContentItem::OutputText { .. })) } _ => false, - })) + } } fn conversation_modalities_for_has_image(has_image_context: bool) -> Vec { @@ -6194,7 +6230,7 @@ fn build_thread_from_snapshot( Thread { id: thread_id.to_string(), preview: String::new(), - conversation_modalities: Some(vec![InputModality::Text]), + conversation_modalities: None, model_provider: config_snapshot.model_provider_id.clone(), created_at: now, updated_at: now, @@ -6446,7 +6482,10 @@ mod tests { } fs::write(&path, contents)?; - assert_eq!(read_rollout_has_image_context(path.as_path()).await?, true); + assert_eq!( + read_rollout_has_image_context(path.as_path()).await?, + Some(true) + ); Ok(()) } } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5d0c8c239f..da1485f3f3 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -222,8 +222,10 @@ use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::models::ContentItem; use codex_protocol::models::DeveloperInstructions; +use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; +use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::InitialHistory; @@ -232,6 +234,8 @@ use codex_utils_readiness::Readiness; use codex_utils_readiness::ReadinessFlag; use tokio::sync::watch; +const BLIND_IMAGE_PLACEHOLDER_TEXT: &str = "Placeholder for what used to be an image."; + /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. pub struct Codex { @@ -3859,7 +3863,19 @@ pub(crate) async fn run_turn( } // Construct the input that we will send to the model. - let sampling_request_input: Vec = { sess.clone_history().await.for_prompt() }; + let mut sampling_request_input: Vec = { + sess.clone_history().await.for_prompt() + }; + if !turn_context + .model_info + .input_modalities + .contains(&InputModality::Image) + { + let (sanitized, replaced) = substitute_blind_images(sampling_request_input); + if replaced { + sampling_request_input = sanitized; + } + } let sampling_request_input_messages = sampling_request_input .iter() @@ -4070,6 +4086,46 @@ fn codex_apps_connector_id(tool: &crate::mcp_connection_manager::ToolInfo) -> Op tool.connector_id.as_deref() } +fn substitute_blind_images(mut items: Vec) -> (Vec, bool) { + let mut replaced = false; + for item in &mut items { + match item { + ResponseItem::Message { content, .. } => { + let mut new_content = Vec::with_capacity(content.len()); + for content_item in content.drain(..) { + match content_item { + ContentItem::InputImage { .. } => { + replaced = true; + new_content.push(ContentItem::InputText { + text: BLIND_IMAGE_PLACEHOLDER_TEXT.to_string(), + }); + } + other => new_content.push(other), + } + } + *content = new_content; + } + ResponseItem::FunctionCallOutput { output, .. } => { + if let Some(content_items) = output.content_items_mut() { + for content_item in content_items.iter_mut() { + if matches!( + content_item, + FunctionCallOutputContentItem::InputImage { .. } + ) { + *content_item = FunctionCallOutputContentItem::InputText { + text: BLIND_IMAGE_PLACEHOLDER_TEXT.to_string(), + }; + replaced = true; + } + } + } + } + _ => {} + } + } + (items, replaced) +} + struct SamplingRequestToolSelection<'a> { explicit_app_paths: &'a [String], skill_name_counts_lower: &'a HashMap, diff --git a/codex-rs/state/migrations/0009_thread_model_history.sql b/codex-rs/state/migrations/0009_thread_model_history.sql new file mode 100644 index 0000000000..f0037411c2 --- /dev/null +++ b/codex-rs/state/migrations/0009_thread_model_history.sql @@ -0,0 +1,2 @@ +ALTER TABLE threads ADD COLUMN initial_model TEXT; +ALTER TABLE threads ADD COLUMN model_history TEXT; diff --git a/codex-rs/state/migrations/0007_thread_conversation_modalities.sql b/codex-rs/state/migrations/0010_thread_conversation_modalities.sql similarity index 100% rename from codex-rs/state/migrations/0007_thread_conversation_modalities.sql rename to codex-rs/state/migrations/0010_thread_conversation_modalities.sql diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index c4fe209b70..045110b5c1 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -1,4 +1,5 @@ use crate::model::ThreadMetadata; +use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::RolloutItem; @@ -10,6 +11,7 @@ use serde::Serialize; use serde_json::Value; const IMAGE_ONLY_USER_MESSAGE_PLACEHOLDER: &str = "[Image]"; +const MODEL_HISTORY_LIMIT: usize = 10; /// Apply a rollout item to the metadata structure. pub fn apply_rollout_item( @@ -57,6 +59,7 @@ fn apply_turn_context(metadata: &mut ThreadMetadata, turn_ctx: &TurnContextItem) metadata.cwd = turn_ctx.cwd.clone(); metadata.sandbox_policy = enum_to_string(&turn_ctx.sandbox_policy); metadata.approval_mode = enum_to_string(&turn_ctx.approval_policy); + update_model_history(metadata, turn_ctx.model.as_str()); } fn apply_event_msg(metadata: &mut ThreadMetadata, event: &EventMsg) { @@ -86,6 +89,9 @@ fn apply_event_msg(metadata: &mut ThreadMetadata, event: &EventMsg) { codex_protocol::openai_models::INPUT_MODALITY_TEXT_MASK | codex_protocol::openai_models::INPUT_MODALITY_IMAGE_MASK, ); + } else if metadata.conversation_modalities.is_none() { + metadata.conversation_modalities = + Some(codex_protocol::openai_models::INPUT_MODALITY_TEXT_MASK); } } _ => {} @@ -98,10 +104,44 @@ fn apply_response_item(metadata: &mut ThreadMetadata, item: &ResponseItem) { codex_protocol::openai_models::INPUT_MODALITY_TEXT_MASK | codex_protocol::openai_models::INPUT_MODALITY_IMAGE_MASK, ); + } else if metadata.conversation_modalities.is_none() && response_item_has_input_text(item) { + metadata.conversation_modalities = + Some(codex_protocol::openai_models::INPUT_MODALITY_TEXT_MASK); } // Title and first_user_message are derived from EventMsg::UserMessage only. } +fn response_item_has_input_text(item: &ResponseItem) -> bool { + match item { + ResponseItem::Message { content, .. } => { + content + .iter() + .any(|item| matches!(item, ContentItem::InputText { .. } | ContentItem::OutputText { .. })) + } + _ => false, + } +} + +fn update_model_history(metadata: &mut ThreadMetadata, model: &str) { + if model.is_empty() { + return; + } + + if metadata.initial_model.is_none() { + metadata.initial_model = Some(model.to_string()); + } + + let history = metadata.model_history.get_or_insert_with(Vec::new); + if let Some(pos) = history.iter().position(|slug| slug == model) { + history.remove(pos); + } + history.push(model.to_string()); + if history.len() > MODEL_HISTORY_LIMIT { + let overflow = history.len() - MODEL_HISTORY_LIMIT; + history.drain(0..overflow); + } +} + fn strip_user_message_prefix(text: &str) -> &str { match text.find(USER_MESSAGE_BEGIN) { Some(idx) => text[idx + USER_MESSAGE_BEGIN.len()..].trim(), @@ -242,6 +282,8 @@ mod tests { tokens_used: 1, first_user_message: None, conversation_modalities: None, + initial_model: None, + model_history: None, archived_at: None, git_sha: None, git_branch: None, diff --git a/codex-rs/state/src/model/thread_metadata.rs b/codex-rs/state/src/model/thread_metadata.rs index b428d2ef87..8422324277 100644 --- a/codex-rs/state/src/model/thread_metadata.rs +++ b/codex-rs/state/src/model/thread_metadata.rs @@ -6,6 +6,7 @@ use codex_protocol::ThreadId; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; +use serde_json::Value; use sqlx::Row; use sqlx::sqlite::SqliteRow; use std::path::PathBuf; @@ -80,6 +81,10 @@ pub struct ThreadMetadata { pub first_user_message: Option, /// Tri-state conversation modalities bitmask (NULL means unknown). pub conversation_modalities: Option, + /// First model selected in this thread, if known. + pub initial_model: Option, + /// Bounded history of model slugs in recency order (JSON array). + pub model_history: Option>, /// The archive timestamp, if the thread is archived. pub archived_at: Option>, /// The git commit SHA, if known. @@ -123,6 +128,10 @@ pub struct ThreadMetadataBuilder { pub git_origin_url: Option, /// Tri-state conversation modalities bitmask, if known. pub conversation_modalities: Option, + /// First model selected in this thread, if known. + pub initial_model: Option, + /// Bounded history of model slugs in recency order. + pub model_history: Option>, } impl ThreadMetadataBuilder { @@ -148,9 +157,9 @@ impl ThreadMetadataBuilder { git_sha: None, git_branch: None, git_origin_url: None, - conversation_modalities: Some( - codex_protocol::openai_models::INPUT_MODALITY_TEXT_MASK, - ), + conversation_modalities: None, + initial_model: None, + model_history: None, } } @@ -181,9 +190,9 @@ impl ThreadMetadataBuilder { approval_mode, tokens_used: 0, first_user_message: None, - conversation_modalities: self.conversation_modalities.or(Some( - codex_protocol::openai_models::INPUT_MODALITY_TEXT_MASK, - )), + conversation_modalities: self.conversation_modalities, + initial_model: self.initial_model.clone(), + model_history: self.model_history.clone(), archived_at: self.archived_at.map(canonicalize_datetime), git_sha: self.git_sha.clone(), git_branch: self.git_branch.clone(), @@ -238,6 +247,12 @@ impl ThreadMetadata { if self.conversation_modalities != other.conversation_modalities { diffs.push("conversation_modalities"); } + if self.initial_model != other.initial_model { + diffs.push("initial_model"); + } + if self.model_history != other.model_history { + diffs.push("model_history"); + } if self.archived_at != other.archived_at { diffs.push("archived_at"); } @@ -274,6 +289,8 @@ pub(crate) struct ThreadRow { tokens_used: i64, first_user_message: String, conversation_modalities: Option, + initial_model: Option, + model_history: Option>, archived_at: Option, git_sha: Option, git_branch: Option, @@ -297,6 +314,8 @@ impl ThreadRow { tokens_used: row.try_get("tokens_used")?, first_user_message: row.try_get("first_user_message")?, conversation_modalities: row.try_get("conversation_modalities")?, + initial_model: row.try_get("initial_model")?, + model_history: parse_model_history(row.try_get("model_history")?)?, archived_at: row.try_get("archived_at")?, git_sha: row.try_get("git_sha")?, git_branch: row.try_get("git_branch")?, @@ -324,6 +343,8 @@ impl TryFrom for ThreadMetadata { tokens_used, first_user_message, conversation_modalities, + initial_model, + model_history, archived_at, git_sha, git_branch, @@ -344,6 +365,8 @@ impl TryFrom for ThreadMetadata { tokens_used, first_user_message: (!first_user_message.is_empty()).then_some(first_user_message), conversation_modalities, + initial_model, + model_history, archived_at: archived_at.map(epoch_seconds_to_datetime).transpose()?, git_sha, git_branch, @@ -352,6 +375,26 @@ impl TryFrom for ThreadMetadata { } } +fn parse_model_history(raw: Option) -> Result>> { + let Some(raw) = raw else { + return Ok(None); + }; + if raw.trim().is_empty() { + return Ok(None); + } + let value: Value = serde_json::from_str(&raw)?; + match value { + Value::Array(items) => Ok(Some( + items + .into_iter() + .filter_map(|item| item.as_str().map(|text| text.to_string())) + .collect(), + )), + Value::Null => Ok(None), + _ => Err(anyhow::anyhow!("invalid model_history payload")), + } +} + pub(crate) fn anchor_from_item(item: &ThreadMetadata, sort_key: SortKey) -> Option { let id = Uuid::parse_str(&item.id.to_string()).ok()?; let ts = match sort_key { diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index c4131ab977..ad40f74e69 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -184,6 +184,8 @@ SELECT tokens_used, first_user_message, conversation_modalities, + initial_model, + model_history, archived_at, git_sha, git_branch, @@ -304,6 +306,8 @@ SELECT tokens_used, first_user_message, conversation_modalities, + initial_model, + model_history, archived_at, git_sha, git_branch, @@ -443,6 +447,11 @@ FROM threads /// Insert or replace thread metadata directly. pub async fn upsert_thread(&self, metadata: &crate::ThreadMetadata) -> anyhow::Result<()> { + let model_history = metadata + .model_history + .as_ref() + .map(serde_json::to_string) + .transpose()?; sqlx::query( r#" INSERT INTO threads ( @@ -460,12 +469,14 @@ INSERT INTO threads ( tokens_used, first_user_message, conversation_modalities, + initial_model, + model_history, archived, archived_at, git_sha, git_branch, git_origin_url -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET rollout_path = excluded.rollout_path, created_at = excluded.created_at, @@ -480,6 +491,8 @@ ON CONFLICT(id) DO UPDATE SET tokens_used = excluded.tokens_used, first_user_message = excluded.first_user_message, conversation_modalities = excluded.conversation_modalities, + initial_model = excluded.initial_model, + model_history = excluded.model_history, archived = excluded.archived, archived_at = excluded.archived_at, git_sha = excluded.git_sha, @@ -501,6 +514,8 @@ ON CONFLICT(id) DO UPDATE SET .bind(metadata.tokens_used) .bind(metadata.first_user_message.as_deref().unwrap_or_default()) .bind(metadata.conversation_modalities) + .bind(metadata.initial_model.as_deref()) + .bind(model_history) .bind(metadata.archived_at.is_some()) .bind(metadata.archived_at.map(datetime_to_epoch_seconds)) .bind(metadata.git_sha.as_deref()) @@ -1430,6 +1445,8 @@ mod tests { conversation_modalities: Some( codex_protocol::openai_models::INPUT_MODALITY_TEXT_MASK, ), + initial_model: Some("gpt-5.1-codex".to_string()), + model_history: Some(vec!["gpt-5.1-codex".to_string()]), archived_at: None, git_sha: None, git_branch: None, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 644bcae8c7..2963693433 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -3152,13 +3152,6 @@ impl ChatWidget { /// When the model does not advertise image support, we keep the draft unchanged and surface a /// warning event so users can switch models or remove attachments. pub(crate) fn attach_image(&mut self, path: PathBuf) { - if !self.current_model_supports_images() { - self.add_to_history(history_cell::new_warning_event( - self.image_inputs_not_supported_message(), - )); - self.request_redraw(); - return; - } tracing::info!("attach_image path={path:?}"); self.bottom_pane.attach_image(path); self.request_redraw(); @@ -3646,16 +3639,6 @@ impl ChatWidget { if text.is_empty() && local_images.is_empty() { return; } - if !local_images.is_empty() && !self.current_model_supports_images() { - self.restore_blocked_image_submission( - text, - text_elements, - local_images, - mention_bindings, - ); - return; - } - let mut items: Vec = Vec::new(); // Special-case: "!cmd" executes a local shell command instead of sending to the model. @@ -6075,34 +6058,8 @@ impl ChatWidget { .unwrap_or(false) } - /// Return whether the effective model currently advertises image-input support. - /// - /// We intentionally default to `true` when model metadata cannot be read so transient catalog - /// failures do not hard-block user input in the UI. - fn current_model_supports_images(&self) -> bool { - let model = self.current_model(); - self.models_manager - .try_list_models(&self.config) - .ok() - .and_then(|models| { - models - .into_iter() - .find(|preset| preset.model == model) - .map(|preset| preset.input_modalities.contains(&InputModality::Image)) - }) - .unwrap_or(true) - } - fn sync_image_paste_enabled(&mut self) { - let enabled = self.current_model_supports_images(); - self.bottom_pane.set_image_paste_enabled(enabled); - } - - fn image_inputs_not_supported_message(&self) -> String { - format!( - "Model {} does not support image inputs. Remove images or switch models.", - self.current_model() - ) + self.bottom_pane.set_image_paste_enabled(true); } #[allow(dead_code)] // Used in tests