add deferred image content reads

This commit is contained in:
Roy Han
2026-05-05 14:24:25 -07:00
committed by rhan-oai
parent 5310c647ee
commit 9689d19348
16 changed files with 451 additions and 58 deletions
@@ -1047,6 +1047,9 @@ impl MessageProcessor {
ClientRequest::ThreadTurnsItemsList { params, .. } => {
self.thread_processor.thread_turns_items_list(params).await
}
ClientRequest::ThreadItemContentRead { params, .. } => {
self.thread_processor.thread_item_content_read(params).await
}
ClientRequest::ThreadShellCommand { params, .. } => {
self.thread_processor
.thread_shell_command(&request_id, params)
@@ -70,9 +70,11 @@ use codex_app_server_protocol::GitInfo as ApiGitInfo;
use codex_app_server_protocol::HookMetadata;
use codex_app_server_protocol::HooksListParams;
use codex_app_server_protocol::HooksListResponse;
use codex_app_server_protocol::ImageGenerationContent;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::InitializeResponse;
use codex_app_server_protocol::JSONRPCErrorError;
use codex_app_server_protocol::LargeContentMode;
use codex_app_server_protocol::ListMcpServerStatusParams;
use codex_app_server_protocol::ListMcpServerStatusResponse;
use codex_app_server_protocol::LoginAccountParams;
@@ -173,6 +175,8 @@ use codex_app_server_protocol::ThreadIncrementElicitationResponse;
use codex_app_server_protocol::ThreadInjectItemsParams;
use codex_app_server_protocol::ThreadInjectItemsResponse;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadItemContentReadParams;
use codex_app_server_protocol::ThreadItemContentReadResponse;
use codex_app_server_protocol::ThreadListCwdFilter;
use codex_app_server_protocol::ThreadListParams;
use codex_app_server_protocol::ThreadListResponse;
@@ -538,6 +538,7 @@ pub(super) async fn handle_pending_thread_resume_request(
active_turn.as_ref(),
);
}
super::thread_processor::apply_large_content_mode_to_thread(&mut thread, pending.large_content);
let thread_status = thread_watch_manager
.loaded_status_for_thread(&thread.id)
@@ -546,6 +546,15 @@ impl ThreadRequestProcessor {
.map(|response| Some(response.into()))
}
pub(crate) async fn thread_item_content_read(
&self,
params: ThreadItemContentReadParams,
) -> Result<Option<ClientResponsePayload>, JSONRPCErrorError> {
self.thread_item_content_read_response_inner(params)
.await
.map(|response| Some(response.into()))
}
pub(crate) async fn thread_shell_command(
&self,
request_id: &ConnectionRequestId,
@@ -2034,6 +2043,7 @@ impl ThreadRequestProcessor {
cursor,
limit,
sort_direction,
large_content,
} = params;
let thread_uuid = ThreadId::from_string(&thread_id)
@@ -2048,22 +2058,8 @@ impl ThreadRequestProcessor {
// every request. Rollback and compaction events can change earlier turns, so
// the server has to rebuild the full turn list until turn metadata is indexed
// separately.
let loaded_thread = self.thread_manager.get_thread(thread_uuid).await.ok();
let has_live_running_thread = match loaded_thread.as_ref() {
Some(thread) => matches!(thread.agent_status().await, AgentStatus::Running),
None => false,
};
let active_turn = if loaded_thread.is_some() {
// Persisted history may not yet include the currently running turn. The
// app-server listener has already projected live turn events into ThreadState,
// so merge that in-memory snapshot before paginating.
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
let state = thread_state.lock().await;
state.active_turn_snapshot()
} else {
None
};
let turns = reconstruct_thread_turns_for_turns_list(
let (has_live_running_thread, active_turn) = self.live_turn_read_context(thread_uuid).await;
let mut turns = reconstruct_thread_turns_for_turns_list(
&items,
self.thread_watch_manager
.loaded_status_for_thread(&thread_uuid.to_string())
@@ -2071,6 +2067,7 @@ impl ThreadRequestProcessor {
has_live_running_thread,
active_turn,
);
apply_large_content_mode_to_turns(&mut turns, large_content.unwrap_or_default());
let page = paginate_thread_turns(
turns,
cursor.as_deref(),
@@ -2094,6 +2091,7 @@ impl ThreadRequestProcessor {
cursor,
limit,
sort_direction,
large_content,
} = params;
let thread_uuid = ThreadId::from_string(&thread_id)
@@ -2102,18 +2100,7 @@ impl ThreadRequestProcessor {
.load_thread_turns_list_history(thread_uuid)
.await
.map_err(thread_read_view_error)?;
let loaded_thread = self.thread_manager.get_thread(thread_uuid).await.ok();
let has_live_running_thread = match loaded_thread.as_ref() {
Some(thread) => matches!(thread.agent_status().await, AgentStatus::Running),
None => false,
};
let active_turn = if loaded_thread.is_some() {
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
let state = thread_state.lock().await;
state.active_turn_snapshot()
} else {
None
};
let (has_live_running_thread, active_turn) = self.live_turn_read_context(thread_uuid).await;
let turns = reconstruct_thread_turns_for_turns_list(
&items,
self.thread_watch_manager
@@ -2122,11 +2109,12 @@ impl ThreadRequestProcessor {
has_live_running_thread,
active_turn,
);
let turn_items = turns
let mut turn_items = turns
.into_iter()
.find(|turn| turn.id == turn_id)
.ok_or_else(|| invalid_request(format!("turn not found: {turn_id}")))?
.items;
apply_large_content_mode_to_items(&mut turn_items, large_content.unwrap_or_default());
let page = paginate_thread_items(
turn_items,
cursor.as_deref(),
@@ -2140,6 +2128,82 @@ impl ThreadRequestProcessor {
})
}
async fn thread_item_content_read_response_inner(
&self,
params: ThreadItemContentReadParams,
) -> Result<ThreadItemContentReadResponse, JSONRPCErrorError> {
let ThreadItemContentReadParams {
thread_id,
turn_id,
item_id,
content_id,
} = params;
let thread_uuid = ThreadId::from_string(&thread_id)
.map_err(|err| invalid_request(format!("invalid thread id: {err}")))?;
let items = self
.load_thread_turns_list_history(thread_uuid)
.await
.map_err(thread_read_view_error)?;
let (has_live_running_thread, active_turn) = self.live_turn_read_context(thread_uuid).await;
let turns = reconstruct_thread_turns_for_turns_list(
&items,
self.thread_watch_manager
.loaded_status_for_thread(&thread_uuid.to_string())
.await,
has_live_running_thread,
active_turn,
);
let turn = turns
.into_iter()
.find(|turn| turn.id == turn_id)
.ok_or_else(|| invalid_request(format!("turn not found: {turn_id}")))?;
let item = turn
.items
.into_iter()
.find(|item| item.id() == item_id)
.ok_or_else(|| invalid_request(format!("item not found: {item_id}")))?;
let ThreadItem::ImageGeneration {
content, result, ..
} = item
else {
return Err(invalid_request(format!(
"item {item_id} does not contain deferred content"
)));
};
if content_id != IMAGE_GENERATION_RESULT_CONTENT_ID {
return Err(invalid_request(format!("content not found: {content_id}")));
}
let ImageGenerationContent::Inline {
mime_type,
byte_length,
..
} = content
else {
return Err(internal_error(format!(
"image generation item {item_id} was unexpectedly already deferred"
)));
};
Ok(ThreadItemContentReadResponse {
mime_type,
data_base64: result,
byte_length,
})
}
async fn live_turn_read_context(&self, thread_id: ThreadId) -> (bool, Option<Turn>) {
let Ok(thread) = self.thread_manager.get_thread(thread_id).await else {
return (false, None);
};
let has_live_running_thread = matches!(thread.agent_status().await, AgentStatus::Running);
// Persisted history may not yet include the currently running turn. The
// app-server listener has already projected live turn events into ThreadState,
// so merge that in-memory snapshot when serving history reads.
let thread_state = self.thread_state_manager.thread_state(thread_id).await;
let state = thread_state.lock().await;
(has_live_running_thread, state.active_turn_snapshot())
}
async fn load_thread_turns_list_history(
&self,
thread_id: ThreadId,
@@ -2334,6 +2398,7 @@ impl ThreadRequestProcessor {
developer_instructions,
personality,
exclude_turns,
large_content,
persist_extended_history: _persist_extended_history,
} = params;
let include_turns = !exclude_turns;
@@ -2458,6 +2523,7 @@ impl ThreadRequestProcessor {
return Ok(());
}
};
apply_large_content_mode_to_thread(&mut thread, large_content.unwrap_or_default());
self.thread_watch_manager
.upsert_thread(thread.clone())
@@ -2691,6 +2757,7 @@ impl ThreadRequestProcessor {
emit_thread_goal_update,
thread_goal_state_db,
include_turns: !params.exclude_turns,
large_content: params.large_content.unwrap_or_default(),
}),
);
if listener_command_tx.send(command).is_err() {
@@ -3325,6 +3392,7 @@ const THREAD_TURNS_DEFAULT_LIMIT: usize = 25;
const THREAD_TURNS_MAX_LIMIT: usize = 100;
const THREAD_ITEMS_DEFAULT_LIMIT: usize = 100;
const THREAD_ITEMS_MAX_LIMIT: usize = 500;
const IMAGE_GENERATION_RESULT_CONTENT_ID: &str = "result";
fn thread_backwards_cursor_for_sort_key(
thread: &StoredThread,
@@ -3570,6 +3638,45 @@ fn parse_thread_items_cursor(cursor: &str) -> Result<ThreadItemsCursor, JSONRPCE
})
}
pub(super) fn apply_large_content_mode_to_thread(thread: &mut Thread, mode: LargeContentMode) {
apply_large_content_mode_to_turns(&mut thread.turns, mode);
}
fn apply_large_content_mode_to_turns(turns: &mut [Turn], mode: LargeContentMode) {
for turn in turns {
apply_large_content_mode_to_items(&mut turn.items, mode);
}
}
fn apply_large_content_mode_to_items(items: &mut [ThreadItem], mode: LargeContentMode) {
if matches!(mode, LargeContentMode::Inline) {
return;
}
for item in items {
if let ThreadItem::ImageGeneration {
content, result, ..
} = item
&& let ImageGenerationContent::Inline {
mime_type,
byte_length,
width,
height,
..
} = content
{
*content = ImageGenerationContent::Deferred {
content_id: IMAGE_GENERATION_RESULT_CONTENT_ID.to_string(),
mime_type: mime_type.clone(),
byte_length: *byte_length,
width: *width,
height: *height,
};
result.clear();
}
}
}
fn reconstruct_thread_turns_for_turns_list(
items: &[RolloutItem],
loaded_status: ThreadStatus,
@@ -525,6 +525,7 @@ mod thread_processor_behavior_tests {
developer_instructions: None,
personality: None,
exclude_turns: false,
large_content: None,
persist_extended_history: false,
};
let config_snapshot = ThreadConfigSnapshot {
+2
View File
@@ -1,5 +1,6 @@
use crate::outgoing_message::ConnectionId;
use crate::outgoing_message::ConnectionRequestId;
use codex_app_server_protocol::LargeContentMode;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ThreadGoal;
use codex_app_server_protocol::ThreadHistoryBuilder;
@@ -33,6 +34,7 @@ pub(crate) struct PendingThreadResumeRequest {
pub(crate) emit_thread_goal_update: bool,
pub(crate) thread_goal_state_db: Option<StateDbHandle>,
pub(crate) include_turns: bool,
pub(crate) large_content: LargeContentMode,
}
// ThreadListenerCommand is used to perform operations in the context of the thread listener, for serialization purposes.