mirror of
https://github.com/openai/codex.git
synced 2026-11-20 22:40:37 +00:00
add deferred image content reads
This commit is contained in:
@@ -1047,6 +1047,9 @@ impl MessageProcessor {
|
||||
ClientRequest::ThreadTurnsItemsList { params, .. } => {
|
||||
self.thread_processor.thread_turns_items_list(params).await
|
||||
}
|
||||
ClientRequest::ThreadItemContentRead { params, .. } => {
|
||||
self.thread_processor.thread_item_content_read(params).await
|
||||
}
|
||||
ClientRequest::ThreadShellCommand { params, .. } => {
|
||||
self.thread_processor
|
||||
.thread_shell_command(&request_id, params)
|
||||
|
||||
@@ -70,9 +70,11 @@ use codex_app_server_protocol::GitInfo as ApiGitInfo;
|
||||
use codex_app_server_protocol::HookMetadata;
|
||||
use codex_app_server_protocol::HooksListParams;
|
||||
use codex_app_server_protocol::HooksListResponse;
|
||||
use codex_app_server_protocol::ImageGenerationContent;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::InitializeResponse;
|
||||
use codex_app_server_protocol::JSONRPCErrorError;
|
||||
use codex_app_server_protocol::LargeContentMode;
|
||||
use codex_app_server_protocol::ListMcpServerStatusParams;
|
||||
use codex_app_server_protocol::ListMcpServerStatusResponse;
|
||||
use codex_app_server_protocol::LoginAccountParams;
|
||||
@@ -173,6 +175,8 @@ use codex_app_server_protocol::ThreadIncrementElicitationResponse;
|
||||
use codex_app_server_protocol::ThreadInjectItemsParams;
|
||||
use codex_app_server_protocol::ThreadInjectItemsResponse;
|
||||
use codex_app_server_protocol::ThreadItem;
|
||||
use codex_app_server_protocol::ThreadItemContentReadParams;
|
||||
use codex_app_server_protocol::ThreadItemContentReadResponse;
|
||||
use codex_app_server_protocol::ThreadListCwdFilter;
|
||||
use codex_app_server_protocol::ThreadListParams;
|
||||
use codex_app_server_protocol::ThreadListResponse;
|
||||
|
||||
@@ -538,6 +538,7 @@ pub(super) async fn handle_pending_thread_resume_request(
|
||||
active_turn.as_ref(),
|
||||
);
|
||||
}
|
||||
super::thread_processor::apply_large_content_mode_to_thread(&mut thread, pending.large_content);
|
||||
|
||||
let thread_status = thread_watch_manager
|
||||
.loaded_status_for_thread(&thread.id)
|
||||
|
||||
@@ -546,6 +546,15 @@ impl ThreadRequestProcessor {
|
||||
.map(|response| Some(response.into()))
|
||||
}
|
||||
|
||||
pub(crate) async fn thread_item_content_read(
|
||||
&self,
|
||||
params: ThreadItemContentReadParams,
|
||||
) -> Result<Option<ClientResponsePayload>, JSONRPCErrorError> {
|
||||
self.thread_item_content_read_response_inner(params)
|
||||
.await
|
||||
.map(|response| Some(response.into()))
|
||||
}
|
||||
|
||||
pub(crate) async fn thread_shell_command(
|
||||
&self,
|
||||
request_id: &ConnectionRequestId,
|
||||
@@ -2034,6 +2043,7 @@ impl ThreadRequestProcessor {
|
||||
cursor,
|
||||
limit,
|
||||
sort_direction,
|
||||
large_content,
|
||||
} = params;
|
||||
|
||||
let thread_uuid = ThreadId::from_string(&thread_id)
|
||||
@@ -2048,22 +2058,8 @@ impl ThreadRequestProcessor {
|
||||
// every request. Rollback and compaction events can change earlier turns, so
|
||||
// the server has to rebuild the full turn list until turn metadata is indexed
|
||||
// separately.
|
||||
let loaded_thread = self.thread_manager.get_thread(thread_uuid).await.ok();
|
||||
let has_live_running_thread = match loaded_thread.as_ref() {
|
||||
Some(thread) => matches!(thread.agent_status().await, AgentStatus::Running),
|
||||
None => false,
|
||||
};
|
||||
let active_turn = if loaded_thread.is_some() {
|
||||
// Persisted history may not yet include the currently running turn. The
|
||||
// app-server listener has already projected live turn events into ThreadState,
|
||||
// so merge that in-memory snapshot before paginating.
|
||||
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
|
||||
let state = thread_state.lock().await;
|
||||
state.active_turn_snapshot()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let turns = reconstruct_thread_turns_for_turns_list(
|
||||
let (has_live_running_thread, active_turn) = self.live_turn_read_context(thread_uuid).await;
|
||||
let mut turns = reconstruct_thread_turns_for_turns_list(
|
||||
&items,
|
||||
self.thread_watch_manager
|
||||
.loaded_status_for_thread(&thread_uuid.to_string())
|
||||
@@ -2071,6 +2067,7 @@ impl ThreadRequestProcessor {
|
||||
has_live_running_thread,
|
||||
active_turn,
|
||||
);
|
||||
apply_large_content_mode_to_turns(&mut turns, large_content.unwrap_or_default());
|
||||
let page = paginate_thread_turns(
|
||||
turns,
|
||||
cursor.as_deref(),
|
||||
@@ -2094,6 +2091,7 @@ impl ThreadRequestProcessor {
|
||||
cursor,
|
||||
limit,
|
||||
sort_direction,
|
||||
large_content,
|
||||
} = params;
|
||||
|
||||
let thread_uuid = ThreadId::from_string(&thread_id)
|
||||
@@ -2102,18 +2100,7 @@ impl ThreadRequestProcessor {
|
||||
.load_thread_turns_list_history(thread_uuid)
|
||||
.await
|
||||
.map_err(thread_read_view_error)?;
|
||||
let loaded_thread = self.thread_manager.get_thread(thread_uuid).await.ok();
|
||||
let has_live_running_thread = match loaded_thread.as_ref() {
|
||||
Some(thread) => matches!(thread.agent_status().await, AgentStatus::Running),
|
||||
None => false,
|
||||
};
|
||||
let active_turn = if loaded_thread.is_some() {
|
||||
let thread_state = self.thread_state_manager.thread_state(thread_uuid).await;
|
||||
let state = thread_state.lock().await;
|
||||
state.active_turn_snapshot()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (has_live_running_thread, active_turn) = self.live_turn_read_context(thread_uuid).await;
|
||||
let turns = reconstruct_thread_turns_for_turns_list(
|
||||
&items,
|
||||
self.thread_watch_manager
|
||||
@@ -2122,11 +2109,12 @@ impl ThreadRequestProcessor {
|
||||
has_live_running_thread,
|
||||
active_turn,
|
||||
);
|
||||
let turn_items = turns
|
||||
let mut turn_items = turns
|
||||
.into_iter()
|
||||
.find(|turn| turn.id == turn_id)
|
||||
.ok_or_else(|| invalid_request(format!("turn not found: {turn_id}")))?
|
||||
.items;
|
||||
apply_large_content_mode_to_items(&mut turn_items, large_content.unwrap_or_default());
|
||||
let page = paginate_thread_items(
|
||||
turn_items,
|
||||
cursor.as_deref(),
|
||||
@@ -2140,6 +2128,82 @@ impl ThreadRequestProcessor {
|
||||
})
|
||||
}
|
||||
|
||||
async fn thread_item_content_read_response_inner(
|
||||
&self,
|
||||
params: ThreadItemContentReadParams,
|
||||
) -> Result<ThreadItemContentReadResponse, JSONRPCErrorError> {
|
||||
let ThreadItemContentReadParams {
|
||||
thread_id,
|
||||
turn_id,
|
||||
item_id,
|
||||
content_id,
|
||||
} = params;
|
||||
|
||||
let thread_uuid = ThreadId::from_string(&thread_id)
|
||||
.map_err(|err| invalid_request(format!("invalid thread id: {err}")))?;
|
||||
let items = self
|
||||
.load_thread_turns_list_history(thread_uuid)
|
||||
.await
|
||||
.map_err(thread_read_view_error)?;
|
||||
let (has_live_running_thread, active_turn) = self.live_turn_read_context(thread_uuid).await;
|
||||
let turns = reconstruct_thread_turns_for_turns_list(
|
||||
&items,
|
||||
self.thread_watch_manager
|
||||
.loaded_status_for_thread(&thread_uuid.to_string())
|
||||
.await,
|
||||
has_live_running_thread,
|
||||
active_turn,
|
||||
);
|
||||
let turn = turns
|
||||
.into_iter()
|
||||
.find(|turn| turn.id == turn_id)
|
||||
.ok_or_else(|| invalid_request(format!("turn not found: {turn_id}")))?;
|
||||
let item = turn
|
||||
.items
|
||||
.into_iter()
|
||||
.find(|item| item.id() == item_id)
|
||||
.ok_or_else(|| invalid_request(format!("item not found: {item_id}")))?;
|
||||
let ThreadItem::ImageGeneration {
|
||||
content, result, ..
|
||||
} = item
|
||||
else {
|
||||
return Err(invalid_request(format!(
|
||||
"item {item_id} does not contain deferred content"
|
||||
)));
|
||||
};
|
||||
if content_id != IMAGE_GENERATION_RESULT_CONTENT_ID {
|
||||
return Err(invalid_request(format!("content not found: {content_id}")));
|
||||
}
|
||||
let ImageGenerationContent::Inline {
|
||||
mime_type,
|
||||
byte_length,
|
||||
..
|
||||
} = content
|
||||
else {
|
||||
return Err(internal_error(format!(
|
||||
"image generation item {item_id} was unexpectedly already deferred"
|
||||
)));
|
||||
};
|
||||
Ok(ThreadItemContentReadResponse {
|
||||
mime_type,
|
||||
data_base64: result,
|
||||
byte_length,
|
||||
})
|
||||
}
|
||||
|
||||
async fn live_turn_read_context(&self, thread_id: ThreadId) -> (bool, Option<Turn>) {
|
||||
let Ok(thread) = self.thread_manager.get_thread(thread_id).await else {
|
||||
return (false, None);
|
||||
};
|
||||
let has_live_running_thread = matches!(thread.agent_status().await, AgentStatus::Running);
|
||||
// Persisted history may not yet include the currently running turn. The
|
||||
// app-server listener has already projected live turn events into ThreadState,
|
||||
// so merge that in-memory snapshot when serving history reads.
|
||||
let thread_state = self.thread_state_manager.thread_state(thread_id).await;
|
||||
let state = thread_state.lock().await;
|
||||
(has_live_running_thread, state.active_turn_snapshot())
|
||||
}
|
||||
|
||||
async fn load_thread_turns_list_history(
|
||||
&self,
|
||||
thread_id: ThreadId,
|
||||
@@ -2334,6 +2398,7 @@ impl ThreadRequestProcessor {
|
||||
developer_instructions,
|
||||
personality,
|
||||
exclude_turns,
|
||||
large_content,
|
||||
persist_extended_history: _persist_extended_history,
|
||||
} = params;
|
||||
let include_turns = !exclude_turns;
|
||||
@@ -2458,6 +2523,7 @@ impl ThreadRequestProcessor {
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
apply_large_content_mode_to_thread(&mut thread, large_content.unwrap_or_default());
|
||||
|
||||
self.thread_watch_manager
|
||||
.upsert_thread(thread.clone())
|
||||
@@ -2691,6 +2757,7 @@ impl ThreadRequestProcessor {
|
||||
emit_thread_goal_update,
|
||||
thread_goal_state_db,
|
||||
include_turns: !params.exclude_turns,
|
||||
large_content: params.large_content.unwrap_or_default(),
|
||||
}),
|
||||
);
|
||||
if listener_command_tx.send(command).is_err() {
|
||||
@@ -3325,6 +3392,7 @@ const THREAD_TURNS_DEFAULT_LIMIT: usize = 25;
|
||||
const THREAD_TURNS_MAX_LIMIT: usize = 100;
|
||||
const THREAD_ITEMS_DEFAULT_LIMIT: usize = 100;
|
||||
const THREAD_ITEMS_MAX_LIMIT: usize = 500;
|
||||
const IMAGE_GENERATION_RESULT_CONTENT_ID: &str = "result";
|
||||
|
||||
fn thread_backwards_cursor_for_sort_key(
|
||||
thread: &StoredThread,
|
||||
@@ -3488,11 +3556,9 @@ fn paginate_thread_items(
|
||||
.as_ref()
|
||||
.and_then(|anchor| items.iter().position(|item| item.id() == anchor.item_id));
|
||||
if anchor.is_some() && anchor_index.is_none() {
|
||||
return Err(JSONRPCErrorError {
|
||||
code: INVALID_REQUEST_ERROR_CODE,
|
||||
message: "invalid cursor: anchor item is no longer present".to_string(),
|
||||
data: None,
|
||||
});
|
||||
return Err(invalid_request(
|
||||
"invalid cursor: anchor item is no longer present",
|
||||
));
|
||||
}
|
||||
|
||||
let mut keyed_items: Vec<_> = items.into_iter().enumerate().collect();
|
||||
@@ -3555,19 +3621,50 @@ fn serialize_thread_items_cursor(
|
||||
item_id: item_id.to_string(),
|
||||
include_anchor,
|
||||
})
|
||||
.map_err(|err| JSONRPCErrorError {
|
||||
code: INTERNAL_ERROR_CODE,
|
||||
message: format!("failed to serialize cursor: {err}"),
|
||||
data: None,
|
||||
})
|
||||
.map_err(|err| internal_error(format!("failed to serialize cursor: {err}")))
|
||||
}
|
||||
|
||||
fn parse_thread_items_cursor(cursor: &str) -> Result<ThreadItemsCursor, JSONRPCErrorError> {
|
||||
serde_json::from_str(cursor).map_err(|_| JSONRPCErrorError {
|
||||
code: INVALID_REQUEST_ERROR_CODE,
|
||||
message: format!("invalid cursor: {cursor}"),
|
||||
data: None,
|
||||
})
|
||||
serde_json::from_str(cursor).map_err(|_| invalid_request(format!("invalid cursor: {cursor}")))
|
||||
}
|
||||
|
||||
pub(super) fn apply_large_content_mode_to_thread(thread: &mut Thread, mode: LargeContentMode) {
|
||||
apply_large_content_mode_to_turns(&mut thread.turns, mode);
|
||||
}
|
||||
|
||||
fn apply_large_content_mode_to_turns(turns: &mut [Turn], mode: LargeContentMode) {
|
||||
for turn in turns {
|
||||
apply_large_content_mode_to_items(&mut turn.items, mode);
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_large_content_mode_to_items(items: &mut [ThreadItem], mode: LargeContentMode) {
|
||||
if matches!(mode, LargeContentMode::Inline) {
|
||||
return;
|
||||
}
|
||||
|
||||
for item in items {
|
||||
if let ThreadItem::ImageGeneration {
|
||||
content, result, ..
|
||||
} = item
|
||||
&& let ImageGenerationContent::Inline {
|
||||
mime_type,
|
||||
byte_length,
|
||||
width,
|
||||
height,
|
||||
..
|
||||
} = content
|
||||
{
|
||||
*content = ImageGenerationContent::Deferred {
|
||||
content_id: IMAGE_GENERATION_RESULT_CONTENT_ID.to_string(),
|
||||
mime_type: mime_type.clone(),
|
||||
byte_length: *byte_length,
|
||||
width: *width,
|
||||
height: *height,
|
||||
};
|
||||
result.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn reconstruct_thread_turns_for_turns_list(
|
||||
|
||||
@@ -525,6 +525,7 @@ mod thread_processor_behavior_tests {
|
||||
developer_instructions: None,
|
||||
personality: None,
|
||||
exclude_turns: false,
|
||||
large_content: None,
|
||||
persist_extended_history: false,
|
||||
};
|
||||
let config_snapshot = ThreadConfigSnapshot {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::outgoing_message::ConnectionId;
|
||||
use crate::outgoing_message::ConnectionRequestId;
|
||||
use codex_app_server_protocol::LargeContentMode;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::ThreadGoal;
|
||||
use codex_app_server_protocol::ThreadHistoryBuilder;
|
||||
@@ -33,6 +34,7 @@ pub(crate) struct PendingThreadResumeRequest {
|
||||
pub(crate) emit_thread_goal_update: bool,
|
||||
pub(crate) thread_goal_state_db: Option<StateDbHandle>,
|
||||
pub(crate) include_turns: bool,
|
||||
pub(crate) large_content: LargeContentMode,
|
||||
}
|
||||
|
||||
// ThreadListenerCommand is used to perform operations in the context of the thread listener, for serialization purposes.
|
||||
|
||||
Reference in New Issue
Block a user