mirror of
https://github.com/openai/codex.git
synced 2026-10-19 17:28:46 +00:00
core: support extension-owned turn items (#31283)
## Description
This PR adds a `codex-extension-items` crate for extension-owned
`TurnItem` schemas, and updates standalone image generation to start
using it via `TurnItem::Extension`.
This gives us a way to prevent Core from having to be aware of all
extension items. App-server still exposes the existing public
`ThreadItem::ImageGeneration` shape, now by wrapping the same shared
`image_generation::ImageGenerationItem` type.
The new `codex-extension-items` crate is necessary because the image gen
extension item is used by:
- `codex-image-generation-extension`, which produces it.
- `codex-tools / core`, which carry it generically.
- `codex-protocol`, which serializes it into lifecycle events and
rollouts.
- `app-server protocol`, which wraps it in public
`ThreadItem::ImageGeneration`
```
extension implementation
↓
codex-extension-items
↓
protocol / tools / app-server
```
We keep the hosted Responses API image generation as
`TurnItem::ImageGeneration` because core still owns its persistence and
legacy fanout.
### Before
Standalone image generation is implemented as an extension, but its item
representation previously lived in the core protocol. This sets the
precedent that core is aware of all extension items, which would be good
to avoid.
```
image-gen extension
→ constructs codex_protocol::ImageGenerationItem
→ emits ExtensionTurnItem::ImageGeneration
→ core matches ImageGeneration specially
→ protocol stores TurnItem::ImageGeneration
```
### After
```
image-gen extension
→ constructs extension-owned ImageGenerationItem
→ emits generic ExtensionItem
→ core transports/persists it generically
→ app-server wraps ImageGenerationItem as ThreadItem::ImageGeneration
```
Future extension items can have typed app-server APIs without adding a
new `TurnItem` variant, `ExtensionTurnItem` variant, or core emitter
match arm.
## What changed
- Added `codex-extension-items` with the closed `ExtensionItem` enum and
shared `image_generation::ImageGenerationItem` schema.
- Added generic `TurnItem::Extension(ExtensionItem)` and
`ExtensionTurnItem::Extension { item, legacy_events }` paths.
- Updated standalone image generation to emit a typed extension item and
provide its existing legacy `ImageGenerationBegin` /
`ImageGenerationEnd` events explicitly.
- Kept canonical lifecycle ordering: core emits `ItemStarted` /
`ItemCompleted` before extension-provided legacy events.
## Follow-up
Standalone web search still uses its typed special-case path. Migrating
it later would let `ExtensionTurnItem` collapse into a single
extension-item struct.
This commit is contained in:
@@ -22,6 +22,7 @@ use crate::protocol::v2::TurnItemsView;
|
||||
use crate::protocol::v2::TurnStatus;
|
||||
use crate::protocol::v2::UserInput;
|
||||
use crate::protocol::v2::WebSearchAction;
|
||||
use codex_extension_items::image_generation::ImageGenerationItem;
|
||||
use codex_protocol::items::parse_hook_prompt_message;
|
||||
use codex_protocol::models::MessagePhase;
|
||||
use codex_protocol::protocol::AgentReasoningEvent;
|
||||
@@ -592,7 +593,8 @@ impl ThreadHistoryBuilder {
|
||||
| codex_protocol::items::TurnItem::CommandExecution(_)
|
||||
| codex_protocol::items::TurnItem::DynamicToolCall(_)
|
||||
| codex_protocol::items::TurnItem::CollabAgentToolCall(_)
|
||||
| codex_protocol::items::TurnItem::SubAgentActivity(_) => true,
|
||||
| codex_protocol::items::TurnItem::SubAgentActivity(_)
|
||||
| codex_protocol::items::TurnItem::Extension(_) => true,
|
||||
codex_protocol::items::TurnItem::UserMessage(_)
|
||||
| codex_protocol::items::TurnItem::HookPrompt(_)
|
||||
| codex_protocol::items::TurnItem::AgentMessage(_)
|
||||
@@ -827,24 +829,24 @@ impl ThreadHistoryBuilder {
|
||||
}
|
||||
|
||||
fn handle_image_generation_begin(&mut self, payload: &ImageGenerationBeginEvent) {
|
||||
let item = ThreadItem::ImageGeneration {
|
||||
let item = ThreadItem::ImageGeneration(ImageGenerationItem {
|
||||
id: payload.call_id.clone(),
|
||||
status: String::new(),
|
||||
revised_prompt: None,
|
||||
result: String::new(),
|
||||
saved_path: None,
|
||||
};
|
||||
});
|
||||
self.upsert_item_in_current_turn(item);
|
||||
}
|
||||
|
||||
fn handle_image_generation_end(&mut self, payload: &ImageGenerationEndEvent) {
|
||||
let item = ThreadItem::ImageGeneration {
|
||||
let item = ThreadItem::ImageGeneration(ImageGenerationItem {
|
||||
id: payload.call_id.clone(),
|
||||
status: payload.status.clone(),
|
||||
revised_prompt: payload.revised_prompt.clone(),
|
||||
result: payload.result.clone(),
|
||||
saved_path: payload.saved_path.clone(),
|
||||
};
|
||||
});
|
||||
self.upsert_item_in_current_turn(item);
|
||||
}
|
||||
|
||||
@@ -1548,6 +1550,7 @@ impl From<&PendingTurn> for Turn {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::protocol::v2::CommandExecutionSource;
|
||||
use codex_extension_items::ExtensionItem as CoreExtensionItem;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem as CoreDynamicToolCallOutputContentItem;
|
||||
use codex_protocol::items::CommandExecutionItem as CoreCommandExecutionItem;
|
||||
@@ -1846,6 +1849,60 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebuilds_extension_image_generation_item_from_persisted_completion() {
|
||||
let turn_id = "turn-1";
|
||||
let thread_id = ThreadId::new();
|
||||
let saved_path = test_path_buf("/tmp/image-1.png").abs();
|
||||
let events = vec![
|
||||
EventMsg::TurnStarted(TurnStartedEvent {
|
||||
turn_id: turn_id.to_string(),
|
||||
trace_id: None,
|
||||
started_at: None,
|
||||
model_context_window: None,
|
||||
collaboration_mode_kind: Default::default(),
|
||||
}),
|
||||
EventMsg::ItemCompleted(ItemCompletedEvent {
|
||||
thread_id,
|
||||
turn_id: turn_id.to_string(),
|
||||
item: CoreTurnItem::Extension(CoreExtensionItem::ImageGeneration(
|
||||
ImageGenerationItem {
|
||||
id: "image-1".to_string(),
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some("A blue square".to_string()),
|
||||
result: "cG5n".to_string(),
|
||||
saved_path: Some(saved_path.clone()),
|
||||
},
|
||||
)),
|
||||
completed_at_ms: 1_000,
|
||||
}),
|
||||
EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
turn_id: turn_id.to_string(),
|
||||
last_agent_message: None,
|
||||
completed_at: None,
|
||||
duration_ms: None,
|
||||
time_to_first_token_ms: None,
|
||||
}),
|
||||
];
|
||||
let items = events
|
||||
.into_iter()
|
||||
.map(RolloutItem::EventMsg)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let turns = build_turns_from_rollout_items(&items);
|
||||
|
||||
assert_eq!(
|
||||
turns[0].items,
|
||||
vec![ThreadItem::ImageGeneration(ImageGenerationItem {
|
||||
id: "image-1".to_string(),
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some("A blue square".to_string()),
|
||||
result: "cG5n".to_string(),
|
||||
saved_path: Some(saved_path),
|
||||
})]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebuilds_command_execution_item_from_persisted_completion() {
|
||||
let turn_id = "turn-1";
|
||||
@@ -2058,13 +2115,13 @@ mod tests {
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
},
|
||||
ThreadItem::ImageGeneration {
|
||||
ThreadItem::ImageGeneration(ImageGenerationItem {
|
||||
id: "ig_123".into(),
|
||||
status: "completed".into(),
|
||||
revised_prompt: Some("final prompt".into()),
|
||||
result: "Zm9v".into(),
|
||||
saved_path: Some(test_path_buf("/tmp/ig_123.png").abs()),
|
||||
},
|
||||
}),
|
||||
],
|
||||
}
|
||||
);
|
||||
|
||||
@@ -11,6 +11,8 @@ use super::shared::v2_enum_from_core;
|
||||
use crate::protocol::item_builders::command_actions_for_path_uri;
|
||||
use crate::protocol::item_builders::convert_patch_changes;
|
||||
use codex_experimental_api_macros::ExperimentalApi;
|
||||
use codex_extension_items::ExtensionItem;
|
||||
pub use codex_extension_items::image_generation::ImageGenerationItem;
|
||||
use codex_protocol::approvals::GuardianAssessmentAction as CoreGuardianAssessmentAction;
|
||||
use codex_protocol::approvals::GuardianAssessmentDecisionSource as CoreGuardianAssessmentDecisionSource;
|
||||
use codex_protocol::approvals::GuardianCommandSource as CoreGuardianCommandSource;
|
||||
@@ -246,7 +248,10 @@ pub enum ThreadItem {
|
||||
#[ts(rename_all = "camelCase")]
|
||||
/// EXPERIMENTAL - proposed plan item content. The completed plan item is
|
||||
/// authoritative and may not match the concatenation of `PlanDelta` text.
|
||||
Plan { id: String, text: String },
|
||||
Plan {
|
||||
id: String,
|
||||
text: String,
|
||||
},
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase")]
|
||||
Reasoning {
|
||||
@@ -373,26 +378,24 @@ pub enum ThreadItem {
|
||||
#[ts(type = "number")]
|
||||
duration_ms: u64,
|
||||
},
|
||||
ImageGeneration(ImageGenerationItem),
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase")]
|
||||
ImageGeneration {
|
||||
EnteredReviewMode {
|
||||
id: String,
|
||||
status: String,
|
||||
revised_prompt: Option<String>,
|
||||
result: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional)]
|
||||
saved_path: Option<AbsolutePathBuf>,
|
||||
review: String,
|
||||
},
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase")]
|
||||
EnteredReviewMode { id: String, review: String },
|
||||
ExitedReviewMode {
|
||||
id: String,
|
||||
review: String,
|
||||
},
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase")]
|
||||
ExitedReviewMode { id: String, review: String },
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase")]
|
||||
ContextCompaction { id: String },
|
||||
ContextCompaction {
|
||||
id: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
@@ -432,10 +435,10 @@ impl ThreadItem {
|
||||
| ThreadItem::WebSearch { id, .. }
|
||||
| ThreadItem::ImageView { id, .. }
|
||||
| ThreadItem::Sleep { id, .. }
|
||||
| ThreadItem::ImageGeneration { id, .. }
|
||||
| ThreadItem::EnteredReviewMode { id, .. }
|
||||
| ThreadItem::ExitedReviewMode { id, .. }
|
||||
| ThreadItem::ContextCompaction { id, .. } => id,
|
||||
ThreadItem::ImageGeneration(item) => &item.id,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -931,13 +934,18 @@ impl From<CoreTurnItem> for ThreadItem {
|
||||
id: sleep.id,
|
||||
duration_ms: sleep.duration_ms,
|
||||
},
|
||||
CoreTurnItem::ImageGeneration(image) => ThreadItem::ImageGeneration {
|
||||
id: image.id,
|
||||
status: image.status,
|
||||
revised_prompt: image.revised_prompt,
|
||||
result: image.result,
|
||||
saved_path: image.saved_path,
|
||||
CoreTurnItem::Extension(extension) => match extension {
|
||||
ExtensionItem::ImageGeneration(item) => ThreadItem::ImageGeneration(item),
|
||||
},
|
||||
CoreTurnItem::ImageGeneration(image) => {
|
||||
ThreadItem::ImageGeneration(ImageGenerationItem {
|
||||
id: image.id,
|
||||
status: image.status,
|
||||
revised_prompt: image.revised_prompt,
|
||||
result: image.result,
|
||||
saved_path: image.saved_path,
|
||||
})
|
||||
}
|
||||
CoreTurnItem::FileChange(file_change) => ThreadItem::FileChange {
|
||||
id: file_change.id,
|
||||
changes: convert_patch_changes(&file_change.changes),
|
||||
|
||||
Reference in New Issue
Block a user