core: support extension-owned turn items (#31283)

## Description

This PR adds a `codex-extension-items` crate for extension-owned
`TurnItem` schemas, and updates standalone image generation to start
using it via `TurnItem::Extension`.

This gives us a way to prevent Core from having to be aware of all
extension items. App-server still exposes the existing public
`ThreadItem::ImageGeneration` shape, now by wrapping the same shared
`image_generation::ImageGenerationItem` type.

The new `codex-extension-items` crate is necessary because the image gen
extension item is used by:
- `codex-image-generation-extension`, which produces it.
- `codex-tools / core`, which carry it generically.
- `codex-protocol`, which serializes it into lifecycle events and
rollouts.
- `app-server protocol`, which wraps it in public
`ThreadItem::ImageGeneration`

```
extension implementation
        ↓
codex-extension-items
        ↓
protocol / tools / app-server
```

We keep the hosted Responses API image generation as
`TurnItem::ImageGeneration` because core still owns its persistence and
legacy fanout.

### Before

Standalone image generation is implemented as an extension, but its item
representation previously lived in the core protocol. This sets the
precedent that core is aware of all extension items, which would be good
to avoid.

```
image-gen extension
→ constructs codex_protocol::ImageGenerationItem
→ emits ExtensionTurnItem::ImageGeneration
→ core matches ImageGeneration specially
→ protocol stores TurnItem::ImageGeneration
```

### After
```
image-gen extension
→ constructs extension-owned ImageGenerationItem
→ emits generic ExtensionItem
→ core transports/persists it generically
→ app-server wraps ImageGenerationItem as ThreadItem::ImageGeneration
```

Future extension items can have typed app-server APIs without adding a
new `TurnItem` variant, `ExtensionTurnItem` variant, or core emitter
match arm.

## What changed

- Added `codex-extension-items` with the closed `ExtensionItem` enum and
shared `image_generation::ImageGenerationItem` schema.
- Added generic `TurnItem::Extension(ExtensionItem)` and
`ExtensionTurnItem::Extension { item, legacy_events }` paths.
- Updated standalone image generation to emit a typed extension item and
provide its existing legacy `ImageGenerationBegin` /
`ImageGenerationEnd` events explicitly.
- Kept canonical lifecycle ordering: core emits `ItemStarted` /
`ItemCompleted` before extension-provided legacy events.

## Follow-up

Standalone web search still uses its typed special-case path. Migrating
it later would let `ExtensionTurnItem` collapse into a single
extension-item struct.
This commit is contained in:
Owen Lin
2026-07-08 03:55:13 +00:00
committed by GitHub
parent aaa30f79c2
commit f1affbac5e
33 changed files with 484 additions and 272 deletions
@@ -2,6 +2,8 @@ use std::sync::Arc;
use std::sync::Weak;
use codex_protocol::items::TurnItem;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_tools::ConversationHistory;
use codex_tools::ExtensionTurnItem;
use codex_tools::ToolCall as ExtensionToolCall;
@@ -16,7 +18,6 @@ use crate::sandboxing::SandboxPermissions;
use crate::session::session::Session;
use crate::session::turn_context::TurnContext;
use crate::stream_events_utils::TurnItemContributorPolicy;
use crate::stream_events_utils::apply_turn_item_contributors;
use crate::stream_events_utils::finalize_turn_item;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolPayload;
@@ -69,10 +70,14 @@ struct CoreTurnItemEmitter {
turn: Weak<TurnContext>,
}
fn extension_turn_item(item: ExtensionTurnItem) -> TurnItem {
match item {
ExtensionTurnItem::WebSearch(item) => TurnItem::WebSearch(item),
ExtensionTurnItem::ImageGeneration(item) => TurnItem::ImageGeneration(item),
async fn emit_legacy_events(session: &Session, turn: &TurnContext, legacy_events: Vec<EventMsg>) {
for msg in legacy_events {
session
.send_event_raw(Event {
id: turn.sub_id.clone(),
msg,
})
.await;
}
}
@@ -82,9 +87,15 @@ impl TurnItemEmitter for CoreTurnItemEmitter {
let (Some(session), Some(turn)) = (self.session.upgrade(), self.turn.upgrade()) else {
return;
};
session
.emit_turn_item_started(turn.as_ref(), &extension_turn_item(item))
.await;
let (item, legacy_events) = match item {
ExtensionTurnItem::WebSearch(item) => (TurnItem::WebSearch(item), Vec::new()),
ExtensionTurnItem::Extension {
item,
legacy_events,
} => (TurnItem::Extension(item), legacy_events),
};
session.emit_turn_item_started(turn.as_ref(), &item).await;
emit_legacy_events(session.as_ref(), turn.as_ref(), legacy_events).await;
})
}
@@ -93,17 +104,11 @@ impl TurnItemEmitter for CoreTurnItemEmitter {
let (Some(session), Some(turn)) = (self.session.upgrade(), self.turn.upgrade()) else {
return;
};
let item = match item {
ExtensionTurnItem::ImageGeneration(item) => {
let mut item = TurnItem::ImageGeneration(item);
apply_turn_item_contributors(
session.as_ref(),
turn.extension_data.as_ref(),
&mut item,
)
.await;
item
}
let (item, legacy_events) = match item {
ExtensionTurnItem::Extension {
item,
legacy_events,
} => (TurnItem::Extension(item), legacy_events),
ExtensionTurnItem::WebSearch(item) => {
let mut item = TurnItem::WebSearch(item);
finalize_turn_item(
@@ -115,10 +120,11 @@ impl TurnItemEmitter for CoreTurnItemEmitter {
== codex_protocol::config_types::ModeKind::Plan,
)
.await;
item
(item, Vec::new())
}
};
session.emit_turn_item_completed(turn.as_ref(), item).await;
emit_legacy_events(session.as_ref(), turn.as_ref(), legacy_events).await;
})
}
}
@@ -173,14 +179,16 @@ async fn to_extension_call(invocation: &ToolInvocation) -> ExtensionToolCall {
mod tests {
use std::sync::Arc;
use codex_extension_api::ExtensionData;
use codex_extension_api::TurnItemContributor;
use codex_extension_items::ExtensionItem;
use codex_extension_items::image_generation::ImageGenerationItem;
use codex_protocol::items::TurnItem;
use codex_protocol::items::WebSearchItem;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::models::WebSearchAction;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::ImageGenerationBeginEvent;
use codex_protocol::protocol::ImageGenerationEndEvent;
use codex_tools::ExtensionTurnItem;
use codex_utils_absolute_path::test_support::PathExt;
use codex_utils_absolute_path::test_support::test_path_buf;
@@ -447,147 +455,64 @@ mod tests {
assert_eq!(end.action, expected.action);
}
struct ImageGenerationExtensionExecutor;
#[derive(Debug)]
struct ExtensionTurnItemContributorRan;
struct RecordExtensionTurnItemContributor;
impl TurnItemContributor for RecordExtensionTurnItemContributor {
fn contribute<'a>(
&'a self,
_thread_store: &'a ExtensionData,
turn_store: &'a ExtensionData,
_item: &'a mut TurnItem,
) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> {
Box::pin(async move {
turn_store.insert(ExtensionTurnItemContributorRan);
Ok(())
})
}
}
#[tokio::test]
async fn extension_completion_runs_turn_item_contributors() {
let (mut session, turn) = crate::session::tests::make_session_and_context().await;
let mut builder = codex_extension_api::ExtensionRegistryBuilder::new();
builder.turn_item_contributor(Arc::new(RecordExtensionTurnItemContributor));
session.services.extensions = Arc::new(builder.build());
let session = Arc::new(session);
let turn = Arc::new(turn);
let emitter = CoreTurnItemEmitter {
session: Arc::downgrade(&session),
turn: Arc::downgrade(&turn),
};
codex_tools::TurnItemEmitter::emit_completed(
&emitter,
ExtensionTurnItem::WebSearch(WebSearchItem {
id: "search-1".to_string(),
query: "contributors".to_string(),
action: WebSearchAction::Other,
}),
)
.await;
assert!(
turn.extension_data
.get::<ExtensionTurnItemContributorRan>()
.is_some()
);
}
impl codex_extension_api::ToolExecutor<codex_tools::ToolCall> for ImageGenerationExtensionExecutor {
fn tool_name(&self) -> codex_tools::ToolName {
codex_tools::ToolName::namespaced("image_gen", "imagegen")
}
fn spec(&self) -> codex_tools::ToolSpec {
codex_tools::ToolSpec::Function(codex_tools::ResponsesApiTool {
name: "imagegen".to_string(),
description: "Generates an image.".to_string(),
strict: false,
parameters: codex_tools::JsonSchema::default(),
output_schema: None,
defer_loading: None,
})
}
fn handle(&self, call: codex_tools::ToolCall) -> codex_tools::ToolExecutorFuture<'_> {
Box::pin(self.handle_call(call))
}
}
impl ImageGenerationExtensionExecutor {
async fn handle_call(
&self,
call: codex_tools::ToolCall,
) -> Result<Box<dyn codex_tools::ToolOutput>, codex_tools::FunctionCallError> {
call.turn_item_emitter
.emit_started(ExtensionTurnItem::ImageGeneration(
codex_protocol::items::ImageGenerationItem {
id: call.call_id.clone(),
status: "in_progress".to_string(),
revised_prompt: None,
result: String::new(),
saved_path: None,
},
))
.await;
call.turn_item_emitter
.emit_completed(ExtensionTurnItem::ImageGeneration(
codex_protocol::items::ImageGenerationItem {
id: call.call_id,
status: "completed".to_string(),
revised_prompt: Some("A tiny blue square".to_string()),
result: "cG5n".to_string(),
saved_path: Some(test_path_buf("/tmp/extension-claimed.png").abs()),
},
))
.await;
Ok(
Box::new(codex_tools::JsonToolOutput::new(json!({ "ok": true })))
as Box<dyn codex_tools::ToolOutput>,
)
}
}
#[tokio::test]
async fn image_generation_publication_preserves_extension_saved_path() {
let (session, turn, rx) = crate::session::tests::make_session_and_context_with_rx().await;
let handler = ExtensionToolAdapter::new(Arc::new(ImageGenerationExtensionExecutor));
let expected_path = test_path_buf("/tmp/extension-claimed.png").abs();
let default_path = crate::stream_events_utils::image_generation_artifact_path(
&turn.config.codex_home,
&session.thread_id.to_string(),
"call-image",
);
let step_context = StepContext::for_test(Arc::clone(&turn));
let invocation = ToolInvocation {
session,
step_context,
turn,
cancellation_token: tokio_util::sync::CancellationToken::new(),
tracker: Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())),
call_id: "call-image".to_string(),
tool_name: codex_tools::ToolName::namespaced("image_gen", "imagegen"),
source: ToolCallSource::Direct,
payload: ToolPayload::Function {
arguments: "{}".to_string(),
},
let emitter = CoreTurnItemEmitter {
session: Arc::downgrade(&session),
turn: Arc::downgrade(&turn),
};
crate::tools::registry::ToolExecutor::handle(&handler, invocation)
.await
.expect("extension call should succeed");
let expected_started_item = ExtensionItem::ImageGeneration(ImageGenerationItem {
id: "call-image".to_string(),
status: "in_progress".to_string(),
revised_prompt: None,
result: String::new(),
saved_path: None,
});
let expected_completed_item = ExtensionItem::ImageGeneration(ImageGenerationItem {
id: "call-image".to_string(),
status: "completed".to_string(),
revised_prompt: Some("A tiny blue square".to_string()),
result: "cG5n".to_string(),
saved_path: Some(expected_path.clone()),
});
codex_tools::TurnItemEmitter::emit_started(
&emitter,
ExtensionTurnItem::Extension {
item: expected_started_item.clone(),
legacy_events: vec![EventMsg::ImageGenerationBegin(ImageGenerationBeginEvent {
call_id: "call-image".to_string(),
})],
},
)
.await;
codex_tools::TurnItemEmitter::emit_completed(
&emitter,
ExtensionTurnItem::Extension {
item: expected_completed_item.clone(),
legacy_events: vec![EventMsg::ImageGenerationEnd(ImageGenerationEndEvent {
call_id: "call-image".to_string(),
status: "completed".to_string(),
revised_prompt: Some("A tiny blue square".to_string()),
result: "cG5n".to_string(),
saved_path: Some(expected_path.clone()),
})],
},
)
.await;
let started = rx.recv().await.expect("item started event");
let EventMsg::ItemStarted(started) = started.msg else {
panic!("expected item started event");
};
let TurnItem::ImageGeneration(started_item) = started.item else {
panic!("expected image generation item");
let TurnItem::Extension(started_item) = started.item else {
panic!("expected extension item");
};
let begin = rx.recv().await.expect("legacy image start event");
assert!(matches!(begin.msg, EventMsg::ImageGenerationBegin(_)));
@@ -595,32 +520,14 @@ mod tests {
let EventMsg::ItemCompleted(completed) = completed.msg else {
panic!("expected item completed event");
};
let TurnItem::ImageGeneration(completed_item) = completed.item else {
panic!("expected image generation item");
let TurnItem::Extension(completed_item) = completed.item else {
panic!("expected extension item");
};
let end = rx.recv().await.expect("legacy image end event");
assert!(matches!(end.msg, EventMsg::ImageGenerationEnd(_)));
assert_eq!(
started_item,
codex_protocol::items::ImageGenerationItem {
id: "call-image".to_string(),
status: "in_progress".to_string(),
revised_prompt: None,
result: String::new(),
saved_path: None,
}
);
assert_eq!(
completed_item,
codex_protocol::items::ImageGenerationItem {
id: "call-image".to_string(),
status: "completed".to_string(),
revised_prompt: Some("A tiny blue square".to_string()),
result: "cG5n".to_string(),
saved_path: Some(expected_path.clone()),
}
);
assert_eq!(started_item, expected_started_item);
assert_eq!(completed_item, expected_completed_item);
assert!(!default_path.exists());
}
}