From f1affbac5e5164b2bae825e9b39e9868bc4e0be2 Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Tue, 7 Jul 2026 20:55:13 -0700 Subject: [PATCH] core: support extension-owned turn items (#31283) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description This PR adds a `codex-extension-items` crate for extension-owned `TurnItem` schemas, and updates standalone image generation to start using it via `TurnItem::Extension`. This gives us a way to prevent Core from having to be aware of all extension items. App-server still exposes the existing public `ThreadItem::ImageGeneration` shape, now by wrapping the same shared `image_generation::ImageGenerationItem` type. The new `codex-extension-items` crate is necessary because the image gen extension item is used by: - `codex-image-generation-extension`, which produces it. - `codex-tools / core`, which carry it generically. - `codex-protocol`, which serializes it into lifecycle events and rollouts. - `app-server protocol`, which wraps it in public `ThreadItem::ImageGeneration` ``` extension implementation ↓ codex-extension-items ↓ protocol / tools / app-server ``` We keep the hosted Responses API image generation as `TurnItem::ImageGeneration` because core still owns its persistence and legacy fanout. ### Before Standalone image generation is implemented as an extension, but its item representation previously lived in the core protocol. This sets the precedent that core is aware of all extension items, which would be good to avoid. ``` image-gen extension → constructs codex_protocol::ImageGenerationItem → emits ExtensionTurnItem::ImageGeneration → core matches ImageGeneration specially → protocol stores TurnItem::ImageGeneration ``` ### After ``` image-gen extension → constructs extension-owned ImageGenerationItem → emits generic ExtensionItem → core transports/persists it generically → app-server wraps ImageGenerationItem as ThreadItem::ImageGeneration ``` Future extension items can have typed app-server APIs without adding a new `TurnItem` variant, `ExtensionTurnItem` variant, or core emitter match arm. ## What changed - Added `codex-extension-items` with the closed `ExtensionItem` enum and shared `image_generation::ImageGenerationItem` schema. - Added generic `TurnItem::Extension(ExtensionItem)` and `ExtensionTurnItem::Extension { item, legacy_events }` paths. - Updated standalone image generation to emit a typed extension item and provide its existing legacy `ImageGenerationBegin` / `ImageGenerationEnd` events explicitly. - Kept canonical lifecycle ordering: core emits `ItemStarted` / `ItemCompleted` before extension-provided legacy events. ## Follow-up Standalone web search still uses its typed special-case path. Migrating it later would let `ExtensionTurnItem` collapse into a single extension-item struct. --- codex-rs/Cargo.lock | 17 ++ codex-rs/Cargo.toml | 2 + .../analytics/src/analytics_client_tests.rs | 5 +- codex-rs/analytics/src/reducer.rs | 22 +- codex-rs/app-server-protocol/Cargo.toml | 1 + .../schema/typescript/ImageGenerationItem.ts | 6 + .../schema/typescript/index.ts | 1 + .../schema/typescript/v2/ThreadItem.ts | 4 +- .../src/protocol/thread_history.rs | 71 ++++- .../src/protocol/v2/item.rs | 48 ++-- .../thread_resume_redaction.rs | 7 +- .../tests/suite/v2/imagegen_extension.rs | 11 +- .../tests/suite/v2/thread_resume.rs | 11 +- codex-rs/core/Cargo.toml | 1 + .../src/tools/handlers/extension_tools.rs | 247 ++++++------------ codex-rs/ext/image-generation/Cargo.toml | 1 + codex-rs/ext/image-generation/src/tool.rs | 74 ++++-- codex-rs/ext/items/BUILD.bazel | 6 + codex-rs/ext/items/Cargo.toml | 23 ++ codex-rs/ext/items/src/image_generation.rs | 21 ++ codex-rs/ext/items/src/lib.rs | 51 ++++ codex-rs/ext/items/src/tests.rs | 57 ++++ codex-rs/protocol/Cargo.toml | 1 + codex-rs/protocol/src/items.rs | 11 + codex-rs/protocol/src/legacy_events.rs | 1 + codex-rs/rollout/src/persistence_metrics.rs | 1 + codex-rs/tools/Cargo.toml | 1 + codex-rs/tools/src/tool_call.rs | 17 +- codex-rs/tui/src/app/agent_status_feed.rs | 2 +- codex-rs/tui/src/chatwidget/protocol.rs | 2 +- codex-rs/tui/src/chatwidget/replay.rs | 15 +- codex-rs/tui/src/chatwidget/tests/helpers.rs | 5 +- codex-rs/tui/src/thread_transcript.rs | 13 +- 33 files changed, 484 insertions(+), 272 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts create mode 100644 codex-rs/ext/items/BUILD.bazel create mode 100644 codex-rs/ext/items/Cargo.toml create mode 100644 codex-rs/ext/items/src/image_generation.rs create mode 100644 codex-rs/ext/items/src/lib.rs create mode 100644 codex-rs/ext/items/src/tests.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index f6879c58e6..9e250998ff 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2108,6 +2108,7 @@ dependencies = [ "anyhow", "clap", "codex-experimental-api-macros", + "codex-extension-items", "codex-protocol", "codex-shell-command", "codex-utils-absolute-path", @@ -2642,6 +2643,7 @@ dependencies = [ "codex-exec-server", "codex-execpolicy", "codex-extension-api", + "codex-extension-items", "codex-features", "codex-feedback", "codex-file-system", @@ -3014,6 +3016,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "codex-extension-items" +version = "0.0.0" +dependencies = [ + "codex-utils-absolute-path", + "pretty_assertions", + "schemars 0.8.22", + "serde", + "serde_json", + "ts-rs", +] + [[package]] name = "codex-external-agent-migration" version = "0.0.0" @@ -3236,6 +3250,7 @@ dependencies = [ "codex-core", "codex-exec-server", "codex-extension-api", + "codex-extension-items", "codex-login", "codex-model-provider", "codex-model-provider-info", @@ -3715,6 +3730,7 @@ dependencies = [ "chrono", "codex-async-utils", "codex-execpolicy", + "codex-extension-items", "codex-network-proxy", "codex-utils-absolute-path", "codex-utils-image", @@ -4072,6 +4088,7 @@ version = "0.0.0" dependencies = [ "codex-code-mode", "codex-connectors", + "codex-extension-items", "codex-features", "codex-file-system", "codex-protocol", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index ad08ce8e02..0c1a4f9711 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -54,6 +54,7 @@ members = [ "ext/goal", "ext/guardian", "ext/image-generation", + "ext/items", "ext/memories", "ext/mcp", "ext/skills", @@ -180,6 +181,7 @@ codex-exec-server-protocol = { path = "exec-server-protocol" } codex-exec-server = { path = "exec-server" } codex-execpolicy = { path = "execpolicy" } codex-extension-api = { path = "ext/extension-api" } +codex-extension-items = { path = "ext/items" } codex-goal-extension = { path = "ext/goal" } codex-guardian = { path = "ext/guardian" } codex-image-generation-extension = { path = "ext/image-generation" } diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 0f759fd938..0789c9de48 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -103,6 +103,7 @@ use codex_app_server_protocol::GuardianApprovalReview; use codex_app_server_protocol::GuardianApprovalReviewAction; use codex_app_server_protocol::GuardianApprovalReviewStatus; use codex_app_server_protocol::GuardianCommandSource as AppServerGuardianCommandSource; +use codex_app_server_protocol::ImageGenerationItem; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::ItemCompletedNotification; @@ -4405,13 +4406,13 @@ async fn turn_event_counts_completed_tool_items() { query: "codex".to_string(), action: None, }, - ThreadItem::ImageGeneration { + ThreadItem::ImageGeneration(ImageGenerationItem { id: "image-1".to_string(), status: "completed".to_string(), revised_prompt: None, result: "ok".to_string(), saved_path: None, - }, + }), ]; for item in completed_tool_items { diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 29f21bac52..fa742b74db 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -403,7 +403,7 @@ impl TurnToolCounts { self.subagent_tool_call += 1; } ThreadItem::WebSearch { .. } => self.web_search += 1, - ThreadItem::ImageGeneration { .. } => self.image_generation += 1, + ThreadItem::ImageGeneration(_) => self.image_generation += 1, ThreadItem::UserMessage { .. } | ThreadItem::HookPrompt { .. } | ThreadItem::AgentMessage { .. } @@ -1733,8 +1733,8 @@ fn tracked_tool_item_id(item: &ThreadItem) -> Option<&str> { | ThreadItem::McpToolCall { id, .. } | ThreadItem::DynamicToolCall { id, .. } | ThreadItem::CollabAgentToolCall { id, .. } - | ThreadItem::WebSearch { id, .. } - | ThreadItem::ImageGeneration { id, .. } => Some(id), + | ThreadItem::WebSearch { id, .. } => Some(id), + ThreadItem::ImageGeneration(item) => Some(&item.id), ThreadItem::UserMessage { .. } | ThreadItem::HookPrompt { .. } | ThreadItem::AgentMessage { .. } @@ -2057,18 +2057,12 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { }, })) } - ThreadItem::ImageGeneration { - id, - status, - revised_prompt, - saved_path, - .. - } => { - let (terminal_status, failure_kind) = image_generation_outcome(status.as_str()); + ThreadItem::ImageGeneration(item) => { + let (terminal_status, failure_kind) = image_generation_outcome(item.status.as_str()); let base = tool_item_base( thread_id, turn_id, - id.clone(), + item.id.clone(), "image_generation".to_string(), ToolItemOutcome { terminal_status, @@ -2089,8 +2083,8 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option { event_type: "codex_image_generation_event", event_params: CodexImageGenerationEventParams { base, - revised_prompt_present: revised_prompt.is_some(), - saved_path_present: saved_path.is_some(), + revised_prompt_present: item.revised_prompt.is_some(), + saved_path_present: item.saved_path.is_some(), }, }, )) diff --git a/codex-rs/app-server-protocol/Cargo.toml b/codex-rs/app-server-protocol/Cargo.toml index dc69e2d4bf..1f0cf450b3 100644 --- a/codex-rs/app-server-protocol/Cargo.toml +++ b/codex-rs/app-server-protocol/Cargo.toml @@ -16,6 +16,7 @@ workspace = true anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } codex-experimental-api-macros = { workspace = true } +codex-extension-items = { workspace = true } codex-protocol = { workspace = true } codex-shell-command = { workspace = true } codex-utils-absolute-path = { workspace = true } diff --git a/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts new file mode 100644 index 0000000000..26cd6285bc --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts @@ -0,0 +1,6 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { AbsolutePathBuf } from "./AbsolutePathBuf"; + +export type ImageGenerationItem = { id: string, status: string, revisedPrompt: string | null, result: string, savedPath?: AbsolutePathBuf, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/index.ts b/codex-rs/app-server-protocol/schema/typescript/index.ts index dcfecf1282..092aa3c5e4 100644 --- a/codex-rs/app-server-protocol/schema/typescript/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/index.ts @@ -37,6 +37,7 @@ export type { GitDiffToRemoteParams } from "./GitDiffToRemoteParams"; export type { GitDiffToRemoteResponse } from "./GitDiffToRemoteResponse"; export type { GitSha } from "./GitSha"; export type { ImageDetail } from "./ImageDetail"; +export type { ImageGenerationItem } from "./ImageGenerationItem"; export type { InitializeCapabilities } from "./InitializeCapabilities"; export type { InitializeParams } from "./InitializeParams"; export type { InitializeResponse } from "./InitializeResponse"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index c611ab071c..4cb8330b79 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -1,7 +1,7 @@ // GENERATED CODE! DO NOT MODIFY BY HAND! // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { AbsolutePathBuf } from "../AbsolutePathBuf"; +import type { ImageGenerationItem } from "../ImageGenerationItem"; import type { LegacyAppPathString } from "../LegacyAppPathString"; import type { MessagePhase } from "../MessagePhase"; import type { ReasoningEffort } from "../ReasoningEffort"; @@ -105,4 +105,4 @@ reasoningEffort: ReasoningEffort | null, /** * Last known status of the target agents, when available. */ -agentsStates: { [key in string]?: CollabAgentState }, } | { "type": "subAgentActivity", id: string, kind: SubAgentActivityKind, agentThreadId: string, agentPath: string, } | { "type": "webSearch", id: string, query: string, action: WebSearchAction | null, } | { "type": "imageView", id: string, path: LegacyAppPathString, } | { "type": "sleep", id: string, durationMs: number, } | { "type": "imageGeneration", id: string, status: string, revisedPrompt: string | null, result: string, savedPath?: AbsolutePathBuf, } | { "type": "enteredReviewMode", id: string, review: string, } | { "type": "exitedReviewMode", id: string, review: string, } | { "type": "contextCompaction", id: string, }; +agentsStates: { [key in string]?: CollabAgentState }, } | { "type": "subAgentActivity", id: string, kind: SubAgentActivityKind, agentThreadId: string, agentPath: string, } | { "type": "webSearch", id: string, query: string, action: WebSearchAction | null, } | { "type": "imageView", id: string, path: LegacyAppPathString, } | { "type": "sleep", id: string, durationMs: number, } | { "type": "imageGeneration" } & ImageGenerationItem | { "type": "enteredReviewMode", id: string, review: string, } | { "type": "exitedReviewMode", id: string, review: string, } | { "type": "contextCompaction", id: string, }; diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 2d9c6b6a1a..ca3849bb38 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -22,6 +22,7 @@ use crate::protocol::v2::TurnItemsView; use crate::protocol::v2::TurnStatus; use crate::protocol::v2::UserInput; use crate::protocol::v2::WebSearchAction; +use codex_extension_items::image_generation::ImageGenerationItem; use codex_protocol::items::parse_hook_prompt_message; use codex_protocol::models::MessagePhase; use codex_protocol::protocol::AgentReasoningEvent; @@ -592,7 +593,8 @@ impl ThreadHistoryBuilder { | codex_protocol::items::TurnItem::CommandExecution(_) | codex_protocol::items::TurnItem::DynamicToolCall(_) | codex_protocol::items::TurnItem::CollabAgentToolCall(_) - | codex_protocol::items::TurnItem::SubAgentActivity(_) => true, + | codex_protocol::items::TurnItem::SubAgentActivity(_) + | codex_protocol::items::TurnItem::Extension(_) => true, codex_protocol::items::TurnItem::UserMessage(_) | codex_protocol::items::TurnItem::HookPrompt(_) | codex_protocol::items::TurnItem::AgentMessage(_) @@ -827,24 +829,24 @@ impl ThreadHistoryBuilder { } fn handle_image_generation_begin(&mut self, payload: &ImageGenerationBeginEvent) { - let item = ThreadItem::ImageGeneration { + let item = ThreadItem::ImageGeneration(ImageGenerationItem { id: payload.call_id.clone(), status: String::new(), revised_prompt: None, result: String::new(), saved_path: None, - }; + }); self.upsert_item_in_current_turn(item); } fn handle_image_generation_end(&mut self, payload: &ImageGenerationEndEvent) { - let item = ThreadItem::ImageGeneration { + let item = ThreadItem::ImageGeneration(ImageGenerationItem { id: payload.call_id.clone(), status: payload.status.clone(), revised_prompt: payload.revised_prompt.clone(), result: payload.result.clone(), saved_path: payload.saved_path.clone(), - }; + }); self.upsert_item_in_current_turn(item); } @@ -1548,6 +1550,7 @@ impl From<&PendingTurn> for Turn { mod tests { use super::*; use crate::protocol::v2::CommandExecutionSource; + use codex_extension_items::ExtensionItem as CoreExtensionItem; use codex_protocol::ThreadId; use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem as CoreDynamicToolCallOutputContentItem; use codex_protocol::items::CommandExecutionItem as CoreCommandExecutionItem; @@ -1846,6 +1849,60 @@ mod tests { ); } + #[test] + fn rebuilds_extension_image_generation_item_from_persisted_completion() { + let turn_id = "turn-1"; + let thread_id = ThreadId::new(); + let saved_path = test_path_buf("/tmp/image-1.png").abs(); + let events = vec![ + EventMsg::TurnStarted(TurnStartedEvent { + turn_id: turn_id.to_string(), + trace_id: None, + started_at: None, + model_context_window: None, + collaboration_mode_kind: Default::default(), + }), + EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id, + turn_id: turn_id.to_string(), + item: CoreTurnItem::Extension(CoreExtensionItem::ImageGeneration( + ImageGenerationItem { + id: "image-1".to_string(), + status: "completed".to_string(), + revised_prompt: Some("A blue square".to_string()), + result: "cG5n".to_string(), + saved_path: Some(saved_path.clone()), + }, + )), + completed_at_ms: 1_000, + }), + EventMsg::TurnComplete(TurnCompleteEvent { + turn_id: turn_id.to_string(), + last_agent_message: None, + completed_at: None, + duration_ms: None, + time_to_first_token_ms: None, + }), + ]; + let items = events + .into_iter() + .map(RolloutItem::EventMsg) + .collect::>(); + + let turns = build_turns_from_rollout_items(&items); + + assert_eq!( + turns[0].items, + vec![ThreadItem::ImageGeneration(ImageGenerationItem { + id: "image-1".to_string(), + status: "completed".to_string(), + revised_prompt: Some("A blue square".to_string()), + result: "cG5n".to_string(), + saved_path: Some(saved_path), + })] + ); + } + #[test] fn rebuilds_command_execution_item_from_persisted_completion() { let turn_id = "turn-1"; @@ -2058,13 +2115,13 @@ mod tests { text_elements: Vec::new(), }], }, - ThreadItem::ImageGeneration { + ThreadItem::ImageGeneration(ImageGenerationItem { id: "ig_123".into(), status: "completed".into(), revised_prompt: Some("final prompt".into()), result: "Zm9v".into(), saved_path: Some(test_path_buf("/tmp/ig_123.png").abs()), - }, + }), ], } ); diff --git a/codex-rs/app-server-protocol/src/protocol/v2/item.rs b/codex-rs/app-server-protocol/src/protocol/v2/item.rs index cd04752449..2fea01da6a 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/item.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/item.rs @@ -11,6 +11,8 @@ use super::shared::v2_enum_from_core; use crate::protocol::item_builders::command_actions_for_path_uri; use crate::protocol::item_builders::convert_patch_changes; use codex_experimental_api_macros::ExperimentalApi; +use codex_extension_items::ExtensionItem; +pub use codex_extension_items::image_generation::ImageGenerationItem; use codex_protocol::approvals::GuardianAssessmentAction as CoreGuardianAssessmentAction; use codex_protocol::approvals::GuardianAssessmentDecisionSource as CoreGuardianAssessmentDecisionSource; use codex_protocol::approvals::GuardianCommandSource as CoreGuardianCommandSource; @@ -246,7 +248,10 @@ pub enum ThreadItem { #[ts(rename_all = "camelCase")] /// EXPERIMENTAL - proposed plan item content. The completed plan item is /// authoritative and may not match the concatenation of `PlanDelta` text. - Plan { id: String, text: String }, + Plan { + id: String, + text: String, + }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] Reasoning { @@ -373,26 +378,24 @@ pub enum ThreadItem { #[ts(type = "number")] duration_ms: u64, }, + ImageGeneration(ImageGenerationItem), #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] - ImageGeneration { + EnteredReviewMode { id: String, - status: String, - revised_prompt: Option, - result: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - #[ts(optional)] - saved_path: Option, + review: String, }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] - EnteredReviewMode { id: String, review: String }, + ExitedReviewMode { + id: String, + review: String, + }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] - ExitedReviewMode { id: String, review: String }, - #[serde(rename_all = "camelCase")] - #[ts(rename_all = "camelCase")] - ContextCompaction { id: String }, + ContextCompaction { + id: String, + }, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] @@ -432,10 +435,10 @@ impl ThreadItem { | ThreadItem::WebSearch { id, .. } | ThreadItem::ImageView { id, .. } | ThreadItem::Sleep { id, .. } - | ThreadItem::ImageGeneration { id, .. } | ThreadItem::EnteredReviewMode { id, .. } | ThreadItem::ExitedReviewMode { id, .. } | ThreadItem::ContextCompaction { id, .. } => id, + ThreadItem::ImageGeneration(item) => &item.id, } } } @@ -931,13 +934,18 @@ impl From for ThreadItem { id: sleep.id, duration_ms: sleep.duration_ms, }, - CoreTurnItem::ImageGeneration(image) => ThreadItem::ImageGeneration { - id: image.id, - status: image.status, - revised_prompt: image.revised_prompt, - result: image.result, - saved_path: image.saved_path, + CoreTurnItem::Extension(extension) => match extension { + ExtensionItem::ImageGeneration(item) => ThreadItem::ImageGeneration(item), }, + CoreTurnItem::ImageGeneration(image) => { + ThreadItem::ImageGeneration(ImageGenerationItem { + id: image.id, + status: image.status, + revised_prompt: image.revised_prompt, + result: image.result, + saved_path: image.saved_path, + }) + } CoreTurnItem::FileChange(file_change) => ThreadItem::FileChange { id: file_change.id, changes: convert_patch_changes(&file_change.changes), diff --git a/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs b/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs index 90d25ff593..598e0bde08 100644 --- a/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs +++ b/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs @@ -32,7 +32,7 @@ pub(super) fn redact_thread_resume_payloads(turns: &mut [Turn]) { } true } - ThreadItem::ImageGeneration { .. } => false, + ThreadItem::ImageGeneration(_) => false, _ => true, }); } @@ -52,6 +52,7 @@ fn redacted_mcp_tool_call_result() -> McpToolCallResult { #[cfg(test)] mod tests { use super::*; + use codex_app_server_protocol::ImageGenerationItem; use codex_app_server_protocol::McpToolCallAppContext; use codex_app_server_protocol::McpToolCallError; use codex_app_server_protocol::McpToolCallStatus; @@ -100,13 +101,13 @@ mod tests { error: None, duration_ms: Some(8), }, - ThreadItem::ImageGeneration { + ThreadItem::ImageGeneration(ImageGenerationItem { id: "ig-1".to_string(), status: "completed".to_string(), revised_prompt: Some("revised".to_string()), result: "base64-result".to_string(), saved_path: Some(test_path_buf("/tmp/ig-1.png").abs()), - }, + }), ]); redact_thread_resume_payloads(&mut thread.turns); diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs index 8ce38bd729..0c3799ca11 100644 --- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs @@ -7,6 +7,7 @@ use app_test_support::ChatGptAuthFixture; use app_test_support::TestAppServer; use app_test_support::to_response; use app_test_support::write_chatgpt_auth; +use codex_app_server_protocol::ImageGenerationItem; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; @@ -107,13 +108,13 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul ) .await??; - let ThreadItem::ImageGeneration { + let ThreadItem::ImageGeneration(ImageGenerationItem { status, revised_prompt, result, saved_path: Some(saved_path), .. - } = completed.item + }) = completed.item else { panic!("expected completed image generation item with saved path"); }; @@ -205,13 +206,13 @@ async fn standalone_image_generation_failure_emits_terminal_item() -> Result<()> .await??; assert_eq!( completed.item, - ThreadItem::ImageGeneration { + ThreadItem::ImageGeneration(ImageGenerationItem { id: call_id.to_string(), status: "failed".to_string(), revised_prompt: Some("paint a blue whale".to_string()), result: String::new(), saved_path: None, - } + }) ); timeout( @@ -537,7 +538,7 @@ async fn wait_for_image_generation_completed( .params .context("item/completed notification should include params")?, )?; - if matches!(&completed.item, ThreadItem::ImageGeneration { .. }) { + if matches!(&completed.item, ThreadItem::ImageGeneration(_)) { return Ok(completed); } } diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index f8fcd84486..a9c3a1f700 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -889,7 +889,7 @@ async fn thread_resume_redacts_payloads_for_chatgpt_remote_clients() -> Result<( !remote_turn .items .iter() - .any(|item| matches!(item, ThreadItem::ImageGeneration { .. })), + .any(|item| matches!(item, ThreadItem::ImageGeneration(_))), "remote resume should drop image generation items for {client_name}" ); } @@ -943,12 +943,9 @@ async fn thread_resume_redacts_payloads_for_chatgpt_remote_clients() -> Result<( assert!( normal_turn.items.iter().any(|item| matches!( item, - ThreadItem::ImageGeneration { - result, - revised_prompt, - .. - } if result == "base64-image-result" - && revised_prompt.as_deref() == Some("secret revised prompt") + ThreadItem::ImageGeneration(item) + if item.result == "base64-image-result" + && item.revised_prompt.as_deref() == Some("secret revised prompt") )), "normal resume should keep image generation items" ); diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index c617684f78..5dc2586b6a 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -134,6 +134,7 @@ openssl-sys = { workspace = true, features = ["vendored"] } codex-shell-escalation = { workspace = true } [dev-dependencies] +codex-extension-items = { workspace = true } assert_cmd = { workspace = true } assert_matches = { workspace = true } codex-image-generation-extension = { workspace = true } diff --git a/codex-rs/core/src/tools/handlers/extension_tools.rs b/codex-rs/core/src/tools/handlers/extension_tools.rs index 1aa7754a18..716f4d3dd8 100644 --- a/codex-rs/core/src/tools/handlers/extension_tools.rs +++ b/codex-rs/core/src/tools/handlers/extension_tools.rs @@ -2,6 +2,8 @@ use std::sync::Arc; use std::sync::Weak; use codex_protocol::items::TurnItem; +use codex_protocol::protocol::Event; +use codex_protocol::protocol::EventMsg; use codex_tools::ConversationHistory; use codex_tools::ExtensionTurnItem; use codex_tools::ToolCall as ExtensionToolCall; @@ -16,7 +18,6 @@ use crate::sandboxing::SandboxPermissions; use crate::session::session::Session; use crate::session::turn_context::TurnContext; use crate::stream_events_utils::TurnItemContributorPolicy; -use crate::stream_events_utils::apply_turn_item_contributors; use crate::stream_events_utils::finalize_turn_item; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolPayload; @@ -69,10 +70,14 @@ struct CoreTurnItemEmitter { turn: Weak, } -fn extension_turn_item(item: ExtensionTurnItem) -> TurnItem { - match item { - ExtensionTurnItem::WebSearch(item) => TurnItem::WebSearch(item), - ExtensionTurnItem::ImageGeneration(item) => TurnItem::ImageGeneration(item), +async fn emit_legacy_events(session: &Session, turn: &TurnContext, legacy_events: Vec) { + for msg in legacy_events { + session + .send_event_raw(Event { + id: turn.sub_id.clone(), + msg, + }) + .await; } } @@ -82,9 +87,15 @@ impl TurnItemEmitter for CoreTurnItemEmitter { let (Some(session), Some(turn)) = (self.session.upgrade(), self.turn.upgrade()) else { return; }; - session - .emit_turn_item_started(turn.as_ref(), &extension_turn_item(item)) - .await; + let (item, legacy_events) = match item { + ExtensionTurnItem::WebSearch(item) => (TurnItem::WebSearch(item), Vec::new()), + ExtensionTurnItem::Extension { + item, + legacy_events, + } => (TurnItem::Extension(item), legacy_events), + }; + session.emit_turn_item_started(turn.as_ref(), &item).await; + emit_legacy_events(session.as_ref(), turn.as_ref(), legacy_events).await; }) } @@ -93,17 +104,11 @@ impl TurnItemEmitter for CoreTurnItemEmitter { let (Some(session), Some(turn)) = (self.session.upgrade(), self.turn.upgrade()) else { return; }; - let item = match item { - ExtensionTurnItem::ImageGeneration(item) => { - let mut item = TurnItem::ImageGeneration(item); - apply_turn_item_contributors( - session.as_ref(), - turn.extension_data.as_ref(), - &mut item, - ) - .await; - item - } + let (item, legacy_events) = match item { + ExtensionTurnItem::Extension { + item, + legacy_events, + } => (TurnItem::Extension(item), legacy_events), ExtensionTurnItem::WebSearch(item) => { let mut item = TurnItem::WebSearch(item); finalize_turn_item( @@ -115,10 +120,11 @@ impl TurnItemEmitter for CoreTurnItemEmitter { == codex_protocol::config_types::ModeKind::Plan, ) .await; - item + (item, Vec::new()) } }; session.emit_turn_item_completed(turn.as_ref(), item).await; + emit_legacy_events(session.as_ref(), turn.as_ref(), legacy_events).await; }) } } @@ -173,14 +179,16 @@ async fn to_extension_call(invocation: &ToolInvocation) -> ExtensionToolCall { mod tests { use std::sync::Arc; - use codex_extension_api::ExtensionData; - use codex_extension_api::TurnItemContributor; + use codex_extension_items::ExtensionItem; + use codex_extension_items::image_generation::ImageGenerationItem; use codex_protocol::items::TurnItem; use codex_protocol::items::WebSearchItem; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::models::WebSearchAction; use codex_protocol::protocol::EventMsg; + use codex_protocol::protocol::ImageGenerationBeginEvent; + use codex_protocol::protocol::ImageGenerationEndEvent; use codex_tools::ExtensionTurnItem; use codex_utils_absolute_path::test_support::PathExt; use codex_utils_absolute_path::test_support::test_path_buf; @@ -447,147 +455,64 @@ mod tests { assert_eq!(end.action, expected.action); } - struct ImageGenerationExtensionExecutor; - - #[derive(Debug)] - struct ExtensionTurnItemContributorRan; - - struct RecordExtensionTurnItemContributor; - - impl TurnItemContributor for RecordExtensionTurnItemContributor { - fn contribute<'a>( - &'a self, - _thread_store: &'a ExtensionData, - turn_store: &'a ExtensionData, - _item: &'a mut TurnItem, - ) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> { - Box::pin(async move { - turn_store.insert(ExtensionTurnItemContributorRan); - Ok(()) - }) - } - } - - #[tokio::test] - async fn extension_completion_runs_turn_item_contributors() { - let (mut session, turn) = crate::session::tests::make_session_and_context().await; - let mut builder = codex_extension_api::ExtensionRegistryBuilder::new(); - builder.turn_item_contributor(Arc::new(RecordExtensionTurnItemContributor)); - session.services.extensions = Arc::new(builder.build()); - let session = Arc::new(session); - let turn = Arc::new(turn); - let emitter = CoreTurnItemEmitter { - session: Arc::downgrade(&session), - turn: Arc::downgrade(&turn), - }; - - codex_tools::TurnItemEmitter::emit_completed( - &emitter, - ExtensionTurnItem::WebSearch(WebSearchItem { - id: "search-1".to_string(), - query: "contributors".to_string(), - action: WebSearchAction::Other, - }), - ) - .await; - - assert!( - turn.extension_data - .get::() - .is_some() - ); - } - - impl codex_extension_api::ToolExecutor for ImageGenerationExtensionExecutor { - fn tool_name(&self) -> codex_tools::ToolName { - codex_tools::ToolName::namespaced("image_gen", "imagegen") - } - - fn spec(&self) -> codex_tools::ToolSpec { - codex_tools::ToolSpec::Function(codex_tools::ResponsesApiTool { - name: "imagegen".to_string(), - description: "Generates an image.".to_string(), - strict: false, - parameters: codex_tools::JsonSchema::default(), - output_schema: None, - defer_loading: None, - }) - } - - fn handle(&self, call: codex_tools::ToolCall) -> codex_tools::ToolExecutorFuture<'_> { - Box::pin(self.handle_call(call)) - } - } - - impl ImageGenerationExtensionExecutor { - async fn handle_call( - &self, - call: codex_tools::ToolCall, - ) -> Result, codex_tools::FunctionCallError> { - call.turn_item_emitter - .emit_started(ExtensionTurnItem::ImageGeneration( - codex_protocol::items::ImageGenerationItem { - id: call.call_id.clone(), - status: "in_progress".to_string(), - revised_prompt: None, - result: String::new(), - saved_path: None, - }, - )) - .await; - call.turn_item_emitter - .emit_completed(ExtensionTurnItem::ImageGeneration( - codex_protocol::items::ImageGenerationItem { - id: call.call_id, - status: "completed".to_string(), - revised_prompt: Some("A tiny blue square".to_string()), - result: "cG5n".to_string(), - saved_path: Some(test_path_buf("/tmp/extension-claimed.png").abs()), - }, - )) - .await; - Ok( - Box::new(codex_tools::JsonToolOutput::new(json!({ "ok": true }))) - as Box, - ) - } - } - #[tokio::test] async fn image_generation_publication_preserves_extension_saved_path() { let (session, turn, rx) = crate::session::tests::make_session_and_context_with_rx().await; - let handler = ExtensionToolAdapter::new(Arc::new(ImageGenerationExtensionExecutor)); let expected_path = test_path_buf("/tmp/extension-claimed.png").abs(); let default_path = crate::stream_events_utils::image_generation_artifact_path( &turn.config.codex_home, &session.thread_id.to_string(), "call-image", ); - let step_context = StepContext::for_test(Arc::clone(&turn)); - let invocation = ToolInvocation { - session, - step_context, - turn, - cancellation_token: tokio_util::sync::CancellationToken::new(), - tracker: Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())), - call_id: "call-image".to_string(), - tool_name: codex_tools::ToolName::namespaced("image_gen", "imagegen"), - source: ToolCallSource::Direct, - payload: ToolPayload::Function { - arguments: "{}".to_string(), - }, + let emitter = CoreTurnItemEmitter { + session: Arc::downgrade(&session), + turn: Arc::downgrade(&turn), }; - - crate::tools::registry::ToolExecutor::handle(&handler, invocation) - .await - .expect("extension call should succeed"); + let expected_started_item = ExtensionItem::ImageGeneration(ImageGenerationItem { + id: "call-image".to_string(), + status: "in_progress".to_string(), + revised_prompt: None, + result: String::new(), + saved_path: None, + }); + let expected_completed_item = ExtensionItem::ImageGeneration(ImageGenerationItem { + id: "call-image".to_string(), + status: "completed".to_string(), + revised_prompt: Some("A tiny blue square".to_string()), + result: "cG5n".to_string(), + saved_path: Some(expected_path.clone()), + }); + codex_tools::TurnItemEmitter::emit_started( + &emitter, + ExtensionTurnItem::Extension { + item: expected_started_item.clone(), + legacy_events: vec![EventMsg::ImageGenerationBegin(ImageGenerationBeginEvent { + call_id: "call-image".to_string(), + })], + }, + ) + .await; + codex_tools::TurnItemEmitter::emit_completed( + &emitter, + ExtensionTurnItem::Extension { + item: expected_completed_item.clone(), + legacy_events: vec![EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { + call_id: "call-image".to_string(), + status: "completed".to_string(), + revised_prompt: Some("A tiny blue square".to_string()), + result: "cG5n".to_string(), + saved_path: Some(expected_path.clone()), + })], + }, + ) + .await; let started = rx.recv().await.expect("item started event"); let EventMsg::ItemStarted(started) = started.msg else { panic!("expected item started event"); }; - let TurnItem::ImageGeneration(started_item) = started.item else { - panic!("expected image generation item"); + let TurnItem::Extension(started_item) = started.item else { + panic!("expected extension item"); }; let begin = rx.recv().await.expect("legacy image start event"); assert!(matches!(begin.msg, EventMsg::ImageGenerationBegin(_))); @@ -595,32 +520,14 @@ mod tests { let EventMsg::ItemCompleted(completed) = completed.msg else { panic!("expected item completed event"); }; - let TurnItem::ImageGeneration(completed_item) = completed.item else { - panic!("expected image generation item"); + let TurnItem::Extension(completed_item) = completed.item else { + panic!("expected extension item"); }; let end = rx.recv().await.expect("legacy image end event"); assert!(matches!(end.msg, EventMsg::ImageGenerationEnd(_))); - assert_eq!( - started_item, - codex_protocol::items::ImageGenerationItem { - id: "call-image".to_string(), - status: "in_progress".to_string(), - revised_prompt: None, - result: String::new(), - saved_path: None, - } - ); - assert_eq!( - completed_item, - codex_protocol::items::ImageGenerationItem { - id: "call-image".to_string(), - status: "completed".to_string(), - revised_prompt: Some("A tiny blue square".to_string()), - result: "cG5n".to_string(), - saved_path: Some(expected_path.clone()), - } - ); + assert_eq!(started_item, expected_started_item); + assert_eq!(completed_item, expected_completed_item); assert!(!default_path.exists()); } } diff --git a/codex-rs/ext/image-generation/Cargo.toml b/codex-rs/ext/image-generation/Cargo.toml index c5139ac12b..ad10b2b0a6 100644 --- a/codex-rs/ext/image-generation/Cargo.toml +++ b/codex-rs/ext/image-generation/Cargo.toml @@ -18,6 +18,7 @@ codex-api = { workspace = true } codex-core = { workspace = true } codex-exec-server = { workspace = true } codex-extension-api = { workspace = true } +codex-extension-items = { workspace = true } codex-login = { workspace = true } codex-model-provider = { workspace = true } codex-model-provider-info = { workspace = true } diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 3bd4dc5348..06a4351a42 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -23,7 +23,8 @@ use codex_extension_api::ToolOutput; use codex_extension_api::ToolPayload; use codex_extension_api::ToolSpec; use codex_extension_api::parse_tool_input_schema; -use codex_protocol::items::ImageGenerationItem; +use codex_extension_items::ExtensionItem; +use codex_extension_items::image_generation::ImageGenerationItem; use codex_protocol::models::ContentItem; use codex_protocol::models::DEFAULT_IMAGE_DETAIL; use codex_protocol::models::FunctionCallOutputBody; @@ -31,6 +32,9 @@ use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::ImageGenerationBeginEvent; +use codex_protocol::protocol::ImageGenerationEndEvent; use codex_tools::ResponsesApiNamespace; use codex_tools::ResponsesApiNamespaceTool; use codex_tools::ResponsesApiTool; @@ -86,6 +90,23 @@ struct ImagegenArgs { num_last_images_to_include: Option, } +fn legacy_end_event(item: &ImageGenerationItem) -> EventMsg { + EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { + call_id: item.id.clone(), + status: item.status.clone(), + revised_prompt: item.revised_prompt.clone(), + result: item.result.clone(), + saved_path: item.saved_path.clone(), + }) +} + +fn extension_turn_item(item: ImageGenerationItem, legacy_event: EventMsg) -> ExtensionTurnItem { + ExtensionTurnItem::Extension { + item: ExtensionItem::ImageGeneration(item), + legacy_events: vec![legacy_event], + } +} + impl ToolExecutor for ImageGenerationTool { /// Keeps the tool in the existing image-generation Responses namespace. fn tool_name(&self) -> ToolName { @@ -115,13 +136,18 @@ impl ImageGenerationTool { request_for_call_args(&args, call.conversation_history.items(), &call.environments) .await?; call.turn_item_emitter - .emit_started(ExtensionTurnItem::ImageGeneration(ImageGenerationItem { - id: call.call_id.clone(), - status: "in_progress".to_string(), - revised_prompt: None, - result: String::new(), - saved_path: None, - })) + .emit_started(extension_turn_item( + ImageGenerationItem { + id: call.call_id.clone(), + status: "in_progress".to_string(), + revised_prompt: None, + result: String::new(), + saved_path: None, + }, + EventMsg::ImageGenerationBegin(ImageGenerationBeginEvent { + call_id: call.call_id.clone(), + }), + )) .await; let result = match request { ImageRequest::Generate(request) => self.backend.generate(request).await, @@ -139,14 +165,16 @@ impl ImageGenerationTool { let result = match result { Ok(result) => result, Err(message) => { + let item = ImageGenerationItem { + id: call.call_id.clone(), + status: "failed".to_string(), + revised_prompt: Some(args.prompt), + result: String::new(), + saved_path: None, + }; + let legacy_event = legacy_end_event(&item); call.turn_item_emitter - .emit_completed(ExtensionTurnItem::ImageGeneration(ImageGenerationItem { - id: call.call_id.clone(), - status: "failed".to_string(), - revised_prompt: Some(args.prompt.clone()), - result: String::new(), - saved_path: None, - })) + .emit_completed(extension_turn_item(item, legacy_event)) .await; return Err(FunctionCallError::RespondToModel(message)); } @@ -176,14 +204,16 @@ impl ImageGenerationTool { }, None => None, }; + let item = ImageGenerationItem { + id: call.call_id.clone(), + status: "completed".to_string(), + revised_prompt: Some(args.prompt), + result: result.clone(), + saved_path: saved_path.clone(), + }; + let legacy_event = legacy_end_event(&item); call.turn_item_emitter - .emit_completed(ExtensionTurnItem::ImageGeneration(ImageGenerationItem { - id: call.call_id.clone(), - status: "completed".to_string(), - revised_prompt: Some(args.prompt), - result: result.clone(), - saved_path: saved_path.clone(), - })) + .emit_completed(extension_turn_item(item, legacy_event)) .await; let output_hint = saved_path.as_ref().and_then(|output_path| { let output_dir = output_path.parent()?; diff --git a/codex-rs/ext/items/BUILD.bazel b/codex-rs/ext/items/BUILD.bazel new file mode 100644 index 0000000000..9bc0c2b673 --- /dev/null +++ b/codex-rs/ext/items/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "items", + crate_name = "codex_extension_items", +) diff --git a/codex-rs/ext/items/Cargo.toml b/codex-rs/ext/items/Cargo.toml new file mode 100644 index 0000000000..fdf848eddf --- /dev/null +++ b/codex-rs/ext/items/Cargo.toml @@ -0,0 +1,23 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-extension-items" +version.workspace = true + +[lib] +name = "codex_extension_items" +path = "src/lib.rs" +doctest = false + +[lints] +workspace = true + +[dependencies] +codex-utils-absolute-path = { workspace = true } +schemars = { workspace = true } +serde = { workspace = true, features = ["derive"] } +ts-rs = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +serde_json = { workspace = true } diff --git a/codex-rs/ext/items/src/image_generation.rs b/codex-rs/ext/items/src/image_generation.rs new file mode 100644 index 0000000000..7b740421f9 --- /dev/null +++ b/codex-rs/ext/items/src/image_generation.rs @@ -0,0 +1,21 @@ +use codex_utils_absolute_path::AbsolutePathBuf; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use ts_rs::TS; + +// Standalone image-generation item owned by the image extension. This is also +// the field-level representation exposed by app-server; core and rollout +// persistence only carry it inside an ExtensionItem envelope. +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] +#[serde(rename_all = "camelCase")] +#[ts(rename_all = "camelCase")] +pub struct ImageGenerationItem { + pub id: String, + pub status: String, + pub revised_prompt: Option, + pub result: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub saved_path: Option, +} diff --git a/codex-rs/ext/items/src/lib.rs b/codex-rs/ext/items/src/lib.rs new file mode 100644 index 0000000000..27eeb9abb0 --- /dev/null +++ b/codex-rs/ext/items/src/lib.rs @@ -0,0 +1,51 @@ +//! Typed display items owned by Codex extensions. +//! +//! This crate intentionally sits below `codex-protocol` so core can carry +//! extension items without owning each extension's display schema. + +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use ts_rs::TS; + +pub mod image_generation; + +/// Canonical extension-owned turn item carried through core lifecycle events. +/// +/// The item is serialized as a flattened, namespaced envelope: +/// +/// ```json +/// { +/// "kind": "image_gen.generation", +/// "id": "call-id", +/// "status": "completed", +/// "revisedPrompt": "A blue square", +/// "result": "cG5n", +/// "savedPath": "/tmp/image.png" +/// } +/// ``` +/// +/// `kind` values follow `.`. Adding a variant +/// also requires app-server to add its typed public wrapper. +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] +#[serde(tag = "kind")] +#[ts(tag = "kind")] +pub enum ExtensionItem { + #[serde(rename = "image_gen.generation")] + #[ts(rename = "image_gen.generation")] + ImageGeneration(image_generation::ImageGenerationItem), +} + +impl ExtensionItem { + /// Returns the stable item identifier without exposing variant fields to + /// core or rollout persistence. + pub fn id(&self) -> &str { + match self { + Self::ImageGeneration(item) => &item.id, + } + } +} + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/codex-rs/ext/items/src/tests.rs b/codex-rs/ext/items/src/tests.rs new file mode 100644 index 0000000000..88575b4bfe --- /dev/null +++ b/codex-rs/ext/items/src/tests.rs @@ -0,0 +1,57 @@ +use pretty_assertions::assert_eq; +use serde_json::json; + +use super::ExtensionItem; +use super::image_generation::ImageGenerationItem; + +fn completed_image_generation_item() -> ExtensionItem { + ExtensionItem::ImageGeneration(ImageGenerationItem { + id: "image-1".to_string(), + status: "completed".to_string(), + revised_prompt: Some("A blue square".to_string()), + result: "cG5n".to_string(), + saved_path: None, + }) +} + +#[test] +fn image_generation_item_preserves_stable_wire_shape() { + let item = completed_image_generation_item(); + let value = serde_json::to_value(&item).expect("serialize extension item"); + + assert_eq!( + value, + json!({ + "kind": "image_gen.generation", + "id": "image-1", + "status": "completed", + "revisedPrompt": "A blue square", + "result": "cG5n", + }) + ); + assert_eq!( + serde_json::from_value::(value).expect("deserialize extension item"), + item + ); +} + +#[test] +fn unknown_extension_kind_is_rejected() { + let value = json!({ + "kind": "image_gen.unknown", + "id": "image-1", + }); + + assert!(serde_json::from_value::(value).is_err()); +} + +#[test] +fn malformed_known_extension_payload_is_rejected() { + let value = json!({ + "kind": "image_gen.generation", + "id": "image-1", + "status": "completed", + }); + + assert!(serde_json::from_value::(value).is_err()); +} diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index 5d1ff57f55..23d065d25d 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -17,6 +17,7 @@ chardetng = { workspace = true } chrono = { workspace = true, features = ["serde"] } codex-async-utils = { workspace = true } codex-execpolicy = { workspace = true } +codex-extension-items = { workspace = true } codex-network-proxy = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-image = { workspace = true } diff --git a/codex-rs/protocol/src/items.rs b/codex-rs/protocol/src/items.rs index 2c9d073f0d..2c29306cda 100644 --- a/codex-rs/protocol/src/items.rs +++ b/codex-rs/protocol/src/items.rs @@ -20,6 +20,7 @@ use crate::protocol::SubAgentActivityKind; use crate::user_input::ByteRange; use crate::user_input::TextElement; use crate::user_input::UserInput; +use codex_extension_items::ExtensionItem; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_path_uri::PathUri; use quick_xml::de::from_str as from_xml_str; @@ -49,6 +50,15 @@ pub enum TurnItem { WebSearch(WebSearchItem), ImageView(ImageViewItem), Sleep(SleepItem), + /// Item whose schema and lifecycle details are owned by an extension. + /// + /// Standalone image generation uses this path. App-server wraps the same + /// typed item in its public image-generation variant. + Extension(ExtensionItem), + /// Hosted Responses API image-generation item handled directly by core. + /// + /// This remains separate from [`Self::Extension`] because core still owns + /// hosted image persistence and legacy-event fanout. ImageGeneration(ImageGenerationItem), FileChange(FileChangeItem), McpToolCall(McpToolCallItem), @@ -605,6 +615,7 @@ impl TurnItem { TurnItem::WebSearch(item) => item.id.clone(), TurnItem::ImageView(item) => item.id.clone(), TurnItem::Sleep(item) => item.id.clone(), + TurnItem::Extension(item) => item.id().to_string(), TurnItem::ImageGeneration(item) => item.id.clone(), TurnItem::FileChange(item) => item.id.clone(), TurnItem::McpToolCall(item) => item.id.clone(), diff --git a/codex-rs/protocol/src/legacy_events.rs b/codex-rs/protocol/src/legacy_events.rs index c12a2747a2..12c2b98d1e 100644 --- a/codex-rs/protocol/src/legacy_events.rs +++ b/codex-rs/protocol/src/legacy_events.rs @@ -487,6 +487,7 @@ impl TurnItem { })] } TurnItem::Sleep(_) => Vec::new(), + TurnItem::Extension(_) => Vec::new(), TurnItem::ImageGeneration(item) => vec![item.as_legacy_event()], TurnItem::FileChange(item) => item .as_legacy_end_event(String::new()) diff --git a/codex-rs/rollout/src/persistence_metrics.rs b/codex-rs/rollout/src/persistence_metrics.rs index 8c2a7e18d2..e8510a0d93 100644 --- a/codex-rs/rollout/src/persistence_metrics.rs +++ b/codex-rs/rollout/src/persistence_metrics.rs @@ -254,6 +254,7 @@ fn turn_item_type(item: &TurnItem) -> &'static str { TurnItem::WebSearch(_) => "web_search", TurnItem::ImageView(_) => "image_view", TurnItem::Sleep(_) => "sleep", + TurnItem::Extension(_) => "extension", TurnItem::ImageGeneration(_) => "image_generation", TurnItem::FileChange(_) => "file_change", TurnItem::McpToolCall(_) => "mcp_tool_call", diff --git a/codex-rs/tools/Cargo.toml b/codex-rs/tools/Cargo.toml index b5f838ca36..76a5de5115 100644 --- a/codex-rs/tools/Cargo.toml +++ b/codex-rs/tools/Cargo.toml @@ -12,6 +12,7 @@ codex-code-mode = { workspace = true } codex-connectors = { workspace = true } codex-features = { workspace = true } codex-file-system = { workspace = true } +codex-extension-items = { workspace = true } codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-output-truncation = { workspace = true } diff --git a/codex-rs/tools/src/tool_call.rs b/codex-rs/tools/src/tool_call.rs index 0bcee56223..8810ea4376 100644 --- a/codex-rs/tools/src/tool_call.rs +++ b/codex-rs/tools/src/tool_call.rs @@ -1,11 +1,12 @@ use crate::FunctionCallError; use crate::ToolName; use crate::ToolPayload; +use codex_extension_items::ExtensionItem; use codex_file_system::ExecutorFileSystem; use codex_file_system::FileSystemSandboxContext; -use codex_protocol::items::ImageGenerationItem; use codex_protocol::items::WebSearchItem; use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::EventMsg; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_output_truncation::TruncationPolicy; use std::future::Future; @@ -34,10 +35,20 @@ impl ConversationHistory { pub type TurnItemEmissionFuture<'a> = Pin + Send + 'a>>; /// Visible turn items that an extension may publish into the host lifecycle. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug)] pub enum ExtensionTurnItem { WebSearch(WebSearchItem), - ImageGeneration(ImageGenerationItem), + /// Canonical extension item plus compatibility events derived by its owner. + /// + /// Core intentionally does not inspect extension-owned payloads, so it + /// cannot derive their legacy fanout. It emits the canonical lifecycle + /// event first, then these extension-provided events. Core also skips + /// global turn-item contributors here so extensions cannot mutate items + /// owned by other extensions. + Extension { + item: ExtensionItem, + legacy_events: Vec, + }, } /// Host-provided capability for extension tools to emit visible turn items. diff --git a/codex-rs/tui/src/app/agent_status_feed.rs b/codex-rs/tui/src/app/agent_status_feed.rs index eb14551d5a..91f8784f6d 100644 --- a/codex-rs/tui/src/app/agent_status_feed.rs +++ b/codex-rs/tui/src/app/agent_status_feed.rs @@ -185,7 +185,7 @@ fn activity_summary(item: &ThreadItem) -> Option { let path = path.render_for_ui(); return bounded_summary(&format!("Viewed {path}")); } - ThreadItem::ImageGeneration { .. } => return Some("Generated an image".to_string()), + ThreadItem::ImageGeneration(_) => return Some("Generated an image".to_string()), ThreadItem::EnteredReviewMode { .. } => return Some("Entered review mode".to_string()), ThreadItem::ExitedReviewMode { .. } => return Some("Exited review mode".to_string()), ThreadItem::ContextCompaction { .. } => return Some("Compacted context".to_string()), diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index 0c5b0ed3ee..df03fb1c74 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -293,7 +293,7 @@ impl ChatWidget { ThreadItem::WebSearch { id, .. } => { self.on_web_search_begin(id); } - ThreadItem::ImageGeneration { .. } => { + ThreadItem::ImageGeneration(_) => { self.on_image_generation_begin(); } ThreadItem::CollabAgentToolCall { diff --git a/codex-rs/tui/src/chatwidget/replay.rs b/codex-rs/tui/src/chatwidget/replay.rs index 3ab5b3486b..8b454fa0c4 100644 --- a/codex-rs/tui/src/chatwidget/replay.rs +++ b/codex-rs/tui/src/chatwidget/replay.rs @@ -150,14 +150,13 @@ impl ChatWidget { ThreadItem::ImageView { id: _, path } => { self.on_view_image_tool_call(path); } - ThreadItem::ImageGeneration { - id, - status, - revised_prompt, - saved_path, - .. - } => { - self.on_image_generation_end(id, status, revised_prompt, saved_path); + ThreadItem::ImageGeneration(item) => { + self.on_image_generation_end( + item.id, + item.status, + item.revised_prompt, + item.saved_path, + ); } ThreadItem::EnteredReviewMode { review, .. } => { if from_replay { diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index 5f890ab000..f0d808ad3d 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -1,4 +1,5 @@ use super::*; +use codex_app_server_protocol::ImageGenerationItem; use codex_app_server_protocol::PluginAvailability; use pretty_assertions::assert_eq; @@ -707,13 +708,13 @@ pub(super) fn handle_image_generation_end( thread_id: thread_id(chat), turn_id: "turn-1".to_string(), completed_at_ms: 0, - item: AppServerThreadItem::ImageGeneration { + item: AppServerThreadItem::ImageGeneration(ImageGenerationItem { id: call_id.into(), status: status.into(), revised_prompt, result: String::new(), saved_path, - }, + }), }), /*replay_kind*/ None, ); diff --git a/codex-rs/tui/src/thread_transcript.rs b/codex-rs/tui/src/thread_transcript.rs index cdffaab27c..2e43f40d95 100644 --- a/codex-rs/tui/src/thread_transcript.rs +++ b/codex-rs/tui/src/thread_transcript.rs @@ -206,14 +206,17 @@ fn fallback_transcript_cell(item: &ThreadItem) -> Option { let path = path.render_for_ui(); vec![format!("image: {path}").dim().into()] } - ThreadItem::ImageGeneration { - status, saved_path, .. - } => { - let saved = saved_path + ThreadItem::ImageGeneration(item) => { + let saved = item + .saved_path .as_ref() .map(|path| format!(" · {}", path.as_path().display())) .unwrap_or_default(); - vec![format!("image generation: {status}{saved}").dim().into()] + vec![ + format!("image generation: {}{saved}", item.status) + .dim() + .into(), + ] } ThreadItem::EnteredReviewMode { review, .. } => { vec![vec!["review started: ".dim(), review.clone().into()].into()]