diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 0dcd53a4e6..ac8dd51de8 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1970,6 +1970,7 @@ dependencies = [ "base64 0.22.1", "caseless", "chrono", + "codex-attachment-store", "codex-extension-api", "codex-protocol", "codex-state", @@ -3757,6 +3758,7 @@ version = "0.0.0" dependencies = [ "base64 0.22.1", "codex-api", + "codex-attachment-store", "codex-core", "codex-exec-server", "codex-extension-api", @@ -4618,6 +4620,7 @@ dependencies = [ "assert_matches", "blake3", "codex-analytics", + "codex-attachment-store", "codex-config", "codex-exec-server", "codex-extension-api", @@ -4771,6 +4774,7 @@ name = "codex-tools" version = "0.0.0" dependencies = [ "bitflags 2.13.1", + "codex-attachment-store", "codex-code-mode", "codex-connectors", "codex-extension-items", diff --git a/codex-rs/analytics/src/tests/suite/tools_tests.rs b/codex-rs/analytics/src/tests/suite/tools_tests.rs index 472cd83b9d..f43d0c88ba 100644 --- a/codex-rs/analytics/src/tests/suite/tools_tests.rs +++ b/codex-rs/analytics/src/tests/suite/tools_tests.rs @@ -40,6 +40,7 @@ use codex_app_server_protocol::CollabAgentToolCallStatus; use codex_app_server_protocol::CommandAction; use codex_app_server_protocol::CommandExecutionSource; use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::GeneratedImageRef; use codex_app_server_protocol::ImageGenerationItem; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; @@ -875,7 +876,9 @@ async fn image_generation_events_preserve_transparent_background_metadata() { id: "image-1".to_string(), status: status.to_string(), revised_prompt: None, - result: "ok".to_string(), + image: GeneratedImageRef::Inline { + result: "ok".to_string(), + }, transparent_background, failure: None, saved_path: None, diff --git a/codex-rs/analytics/src/tests/suite/turns/events_tests.rs b/codex-rs/analytics/src/tests/suite/turns/events_tests.rs index 5d7bf5d0b4..bfcf6c7b32 100644 --- a/codex-rs/analytics/src/tests/suite/turns/events_tests.rs +++ b/codex-rs/analytics/src/tests/suite/turns/events_tests.rs @@ -25,6 +25,7 @@ use codex_app_server_protocol::CollabAgentTool; use codex_app_server_protocol::CollabAgentToolCallStatus; use codex_app_server_protocol::CommandExecutionStatus; use codex_app_server_protocol::DynamicToolCallStatus; +use codex_app_server_protocol::GeneratedImageRef; use codex_app_server_protocol::ImageGenerationItem; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; @@ -550,7 +551,9 @@ async fn turn_event_counts_completed_tool_items() { id: "image-1".to_string(), status: "completed".to_string(), revised_prompt: None, - result: "ok".to_string(), + image: GeneratedImageRef::Inline { + result: "ok".to_string(), + }, transparent_background: None, failure: None, saved_path: None, diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index ed40f2f769..1bda7da8a6 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -5364,6 +5364,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -5379,9 +5406,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -5418,7 +5442,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index b96da22504..37dfc25506 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -22840,6 +22840,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -22855,9 +22882,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -22894,7 +22918,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 6d7bfd8937..be25aa9d0a 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -20554,6 +20554,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -20569,9 +20596,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -20608,7 +20632,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 31e915e4e8..0daf5a7733 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -1523,6 +1523,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -1538,9 +1565,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -1577,7 +1601,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 0ea615c48c..b9dda6f5f5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -1523,6 +1523,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -1538,9 +1565,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -1577,7 +1601,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index f1f9c0dcb8..6675cce9d1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -1726,6 +1726,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -1741,9 +1768,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -1780,7 +1804,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index b1aff21738..a754510467 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -2340,6 +2340,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -2355,9 +2382,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -2394,7 +2418,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json index 5bb7c64b97..c6c0b0987e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadItemsListResponse.json @@ -1523,6 +1523,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -1538,9 +1565,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -1577,7 +1601,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index bbf8ed7d34..e297f3a3cf 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -2131,6 +2131,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -2146,9 +2173,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -2185,7 +2209,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index fe79128b0a..19e039653c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -2131,6 +2131,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -2146,9 +2173,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -2185,7 +2209,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 9c4975d2ba..cb116237c8 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -2131,6 +2131,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -2146,9 +2173,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -2185,7 +2209,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index b7e1f4cbfa..6886c4dee2 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -2392,6 +2392,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -2407,9 +2434,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -2446,7 +2470,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json index 7a8a97b9d9..e2b4e9b859 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRevertResponse.json @@ -2131,6 +2131,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -2146,9 +2173,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -2185,7 +2209,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 00d698b329..26c772ce40 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -2340,6 +2340,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -2355,9 +2382,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -2394,7 +2418,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index b0cee0b30a..d1453d23f9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -2131,6 +2131,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -2146,9 +2173,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -2185,7 +2209,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json index ea56799c41..03167f2455 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadTurnsListResponse.json @@ -1726,6 +1726,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -1741,9 +1768,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -1780,7 +1804,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index e2039c8c32..3f287eea16 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -2131,6 +2131,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -2146,9 +2173,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -2185,7 +2209,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index a0c543c01f..ac2f9e2a8e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -1726,6 +1726,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -1741,9 +1768,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -1780,7 +1804,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index f2f5c903f3..3919455d5a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -1726,6 +1726,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -1741,9 +1768,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -1780,7 +1804,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 6f5969c8e9..c85c9ff41b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -1726,6 +1726,33 @@ "type": "object" }, { + "anyOf": [ + { + "properties": { + "fileId": { + "type": "string" + } + }, + "required": [ + "fileId" + ], + "title": "FileIdThreadItem", + "type": "object" + }, + { + "properties": { + "result": { + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ResultThreadItem", + "type": "object" + } + ], + "description": "Either inline generated-image data or its durable attachment-store identity.\n\n`File` must remain first so rollouts written during the migration, which contain both an empty `result` and a `fileId`, retain the file reference when deserialized.", "properties": { "failure": { "anyOf": [ @@ -1741,9 +1768,6 @@ "id": { "type": "string" }, - "result": { - "type": "string" - }, "revisedPrompt": { "type": [ "string", @@ -1780,7 +1804,6 @@ }, "required": [ "id", - "result", "status", "type" ], diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index f632cc10f6..a966e2d0b0 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index 64de0f45ca..cee1aa5eb7 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts index 645d275e41..29783fa9fa 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ImageGenerationItem.ts @@ -4,4 +4,4 @@ import type { AbsolutePathBuf } from "./AbsolutePathBuf"; import type { ImageGenerationFailure } from "./ImageGenerationFailure"; -export type ImageGenerationItem = { id: string, status: string, revisedPrompt: string | null, result: string, transparentBackground?: boolean, failure: ImageGenerationFailure | null, savedPath?: AbsolutePathBuf, }; +export type ImageGenerationItem = { id: string, status: string, revisedPrompt: string | null, transparentBackground?: boolean, failure: ImageGenerationFailure | null, savedPath?: AbsolutePathBuf, } & ({ fileId: string, } | { result: string, }); diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 53817bb9e5..2a685e7335 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -27,6 +27,7 @@ use crate::protocol::v2::UserInput; use crate::protocol::v2::WebSearchAction; use crate::protocol::v2::WebSearchItem; use crate::protocol::v2::web_search_action_from_core; +use codex_extension_items::image_generation::GeneratedImageRef; use codex_extension_items::image_generation::ImageGenerationItem; use codex_protocol::items::parse_hook_prompt_message; use codex_protocol::protocol::AgentMessageEvent; @@ -899,7 +900,9 @@ impl ThreadHistoryBuilder { id: payload.call_id.clone(), status: String::new(), revised_prompt: None, - result: String::new(), + image: GeneratedImageRef::Inline { + result: String::new(), + }, transparent_background: None, failure: None, saved_path: None, @@ -914,7 +917,7 @@ impl ThreadHistoryBuilder { id: payload.call_id.clone(), status: payload.status.clone(), revised_prompt: payload.revised_prompt.clone(), - result: payload.result.clone(), + image: payload.image.clone(), transparent_background: payload.transparent_background, failure: payload.failure.clone(), saved_path: payload.saved_path.clone(), @@ -2348,7 +2351,9 @@ mod tests { id: "image-1".to_string(), status: "completed".to_string(), revised_prompt: Some("A blue square".to_string()), - result: "cG5n".to_string(), + image: GeneratedImageRef::Inline { + result: "cG5n".to_string(), + }, transparent_background: Some(true), failure: None, saved_path: Some(saved_path.clone()), @@ -2383,7 +2388,9 @@ mod tests { id: "image-1".to_string(), status: "completed".to_string(), revised_prompt: Some("A blue square".to_string()), - result: "cG5n".to_string(), + image: GeneratedImageRef::Inline { + result: "cG5n".to_string(), + }, transparent_background: Some(true), failure: None, saved_path: Some(saved_path), @@ -2648,7 +2655,7 @@ mod tests { } #[test] - fn replays_image_generation_end_events_into_turn_history() { + fn replays_file_backed_image_generation_end_events_into_turn_history() { let items = vec![ RolloutItem::EventMsg(EventMsg::TurnStarted(TurnStartedEvent { turn_attribution: None, @@ -2671,7 +2678,9 @@ mod tests { call_id: "ig_123".into(), status: "completed".into(), revised_prompt: Some("final prompt".into()), - result: "Zm9v".into(), + image: GeneratedImageRef::File { + file_id: "file_123".into(), + }, transparent_background: Some(true), failure: Some( codex_extension_items::image_generation::ImageGenerationFailure::UsageLimitExceeded { @@ -2719,7 +2728,9 @@ mod tests { id: "ig_123".into(), status: "completed".into(), revised_prompt: Some("final prompt".into()), - result: "Zm9v".into(), + image: GeneratedImageRef::File { + file_id: "file_123".into(), + }, transparent_background: Some(true), failure: Some( codex_extension_items::image_generation::ImageGenerationFailure::UsageLimitExceeded { diff --git a/codex-rs/app-server-protocol/src/protocol/v2/item.rs b/codex-rs/app-server-protocol/src/protocol/v2/item.rs index 65d2838270..0fbc99088b 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/item.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/item.rs @@ -15,6 +15,7 @@ use crate::protocol::item_builders::convert_patch_changes; use crate::protocol::item_builders::review_output_text; use codex_experimental_api_macros::ExperimentalApi; use codex_extension_items::ExtensionItem; +pub use codex_extension_items::image_generation::GeneratedImageRef; pub use codex_extension_items::image_generation::ImageGenerationFailure; pub use codex_extension_items::image_generation::ImageGenerationItem; pub use codex_extension_items::sleep::SleepItem; @@ -1009,7 +1010,9 @@ impl From for ThreadItem { id: image.id, status: image.status, revised_prompt: image.revised_prompt, - result: image.result, + image: GeneratedImageRef::Inline { + result: image.result, + }, transparent_background: None, failure: None, saved_path: image.saved_path, diff --git a/codex-rs/app-server/src/notification_media.rs b/codex-rs/app-server/src/notification_media.rs index a1572cbaf7..2391a0bc22 100644 --- a/codex-rs/app-server/src/notification_media.rs +++ b/codex-rs/app-server/src/notification_media.rs @@ -1,4 +1,5 @@ use codex_app_server_protocol::DynamicToolCallOutputContentItem; +use codex_app_server_protocol::GeneratedImageRef; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::UserInput; @@ -190,7 +191,11 @@ fn without_thread_item_media(mut item: ThreadItem) -> ThreadItem { .is_none() }); } - ThreadItem::ImageGeneration(item) => item.result.clear(), + ThreadItem::ImageGeneration(item) => { + item.image = GeneratedImageRef::Inline { + result: String::new(), + }; + } ThreadItem::HookPrompt { .. } | ThreadItem::AgentMessage { .. } | ThreadItem::Plan { .. } diff --git a/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs b/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs index f35d0341e0..92027dc4ff 100644 --- a/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs +++ b/codex-rs/app-server/src/request_processors/thread_resume_redaction.rs @@ -1,3 +1,5 @@ +#[cfg(test)] +use codex_app_server_protocol::GeneratedImageRef; use codex_app_server_protocol::McpToolCallResult; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::Turn; @@ -108,7 +110,9 @@ mod tests { id: "ig-1".to_string(), status: "completed".to_string(), revised_prompt: Some("revised".to_string()), - result: "base64-result".to_string(), + image: GeneratedImageRef::Inline { + result: "base64-result".to_string(), + }, transparent_background: None, failure: None, saved_path: Some(test_path_buf("/tmp/ig-1.png").abs()), diff --git a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs index b7fc11d2f2..67c6c75304 100644 --- a/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs +++ b/codex-rs/app-server/tests/suite/v2/imagegen_extension.rs @@ -7,6 +7,7 @@ use app_test_support::ChatGptAuthFixture; use app_test_support::MockResponsesConfig; use app_test_support::TestAppServer; use app_test_support::write_chatgpt_auth; +use codex_app_server_protocol::GeneratedImageRef; use codex_app_server_protocol::ImageGenerationFailure; use codex_app_server_protocol::ImageGenerationItem; use codex_app_server_protocol::ImageReference as V2ImageReference; @@ -159,7 +160,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model( let ThreadItem::ImageGeneration(ImageGenerationItem { status, revised_prompt, - result, + image, transparent_background, saved_path: Some(saved_path), .. @@ -169,7 +170,12 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model( }; assert_eq!(status, "completed"); assert_eq!(revised_prompt.as_deref(), Some("paint a blue whale")); - assert_eq!(result, RESULT); + assert_eq!( + image, + GeneratedImageRef::Inline { + result: RESULT.to_string(), + } + ); assert_eq!(transparent_background, Some(false)); assert_eq!(std::fs::read(&saved_path)?, TINY_PNG_BYTES); @@ -315,7 +321,7 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history() -> let thread_id = completed.thread_id.clone(); let ThreadItem::ImageGeneration(ImageGenerationItem { status, - result, + image, transparent_background, .. }) = completed.item @@ -323,7 +329,12 @@ async fn transparent_image_preserves_output_metadata_and_persisted_history() -> panic!("expected completed image-generation item"); }; assert_eq!(status, "completed"); - assert_eq!(result, RESULT); + assert_eq!( + image, + GeneratedImageRef::Inline { + result: RESULT.to_string(), + } + ); assert_eq!(transparent_background, Some(true)); drop(mcp); @@ -568,7 +579,9 @@ async fn standalone_image_generation_failure_emits_terminal_item() -> Result<()> id: call_id.to_string(), status: "failed".to_string(), revised_prompt: Some("paint a blue whale".to_string()), - result: String::new(), + image: GeneratedImageRef::Inline { + result: String::new(), + }, transparent_background: None, failure: None, saved_path: None, diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index df29831ad0..a3870ea7ac 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -28,6 +28,7 @@ use codex_app_server_protocol::CommandExecutionRequestApprovalResponse; use codex_app_server_protocol::DeprecationNoticeNotification; use codex_app_server_protocol::FileChangeApprovalDecision; use codex_app_server_protocol::FileChangeRequestApprovalResponse; +use codex_app_server_protocol::GeneratedImageRef; use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCErrorError; @@ -2781,7 +2782,11 @@ async fn thread_resume_redacts_payloads_for_chatgpt_remote_clients() -> Result<( normal_turn.items.iter().any(|item| matches!( item, ThreadItem::ImageGeneration(item) - if item.result == "base64-image-result" + if matches!( + &item.image, + GeneratedImageRef::Inline { result } + if result == "base64-image-result" + ) && item.revised_prompt.as_deref() == Some("secret revised prompt") )), "normal resume should keep image generation items" @@ -2889,7 +2894,9 @@ fn append_resume_redaction_history( call_id: "ig-1".to_string(), status: "completed".to_string(), revised_prompt: Some("secret revised prompt".to_string()), - result: "base64-image-result".to_string(), + image: GeneratedImageRef::Inline { + result: "base64-image-result".to_string(), + }, transparent_background: None, failure: None, saved_path: Some(test_absolute_path("/tmp/ig-1.png")), diff --git a/codex-rs/attachment-store/src/lib.rs b/codex-rs/attachment-store/src/lib.rs index f29c6a63ae..4c45e792b5 100644 --- a/codex-rs/attachment-store/src/lib.rs +++ b/codex-rs/attachment-store/src/lib.rs @@ -1,5 +1,6 @@ //! Storage-neutral attachment persistence interfaces. +use std::collections::HashMap; use std::error::Error; use std::fmt; use std::future::Future; @@ -42,6 +43,8 @@ pub struct UploadRequest { pub ephemeral: bool, /// Optional name associated with the attachment. pub file_name: Option, + /// Caller-supplied tags used by storage backends for policy decisions. + pub tags: HashMap, /// Attachment bytes to persist. pub data: Vec, } @@ -53,16 +56,19 @@ impl fmt::Debug for UploadRequest { .field("thread_id", &self.thread_id) .field("ephemeral", &self.ephemeral) .field("file_name", &self.file_name) + .field("tags", &self.tags) .field("data", &"") .finish() } } /// Parameters supplied to [`AttachmentStore::resolve`]. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct ResolveRequest<'a> { /// Identifier assigned by the attachment store. pub file_id: &'a str, + /// Caller-supplied tags used by storage backends for policy decisions. + pub tags: HashMap, /// Minimum remaining lifetime required for the returned file URL. /// /// When absent, implementations must omit the file URL and may return cached metadata. When diff --git a/codex-rs/attachment-store/src/lib_tests.rs b/codex-rs/attachment-store/src/lib_tests.rs index 0a3625498d..f93b0efd29 100644 --- a/codex-rs/attachment-store/src/lib_tests.rs +++ b/codex-rs/attachment-store/src/lib_tests.rs @@ -9,6 +9,7 @@ fn attachment_debug_output_redacts_bytes() { thread_id: "thread-1".to_string(), ephemeral: false, file_name: Some("image.png".to_string()), + tags: HashMap::new(), data: b"secret".to_vec(), }; let result = UploadResult::Inline { @@ -18,7 +19,7 @@ fn attachment_debug_output_redacts_bytes() { assert_eq!( (format!("{request:?}"), format!("{result:?}")), ( - r#"UploadRequest { thread_id: "thread-1", ephemeral: false, file_name: Some("image.png"), data: "" }"#.to_string(), + r#"UploadRequest { thread_id: "thread-1", ephemeral: false, file_name: Some("image.png"), tags: {}, data: "" }"#.to_string(), r#"Inline { bytes: "" }"#.to_string(), ) ); @@ -52,6 +53,7 @@ async fn inline_store_preserves_image_bytes() { thread_id: "thread-1".to_string(), ephemeral: false, file_name: Some(file_name.to_string()), + tags: HashMap::new(), data: data.to_vec(), }) .await @@ -70,6 +72,7 @@ async fn inline_store_cannot_resolve_file_references() { let error = InlineAttachmentStore .resolve(ResolveRequest { file_id: "file_123", + tags: HashMap::new(), download_url_ttl: None, }) .await diff --git a/codex-rs/core/src/image_preparation.rs b/codex-rs/core/src/image_preparation.rs index 3142613b04..611f9e3ba1 100644 --- a/codex-rs/core/src/image_preparation.rs +++ b/codex-rs/core/src/image_preparation.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use crate::config::ManagedFeatures; use crate::context::ContextualUserFragment; use crate::context::ImageResizeNotice; @@ -370,6 +372,7 @@ async fn prepare_image( thread_id: origin.thread_id.to_owned(), ephemeral: origin.ephemeral, file_name: None, + tags: HashMap::new(), data: prepared.encoded.bytes.to_vec(), }; *image = match image_store.upload(request).await { diff --git a/codex-rs/core/src/tools/handlers/extension_tools.rs b/codex-rs/core/src/tools/handlers/extension_tools.rs index 70e7ed0164..784f7abfdf 100644 --- a/codex-rs/core/src/tools/handlers/extension_tools.rs +++ b/codex-rs/core/src/tools/handlers/extension_tools.rs @@ -212,6 +212,8 @@ async fn to_extension_call<'a>( session: Arc::downgrade(&invocation.session), turn: Arc::downgrade(&invocation.turn), }), + attachment_store: Arc::clone(&invocation.session.services.image_store), + ephemeral: invocation.turn.config.ephemeral, environments, payload: invocation.payload.clone(), } @@ -222,6 +224,7 @@ mod tests { use std::sync::Arc; use codex_extension_items::ExtensionItem; + use codex_extension_items::image_generation::GeneratedImageRef; use codex_extension_items::image_generation::ImageGenerationItem; use codex_extension_items::web_search::WebSearchItem; use codex_protocol::items::TurnItem; @@ -567,7 +570,9 @@ mod tests { id: "call-image".to_string(), status: "in_progress".to_string(), revised_prompt: None, - result: String::new(), + image: GeneratedImageRef::Inline { + result: String::new(), + }, transparent_background: None, failure: None, saved_path: None, @@ -578,7 +583,9 @@ mod tests { id: "call-image".to_string(), status: "completed".to_string(), revised_prompt: Some("A tiny blue square".to_string()), - result: "cG5n".to_string(), + image: GeneratedImageRef::Inline { + result: "cG5n".to_string(), + }, transparent_background: Some(true), failure: None, saved_path: Some(expected_path.clone()), @@ -603,7 +610,9 @@ mod tests { call_id: "call-image".to_string(), status: "completed".to_string(), revised_prompt: Some("A tiny blue square".to_string()), - result: "cG5n".to_string(), + image: GeneratedImageRef::Inline { + result: "cG5n".to_string(), + }, transparent_background: Some(true), failure: None, saved_path: Some(expected_path.clone()), diff --git a/codex-rs/core/tests/suite/extension_sandbox.rs b/codex-rs/core/tests/suite/extension_sandbox.rs index ee02338961..1d31674af2 100644 --- a/codex-rs/core/tests/suite/extension_sandbox.rs +++ b/codex-rs/core/tests/suite/extension_sandbox.rs @@ -1,5 +1,6 @@ use core_test_support::test_codex::local_requests; use std::collections::BTreeSet; +use std::collections::HashMap; use std::sync::Arc; use anyhow::Context; @@ -9,7 +10,10 @@ use codex_core::config::Config; use codex_core::config::Constrained; use codex_extension_api::ExtensionRegistry; use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_items::ExtensionItem; +use codex_extension_items::image_generation::GeneratedImageRef; use codex_features::Feature; +use codex_history::RolloutItem; use codex_http_client::DestinationPolicy; use codex_http_client::NetworkPolicyController; use codex_image_generation_extension::install as install_image_generation_extension; @@ -20,7 +24,10 @@ use codex_protocol::config_types::ModeKind; use codex_protocol::config_types::Settings; use codex_protocol::config_types::WebSearchMode; use codex_protocol::models::FileSystemPermissions; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ImageReference; use codex_protocol::models::PermissionProfile; +use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::InputModality; use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; @@ -31,6 +38,7 @@ use codex_protocol::permissions::NetworkSandboxPolicy; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; +use codex_protocol::protocol::ThreadHistoryMode; use codex_protocol::protocol::ThreadSettingsOverrides; use codex_protocol::request_permissions::PermissionGrantScope; use codex_protocol::request_permissions::RequestPermissionProfile; @@ -44,6 +52,7 @@ use core_test_support::skip_if_target_windows; use core_test_support::test_codex::test_codex; use core_test_support::test_codex::turn_permission_fields; use core_test_support::wait_for_event; +use core_test_support::wait_for_event_match; use pretty_assertions::assert_eq; use serde_json::json; use test_case::test_case; @@ -52,6 +61,8 @@ use wiremock::ResponseTemplate; use wiremock::matchers::method; use wiremock::matchers::path; +use super::image_rollout::RecordingFileAttachmentStore; + const TINY_PNG_BYTES: &[u8] = &[ 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0, 31, 21, 196, 137, 0, 0, 0, 13, 73, 68, 65, 84, 120, 156, 99, 248, 207, 192, 240, 31, 0, @@ -70,6 +81,150 @@ fn image_generation_extensions( Arc::new(extension_builder.build()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn generated_image_ui_item_is_file_backed_in_rollout() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + Mock::given(method("POST")) + .and(path("/v1/images/generations")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "created": 1, + "data": [{"b64_json": TINY_PNG_BASE64}], + }))) + .expect(1) + .mount(&server) + .await; + + let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); + let extensions = image_generation_extensions(&auth, |_config| None); + let image_store = Arc::new(RecordingFileAttachmentStore::default()); + let image_call_id = "image-generate-file-backed"; + responses::mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace( + image_call_id, + "image_gen", + "imagegen", + &json!({"prompt": "paint a blue whale"}).to_string(), + ), + responses::ev_completed("resp-1"), + ]), + responses::sse(vec![ + responses::ev_response_created("resp-2"), + responses::ev_assistant_message("msg-1", "done"), + responses::ev_completed("resp-2"), + ]), + ], + ) + .await; + let test = test_codex() + .with_auth(auth) + .with_extensions(extensions) + .with_image_store(image_store.clone()) + .with_history_mode(ThreadHistoryMode::Paginated) + .with_model_info_override("gpt-5.4", |model_info| { + model_info.use_responses_lite = true; + model_info.input_modalities = vec![InputModality::Text, InputModality::Image]; + }) + .build(&server) + .await?; + + test.codex + .start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text { + text: "generate an image".to_string(), + text_elements: Vec::new(), + }])) + .await?; + let live_image = + wait_for_event_match(&test.codex, |event| match event { + EventMsg::ItemCompleted(event) => match &event.item { + codex_protocol::items::TurnItem::Extension(ExtensionItem::ImageGeneration( + image, + )) if image.id == image_call_id => Some(image.clone()), + _ => None, + }, + _ => None, + }) + .await; + assert_eq!( + live_image.image, + GeneratedImageRef::File { + file_id: "file_uploaded_image".to_string(), + } + ); + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + test.codex.flush_rollout().await?; + + { + let uploads = image_store.uploads.lock().expect("uploads lock"); + let [item_upload, model_upload] = uploads.as_slice() else { + panic!("expected separate item and model uploads"); + }; + assert_eq!( + item_upload.thread_id, + test.session_configured.thread_id.to_string() + ); + assert!(!item_upload.ephemeral); + assert_eq!( + item_upload.tags, + HashMap::from([("usage".to_string(), "image_gen".to_string())]) + ); + assert_eq!(item_upload.data, TINY_PNG_BYTES); + assert_eq!(model_upload.tags, HashMap::new()); + } + + let rollout_path = test.codex.rollout_path().context("rollout path")?; + let rollout = std::fs::read_to_string(rollout_path)? + .lines() + .filter_map(|line| codex_rollout::parse_rollout_line(line).ok()) + .map(|line| line.item) + .collect::>(); + let image = + rollout.iter().find_map(|item| match item { + RolloutItem::EventMsg(EventMsg::ItemCompleted(event)) => match &event.item { + codex_protocol::items::TurnItem::Extension(ExtensionItem::ImageGeneration( + image, + )) if image.id == image_call_id => Some(image), + _ => None, + }, + _ => None, + }); + let image = image.context("persisted image-generation item")?; + assert_eq!(image, &live_image); + + let output_image = rollout.iter().find_map(|item| match item { + RolloutItem::ResponseItem(envelope) => match &envelope.item { + ResponseItem::FunctionCallOutput { + call_id: Some(call_id), + output, + .. + } if call_id == image_call_id => output.content_items().and_then(|items| { + items.iter().find_map(|item| match item { + FunctionCallOutputContentItem::InputImage { image, .. } => Some(image), + _ => None, + }) + }), + _ => None, + }, + _ => None, + }); + assert_eq!( + output_image, + Some(&ImageReference::File { + file_id: "file_uploaded_image".to_string(), + }) + ); + + Ok(()) +} + #[test_case(false; "file_access")] #[test_case(true; "application_network")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/codex-rs/ext/agent-message-board/Cargo.toml b/codex-rs/ext/agent-message-board/Cargo.toml index f522893f59..ef9f602f7b 100644 --- a/codex-rs/ext/agent-message-board/Cargo.toml +++ b/codex-rs/ext/agent-message-board/Cargo.toml @@ -29,6 +29,7 @@ tracing = { workspace = true } uuid = { workspace = true, features = ["serde", "v7"] } [dev-dependencies] +codex-attachment-store = { workspace = true } codex-utils-output-truncation = { workspace = true } pretty_assertions = { workspace = true } tempfile = { workspace = true } diff --git a/codex-rs/ext/agent-message-board/tests/local_board.rs b/codex-rs/ext/agent-message-board/tests/local_board.rs index f68c2feba4..1ca929c1c5 100644 --- a/codex-rs/ext/agent-message-board/tests/local_board.rs +++ b/codex-rs/ext/agent-message-board/tests/local_board.rs @@ -949,6 +949,8 @@ fn board_tool_call(name: &str, args: serde_json::Value) -> codex_tools::ToolCall source: codex_tools::ToolCallSource::Direct, conversation_history: codex_tools::ConversationHistory::default(), turn_item_emitter: Arc::new(codex_tools::NoopTurnItemEmitter), + attachment_store: Arc::new(codex_attachment_store::InlineAttachmentStore), + ephemeral: false, environments: vec![], payload: codex_tools::ToolPayload::Function { arguments: args.to_string(), diff --git a/codex-rs/ext/goal/tests/goal_extension_backend.rs b/codex-rs/ext/goal/tests/goal_extension_backend.rs index c9a855b555..cc88125491 100644 --- a/codex-rs/ext/goal/tests/goal_extension_backend.rs +++ b/codex-rs/ext/goal/tests/goal_extension_backend.rs @@ -1968,6 +1968,8 @@ fn tool_call(tool_name: &str, call_id: &str, arguments: serde_json::Value) -> To source: ToolCallSource::Direct, conversation_history: codex_extension_api::ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: codex_core::passthrough_image_store(), + ephemeral: false, environments: Vec::new(), payload: ToolPayload::Function { arguments: arguments.to_string(), diff --git a/codex-rs/ext/history-notes/tests/history_notes_extension.rs b/codex-rs/ext/history-notes/tests/history_notes_extension.rs index 02e7a4d41c..143eabe01f 100644 --- a/codex-rs/ext/history-notes/tests/history_notes_extension.rs +++ b/codex-rs/ext/history-notes/tests/history_notes_extension.rs @@ -462,6 +462,8 @@ fn tool_call(tool_name: ToolName, arguments: serde_json::Value) -> ToolCall<'sta source: ToolCallSource::Direct, conversation_history: ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: codex_core::passthrough_image_store(), + ephemeral: false, environments: Vec::new(), payload: ToolPayload::Function { arguments: arguments.to_string(), diff --git a/codex-rs/ext/image-generation/Cargo.toml b/codex-rs/ext/image-generation/Cargo.toml index 4b61557306..0c74fddf12 100644 --- a/codex-rs/ext/image-generation/Cargo.toml +++ b/codex-rs/ext/image-generation/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] base64 = { workspace = true } codex-api = { workspace = true } +codex-attachment-store = { workspace = true } codex-core = { workspace = true } codex-exec-server = { workspace = true } codex-extension-api = { workspace = true } diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 781d1e03ff..5623bb9a13 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::io; use base64::Engine; @@ -6,6 +7,8 @@ use codex_api::ImageBackground; use codex_api::ImageEditRequest; use codex_api::ImageGenerationRequest; use codex_api::ImageQuality; +use codex_attachment_store::UploadRequest; +use codex_attachment_store::UploadResult; use codex_exec_server::CreateDirectoryOptions; use codex_exec_server::LOCAL_FS; use codex_extension_api::ExtensionTurnItem; @@ -19,6 +22,7 @@ use codex_extension_api::ToolPayload; use codex_extension_api::ToolSpec; use codex_extension_api::parse_tool_input_schema; use codex_extension_items::ExtensionItem; +use codex_extension_items::image_generation::GeneratedImageRef; use codex_extension_items::image_generation::ImageGenerationFailure; use codex_extension_items::image_generation::ImageGenerationItem; use codex_protocol::error::CodexErr; @@ -102,7 +106,7 @@ fn legacy_end_event(item: &ImageGenerationItem) -> EventMsg { call_id: item.id.clone(), status: item.status.clone(), revised_prompt: item.revised_prompt.clone(), - result: item.result.clone(), + image: item.image.clone(), transparent_background: item.transparent_background, failure: item.failure.clone(), saved_path: item.saved_path.clone(), @@ -156,7 +160,9 @@ impl ImageGenerationTool { id: call.call_id.clone(), status: "in_progress".to_string(), revised_prompt: None, - result: String::new(), + image: GeneratedImageRef::Inline { + result: String::new(), + }, transparent_background: None, failure: None, saved_path: None, @@ -214,7 +220,9 @@ impl ImageGenerationTool { id: call.call_id.clone(), status: "failed".to_string(), revised_prompt: Some(args.prompt), - result: String::new(), + image: GeneratedImageRef::Inline { + result: String::new(), + }, transparent_background: None, failure, saved_path: None, @@ -240,11 +248,45 @@ impl ImageGenerationTool { &result, ) .await; + // The UI item retains its attachment-store reference, while the model-facing output keeps + // the original image so Core can prepare and upload it separately. + let image = match BASE64_STANDARD.decode(result.trim().as_bytes()) { + Ok(data) => { + let request = UploadRequest { + thread_id: self.thread_id.clone(), + ephemeral: call.ephemeral, + file_name: None, + tags: HashMap::from([("usage".to_string(), "image_gen".to_string())]), + data, + }; + match call.attachment_store.upload(request).await { + Ok(UploadResult::File { file_id }) => GeneratedImageRef::File { file_id }, + Ok(UploadResult::Inline { bytes }) => GeneratedImageRef::Inline { + result: BASE64_STANDARD.encode(bytes), + }, + Err(error) => { + tracing::warn!( + kind = ?error.kind(), + "failed to upload generated image" + ); + GeneratedImageRef::Inline { + result: result.clone(), + } + } + } + } + Err(error) => { + tracing::warn!("failed to decode generated image for upload: {error}"); + GeneratedImageRef::Inline { + result: result.clone(), + } + } + }; let item = ImageGenerationItem { id: call.call_id.clone(), status: "completed".to_string(), revised_prompt: Some(args.prompt), - result: result.clone(), + image, transparent_background, failure: None, saved_path: saved_path.clone(), diff --git a/codex-rs/ext/items/src/image_generation.rs b/codex-rs/ext/items/src/image_generation.rs index 8cc20123c0..8c10d17d50 100644 --- a/codex-rs/ext/items/src/image_generation.rs +++ b/codex-rs/ext/items/src/image_generation.rs @@ -29,7 +29,8 @@ pub struct ImageGenerationItem { pub id: String, pub status: String, pub revised_prompt: Option, - pub result: String, + #[serde(flatten)] + pub image: GeneratedImageRef, #[serde(default)] #[ts(optional)] pub transparent_background: Option, @@ -52,3 +53,21 @@ pub struct ImageGenerationItem { #[ts(skip)] pub generation_id: Option, } + +/// Either inline generated-image data or its durable attachment-store identity. +/// +/// `File` must remain first so rollouts written during the migration, which contain both an empty +/// `result` and a `fileId`, retain the file reference when deserialized. +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] +#[serde(untagged)] +#[ts(untagged)] +pub enum GeneratedImageRef { + File { + #[serde(rename = "fileId")] + #[ts(rename = "fileId")] + file_id: String, + }, + Inline { + result: String, + }, +} diff --git a/codex-rs/ext/items/src/tests.rs b/codex-rs/ext/items/src/tests.rs index 33da46f236..e0d8b07f72 100644 --- a/codex-rs/ext/items/src/tests.rs +++ b/codex-rs/ext/items/src/tests.rs @@ -3,6 +3,7 @@ use serde_json::json; use ts_rs::TS; use super::ExtensionItem; +use super::image_generation::GeneratedImageRef; use super::image_generation::ImageGenerationItem; use super::sleep::SleepItem; use super::web_search::WebSearchAction; @@ -13,7 +14,9 @@ fn completed_image_generation_item() -> ExtensionItem { id: "image-1".to_string(), status: "completed".to_string(), revised_prompt: Some("A blue square".to_string()), - result: "cG5n".to_string(), + image: GeneratedImageRef::Inline { + result: "cG5n".to_string(), + }, transparent_background: None, failure: None, saved_path: None, @@ -54,6 +57,73 @@ fn image_generation_item_preserves_stable_wire_shape() { .expect("deserialize legacy image-generation item without transparency metadata"), item ); + assert_eq!( + serde_json::from_value::(json!({ + "kind": "image_gen.generation", + "id": "image-1", + "status": "completed", + "revisedPrompt": "A blue square", + "result": "cG5n", + "fileId": null, + })) + .expect("deserialize transitional inline image-generation item"), + item + ); +} + +/// Transitional rollouts containing both fields must retain the durable file reference. +#[test] +fn image_generation_item_reads_transitional_file_wire_shape() { + let item = ExtensionItem::ImageGeneration(ImageGenerationItem { + id: "image-1".to_string(), + status: "completed".to_string(), + revised_prompt: Some("A blue square".to_string()), + image: GeneratedImageRef::File { + file_id: "file-1".to_string(), + }, + transparent_background: None, + failure: None, + saved_path: None, + imagegen_request_id: None, + generation_id: None, + }); + + assert_eq!( + serde_json::from_value::(json!({ + "kind": "image_gen.generation", + "id": "image-1", + "status": "completed", + "revisedPrompt": "A blue square", + "result": "", + "fileId": "file-1", + })) + .expect("deserialize transitional file-backed image-generation item"), + item + ); + assert_eq!( + serde_json::from_value::(json!({ + "kind": "image_gen.generation", + "id": "image-1", + "status": "completed", + "revisedPrompt": "A blue square", + "result": "stale-inline-bytes", + "fileId": "file-1", + })) + .expect("prefer file ID when both image representations are populated"), + item + ); + assert_eq!( + serde_json::to_value(&item).expect("serialize file-backed extension item"), + json!({ + "kind": "image_gen.generation", + "id": "image-1", + "status": "completed", + "revisedPrompt": "A blue square", + "fileId": "file-1", + "transparentBackground": null, + "failure": null, + }) + ); } #[test] @@ -84,7 +154,12 @@ fn image_generation_item_preserves_authoritative_transparency() { } #[test] -fn image_generation_transparency_is_optional_in_typescript() { +fn image_generation_typescript_fields_preserve_wire_contracts() { + assert!( + ImageGenerationItem::inline().contains("{ fileId: string, } | { result: string, }") + || ImageGenerationItem::inline().contains("{ result: string, } | { fileId: string, }"), + "v2 image-generation items must require exactly one image reference" + ); assert!( ImageGenerationItem::inline().contains("transparentBackground?: boolean"), "image-generation transparency must remain optional for existing TypeScript clients" diff --git a/codex-rs/ext/memories/src/tests.rs b/codex-rs/ext/memories/src/tests.rs index a4f780d3d4..c12814e92b 100644 --- a/codex-rs/ext/memories/src/tests.rs +++ b/codex-rs/ext/memories/src/tests.rs @@ -230,6 +230,8 @@ async fn add_ad_hoc_note_tool_creates_note_file() { source: ToolCallSource::Direct, conversation_history: codex_extension_api::ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: codex_core::passthrough_image_store(), + ephemeral: false, environments: Vec::new(), payload: payload.clone(), }) @@ -276,6 +278,8 @@ async fn add_ad_hoc_note_tool_rejects_paths_as_filenames() { source: ToolCallSource::Direct, conversation_history: codex_extension_api::ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: codex_core::passthrough_image_store(), + ephemeral: false, environments: Vec::new(), payload, }) @@ -323,6 +327,8 @@ async fn read_tool_reads_memory_file() { source: ToolCallSource::Direct, conversation_history: codex_extension_api::ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: codex_core::passthrough_image_store(), + ephemeral: false, environments: Vec::new(), payload: payload.clone(), }) @@ -440,6 +446,8 @@ async fn search_tool_accepts_multiple_queries() { source: ToolCallSource::Direct, conversation_history: codex_extension_api::ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: codex_core::passthrough_image_store(), + ephemeral: false, environments: Vec::new(), payload: payload.clone(), }) @@ -516,6 +524,8 @@ async fn search_tool_accepts_windowed_all_match_mode() { source: ToolCallSource::Direct, conversation_history: codex_extension_api::ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: codex_core::passthrough_image_store(), + ephemeral: false, environments: Vec::new(), payload: payload.clone(), }) @@ -572,6 +582,8 @@ async fn search_tool_rejects_legacy_single_query() { source: ToolCallSource::Direct, conversation_history: codex_extension_api::ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: codex_core::passthrough_image_store(), + ephemeral: false, environments: Vec::new(), payload, }) diff --git a/codex-rs/ext/skills/Cargo.toml b/codex-rs/ext/skills/Cargo.toml index 96019ce3f9..4d351b57ba 100644 --- a/codex-rs/ext/skills/Cargo.toml +++ b/codex-rs/ext/skills/Cargo.toml @@ -38,6 +38,7 @@ tracing = { workspace = true } [dev-dependencies] assert_matches = { workspace = true } +codex-attachment-store = { workspace = true } codex-models-manager = { workspace = true } codex-utils-cargo-bin = { workspace = true } dunce = { workspace = true } diff --git a/codex-rs/ext/skills/tests/skills_extension.rs b/codex-rs/ext/skills/tests/skills_extension.rs index 1d1c46b401..521f0c03e1 100644 --- a/codex-rs/ext/skills/tests/skills_extension.rs +++ b/codex-rs/ext/skills/tests/skills_extension.rs @@ -5,6 +5,7 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use assert_matches::assert_matches; +use codex_attachment_store::InlineAttachmentStore; use codex_config::ConfigLayerEntry; use codex_config::ConfigLayerSource; use codex_config::ConfigLayerStack; @@ -2085,6 +2086,8 @@ async fn skills_list_only_returns_model_visible_bounded_metadata() -> TestResult source: ToolCallSource::Direct, conversation_history: ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: Arc::new(InlineAttachmentStore), + ephemeral: false, environments: Vec::new(), payload: payload.clone(), }; @@ -2245,6 +2248,8 @@ async fn skills_list_only_returns_model_visible_bounded_metadata() -> TestResult source: ToolCallSource::Direct, conversation_history: ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: Arc::new(InlineAttachmentStore), + ephemeral: false, environments: Vec::new(), payload: insufficient_budget_payload, }) diff --git a/codex-rs/ext/skills/tests/skills_extension/presentation_dedup_tests.rs b/codex-rs/ext/skills/tests/skills_extension/presentation_dedup_tests.rs index 1c9d62a628..c2e011a819 100644 --- a/codex-rs/ext/skills/tests/skills_extension/presentation_dedup_tests.rs +++ b/codex-rs/ext/skills/tests/skills_extension/presentation_dedup_tests.rs @@ -226,6 +226,8 @@ async fn cloud_preference_preserves_aliases_reads_and_executor_fallback() -> Tes source: ToolCallSource::Direct, conversation_history: ConversationHistory::default(), turn_item_emitter: Arc::new(NoopTurnItemEmitter), + attachment_store: Arc::new(InlineAttachmentStore), + ephemeral: false, environments: vec![ToolEnvironment::new( "local".to_string(), PathUri::parse("file:///skills")?, diff --git a/codex-rs/protocol/src/legacy_events.rs b/codex-rs/protocol/src/legacy_events.rs index 4ae3b4e9b9..228d9e893c 100644 --- a/codex-rs/protocol/src/legacy_events.rs +++ b/codex-rs/protocol/src/legacy_events.rs @@ -65,6 +65,7 @@ use crate::protocol::ViewImageToolCallEvent; use crate::protocol::WebSearchBeginEvent; use crate::protocol::WebSearchEndEvent; use crate::user_input::UserInput; +use codex_extension_items::image_generation::GeneratedImageRef; /// Converts canonical item lifecycle events back into the legacy raw event stream used by /// compatibility consumers that have not migrated to `TurnItem`. @@ -463,7 +464,9 @@ impl ImageGenerationItem { call_id: self.id.clone(), status: self.status.clone(), revised_prompt: self.revised_prompt.clone(), - result: self.result.clone(), + image: GeneratedImageRef::Inline { + result: self.result.clone(), + }, transparent_background: None, failure: None, saved_path: self.saved_path.clone(), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index ac01685923..d2db6663a1 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -65,6 +65,7 @@ use crate::turn_input::TurnInputMode; use crate::turn_input::TurnInputRequest; use crate::turn_input::TurnInputSubmission; use crate::turn_input::TurnStartOptions; +use codex_extension_items::image_generation::GeneratedImageRef; use codex_extension_items::image_generation::ImageGenerationFailure; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_path_uri::PathUri; @@ -2924,7 +2925,8 @@ pub struct ImageGenerationEndEvent { #[serde(skip_serializing_if = "Option::is_none")] #[ts(optional)] pub revised_prompt: Option, - pub result: String, + #[serde(flatten)] + pub image: GeneratedImageRef, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] pub transparent_background: Option, @@ -5594,7 +5596,12 @@ mod tests { assert_eq!(event.call_id, "ig-1"); assert_eq!(event.status, "completed"); assert_eq!(event.revised_prompt.as_deref(), Some("A tiny blue square")); - assert_eq!(event.result, "Zm9v"); + assert_eq!( + event.image, + GeneratedImageRef::Inline { + result: "Zm9v".to_string(), + } + ); assert_eq!( event.saved_path.as_ref().map(AbsolutePathBuf::as_path), Some(test_path_buf("/tmp/ig-1.png").as_path()) diff --git a/codex-rs/rollout/src/policy.rs b/codex-rs/rollout/src/policy.rs index eb13ec553e..51997cd91b 100644 --- a/codex-rs/rollout/src/policy.rs +++ b/codex-rs/rollout/src/policy.rs @@ -282,3 +282,7 @@ fn persisted_item_completed_event<'a>( }, } } + +#[cfg(test)] +#[path = "policy_tests.rs"] +mod tests; diff --git a/codex-rs/rollout/src/policy_tests.rs b/codex-rs/rollout/src/policy_tests.rs new file mode 100644 index 0000000000..c8303f1a58 --- /dev/null +++ b/codex-rs/rollout/src/policy_tests.rs @@ -0,0 +1,115 @@ +use codex_extension_items::ExtensionItem; +use codex_extension_items::image_generation::GeneratedImageRef; +use codex_extension_items::image_generation::ImageGenerationItem; +use codex_protocol::ThreadId; +use codex_protocol::items::TurnItem; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::ImageGenerationEndEvent; +use codex_protocol::protocol::ItemCompletedEvent; +use codex_protocol::protocol::ThreadHistoryMode; + +use super::persisted_event_msg; + +fn image_item(image: GeneratedImageRef) -> ImageGenerationItem { + ImageGenerationItem { + id: "image-1".to_string(), + status: "completed".to_string(), + revised_prompt: Some("paint a blue whale".to_string()), + image, + transparent_background: Some(false), + failure: None, + saved_path: None, + imagegen_request_id: None, + generation_id: None, + } +} + +#[test] +fn paginated_image_generation_preserves_file_reference() { + let event = EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id: ThreadId::new(), + turn_id: "turn-1".to_string(), + item: TurnItem::Extension(ExtensionItem::ImageGeneration(image_item( + GeneratedImageRef::File { + file_id: "file-1".to_string(), + }, + ))), + started_at_ms: Some(1), + completed_at_ms: 2, + }); + + let persisted = persisted_event_msg(&event, ThreadHistoryMode::Paginated) + .expect("expected persisted image-generation item"); + let EventMsg::ItemCompleted(persisted) = persisted.as_ref() else { + panic!("expected persisted image-generation item"); + }; + let TurnItem::Extension(ExtensionItem::ImageGeneration(image)) = &persisted.item else { + panic!("expected image-generation extension item"); + }; + + assert_eq!( + image, + &image_item(GeneratedImageRef::File { + file_id: "file-1".to_string(), + }) + ); +} + +#[test] +fn legacy_image_generation_preserves_file_reference() { + let event = EventMsg::ImageGenerationEnd(ImageGenerationEndEvent { + call_id: "image-1".to_string(), + status: "completed".to_string(), + revised_prompt: Some("paint a blue whale".to_string()), + image: GeneratedImageRef::File { + file_id: "file-1".to_string(), + }, + transparent_background: Some(false), + failure: None, + saved_path: None, + }); + + let persisted = persisted_event_msg(&event, ThreadHistoryMode::Legacy) + .expect("expected persisted legacy image-generation event"); + let EventMsg::ImageGenerationEnd(persisted) = persisted.as_ref() else { + panic!("expected persisted legacy image-generation event"); + }; + + assert_eq!( + persisted.image, + GeneratedImageRef::File { + file_id: "file-1".to_string(), + } + ); +} + +#[test] +fn inline_image_generation_result_is_preserved() { + let event = EventMsg::ItemCompleted(ItemCompletedEvent { + thread_id: ThreadId::new(), + turn_id: "turn-1".to_string(), + item: TurnItem::Extension(ExtensionItem::ImageGeneration(image_item( + GeneratedImageRef::Inline { + result: "base64-image".to_string(), + }, + ))), + started_at_ms: Some(1), + completed_at_ms: 2, + }); + + let persisted = persisted_event_msg(&event, ThreadHistoryMode::Paginated) + .expect("expected persisted inline image-generation item"); + let EventMsg::ItemCompleted(persisted) = persisted.as_ref() else { + panic!("expected persisted inline image-generation item"); + }; + let TurnItem::Extension(ExtensionItem::ImageGeneration(image)) = &persisted.item else { + panic!("expected inline image-generation extension item"); + }; + + assert_eq!( + image, + &image_item(GeneratedImageRef::Inline { + result: "base64-image".to_string(), + }) + ); +} diff --git a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs index 48fe3eeff4..80176ddf6c 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/legacy_event.rs @@ -217,7 +217,7 @@ pub(super) fn completed_item( id: event.call_id.clone(), status: event.status.clone(), revised_prompt: event.revised_prompt.clone(), - result: event.result.clone(), + image: event.image.clone(), transparent_background: event.transparent_background, failure: event.failure.clone(), saved_path: event.saved_path.clone(), diff --git a/codex-rs/thread-store/src/local/rollout_migration_tests.rs b/codex-rs/thread-store/src/local/rollout_migration_tests.rs index 4ddcbd596f..d67ad997a1 100644 --- a/codex-rs/thread-store/src/local/rollout_migration_tests.rs +++ b/codex-rs/thread-store/src/local/rollout_migration_tests.rs @@ -9,6 +9,7 @@ use std::os::unix::fs::PermissionsExt; use codex_app_server_protocol::build_turns_from_rollout_items; use codex_extension_items::ExtensionItem; +use codex_extension_items::image_generation::GeneratedImageRef; use codex_extension_items::image_generation::ImageGenerationFailure; use codex_extension_items::image_generation::ImageGenerationItem; use codex_protocol::AgentPath; @@ -532,7 +533,9 @@ async fn migration_preserves_image_generation_failure_metadata() { id: "image-call".to_string(), status: "failed".to_string(), revised_prompt: Some("paint a blue whale".to_string()), - result: String::new(), + image: GeneratedImageRef::Inline { + result: String::new(), + }, transparent_background: None, failure: Some(ImageGenerationFailure::UsageLimitExceeded { limit_id: "image_gen".to_string(), @@ -547,7 +550,7 @@ async fn migration_preserves_image_generation_failure_metadata() { call_id: expected_item.id.clone(), status: expected_item.status.clone(), revised_prompt: expected_item.revised_prompt.clone(), - result: expected_item.result.clone(), + image: expected_item.image.clone(), transparent_background: expected_item.transparent_background, failure: expected_item.failure.clone(), saved_path: expected_item.saved_path.clone(), diff --git a/codex-rs/tools/Cargo.toml b/codex-rs/tools/Cargo.toml index fc75ad22ce..b943f38be2 100644 --- a/codex-rs/tools/Cargo.toml +++ b/codex-rs/tools/Cargo.toml @@ -9,6 +9,7 @@ workspace = true [dependencies] bitflags = { workspace = true } +codex-attachment-store = { workspace = true } codex-code-mode = { workspace = true } codex-connectors = { workspace = true } codex-features = { workspace = true } diff --git a/codex-rs/tools/src/tool_call.rs b/codex-rs/tools/src/tool_call.rs index 95b3360e8e..6b2d7ba1d2 100644 --- a/codex-rs/tools/src/tool_call.rs +++ b/codex-rs/tools/src/tool_call.rs @@ -1,6 +1,7 @@ use crate::FunctionCallError; use crate::ToolName; use crate::ToolPayload; +use codex_attachment_store::AttachmentStore; use codex_extension_items::ExtensionItem; use codex_file_system::EnvironmentAccess; use codex_protocol::models::ResponseItem; @@ -154,6 +155,8 @@ pub struct ToolCall<'call> { pub source: ToolCallSource, pub conversation_history: ConversationHistory, pub turn_item_emitter: Arc, + pub attachment_store: Arc, + pub ephemeral: bool, pub environments: Vec>, pub payload: ToolPayload, } @@ -173,6 +176,8 @@ impl std::fmt::Debug for ToolCall<'_> { .field("source", &self.source) .field("conversation_history", &self.conversation_history) .field("turn_item_emitter", &"") + .field("attachment_store", &"") + .field("ephemeral", &self.ephemeral) .field("environment_count", &self.environments.len()) .field("payload", &self.payload) .finish() diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index 9ce2635d3b..897d9f4d26 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -1,5 +1,6 @@ use super::*; use crate::test_support::boxed_future; +use codex_app_server_protocol::GeneratedImageRef; use codex_app_server_protocol::ImageGenerationItem; use codex_app_server_protocol::PluginAvailability; use codex_utils_absolute_path::test_support::PathExt; @@ -856,7 +857,9 @@ pub(super) fn handle_image_generation_end( id: call_id.into(), status: status.into(), revised_prompt, - result: String::new(), + image: GeneratedImageRef::Inline { + result: String::new(), + }, transparent_background: None, failure: None, saved_path, diff --git a/codex-rs/tui/src/dynamic_tools.rs b/codex-rs/tui/src/dynamic_tools.rs index dd1897bf17..48e677d311 100644 --- a/codex-rs/tui/src/dynamic_tools.rs +++ b/codex-rs/tui/src/dynamic_tools.rs @@ -13,6 +13,7 @@ use codex_app_server_protocol::DynamicToolFunctionSpec; use codex_app_server_protocol::DynamicToolNamespaceSpec; use codex_app_server_protocol::DynamicToolNamespaceTool; use codex_app_server_protocol::DynamicToolSpec; +use codex_app_server_protocol::GeneratedImageRef; use codex_app_server_protocol::GetAccountParams; use codex_app_server_protocol::GetAccountResponse; use codex_app_server_protocol::ImageReference; @@ -1550,7 +1551,14 @@ fn turn_summary(turn: &Turn, include_outputs: bool, output_chars: usize) -> Valu "savedPath": item.saved_path }); if include_outputs { - image["result"] = output_summary(&item.result, output_chars); + match &item.image { + GeneratedImageRef::Inline { result } => { + image["result"] = output_summary(result, output_chars); + } + GeneratedImageRef::File { file_id } => { + image["fileId"] = Value::String(file_id.clone()); + } + } } image } diff --git a/codex-rs/tui/src/dynamic_tools_tests.rs b/codex-rs/tui/src/dynamic_tools_tests.rs index 95ed8a3df2..f6b7a33af2 100644 --- a/codex-rs/tui/src/dynamic_tools_tests.rs +++ b/codex-rs/tui/src/dynamic_tools_tests.rs @@ -400,6 +400,45 @@ fn activity_metadata_is_retained_without_including_outputs() -> color_eyre::Resu Ok(()) } +/// File-backed generated images retain their reference when task output is requested without +/// exposing that output when the caller asks for metadata only. +#[test] +fn file_backed_image_generation_is_retained_as_a_reference() -> color_eyre::Result<()> { + let turn: Turn = serde_json::from_value(json!({ + "id": "turn-1", + "status": "completed", + "items": [{ + "type": "imageGeneration", + "id": "image-1", + "status": "completed", + "fileId": "file-generated-image" + }] + }))?; + + assert_eq!( + turn_summary(&turn, /*include_outputs*/ true, DEFAULT_OUTPUT_CHARS)["items"][0], + json!({ + "type": "imageGeneration", + "id": "image-1", + "status": "completed", + "revisedPrompt": null, + "savedPath": null, + "fileId": "file-generated-image" + }) + ); + assert_eq!( + turn_summary(&turn, /*include_outputs*/ false, DEFAULT_OUTPUT_CHARS)["items"][0], + json!({ + "type": "imageGeneration", + "id": "image-1", + "status": "completed", + "revisedPrompt": null, + "savedPath": null + }) + ); + Ok(()) +} + #[tokio::test] async fn task_management_tools_use_existing_app_server_operations() -> color_eyre::Result<()> { let (codex_home, server, source, target) = test_server(json!({})).await?; diff --git a/codex-rs/tui/src/thread_transcript/other_items_tests.rs b/codex-rs/tui/src/thread_transcript/other_items_tests.rs index e116a6025c..2d27bd7137 100644 --- a/codex-rs/tui/src/thread_transcript/other_items_tests.rs +++ b/codex-rs/tui/src/thread_transcript/other_items_tests.rs @@ -7,6 +7,7 @@ use crate::history_cell::HistoryCell; use crate::test_support::PathBufExt; use crate::test_support::test_path_buf; use codex_app_server_protocol::FileUpdateChange; +use codex_app_server_protocol::GeneratedImageRef; use codex_app_server_protocol::ImageGenerationItem; use codex_app_server_protocol::PatchApplyStatus; use codex_app_server_protocol::PatchChangeKind; @@ -124,7 +125,9 @@ fn tool_and_notice_projection_uses_normal_transcript_presentation() { id: "image-1".to_string(), status: "completed".to_string(), revised_prompt: Some("A diagram of the history pages".to_string()), - result: String::new(), + image: GeneratedImageRef::Inline { + result: String::new(), + }, transparent_background: None, failure: None, saved_path: None, diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py index d6663c65ab..bc32e7ce04 100755 --- a/sdk/python/scripts/update_sdk_artifacts.py +++ b/sdk/python/scripts/update_sdk_artifacts.py @@ -591,6 +591,7 @@ def generate_v2_all(schema_dir: Path, *, format_output: bool = True) -> None: cwd=sdk_root() if format_output else Path(td), ) _preserve_inline_image_class_names(out_path) + _add_legacy_image_generation_thread_item_alias(out_path) _require_nullable_field(out_path, "ChatgptAccount", r"email: str \| None") _require_nullable_field(out_path, "McpResourceReadTarget", r"link_id: Annotated\[[\s\S]*?\]") _preserve_open_enum(out_path, "CodexErrorInfoValue", allow_empty=True) @@ -618,6 +619,22 @@ def _preserve_inline_image_class_names(out_path: Path) -> None: out_path.write_text(source) +def _add_legacy_image_generation_thread_item_alias(out_path: Path) -> None: + """Keep the inline image-generation class import available after splitting the wire type.""" + source = out_path.read_text() + class_start = source.find("class ResultThreadItem(BaseModel):") + if class_start == -1: + raise RuntimeError("Generated SDK is missing ResultThreadItem") + if re.search(r"\bImageGenerationThreadItem\b", source): + raise RuntimeError("Generated SDK already defines ImageGenerationThreadItem") + class_end = source.find("\n\nclass ", class_start) + if class_end == -1: + class_end = len(source) + + alias = "\n\nImageGenerationThreadItem = ResultThreadItem\n" + out_path.write_text(source[:class_end] + alias + source[class_end:]) + + def _require_nullable_field(out_path: Path, class_name: str, field_pattern: str) -> None: """Preserve required-but-nullable fields that codegen gives a None default.""" source = out_path.read_text() diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py index 0faaa9e074..102cae18fd 100644 --- a/sdk/python/src/openai_codex/generated/v2_all.py +++ b/sdk/python/src/openai_codex/generated/v2_all.py @@ -5524,18 +5524,35 @@ class SleepThreadItem(BaseModel): type: Annotated[Literal["sleep"], Field(title="SleepThreadItemType")] -class ImageGenerationThreadItem(BaseModel): +class FileIdThreadItem(BaseModel): model_config = ConfigDict( populate_by_name=True, ) failure: ImageGenerationFailure | None = None id: str - result: str revised_prompt: Annotated[str | None, Field(alias="revisedPrompt")] = None saved_path: Annotated[AbsolutePathBuf | None, Field(alias="savedPath")] = None status: str transparent_background: Annotated[bool | None, Field(alias="transparentBackground")] = None type: Annotated[Literal["imageGeneration"], Field(title="ImageGenerationThreadItemType")] + file_id: Annotated[str, Field(alias="fileId")] + + +class ResultThreadItem(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + failure: ImageGenerationFailure | None = None + id: str + revised_prompt: Annotated[str | None, Field(alias="revisedPrompt")] = None + saved_path: Annotated[AbsolutePathBuf | None, Field(alias="savedPath")] = None + status: str + transparent_background: Annotated[bool | None, Field(alias="transparentBackground")] = None + type: Annotated[Literal["imageGeneration"], Field(title="ImageGenerationThreadItemType")] + result: str + + +ImageGenerationThreadItem = ResultThreadItem class EnteredReviewModeThreadItem(BaseModel): @@ -11647,7 +11664,8 @@ class ThreadItem( | WebSearchThreadItem | ImageViewThreadItem | SleepThreadItem - | ImageGenerationThreadItem + | FileIdThreadItem + | ResultThreadItem | EnteredReviewModeThreadItem | ExitedReviewModeThreadItem | ContextCompactionThreadItem @@ -11672,7 +11690,8 @@ class ThreadItem( | WebSearchThreadItem | ImageViewThreadItem | SleepThreadItem - | ImageGenerationThreadItem + | FileIdThreadItem + | ResultThreadItem | EnteredReviewModeThreadItem | ExitedReviewModeThreadItem | ContextCompactionThreadItem diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py index 5302aa4109..aa55324dec 100644 --- a/sdk/python/tests/test_artifact_workflow_and_binaries.py +++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py @@ -556,6 +556,30 @@ def test_generated_inline_image_class_names_remain_stable() -> None: ) +def test_generated_image_generation_thread_item_alias_remains_available() -> None: + """Existing SDK callers can still construct and validate inline image-generation items.""" + from openai_codex.generated.v2_all import ( + ImageGenerationThreadItem, + ResultThreadItem, + ) + + item = ImageGenerationThreadItem.model_validate( + { + "failure": None, + "id": "image_1", + "result": "aW1hZ2U=", + "revisedPrompt": None, + "savedPath": None, + "status": "completed", + "transparentBackground": None, + "type": "imageGeneration", + } + ) + + assert ImageGenerationThreadItem is ResultThreadItem + assert isinstance(item, ResultThreadItem) + + def test_runtime_package_template_has_no_checked_in_binaries() -> None: runtime_root = ROOT.parent / "python-runtime" / "src" / "codex_cli_bin" assert sorted(