From 329222a4a73a60fee9560b46394c6cd8787214a5 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Mon, 4 May 2026 18:39:45 -0700 Subject: [PATCH] add deferred image content apis --- .../schema/json/ClientRequest.json | 7 + .../schema/json/ServerNotification.json | 111 ++++++ .../codex_app_server_protocol.schemas.json | 118 +++++++ .../codex_app_server_protocol.v2.schemas.json | 118 +++++++ .../json/v2/ItemCompletedNotification.json | 111 ++++++ .../json/v2/ItemStartedNotification.json | 111 ++++++ .../schema/json/v2/ReviewStartResponse.json | 111 ++++++ .../schema/json/v2/ThreadForkResponse.json | 111 ++++++ .../schema/json/v2/ThreadListResponse.json | 111 ++++++ .../json/v2/ThreadMetadataUpdateResponse.json | 111 ++++++ .../schema/json/v2/ThreadReadResponse.json | 111 ++++++ .../schema/json/v2/ThreadResumeParams.json | 7 + .../schema/json/v2/ThreadResumeResponse.json | 111 ++++++ .../json/v2/ThreadRollbackResponse.json | 111 ++++++ .../schema/json/v2/ThreadStartResponse.json | 111 ++++++ .../json/v2/ThreadStartedNotification.json | 111 ++++++ .../json/v2/ThreadUnarchiveResponse.json | 111 ++++++ .../json/v2/TurnCompletedNotification.json | 111 ++++++ .../schema/json/v2/TurnStartResponse.json | 111 ++++++ .../json/v2/TurnStartedNotification.json | 111 ++++++ .../typescript/v2/ImageGenerationContent.ts | 5 + .../schema/typescript/v2/LargeContentMode.ts | 5 + .../schema/typescript/v2/ThreadItem.ts | 3 +- .../schema/typescript/v2/index.ts | 2 + .../src/protocol/common.rs | 15 + .../src/protocol/thread_history.rs | 21 ++ .../app-server-protocol/src/protocol/v2.rs | 153 +++++++++ codex-rs/app-server/README.md | 51 ++- codex-rs/app-server/src/message_processor.rs | 6 + codex-rs/app-server/src/request_processors.rs | 6 + .../request_processors/thread_lifecycle.rs | 1 + .../request_processors/thread_processor.rs | 325 +++++++++++++++++- .../thread_processor_tests.rs | 1 + codex-rs/app-server/src/thread_state.rs | 2 + .../app-server/tests/common/mcp_process.rs | 20 ++ .../app-server/tests/suite/v2/thread_read.rs | 149 ++++++++ .../tests/suite/v2/thread_shell_command.rs | 1 + codex-rs/tui/src/chatwidget/tests/helpers.rs | 7 + 38 files changed, 2780 insertions(+), 19 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ImageGenerationContent.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/LargeContentMode.ts diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index fdd4c155b0..cfd5362d19 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -1478,6 +1478,13 @@ ], "type": "object" }, + "LargeContentMode": { + "enum": [ + "inline", + "deferred" + ], + "type": "string" + }, "ListMcpServerStatusParams": { "properties": { "cursor": { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 82914f3a6f..117fd69068 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -1930,6 +1930,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "ItemCompletedNotification": { "properties": { "item": { @@ -3681,6 +3777,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 6c2ad69e1b..6d8d6a3479 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -10087,6 +10087,102 @@ ], "type": "string" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "InputModality": { "description": "Canonical user-input modality tags advertised by a model.", "oneOf": [ @@ -10228,6 +10324,13 @@ "title": "ItemStartedNotification", "type": "object" }, + "LargeContentMode": { + "enum": [ + "inline", + "deferred" + ], + "type": "string" + }, "ListMcpServerStatusParams": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { @@ -15951,6 +16054,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/v2/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 150b221adf..b9a48e0bf0 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -6696,6 +6696,102 @@ ], "type": "string" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "InitializeCapabilities": { "description": "Client-declared capabilities negotiated during initialize.", "properties": { @@ -6881,6 +6977,13 @@ "title": "ItemStartedNotification", "type": "object" }, + "LargeContentMode": { + "enum": [ + "inline", + "deferred" + ], + "type": "string" + }, "ListMcpServerStatusParams": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { @@ -13837,6 +13940,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 0831483a32..319d06300b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -285,6 +285,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1035,6 +1131,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 16bfeece14..f073e0765e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -285,6 +285,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1035,6 +1131,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 16abcd7806..9114b9ee83 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -422,6 +422,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1179,6 +1275,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 653c5f2387..490406608d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -756,6 +756,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -2005,6 +2101,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 2f5cbb9500..0c0b71977a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -448,6 +448,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1455,6 +1551,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index b9ae59708a..906a9c7391 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -448,6 +448,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1455,6 +1551,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index cda474c294..b99a71a44d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -448,6 +448,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1455,6 +1551,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index 9fe5c7f47f..3b24408510 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -215,6 +215,13 @@ ], "type": "string" }, + "LargeContentMode": { + "enum": [ + "inline", + "deferred" + ], + "type": "string" + }, "LocalShellAction": { "oneOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 27cf47f2fc..1ad0a1618f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -756,6 +756,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -2005,6 +2101,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index e5339f4e99..27d4163f5f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -448,6 +448,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1455,6 +1551,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 7d93606aa4..09b3225fd8 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -756,6 +756,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -2005,6 +2101,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 774686e46a..e3bd0752ea 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -448,6 +448,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1455,6 +1551,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 64179af7e1..acff4d8ac7 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -448,6 +448,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1455,6 +1551,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 0739fa31bc..db1464cfdd 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -422,6 +422,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1179,6 +1275,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index bc5917ef15..da2c5205c5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -422,6 +422,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1179,6 +1275,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 22ad85d906..a82bfcc491 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -422,6 +422,102 @@ ], "type": "object" }, + "ImageGenerationContent": { + "oneOf": [ + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "dataBase64": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "inline" + ], + "title": "InlineImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "dataBase64", + "mimeType", + "type" + ], + "title": "InlineImageGenerationContent", + "type": "object" + }, + { + "properties": { + "byteLength": { + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "contentId": { + "type": "string" + }, + "height": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "mimeType": { + "type": "string" + }, + "type": { + "enum": [ + "deferred" + ], + "title": "DeferredImageGenerationContentType", + "type": "string" + }, + "width": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "byteLength", + "contentId", + "mimeType", + "type" + ], + "title": "DeferredImageGenerationContent", + "type": "object" + } + ] + }, "McpToolCallError": { "properties": { "message": { @@ -1179,6 +1275,21 @@ }, { "properties": { + "content": { + "allOf": [ + { + "$ref": "#/definitions/ImageGenerationContent" + } + ], + "default": { + "byteLength": 0, + "dataBase64": "", + "height": null, + "mimeType": "image/png", + "type": "inline", + "width": null + } + }, "id": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ImageGenerationContent.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ImageGenerationContent.ts new file mode 100644 index 0000000000..435e711837 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ImageGenerationContent.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ImageGenerationContent = { "type": "inline", mimeType: string, dataBase64: string, byteLength: bigint, width: number | null, height: number | null, } | { "type": "deferred", contentId: string, mimeType: string, byteLength: bigint, width: number | null, height: number | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/LargeContentMode.ts b/codex-rs/app-server-protocol/schema/typescript/v2/LargeContentMode.ts new file mode 100644 index 0000000000..8d0bb63df4 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/LargeContentMode.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type LargeContentMode = "inline" | "deferred"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index f7880c9d32..6f0ac9ed37 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -15,6 +15,7 @@ import type { DynamicToolCallOutputContentItem } from "./DynamicToolCallOutputCo import type { DynamicToolCallStatus } from "./DynamicToolCallStatus"; import type { FileUpdateChange } from "./FileUpdateChange"; import type { HookPromptFragment } from "./HookPromptFragment"; +import type { ImageGenerationContent } from "./ImageGenerationContent"; import type { McpToolCallError } from "./McpToolCallError"; import type { McpToolCallResult } from "./McpToolCallResult"; import type { McpToolCallStatus } from "./McpToolCallStatus"; @@ -98,4 +99,4 @@ reasoningEffort: ReasoningEffort | null, /** * Last known status of the target agents, when available. */ -agentsStates: { [key in string]?: CollabAgentState }, } | { "type": "webSearch", id: string, query: string, action: WebSearchAction | null, } | { "type": "imageView", id: string, path: AbsolutePathBuf, } | { "type": "imageGeneration", id: string, status: string, revisedPrompt: string | null, result: string, savedPath?: AbsolutePathBuf, } | { "type": "enteredReviewMode", id: string, review: string, } | { "type": "exitedReviewMode", id: string, review: string, } | { "type": "contextCompaction", id: string, }; +agentsStates: { [key in string]?: CollabAgentState }, } | { "type": "webSearch", id: string, query: string, action: WebSearchAction | null, } | { "type": "imageView", id: string, path: AbsolutePathBuf, } | { "type": "imageGeneration", id: string, status: string, revisedPrompt: string | null, content: ImageGenerationContent, result: string, savedPath?: AbsolutePathBuf, } | { "type": "enteredReviewMode", id: string, review: string, } | { "type": "exitedReviewMode", id: string, review: string, } | { "type": "contextCompaction", id: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index d369ba3423..00eac63a1d 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -171,10 +171,12 @@ export type { HookStartedNotification } from "./HookStartedNotification"; export type { HooksListEntry } from "./HooksListEntry"; export type { HooksListParams } from "./HooksListParams"; export type { HooksListResponse } from "./HooksListResponse"; +export type { ImageGenerationContent } from "./ImageGenerationContent"; export type { ItemCompletedNotification } from "./ItemCompletedNotification"; export type { ItemGuardianApprovalReviewCompletedNotification } from "./ItemGuardianApprovalReviewCompletedNotification"; export type { ItemGuardianApprovalReviewStartedNotification } from "./ItemGuardianApprovalReviewStartedNotification"; export type { ItemStartedNotification } from "./ItemStartedNotification"; +export type { LargeContentMode } from "./LargeContentMode"; export type { ListMcpServerStatusParams } from "./ListMcpServerStatusParams"; export type { ListMcpServerStatusResponse } from "./ListMcpServerStatusResponse"; export type { LoginAccountParams } from "./LoginAccountParams"; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index c5a7d61f01..3df5326d70 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -571,6 +571,20 @@ client_request_definitions! { serialization: None, response: v2::ThreadTurnsListResponse, }, + #[experimental("thread/turns/items/list")] + ThreadTurnsItemsList => "thread/turns/items/list" { + params: v2::ThreadTurnsItemsListParams, + // Explicitly concurrent: this primarily reads append-only rollout storage. + serialization: None, + response: v2::ThreadTurnsItemsListResponse, + }, + #[experimental("thread/item/content/read")] + ThreadItemContentRead => "thread/item/content/read" { + params: v2::ThreadItemContentReadParams, + // Explicitly concurrent: this primarily reads append-only rollout storage. + serialization: None, + response: v2::ThreadItemContentReadResponse, + }, /// Append raw Responses API items to the thread history without starting a user turn. ThreadInjectItems => "thread/inject_items" { params: v2::ThreadInjectItemsParams, @@ -1792,6 +1806,7 @@ mod tests { cursor: None, limit: None, sort_direction: None, + large_content: None, }, }; assert_eq!(thread_turns_list.serialization_scope(), None); diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 57eeedec34..0cc9413ef9 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -585,6 +585,13 @@ impl ThreadHistoryBuilder { id: payload.call_id.clone(), status: String::new(), revised_prompt: None, + content: crate::protocol::v2::ImageGenerationContent::Inline { + mime_type: "image/png".to_string(), + data_base64: String::new(), + byte_length: 0, + width: None, + height: None, + }, result: String::new(), saved_path: None, }; @@ -596,6 +603,13 @@ impl ThreadHistoryBuilder { id: payload.call_id.clone(), status: payload.status.clone(), revised_prompt: payload.revised_prompt.clone(), + content: crate::protocol::v2::ImageGenerationContent::Inline { + mime_type: "image/png".to_string(), + data_base64: payload.result.clone(), + byte_length: crate::protocol::v2::image_generation_byte_length(&payload.result), + width: None, + height: None, + }, result: payload.result.clone(), saved_path: payload.saved_path.clone(), }; @@ -1464,6 +1478,13 @@ mod tests { id: "ig_123".into(), status: "completed".into(), revised_prompt: Some("final prompt".into()), + content: crate::protocol::v2::ImageGenerationContent::Inline { + mime_type: "image/png".into(), + data_base64: "Zm9v".into(), + byte_length: 3, + width: None, + height: None, + }, result: "Zm9v".into(), saved_path: Some(test_path_buf("/tmp/ig_123.png").abs()), }, diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index aea80029a8..354c2d7fd0 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -3763,6 +3763,11 @@ pub struct ThreadResumeParams { #[experimental("thread/resume.excludeTurns")] #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub exclude_turns: bool, + /// Controls whether large item payloads are embedded in returned turns or + /// replaced with deferred-content metadata. + #[experimental("thread/resume.largeContent")] + #[ts(optional = nullable)] + pub large_content: Option, /// If true, persist additional EventMsg variants to the rollout file. /// However, `thread/read`, `thread/resume`, and `thread/fork` still only /// return the limited form of thread history for scalability reasons. @@ -4460,6 +4465,10 @@ pub struct ThreadTurnsListParams { /// Optional turn pagination direction; defaults to descending. #[ts(optional = nullable)] pub sort_direction: Option, + /// Controls whether large item payloads are embedded in returned turns or + /// replaced with deferred-content metadata. + #[ts(optional = nullable)] + pub large_content: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -4477,6 +4486,70 @@ pub struct ThreadTurnsListResponse { pub backwards_cursor: Option, } +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS, Default)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum LargeContentMode { + #[default] + Inline, + Deferred, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadTurnsItemsListParams { + pub thread_id: String, + pub turn_id: String, + /// Opaque cursor to pass to the next call to continue after the last item. + #[ts(optional = nullable)] + pub cursor: Option, + /// Optional item page size. + #[ts(optional = nullable)] + pub limit: Option, + /// Optional item pagination direction; defaults to ascending. + #[ts(optional = nullable)] + pub sort_direction: Option, + /// Controls whether large item payloads are embedded in returned items or + /// replaced with deferred-content metadata. + #[ts(optional = nullable)] + pub large_content: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadTurnsItemsListResponse { + pub data: Vec, + /// Opaque cursor to pass to the next call to continue after the last item. + /// if None, there are no more items to return. + pub next_cursor: Option, + /// Opaque cursor to pass as `cursor` when reversing `sortDirection`. + /// This is only populated when the page contains at least one item. + /// Use it with the opposite `sortDirection` to include the anchor item again + /// and catch updates to that item. + pub backwards_cursor: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadItemContentReadParams { + pub thread_id: String, + pub turn_id: String, + pub item_id: String, + pub content_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadItemContentReadResponse { + pub mime_type: String, + pub data_base64: String, + pub byte_length: u64, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -6017,6 +6090,8 @@ pub enum ThreadItem { id: String, status: String, revised_prompt: Option, + #[serde(default)] + content: ImageGenerationContent, result: String, #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] @@ -6033,6 +6108,52 @@ pub enum ThreadItem { ContextCompaction { id: String }, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(tag = "type", rename_all = "camelCase")] +#[ts(tag = "type", export_to = "v2/")] +pub enum ImageGenerationContent { + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + Inline { + mime_type: String, + data_base64: String, + byte_length: u64, + width: Option, + height: Option, + }, + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + Deferred { + content_id: String, + mime_type: String, + byte_length: u64, + width: Option, + height: Option, + }, +} + +impl Default for ImageGenerationContent { + fn default() -> Self { + Self::Inline { + mime_type: "image/png".to_string(), + data_base64: String::new(), + byte_length: 0, + width: None, + height: None, + } + } +} + +pub(crate) fn image_generation_byte_length(data_base64: &str) -> u64 { + let padding_len = data_base64 + .as_bytes() + .iter() + .rev() + .take_while(|byte| **byte == b'=') + .count(); + ((data_base64.len() / 4) * 3).saturating_sub(padding_len) as u64 +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase", export_to = "v2/")] @@ -6494,6 +6615,13 @@ impl From for ThreadItem { id: image.id, status: image.status, revised_prompt: image.revised_prompt, + content: ImageGenerationContent::Inline { + mime_type: "image/png".to_string(), + data_base64: image.result.clone(), + byte_length: image_generation_byte_length(&image.result), + width: None, + height: None, + }, result: image.result, saved_path: image.saved_path, }, @@ -8750,6 +8878,31 @@ mod tests { assert_eq!(decoded, response); } + #[test] + fn image_generation_defaults_missing_content_for_legacy_payloads() { + let item: ThreadItem = serde_json::from_value(json!({ + "type": "imageGeneration", + "id": "ig_123", + "status": "completed", + "revisedPrompt": null, + "result": "Zm9v", + "savedPath": null + })) + .expect("legacy image generation item should deserialize"); + + assert_eq!( + item, + ThreadItem::ImageGeneration { + id: "ig_123".to_string(), + status: "completed".to_string(), + revised_prompt: None, + content: ImageGenerationContent::default(), + result: "Zm9v".to_string(), + saved_path: None, + } + ); + } + #[test] fn fs_read_file_params_round_trip() { let params = FsReadFileParams { diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index df41d859cd..a618d60b13 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -143,13 +143,15 @@ Example with notification opt-out: ## API Overview - `thread/start` — create a new thread; emits `thread/started` (including the current `thread.status`) and auto-subscribes you to turn/item events for that thread. When the request includes a `cwd` and the resolved sandbox is `workspace-write` or full access, app-server also marks that project as trusted in the user `config.toml`. Pass `sessionStartSource: "clear"` when starting a replacement thread after clearing the current session so `SessionStart` hooks receive `source: "clear"` instead of the default `"startup"`. For permissions, prefer experimental `permissions` profile selection; the legacy `sandbox` shorthand is still accepted but cannot be combined with `permissions`. Experimental `environments` selects the sticky execution environments for turns on the thread; omit it to use the server default, pass `[]` to disable environments, or pass explicit environment ids with per-environment `cwd`. -- `thread/resume` — reopen an existing thread by id so subsequent `turn/start` calls append to it. Accepts the same permission override rules as `thread/start`. +- `thread/resume` — reopen an existing thread by id so subsequent `turn/start` calls append to it. Accepts the same permission override rules as `thread/start`. Experimental clients can pass `largeContent: "deferred"` to replace large item payloads such as generated-image bytes with metadata placeholders. - `thread/fork` — fork an existing thread into a new thread id by copying the stored history; if the source thread is currently mid-turn, the fork records the same interruption marker as `turn/interrupt` instead of inheriting an unmarked partial turn suffix. The returned `thread.forkedFromId` points at the source thread when known. Accepts `ephemeral: true` for an in-memory temporary fork, emits `thread/started` (including the current `thread.status`), and auto-subscribes you to turn/item events for the new thread. Experimental clients can pass `excludeTurns: true` when they plan to page fork history via `thread/turns/list` instead of receiving the full turn array immediately. Accepts the same permission override rules as `thread/start`. - `thread/start`, `thread/resume`, and `thread/fork` responses include the legacy `sandbox` compatibility projection. Experimental clients can read response `permissionProfile` for the exact active runtime permissions and `activePermissionProfile` for the named or implicit built-in profile identity/provenance when known. - `thread/list` — page through stored rollouts; supports cursor-based pagination and optional `modelProviders`, `sourceKinds`, `archived`, `cwd`, and `searchTerm` filters. Each returned `thread` includes `status` (`ThreadStatus`), defaulting to `notLoaded` when the thread is not currently loaded. - `thread/loaded/list` — list the thread ids currently loaded in memory. - `thread/read` — read a stored thread by id without resuming it; optionally include turns via `includeTurns`. The returned `thread` includes `status` (`ThreadStatus`), defaulting to `notLoaded` when the thread is not currently loaded. -- `thread/turns/list` — experimental; page through a stored thread’s turn history without resuming it; supports cursor-based pagination with `sortDirection`, `nextCursor`, and `backwardsCursor`. +- `thread/turns/list` — experimental; page through a stored thread’s turn history without resuming it; supports cursor-based pagination with `sortDirection`, `nextCursor`, and `backwardsCursor`, plus `largeContent: "deferred"` for generated-image payloads. +- `thread/turns/items/list` — experimental; page through one stored turn’s items with the same cursor model and optional `largeContent: "deferred"` handling. +- `thread/item/content/read` — experimental; fetch deferred large-content bytes for one stored item. - `thread/metadata/update` — patch stored thread metadata in sqlite; currently supports updating persisted `gitInfo` fields and returns the refreshed `thread`. - `thread/memoryMode/set` — experimental; set a thread’s persisted memory eligibility to `"enabled"` or `"disabled"` for either a loaded thread or a stored rollout; returns `{}` on success. - `memory/reset` — experimental; clear the current `CODEX_HOME/memories` directory and reset persisted memory stage data in sqlite while preserving existing thread memory modes; returns `{}` on success. @@ -434,6 +436,51 @@ Use `thread/turns/list` with `capabilities.experimentalApi = true` to page a sto } } ``` +### Example: Defer generated-image content (experimental) + +Pass `largeContent: "deferred"` when listing a turn's items to keep generated-image bytes out of the page response. Deferred `imageGeneration` items retain the legacy `result` field as an empty string and expose a structured `content` descriptor that can be loaded separately. For a just-completed live turn, callers may need to retry a content read briefly while rollout persistence catches up with the live turn view. + +```json +{ "method": "thread/turns/items/list", "id": 25, "params": { + "threadId": "thr_123", + "turnId": "turn_456", + "limit": 100, + "sortDirection": "asc", + "largeContent": "deferred" +} } +{ "id": 25, "result": { + "data": [{ + "type": "imageGeneration", + "id": "ig_789", + "status": "completed", + "revisedPrompt": null, + "content": { + "type": "deferred", + "contentId": "result", + "mimeType": "image/png", + "byteLength": 1234567, + "width": null, + "height": null + }, + "result": "", + "savedPath": null + }], + "nextCursor": null, + "backwardsCursor": "newer-items-cursor-or-null" +} } +{ "method": "thread/item/content/read", "id": 26, "params": { + "threadId": "thr_123", + "turnId": "turn_456", + "itemId": "ig_789", + "contentId": "result" +} } +{ "id": 26, "result": { + "mimeType": "image/png", + "dataBase64": "...", + "byteLength": 1234567 +} } +``` + ### Example: Update stored thread metadata Use `thread/metadata/update` to patch sqlite-backed metadata for a thread without resuming it. Today this supports persisted `gitInfo`; omitted fields are left unchanged, while explicit `null` clears a stored value. diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index b4bce010de..864dd4f645 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -1023,6 +1023,12 @@ impl MessageProcessor { ClientRequest::ThreadTurnsList { params, .. } => { self.thread_processor.thread_turns_list(params).await } + ClientRequest::ThreadTurnsItemsList { params, .. } => { + self.thread_processor.thread_turns_items_list(params).await + } + ClientRequest::ThreadItemContentRead { params, .. } => { + self.thread_processor.thread_item_content_read(params).await + } ClientRequest::ThreadShellCommand { params, .. } => { self.thread_processor .thread_shell_command(&request_id, params) diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index f59ea14402..65c9fd3be2 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -72,9 +72,11 @@ use codex_app_server_protocol::GitInfo as ApiGitInfo; use codex_app_server_protocol::HookMetadata; use codex_app_server_protocol::HooksListParams; use codex_app_server_protocol::HooksListResponse; +use codex_app_server_protocol::ImageGenerationContent; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::InitializeResponse; use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::LargeContentMode; use codex_app_server_protocol::ListMcpServerStatusParams; use codex_app_server_protocol::ListMcpServerStatusResponse; use codex_app_server_protocol::LoginAccountParams; @@ -175,6 +177,8 @@ use codex_app_server_protocol::ThreadIncrementElicitationResponse; use codex_app_server_protocol::ThreadInjectItemsParams; use codex_app_server_protocol::ThreadInjectItemsResponse; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadItemContentReadParams; +use codex_app_server_protocol::ThreadItemContentReadResponse; use codex_app_server_protocol::ThreadListCwdFilter; use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadListResponse; @@ -211,6 +215,8 @@ use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::ThreadStartedNotification; use codex_app_server_protocol::ThreadStatus; +use codex_app_server_protocol::ThreadTurnsItemsListParams; +use codex_app_server_protocol::ThreadTurnsItemsListResponse; use codex_app_server_protocol::ThreadTurnsListParams; use codex_app_server_protocol::ThreadTurnsListResponse; use codex_app_server_protocol::ThreadUnarchiveParams; diff --git a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs index 9dabdea44f..7478eb0d6d 100644 --- a/codex-rs/app-server/src/request_processors/thread_lifecycle.rs +++ b/codex-rs/app-server/src/request_processors/thread_lifecycle.rs @@ -549,6 +549,7 @@ pub(super) async fn handle_pending_thread_resume_request( active_turn.as_ref(), ); } + super::thread_processor::apply_large_content_mode_to_thread(&mut thread, pending.large_content); let thread_status = thread_watch_manager .loaded_status_for_thread(&thread.id) diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index 40be4f34ef..81bbc5112f 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -527,6 +527,24 @@ impl ThreadRequestProcessor { .map(|response| Some(response.into())) } + pub(crate) async fn thread_turns_items_list( + &self, + params: ThreadTurnsItemsListParams, + ) -> Result, JSONRPCErrorError> { + self.thread_turns_items_list_response_inner(params) + .await + .map(|response| Some(response.into())) + } + + pub(crate) async fn thread_item_content_read( + &self, + params: ThreadItemContentReadParams, + ) -> Result, JSONRPCErrorError> { + self.thread_item_content_read_response_inner(params) + .await + .map(|response| Some(response.into())) + } + pub(crate) async fn thread_shell_command( &self, request_id: &ConnectionRequestId, @@ -2131,6 +2149,7 @@ impl ThreadRequestProcessor { cursor, limit, sort_direction, + large_content, } = params; let thread_uuid = ThreadId::from_string(&thread_id) @@ -2145,22 +2164,8 @@ impl ThreadRequestProcessor { // every request. Rollback and compaction events can change earlier turns, so // the server has to rebuild the full turn list until turn metadata is indexed // separately. - let loaded_thread = self.thread_manager.get_thread(thread_uuid).await.ok(); - let has_live_running_thread = match loaded_thread.as_ref() { - Some(thread) => matches!(thread.agent_status().await, AgentStatus::Running), - None => false, - }; - let active_turn = if loaded_thread.is_some() { - // Persisted history may not yet include the currently running turn. The - // app-server listener has already projected live turn events into ThreadState, - // so merge that in-memory snapshot before paginating. - let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; - let state = thread_state.lock().await; - state.active_turn_snapshot() - } else { - None - }; - let turns = reconstruct_thread_turns_for_turns_list( + let (has_live_running_thread, active_turn) = self.live_turn_read_context(thread_uuid).await; + let mut turns = reconstruct_thread_turns_for_turns_list( &items, self.thread_watch_manager .loaded_status_for_thread(&thread_uuid.to_string()) @@ -2168,6 +2173,7 @@ impl ThreadRequestProcessor { has_live_running_thread, active_turn, ); + apply_large_content_mode_to_turns(&mut turns, large_content.unwrap_or_default()); let page = paginate_thread_turns( turns, cursor.as_deref(), @@ -2181,6 +2187,129 @@ impl ThreadRequestProcessor { }) } + async fn thread_turns_items_list_response_inner( + &self, + params: ThreadTurnsItemsListParams, + ) -> Result { + let ThreadTurnsItemsListParams { + thread_id, + turn_id, + cursor, + limit, + sort_direction, + large_content, + } = params; + + let thread_uuid = ThreadId::from_string(&thread_id) + .map_err(|err| invalid_request(format!("invalid thread id: {err}")))?; + let items = self + .load_thread_turns_list_history(thread_uuid) + .await + .map_err(thread_read_view_error)?; + let (has_live_running_thread, active_turn) = self.live_turn_read_context(thread_uuid).await; + let turns = reconstruct_thread_turns_for_turns_list( + &items, + self.thread_watch_manager + .loaded_status_for_thread(&thread_uuid.to_string()) + .await, + has_live_running_thread, + active_turn, + ); + let mut turn_items = turns + .into_iter() + .find(|turn| turn.id == turn_id) + .ok_or_else(|| invalid_request(format!("turn not found: {turn_id}")))? + .items; + apply_large_content_mode_to_items(&mut turn_items, large_content.unwrap_or_default()); + let page = paginate_thread_items( + turn_items, + cursor.as_deref(), + limit, + sort_direction.unwrap_or(SortDirection::Asc), + )?; + Ok(ThreadTurnsItemsListResponse { + data: page.items, + next_cursor: page.next_cursor, + backwards_cursor: page.backwards_cursor, + }) + } + + async fn thread_item_content_read_response_inner( + &self, + params: ThreadItemContentReadParams, + ) -> Result { + let ThreadItemContentReadParams { + thread_id, + turn_id, + item_id, + content_id, + } = params; + + let thread_uuid = ThreadId::from_string(&thread_id) + .map_err(|err| invalid_request(format!("invalid thread id: {err}")))?; + let items = self + .load_thread_turns_list_history(thread_uuid) + .await + .map_err(thread_read_view_error)?; + let (has_live_running_thread, active_turn) = self.live_turn_read_context(thread_uuid).await; + let turns = reconstruct_thread_turns_for_turns_list( + &items, + self.thread_watch_manager + .loaded_status_for_thread(&thread_uuid.to_string()) + .await, + has_live_running_thread, + active_turn, + ); + let turn = turns + .into_iter() + .find(|turn| turn.id == turn_id) + .ok_or_else(|| invalid_request(format!("turn not found: {turn_id}")))?; + let item = turn + .items + .into_iter() + .find(|item| item.id() == item_id) + .ok_or_else(|| invalid_request(format!("item not found: {item_id}")))?; + let ThreadItem::ImageGeneration { + content, result, .. + } = item + else { + return Err(invalid_request(format!( + "item {item_id} does not contain deferred content" + ))); + }; + if content_id != IMAGE_GENERATION_RESULT_CONTENT_ID { + return Err(invalid_request(format!("content not found: {content_id}"))); + } + let ImageGenerationContent::Inline { + mime_type, + byte_length, + .. + } = content + else { + return Err(internal_error(format!( + "image generation item {item_id} was unexpectedly already deferred" + ))); + }; + Ok(ThreadItemContentReadResponse { + mime_type, + data_base64: result, + byte_length, + }) + } + + async fn live_turn_read_context(&self, thread_id: ThreadId) -> (bool, Option) { + let Ok(thread) = self.thread_manager.get_thread(thread_id).await else { + return (false, None); + }; + let has_live_running_thread = matches!(thread.agent_status().await, AgentStatus::Running); + // Persisted history may not yet include the currently running turn. The + // app-server listener has already projected live turn events into ThreadState, + // so merge that in-memory snapshot when serving history reads. + let thread_state = self.thread_state_manager.thread_state(thread_id).await; + let state = thread_state.lock().await; + (has_live_running_thread, state.active_turn_snapshot()) + } + async fn load_thread_turns_list_history( &self, thread_id: ThreadId, @@ -2361,6 +2490,7 @@ impl ThreadRequestProcessor { developer_instructions, personality, exclude_turns, + large_content, persist_extended_history, } = params; let include_turns = !exclude_turns; @@ -2475,6 +2605,7 @@ impl ThreadRequestProcessor { return Ok(()); } }; + apply_large_content_mode_to_thread(&mut thread, large_content.unwrap_or_default()); self.thread_watch_manager .upsert_thread(thread.clone()) @@ -2700,6 +2831,7 @@ impl ThreadRequestProcessor { emit_thread_goal_update, thread_goal_state_db, include_turns: !params.exclude_turns, + large_content: params.large_content.unwrap_or_default(), }), ); if listener_command_tx.send(command).is_err() { @@ -3321,6 +3453,9 @@ impl ThreadRequestProcessor { const THREAD_TURNS_DEFAULT_LIMIT: usize = 25; const THREAD_TURNS_MAX_LIMIT: usize = 100; +const THREAD_ITEMS_DEFAULT_LIMIT: usize = 100; +const THREAD_ITEMS_MAX_LIMIT: usize = 500; +const IMAGE_GENERATION_RESULT_CONTENT_ID: &str = "result"; fn thread_backwards_cursor_for_sort_key( thread: &StoredThread, @@ -3346,6 +3481,12 @@ struct ThreadTurnsPage { pub(super) backwards_cursor: Option, } +struct ThreadItemsPage { + pub(super) items: Vec, + pub(super) next_cursor: Option, + pub(super) backwards_cursor: Option, +} + #[derive(serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] struct ThreadTurnsCursor { @@ -3353,6 +3494,13 @@ struct ThreadTurnsCursor { include_anchor: bool, } +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct ThreadItemsCursor { + item_id: String, + include_anchor: bool, +} + fn paginate_thread_turns( turns: Vec, cursor: Option<&str>, @@ -3457,6 +3605,151 @@ fn parse_thread_turns_cursor(cursor: &str) -> Result, + cursor: Option<&str>, + limit: Option, + sort_direction: SortDirection, +) -> Result { + if items.is_empty() { + return Ok(ThreadItemsPage { + items: Vec::new(), + next_cursor: None, + backwards_cursor: None, + }); + } + + let anchor = cursor.map(parse_thread_items_cursor).transpose()?; + let page_size = limit + .map(|value| value as usize) + .unwrap_or(THREAD_ITEMS_DEFAULT_LIMIT) + .clamp(1, THREAD_ITEMS_MAX_LIMIT); + + let anchor_index = anchor + .as_ref() + .and_then(|anchor| items.iter().position(|item| item.id() == anchor.item_id)); + if anchor.is_some() && anchor_index.is_none() { + return Err(JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message: "invalid cursor: anchor item is no longer present".to_string(), + data: None, + }); + } + + let mut keyed_items: Vec<_> = items.into_iter().enumerate().collect(); + match sort_direction { + SortDirection::Asc => { + if let (Some(anchor), Some(anchor_index)) = (anchor.as_ref(), anchor_index) { + keyed_items.retain(|(index, _)| { + if anchor.include_anchor { + *index >= anchor_index + } else { + *index > anchor_index + } + }); + } + } + SortDirection::Desc => { + keyed_items.reverse(); + if let (Some(anchor), Some(anchor_index)) = (anchor.as_ref(), anchor_index) { + keyed_items.retain(|(index, _)| { + if anchor.include_anchor { + *index <= anchor_index + } else { + *index < anchor_index + } + }); + } + } + } + + let more_items_available = keyed_items.len() > page_size; + keyed_items.truncate(page_size); + let backwards_cursor = keyed_items + .first() + .map(|(_, item)| serialize_thread_items_cursor(item.id(), /*include_anchor*/ true)) + .transpose()?; + let next_cursor = if more_items_available { + keyed_items + .last() + .map(|(_, item)| { + serialize_thread_items_cursor(item.id(), /*include_anchor*/ false) + }) + .transpose()? + } else { + None + }; + let items = keyed_items.into_iter().map(|(_, item)| item).collect(); + + Ok(ThreadItemsPage { + items, + next_cursor, + backwards_cursor, + }) +} + +fn serialize_thread_items_cursor( + item_id: &str, + include_anchor: bool, +) -> Result { + serde_json::to_string(&ThreadItemsCursor { + item_id: item_id.to_string(), + include_anchor, + }) + .map_err(|err| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to serialize cursor: {err}"), + data: None, + }) +} + +fn parse_thread_items_cursor(cursor: &str) -> Result { + serde_json::from_str(cursor).map_err(|_| JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message: format!("invalid cursor: {cursor}"), + data: None, + }) +} + +pub(super) fn apply_large_content_mode_to_thread(thread: &mut Thread, mode: LargeContentMode) { + apply_large_content_mode_to_turns(&mut thread.turns, mode); +} + +fn apply_large_content_mode_to_turns(turns: &mut [Turn], mode: LargeContentMode) { + for turn in turns { + apply_large_content_mode_to_items(&mut turn.items, mode); + } +} + +fn apply_large_content_mode_to_items(items: &mut [ThreadItem], mode: LargeContentMode) { + if matches!(mode, LargeContentMode::Inline) { + return; + } + + for item in items { + if let ThreadItem::ImageGeneration { + content, result, .. + } = item + && let ImageGenerationContent::Inline { + mime_type, + byte_length, + width, + height, + .. + } = content + { + *content = ImageGenerationContent::Deferred { + content_id: IMAGE_GENERATION_RESULT_CONTENT_ID.to_string(), + mime_type: mime_type.clone(), + byte_length: *byte_length, + width: *width, + height: *height, + }; + result.clear(); + } + } +} + fn reconstruct_thread_turns_for_turns_list( items: &[RolloutItem], loaded_status: ThreadStatus, diff --git a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs index 14859d5938..27edeb4fdb 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs @@ -522,6 +522,7 @@ mod thread_processor_behavior_tests { developer_instructions: None, personality: None, exclude_turns: false, + large_content: None, persist_extended_history: false, }; let config_snapshot = ThreadConfigSnapshot { diff --git a/codex-rs/app-server/src/thread_state.rs b/codex-rs/app-server/src/thread_state.rs index dddbcf483b..b4dac56939 100644 --- a/codex-rs/app-server/src/thread_state.rs +++ b/codex-rs/app-server/src/thread_state.rs @@ -1,5 +1,6 @@ use crate::outgoing_message::ConnectionId; use crate::outgoing_message::ConnectionRequestId; +use codex_app_server_protocol::LargeContentMode; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadGoal; use codex_app_server_protocol::ThreadHistoryBuilder; @@ -33,6 +34,7 @@ pub(crate) struct PendingThreadResumeRequest { pub(crate) emit_thread_goal_update: bool, pub(crate) thread_goal_state_db: Option, pub(crate) include_turns: bool, + pub(crate) large_content: LargeContentMode, } // ThreadListenerCommand is used to perform operations in the context of the thread listener, for serialization purposes. diff --git a/codex-rs/app-server/tests/common/mcp_process.rs b/codex-rs/app-server/tests/common/mcp_process.rs index 2abdbd8f7c..2f549ba576 100644 --- a/codex-rs/app-server/tests/common/mcp_process.rs +++ b/codex-rs/app-server/tests/common/mcp_process.rs @@ -70,6 +70,7 @@ use codex_app_server_protocol::ThreadArchiveParams; use codex_app_server_protocol::ThreadCompactStartParams; use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadInjectItemsParams; +use codex_app_server_protocol::ThreadItemContentReadParams; use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadLoadedListParams; use codex_app_server_protocol::ThreadMemoryModeSetParams; @@ -85,6 +86,7 @@ use codex_app_server_protocol::ThreadRollbackParams; use codex_app_server_protocol::ThreadSetNameParams; use codex_app_server_protocol::ThreadShellCommandParams; use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadTurnsItemsListParams; use codex_app_server_protocol::ThreadTurnsListParams; use codex_app_server_protocol::ThreadUnarchiveParams; use codex_app_server_protocol::ThreadUnsubscribeParams; @@ -518,6 +520,24 @@ impl McpProcess { self.send_request("thread/turns/list", params).await } + /// Send a `thread/turns/items/list` JSON-RPC request. + pub async fn send_thread_turns_items_list_request( + &mut self, + params: ThreadTurnsItemsListParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/turns/items/list", params).await + } + + /// Send a `thread/item/content/read` JSON-RPC request. + pub async fn send_thread_item_content_read_request( + &mut self, + params: ThreadItemContentReadParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/item/content/read", params).await + } + /// Send a `model/list` JSON-RPC request. pub async fn send_list_models_request( &mut self, diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs index 8c46a5ad95..3ebe48407c 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_read.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs @@ -9,16 +9,20 @@ use codex_app_server::in_process; use codex_app_server::in_process::InProcessStartArgs; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::ImageGenerationContent; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::LargeContentMode; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::SessionSource; use codex_app_server_protocol::SortDirection; use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadForkResponse; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadItemContentReadParams; +use codex_app_server_protocol::ThreadItemContentReadResponse; use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadListResponse; use codex_app_server_protocol::ThreadNameUpdatedNotification; @@ -31,6 +35,8 @@ use codex_app_server_protocol::ThreadSetNameResponse; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::ThreadStatus; +use codex_app_server_protocol::ThreadTurnsItemsListParams; +use codex_app_server_protocol::ThreadTurnsItemsListResponse; use codex_app_server_protocol::ThreadTurnsListParams; use codex_app_server_protocol::ThreadTurnsListResponse; use codex_app_server_protocol::TurnStartParams; @@ -221,6 +227,7 @@ async fn thread_turns_list_can_page_backward_and_forward() -> Result<()> { cursor: None, limit: Some(2), sort_direction: Some(SortDirection::Desc), + large_content: None, }) .await?; let read_resp: JSONRPCResponse = timeout( @@ -243,6 +250,7 @@ async fn thread_turns_list_can_page_backward_and_forward() -> Result<()> { cursor: Some(next_cursor), limit: Some(10), sort_direction: Some(SortDirection::Desc), + large_content: None, }) .await?; let read_resp: JSONRPCResponse = timeout( @@ -261,6 +269,7 @@ async fn thread_turns_list_can_page_backward_and_forward() -> Result<()> { cursor: Some(backwards_cursor), limit: Some(10), sort_direction: Some(SortDirection::Asc), + large_content: None, }) .await?; let read_resp: JSONRPCResponse = timeout( @@ -274,6 +283,119 @@ async fn thread_turns_list_can_page_backward_and_forward() -> Result<()> { Ok(()) } +#[tokio::test] +async fn thread_turn_items_list_defers_image_generation_content_and_reads_it_back() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let filename_ts = "2025-01-05T12-00-00"; + let conversation_id = create_fake_rollout_with_text_elements( + codex_home.path(), + filename_ts, + "2025-01-05T12:00:00Z", + "make an image", + vec![], + Some("mock_provider"), + /*git_info*/ None, + )?; + let rollout_path = rollout_path(codex_home.path(), filename_ts, &conversation_id); + append_image_generation_end( + rollout_path.as_path(), + "2025-01-05T12:00:01Z", + "ig_123", + "Zm9v", + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let turns_list_id = mcp + .send_thread_turns_list_request(ThreadTurnsListParams { + thread_id: conversation_id.clone(), + cursor: None, + limit: Some(10), + sort_direction: Some(SortDirection::Asc), + large_content: Some(LargeContentMode::Deferred), + }) + .await?; + let turns_list_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turns_list_id)), + ) + .await??; + let ThreadTurnsListResponse { data, .. } = + to_response::(turns_list_resp)?; + let turn = data.first().expect("expected one turn"); + let turn_id = turn.id.clone(); + let deferred_item = turn + .items + .iter() + .find(|item| item.id() == "ig_123") + .expect("expected deferred image generation item"); + assert_eq!( + deferred_item, + &ThreadItem::ImageGeneration { + id: "ig_123".to_string(), + status: "completed".to_string(), + revised_prompt: None, + content: ImageGenerationContent::Deferred { + content_id: "result".to_string(), + mime_type: "image/png".to_string(), + byte_length: 3, + width: None, + height: None, + }, + result: String::new(), + saved_path: None, + } + ); + + let items_list_id = mcp + .send_thread_turns_items_list_request(ThreadTurnsItemsListParams { + thread_id: conversation_id.clone(), + turn_id: turn_id.clone(), + cursor: None, + limit: Some(10), + sort_direction: Some(SortDirection::Asc), + large_content: Some(LargeContentMode::Deferred), + }) + .await?; + let items_list_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(items_list_id)), + ) + .await??; + let ThreadTurnsItemsListResponse { data, .. } = + to_response::(items_list_resp)?; + assert!(data.contains(deferred_item)); + + let content_read_id = mcp + .send_thread_item_content_read_request(ThreadItemContentReadParams { + thread_id: conversation_id, + turn_id, + item_id: "ig_123".to_string(), + content_id: "result".to_string(), + }) + .await?; + let content_read_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(content_read_id)), + ) + .await??; + let content = to_response::(content_read_resp)?; + assert_eq!( + content, + ThreadItemContentReadResponse { + mime_type: "image/png".to_string(), + data_base64: "Zm9v".to_string(), + byte_length: 3, + } + ); + + Ok(()) +} + #[tokio::test] async fn thread_turns_list_reads_store_history_without_rollout_path() -> Result<()> { let codex_home = TempDir::new()?; @@ -328,6 +450,7 @@ async fn thread_turns_list_reads_store_history_without_rollout_path() -> Result< cursor: None, limit: Some(10), sort_direction: Some(SortDirection::Asc), + large_content: None, }, }) .await? @@ -577,6 +700,7 @@ async fn thread_turns_list_rejects_cursor_when_anchor_turn_is_rolled_back() -> R cursor: None, limit: Some(2), sort_direction: Some(SortDirection::Desc), + large_content: None, }) .await?; let read_resp: JSONRPCResponse = timeout( @@ -601,6 +725,7 @@ async fn thread_turns_list_rejects_cursor_when_anchor_turn_is_rolled_back() -> R cursor: Some(backwards_cursor), limit: Some(10), sort_direction: Some(SortDirection::Asc), + large_content: None, }) .await?; let read_err: JSONRPCError = timeout( @@ -957,6 +1082,7 @@ async fn thread_turns_list_rejects_unmaterialized_loaded_thread() -> Result<()> cursor: None, limit: None, sort_direction: None, + large_content: None, }) .await?; let read_err: JSONRPCError = timeout( @@ -1062,6 +1188,29 @@ fn append_user_message(path: &Path, timestamp: &str, text: &str) -> std::io::Res ) } +fn append_image_generation_end( + path: &Path, + timestamp: &str, + call_id: &str, + result: &str, +) -> std::io::Result<()> { + let mut file = std::fs::OpenOptions::new().append(true).open(path)?; + writeln!( + file, + "{}", + json!({ + "timestamp": timestamp, + "type":"event_msg", + "payload": { + "type":"image_generation_end", + "call_id": call_id, + "status": "completed", + "result": result + } + }) + ) +} + fn append_thread_rollback(path: &Path, timestamp: &str, num_turns: u32) -> std::io::Result<()> { let mut file = std::fs::OpenOptions::new().append(true).open(path)?; writeln!( diff --git a/codex-rs/app-server/tests/suite/v2/thread_shell_command.rs b/codex-rs/app-server/tests/suite/v2/thread_shell_command.rs index eebc4077df..27830a4b28 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_shell_command.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_shell_command.rs @@ -151,6 +151,7 @@ async fn thread_shell_command_history_responses_exclude_persisted_command_execut cursor: None, limit: None, sort_direction: Some(SortDirection::Asc), + large_content: None, }) .await?; let turns_list_resp: JSONRPCResponse = timeout( diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index 2631e50659..c853678704 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -824,6 +824,13 @@ pub(super) fn handle_image_generation_end( id: call_id.into(), status: "completed".to_string(), revised_prompt, + content: codex_app_server_protocol::ImageGenerationContent::Inline { + mime_type: "image/png".to_string(), + data_base64: String::new(), + byte_length: 0, + width: None, + height: None, + }, result: String::new(), saved_path, },