diff --git a/codex-rs/app-server-protocol/schema/json/EventMsg.json b/codex-rs/app-server-protocol/schema/json/EventMsg.json index f399912e03..0a754a5711 100644 --- a/codex-rs/app-server-protocol/schema/json/EventMsg.json +++ b/codex-rs/app-server-protocol/schema/json/EventMsg.json @@ -213,6 +213,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupported_input_modality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupported_input_modality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -2993,6 +3018,25 @@ ], "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "LocalShellAction": { "oneOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 0e2f94f4d8..4a1e5ba25b 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -346,6 +346,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -488,6 +513,31 @@ "title": "ModelCapCodexErrorInfo2", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupported_input_modality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupported_input_modality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo2", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -3636,6 +3686,25 @@ ], "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "ItemCompletedNotification": { "properties": { "item": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 9b22ff2207..3161b07990 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -1868,6 +1868,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupported_input_modality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupported_input_modality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -5592,6 +5617,25 @@ } ] }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "InterruptConversationParams": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { @@ -10004,6 +10048,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/v2/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/v1/ForkConversationResponse.json b/codex-rs/app-server-protocol/schema/json/v1/ForkConversationResponse.json index a5838e89e7..9e8929278b 100644 --- a/codex-rs/app-server-protocol/schema/json/v1/ForkConversationResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v1/ForkConversationResponse.json @@ -213,6 +213,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupported_input_modality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupported_input_modality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -2993,6 +3018,25 @@ ], "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "LocalShellAction": { "oneOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v1/ResumeConversationResponse.json b/codex-rs/app-server-protocol/schema/json/v1/ResumeConversationResponse.json index 718b17aa28..b55190b646 100644 --- a/codex-rs/app-server-protocol/schema/json/v1/ResumeConversationResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v1/ResumeConversationResponse.json @@ -213,6 +213,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupported_input_modality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupported_input_modality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -2993,6 +3018,25 @@ ], "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "LocalShellAction": { "oneOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v1/SessionConfiguredNotification.json b/codex-rs/app-server-protocol/schema/json/v1/SessionConfiguredNotification.json index a85b78281b..c5dcae4625 100644 --- a/codex-rs/app-server-protocol/schema/json/v1/SessionConfiguredNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v1/SessionConfiguredNotification.json @@ -213,6 +213,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupported_input_modality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupported_input_modality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -2993,6 +3018,25 @@ ], "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "LocalShellAction": { "oneOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ErrorNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ErrorNotification.json index 17991ebaed..5518f87c27 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ErrorNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ErrorNotification.json @@ -46,6 +46,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -143,6 +168,25 @@ } ] }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "TurnError": { "properties": { "additionalDetails": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index dfbd76a20b..66d0eeeb58 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -65,6 +65,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -345,6 +370,25 @@ ], "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 61b12ceff3..d569a27e70 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -78,6 +78,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -381,6 +406,25 @@ }, "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 50c3d60e37..67a1698d55 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -65,6 +65,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -368,6 +393,25 @@ }, "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index c4fa3b2028..671e7cfa9f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -65,6 +65,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -368,6 +393,25 @@ }, "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 0534f6e16e..388da96aa0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -78,6 +78,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -381,6 +406,25 @@ }, "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index 148bbe7d7d..5ecdebc2d4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -65,6 +65,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -368,6 +393,25 @@ }, "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index c85d7ce97a..4e6347a8a5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -78,6 +78,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -381,6 +406,25 @@ }, "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 44aca775c1..583c5c9540 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -65,6 +65,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -368,6 +393,25 @@ }, "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 055c3fd4a9..6b85934cfb 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -65,6 +65,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -368,6 +393,25 @@ }, "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 798caa5995..ebc556ffe1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -65,6 +65,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -345,6 +370,25 @@ ], "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 65b2a66be0..022586ec48 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -65,6 +65,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -345,6 +370,25 @@ ], "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 1a63c0d7d0..4502148bf7 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -65,6 +65,31 @@ "title": "ModelCapCodexErrorInfo", "type": "object" }, + { + "additionalProperties": false, + "properties": { + "unsupportedInputModality": { + "properties": { + "modality": { + "$ref": "#/definitions/InputModality" + }, + "model": { + "type": "string" + } + }, + "required": [ + "modality", + "model" + ], + "type": "object" + } + }, + "required": [ + "unsupportedInputModality" + ], + "title": "UnsupportedInputModalityCodexErrorInfo", + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -345,6 +370,25 @@ ], "type": "object" }, + "InputModality": { + "description": "Canonical user-input modality tags advertised by a model.", + "oneOf": [ + { + "description": "Plain text turns and tool payloads.", + "enum": [ + "text" + ], + "type": "string" + }, + { + "description": "Image attachments included in user turns.", + "enum": [ + "image" + ], + "type": "string" + } + ] + }, "McpToolCallError": { "properties": { "message": { diff --git a/codex-rs/app-server-protocol/schema/typescript/CodexErrorInfo.ts b/codex-rs/app-server-protocol/schema/typescript/CodexErrorInfo.ts index 20dd2414bb..6860581af5 100644 --- a/codex-rs/app-server-protocol/schema/typescript/CodexErrorInfo.ts +++ b/codex-rs/app-server-protocol/schema/typescript/CodexErrorInfo.ts @@ -1,8 +1,9 @@ // GENERATED CODE! DO NOT MODIFY BY HAND! // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { InputModality } from "./InputModality"; /** * Codex errors that we expose to clients. */ -export type CodexErrorInfo = "context_window_exceeded" | "usage_limit_exceeded" | { "model_cap": { model: string, reset_after_seconds: bigint | null, } } | { "http_connection_failed": { http_status_code: number | null, } } | { "response_stream_connection_failed": { http_status_code: number | null, } } | "internal_server_error" | "unauthorized" | "bad_request" | "sandbox_error" | { "response_stream_disconnected": { http_status_code: number | null, } } | { "response_too_many_failed_attempts": { http_status_code: number | null, } } | "thread_rollback_failed" | "other"; +export type CodexErrorInfo = "context_window_exceeded" | "usage_limit_exceeded" | { "model_cap": { model: string, reset_after_seconds: bigint | null, } } | { "unsupported_input_modality": { model: string, modality: InputModality, } } | { "http_connection_failed": { http_status_code: number | null, } } | { "response_stream_connection_failed": { http_status_code: number | null, } } | "internal_server_error" | "unauthorized" | "bad_request" | "sandbox_error" | { "response_stream_disconnected": { http_status_code: number | null, } } | { "response_too_many_failed_attempts": { http_status_code: number | null, } } | "thread_rollback_failed" | "other"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/CodexErrorInfo.ts b/codex-rs/app-server-protocol/schema/typescript/v2/CodexErrorInfo.ts index a1e65c30c0..2f708b3949 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/CodexErrorInfo.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/CodexErrorInfo.ts @@ -1,6 +1,7 @@ // GENERATED CODE! DO NOT MODIFY BY HAND! // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { InputModality } from "../InputModality"; /** * This translation layer make sure that we expose codex error code in camel case. @@ -8,4 +9,4 @@ * When an upstream HTTP status is available (for example, from the Responses API or a provider), * it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant. */ -export type CodexErrorInfo = "contextWindowExceeded" | "usageLimitExceeded" | { "modelCap": { model: string, reset_after_seconds: bigint | null, } } | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | "other"; +export type CodexErrorInfo = "contextWindowExceeded" | "usageLimitExceeded" | { "modelCap": { model: string, reset_after_seconds: bigint | null, } } | { "unsupportedInputModality": { model: string, modality: InputModality, } } | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | "other"; diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index baafa84677..13897039d9 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -92,6 +92,10 @@ pub enum CodexErrorInfo { model: String, reset_after_seconds: Option, }, + UnsupportedInputModality { + model: String, + modality: InputModality, + }, HttpConnectionFailed { #[serde(rename = "httpStatusCode")] #[ts(rename = "httpStatusCode")] @@ -135,6 +139,9 @@ impl From for CodexErrorInfo { model, reset_after_seconds, }, + CoreCodexErrorInfo::UnsupportedInputModality { model, modality } => { + CodexErrorInfo::UnsupportedInputModality { model, modality } + } CoreCodexErrorInfo::HttpConnectionFailed { http_status_code } => { CodexErrorInfo::HttpConnectionFailed { http_status_code } } @@ -3214,6 +3221,24 @@ mod tests { ); } + #[test] + fn codex_error_info_serializes_unsupported_input_modality_in_camel_case() { + let value = CodexErrorInfo::UnsupportedInputModality { + model: "gpt-5.2-codex".to_string(), + modality: InputModality::Image, + }; + + assert_eq!( + serde_json::to_value(value).unwrap(), + json!({ + "unsupportedInputModality": { + "model": "gpt-5.2-codex", + "modality": "image" + } + }) + ); + } + #[test] fn dynamic_tool_response_serializes_content_items() { let value = serde_json::to_value(DynamicToolCallResponse { diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c522e35b7f..38d1ea4339 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -214,6 +214,7 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::DeveloperInstructions; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; +use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::InitialHistory; @@ -3839,6 +3840,26 @@ fn codex_apps_connector_id(tool: &crate::mcp_connection_manager::ToolInfo) -> Op tool.connector_id.as_deref() } +fn preflight_validate_sampling_input_modalities( + turn_context: &TurnContext, + input: &[ResponseItem], +) -> CodexResult<()> { + let model_supports_images = turn_context + .model_info + .input_modalities + .contains(&InputModality::Image); + let input_contains_images = input.iter().any(ResponseItem::has_input_image); + + if !model_supports_images && input_contains_images { + return Err(CodexErr::UnsupportedInputModality { + model: turn_context.model_info.slug.clone(), + modality: InputModality::Image, + }); + } + + Ok(()) +} + struct SamplingRequestToolSelection<'a> { explicit_app_paths: &'a [String], skill_name_counts_lower: &'a HashMap, @@ -3861,6 +3882,8 @@ async fn run_sampling_request( tool_selection: SamplingRequestToolSelection<'_>, cancellation_token: CancellationToken, ) -> CodexResult { + preflight_validate_sampling_input_modalities(turn_context.as_ref(), &input)?; + let mut mcp_tools = sess .services .mcp_connection_manager diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index 889335824e..aa12762023 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -9,6 +9,7 @@ use chrono::Local; use chrono::Utc; use codex_async_utils::CancelErr; use codex_protocol::ThreadId; +use codex_protocol::openai_models::InputModality; use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::RateLimitSnapshot; @@ -106,6 +107,14 @@ pub enum CodexErr { #[error("{0}")] InvalidRequest(String), + #[error( + "Model {model} does not support {modality} inputs. Remove those inputs or switch models." + )] + UnsupportedInputModality { + model: String, + modality: InputModality, + }, + /// Invalid image. #[error("Image poisoning")] InvalidImageRequest(), @@ -198,6 +207,7 @@ impl CodexErr { | CodexErr::QuotaExceeded | CodexErr::InvalidImageRequest() | CodexErr::InvalidRequest(_) + | CodexErr::UnsupportedInputModality { .. } | CodexErr::RefreshTokenFailed(_) | CodexErr::UnsupportedOperation(_) | CodexErr::Sandbox(_) @@ -593,6 +603,12 @@ impl CodexErr { model: err.model.clone(), reset_after_seconds: err.reset_after_seconds, }, + CodexErr::UnsupportedInputModality { model, modality } => { + CodexErrorInfo::UnsupportedInputModality { + model: model.clone(), + modality: *modality, + } + } CodexErr::RetryLimit(_) => CodexErrorInfo::ResponseTooManyFailedAttempts { http_status_code: self.http_status_code_value(), }, @@ -776,6 +792,21 @@ mod tests { ); } + #[test] + fn unsupported_input_modality_maps_to_protocol() { + let err = CodexErr::UnsupportedInputModality { + model: "gpt-5.2-codex".to_string(), + modality: InputModality::Image, + }; + assert_eq!( + err.to_codex_protocol_error(), + CodexErrorInfo::UnsupportedInputModality { + model: "gpt-5.2-codex".to_string(), + modality: InputModality::Image, + } + ); + } + #[test] fn sandbox_denied_uses_aggregated_output_when_stderr_empty() { let output = ExecToolCallOutput { diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 15114d970b..cfefff0c67 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -72,6 +72,12 @@ pub enum ContentItem { OutputText { text: String }, } +impl ContentItem { + pub fn is_input_image(&self) -> bool { + matches!(self, Self::InputImage { .. }) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "snake_case")] pub enum MessagePhase { @@ -185,6 +191,23 @@ pub enum ResponseItem { Other, } +impl ResponseItem { + /// Returns true when this response item contains at least one input image. + pub fn has_input_image(&self) -> bool { + match self { + Self::Message { content, .. } => content.iter().any(ContentItem::is_input_image), + Self::FunctionCallOutput { output, .. } => { + output.content_items().is_some_and(|items| { + items + .iter() + .any(FunctionCallOutputContentItem::is_input_image) + }) + } + _ => false, + } + } +} + pub const BASE_INSTRUCTIONS_DEFAULT: &str = include_str!("prompts/base_instructions/default.md"); /// Base instructions for the model in a thread. Corresponds to the `instructions` field in the ResponsesAPI. @@ -779,6 +802,12 @@ pub enum FunctionCallOutputContentItem { InputImage { image_url: String }, } +impl FunctionCallOutputContentItem { + pub fn is_input_image(&self) -> bool { + matches!(self, Self::InputImage { .. }) + } +} + /// Converts structured function-call output content into plain text for /// human-readable surfaces. /// @@ -1689,4 +1718,39 @@ mod tests { Ok(()) } + + #[test] + fn response_item_has_input_image_detects_message_images() { + let item = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputText { + text: "hello".to_string(), + }, + ContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + }, + ], + end_turn: None, + phase: None, + }; + assert!(item.has_input_image()); + } + + #[test] + fn response_item_has_input_image_detects_function_output_images() { + let item = ResponseItem::FunctionCallOutput { + call_id: "call_1".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputText { + text: "caption".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BBB".to_string(), + }, + ]), + }; + assert!(item.has_input_image()); + } } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 078ea31d18..240a3e0def 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -34,6 +34,7 @@ use crate::models::ContentItem; use crate::models::ResponseItem; use crate::models::WebSearchAction; use crate::num_format::format_with_separators; +use crate::openai_models::InputModality; use crate::openai_models::ReasoningEffort as ReasoningEffortConfig; use crate::parse_command::ParsedCommand; use crate::plan_tool::UpdatePlanArgs; @@ -960,6 +961,10 @@ pub enum CodexErrorInfo { model: String, reset_after_seconds: Option, }, + UnsupportedInputModality { + model: String, + modality: InputModality, + }, HttpConnectionFailed { http_status_code: Option, },