diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 1557df56f7..8a3613daa8 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -670,6 +670,26 @@ "title": "InputImageContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioContentItem", + "type": "object" + }, { "properties": { "text": { @@ -1251,6 +1271,26 @@ "title": "InputImageFunctionCallOutputContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioFunctionCallOutputContentItem", + "type": "object" + }, { "properties": { "encrypted_content": { @@ -4864,6 +4904,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 42038ba371..db75d4a0d4 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -5562,6 +5562,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 7588992563..4cd4636b9a 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -9012,6 +9012,26 @@ "title": "InputImageContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioContentItem", + "type": "object" + }, { "properties": { "text": { @@ -10628,6 +10648,26 @@ "title": "InputImageFunctionCallOutputContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioFunctionCallOutputContentItem", + "type": "object" + }, { "properties": { "encrypted_content": { @@ -21234,6 +21274,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 7c265ebbff..2ba1134dc6 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -5205,6 +5205,26 @@ "title": "InputImageContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioContentItem", + "type": "object" + }, { "properties": { "text": { @@ -6821,6 +6841,26 @@ "title": "InputImageFunctionCallOutputContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioFunctionCallOutputContentItem", + "type": "object" + }, { "properties": { "encrypted_content": { @@ -19013,6 +19053,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 58e74d1a3c..dd8353e729 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -1378,6 +1378,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 4244c2bb19..5e12386041 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -1378,6 +1378,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index dad669b139..b20400d16a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -97,6 +97,26 @@ "title": "InputImageContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioContentItem", + "type": "object" + }, { "properties": { "text": { @@ -185,6 +205,26 @@ "title": "InputImageFunctionCallOutputContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioFunctionCallOutputContentItem", + "type": "object" + }, { "properties": { "encrypted_content": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 46bcb0ac64..1008400bc9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -1653,6 +1653,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 03b4e5db33..fcdc2c6630 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -2259,6 +2259,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 98c2174473..0809b9c3ee 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -2050,6 +2050,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index b719f1dab3..17bc5bb5b6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -2050,6 +2050,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 8a84fae30e..dd9458d9c9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -2050,6 +2050,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index 0cd7d9ae7b..0ff306cf1b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -159,6 +159,26 @@ "title": "InputImageContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioContentItem", + "type": "object" + }, { "properties": { "text": { @@ -247,6 +267,26 @@ "title": "InputImageFunctionCallOutputContentItem", "type": "object" }, + { + "properties": { + "audio_url": { + "type": "string" + }, + "type": { + "enum": [ + "input_audio" + ], + "title": "InputAudioFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "audio_url", + "type" + ], + "title": "InputAudioFunctionCallOutputContentItem", + "type": "object" + }, { "properties": { "encrypted_content": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 8f339d196e..28081e377a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -2285,6 +2285,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index 1c2f9f5bbc..8bd889b655 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -2050,6 +2050,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 1b10b34d58..e4a280e1ac 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -2259,6 +2259,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index ba1f718b09..35c8e432d6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -2050,6 +2050,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 44ecb9f7b2..144f0d981c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -2050,6 +2050,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index b63f3b6eb6..554a4cff6b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -1653,6 +1653,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index b38792e448..477e57e43e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -472,6 +472,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 55fb943eef..484c34b071 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -1653,6 +1653,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 1de6c7d92c..06fdc6e81e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -1653,6 +1653,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json index 63317f13ba..2a3627cbd0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json @@ -166,6 +166,46 @@ "title": "LocalImageUserInput", "type": "object" }, + { + "properties": { + "type": { + "enum": [ + "audio" + ], + "title": "AudioUserInputType", + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "type", + "url" + ], + "title": "AudioUserInput", + "type": "object" + }, + { + "properties": { + "path": { + "type": "string" + }, + "type": { + "enum": [ + "localAudio" + ], + "title": "LocalAudioUserInputType", + "type": "string" + } + }, + "required": [ + "path", + "type" + ], + "title": "LocalAudioUserInput", + "type": "object" + }, { "properties": { "name": { diff --git a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts index 21cd8d02f3..9e53b5fc25 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ContentItem.ts @@ -3,4 +3,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ImageDetail } from "./ImageDetail"; -export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "output_text", text: string, }; +export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts index cd18908145..6c2ab2afa1 100644 --- a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts @@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail"; * Responses API compatible content items that can be returned by a tool call. * This is a subset of ContentItem with the types we support as function call outputs. */ -export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "encrypted_content", encrypted_content: string, }; +export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "input_audio", audio_url: string, } | { "type": "encrypted_content", encrypted_content: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts b/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts index 2ac37c5228..c268cb4f8d 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/UserInput.ts @@ -8,4 +8,4 @@ export type UserInput = { "type": "text", text: string, /** * UI-defined spans within `text` used to render or persist special elements. */ -text_elements: Array, } | { "type": "image", detail?: ImageDetail, url: string, } | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, }; +text_elements: Array, } | { "type": "image", detail?: ImageDetail, url: string, } | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", url: string, } | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, }; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs index fd2a28ddde..580a57d592 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs @@ -2545,6 +2545,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { path: PathBuf::from("local/image.png"), detail: Some(ImageDetail::Original), }, + CoreUserInput::Audio { + audio_url: "data:audio/wav;base64,AAA".to_string(), + }, + CoreUserInput::LocalAudio { + path: PathBuf::from("local/audio.mp3"), + }, CoreUserInput::Skill { name: "skill-creator".to_string(), path: PathBuf::from("/repo/.codex/skills/skill-creator/SKILL.md"), @@ -2574,6 +2580,12 @@ fn core_turn_item_into_thread_item_converts_supported_variants() { path: PathBuf::from("local/image.png"), detail: Some(ImageDetail::Original), }, + UserInput::Audio { + url: "data:audio/wav;base64,AAA".to_string(), + }, + UserInput::LocalAudio { + path: PathBuf::from("local/audio.mp3"), + }, UserInput::Skill { name: "skill-creator".to_string(), path: PathBuf::from("/repo/.codex/skills/skill-creator/SKILL.md"), @@ -3022,7 +3034,7 @@ fn mcp_tool_call_app_context_serializes_missing_mixed_version_fields_as_null() { } #[test] -fn user_input_into_core_preserves_image_detail() { +fn user_input_into_core_preserves_media_fields() { assert_eq!( UserInput::Image { url: "https://example.com/image.png".to_string(), @@ -3046,6 +3058,26 @@ fn user_input_into_core_preserves_image_detail() { detail: Some(ImageDetail::Original), } ); + + assert_eq!( + UserInput::Audio { + url: "data:audio/wav;base64,AAA".to_string(), + } + .into_core(), + CoreUserInput::Audio { + audio_url: "data:audio/wav;base64,AAA".to_string(), + } + ); + + assert_eq!( + UserInput::LocalAudio { + path: PathBuf::from("local/audio.mp3"), + } + .into_core(), + CoreUserInput::LocalAudio { + path: PathBuf::from("local/audio.mp3"), + } + ); } #[test] diff --git a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs index 62a289cd25..cb8473d49a 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs @@ -308,6 +308,12 @@ pub enum UserInput { detail: Option, path: PathBuf, }, + Audio { + url: String, + }, + LocalAudio { + path: PathBuf, + }, Skill { name: String, path: PathBuf, @@ -333,6 +339,8 @@ impl UserInput { detail, }, UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail }, + UserInput::Audio { url } => CoreUserInput::Audio { audio_url: url }, + UserInput::LocalAudio { path } => CoreUserInput::LocalAudio { path }, UserInput::Skill { name, path } => CoreUserInput::Skill { name, path }, UserInput::Mention { name, path } => CoreUserInput::Mention { name, path }, } @@ -354,6 +362,8 @@ impl From for UserInput { detail, }, CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail }, + CoreUserInput::Audio { audio_url } => UserInput::Audio { url: audio_url }, + CoreUserInput::LocalAudio { path } => UserInput::LocalAudio { path }, CoreUserInput::Skill { name, path } => UserInput::Skill { name, path }, CoreUserInput::Mention { name, path } => UserInput::Mention { name, path }, _ => unreachable!("unsupported user input variant"), @@ -367,6 +377,8 @@ impl UserInput { UserInput::Text { text, .. } => text.chars().count(), UserInput::Image { .. } | UserInput::LocalImage { .. } + | UserInput::Audio { .. } + | UserInput::LocalAudio { .. } | UserInput::Skill { .. } | UserInput::Mention { .. } => 0, } diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs index 482a6ff1dd..ed23ac88b5 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_read.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs @@ -2078,6 +2078,8 @@ fn turn_user_texts(turns: &[codex_app_server_protocol::Turn]) -> Vec<&str> { UserInput::Text { text, .. } => Some(text.as_str()), UserInput::Image { .. } | UserInput::LocalImage { .. } + | UserInput::Audio { .. } + | UserInput::LocalAudio { .. } | UserInput::Skill { .. } | UserInput::Mention { .. } => None, }, diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rs index 2c944ad7f6..749c3f3f4b 100644 --- a/codex-rs/core/src/agent/control.rs +++ b/codex-rs/core/src/agent/control.rs @@ -750,6 +750,10 @@ pub(crate) fn render_input_preview(input: &[UserInput]) -> String { UserInput::LocalImage { path, .. } => { format!("[local_image:{}]", path.display()) } + UserInput::Audio { .. } => "[audio]".to_string(), + UserInput::LocalAudio { path } => { + format!("[local_audio:{}]", path.display()) + } UserInput::Skill { name, path, .. } => { format!("[skill:${name}]({})", path.display()) } diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 97016c0bd4..6d4cbc7ebd 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -252,7 +252,7 @@ fn has_subagent_notification(history_items: &[ResponseItem]) -> bool { ContentItem::InputText { text } | ContentItem::OutputText { text } => { SubagentNotification::matches_text(text) } - ContentItem::InputImage { .. } => false, + ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => false, }) }) } @@ -267,7 +267,7 @@ fn history_contains_text(history_items: &[ResponseItem], needle: &str) -> bool { ContentItem::InputText { text } | ContentItem::OutputText { text } => { text.contains(needle) } - ContentItem::InputImage { .. } => false, + ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => false, }) }) } @@ -290,7 +290,9 @@ fn history_contains_assistant_inter_agent_communication( .as_ref() == Some(expected) } - ContentItem::InputText { .. } | ContentItem::InputImage { .. } => false, + ContentItem::InputText { .. } + | ContentItem::InputImage { .. } + | ContentItem::InputAudio { .. } => false, }) }) } diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 599f518aa0..a082c7d20a 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -484,7 +484,7 @@ pub fn content_items_to_text(content: &[ContentItem]) -> Option { pieces.push(text.as_str()); } } - ContentItem::InputImage { .. } => {} + ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => {} } } if pieces.is_empty() { diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index 58cbfb1218..a0bb9e11ae 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -516,7 +516,7 @@ fn message_text_token_count(item: &ResponseItem) -> usize { ContentItem::InputText { text } | ContentItem::OutputText { text } => { approx_token_count(text) } - ContentItem::InputImage { .. } => 0, + ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => 0, }) .sum() } @@ -556,7 +556,9 @@ fn truncate_message_text_to_token_budget( truncated_content.push(content_item); } } - ContentItem::InputImage { .. } => truncated_content.push(content_item), + ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => { + truncated_content.push(content_item); + } } } diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index 6a474567b3..f3b714c8ef 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -113,6 +113,11 @@ fn parse_user_message(message: &[ContentItem]) -> Option { detail: *detail, }); } + ContentItem::InputAudio { audio_url } => { + content.push(UserInput::Audio { + audio_url: audio_url.clone(), + }); + } ContentItem::OutputText { text } => { warn!("Output text in user message: {}", text); } diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs index 34ea6acad9..bc8eaf8b69 100644 --- a/codex-rs/core/src/guardian/tests.rs +++ b/codex-rs/core/src/guardian/tests.rs @@ -345,7 +345,7 @@ fn response_item_contains_message_text(item: &ResponseItem, needle: &str) -> boo }; content.iter().any(|item| match item { ContentItem::InputText { text } | ContentItem::OutputText { text } => text.contains(needle), - ContentItem::InputImage { .. } => false, + ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => false, }) } diff --git a/codex-rs/core/src/hook_runtime.rs b/codex-rs/core/src/hook_runtime.rs index 3adbea420c..c9fe099332 100644 --- a/codex-rs/core/src/hook_runtime.rs +++ b/codex-rs/core/src/hook_runtime.rs @@ -844,7 +844,9 @@ mod tests { .iter() .map(|item| match item { ContentItem::InputText { text } => text.as_str(), - ContentItem::InputImage { .. } | ContentItem::OutputText { .. } => { + ContentItem::InputImage { .. } + | ContentItem::InputAudio { .. } + | ContentItem::OutputText { .. } => { panic!("expected input text content, got {item:?}") } }) diff --git a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs index f888811052..9e74b01a50 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_spec.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_spec.rs @@ -559,10 +559,14 @@ fn create_collab_input_items_schema() -> JsonSchema { "image_url".to_string(), JsonSchema::string(Some("Image URL when type is image.".to_string())), ), + ( + "audio_url".to_string(), + JsonSchema::string(Some("Audio URL when type is audio.".to_string())), + ), ( "path".to_string(), JsonSchema::string(Some( - "Path when type is local_image/skill, or structured mention target such as app:// or plugin://@ when type is mention." + "Path when type is local_image/local_audio/skill, or structured mention target such as app:// or plugin://@ when type is mention." .to_string(), )), ), diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 3c73aaa5d4..adb2e0261f 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -363,7 +363,9 @@ fn recent_images(history: &[ResponseItem], count: usize) -> Vec { ResponseItem::Message { content, .. } => { image_urls.extend(content.iter().rev().filter_map(|item| match item { ContentItem::InputImage { image_url, .. } => Some(image_url.clone()), - ContentItem::InputText { .. } | ContentItem::OutputText { .. } => None, + ContentItem::InputText { .. } + | ContentItem::InputAudio { .. } + | ContentItem::OutputText { .. } => None, })); } ResponseItem::FunctionCallOutput { @@ -417,6 +419,7 @@ fn output_image_urls(output: &FunctionCallOutputPayload) -> impl Iterator Some(image_url.clone()), FunctionCallOutputContentItem::InputText { .. } + | FunctionCallOutputContentItem::InputAudio { .. } | FunctionCallOutputContentItem::EncryptedContent { .. } => None, }) } diff --git a/codex-rs/ext/skills/src/selection.rs b/codex-rs/ext/skills/src/selection.rs index a708b18d6c..946115e5c3 100644 --- a/codex-rs/ext/skills/src/selection.rs +++ b/codex-rs/ext/skills/src/selection.rs @@ -36,7 +36,11 @@ pub(crate) fn collect_explicit_skill_mentions( blocked_plain_names.insert(name.clone()); select_by_path(catalog, path, &mut seen, &mut selected); } - UserInput::Text { .. } | UserInput::Image { .. } | UserInput::LocalImage { .. } => {} + UserInput::Text { .. } + | UserInput::Image { .. } + | UserInput::LocalImage { .. } + | UserInput::Audio { .. } + | UserInput::LocalAudio { .. } => {} UserInput::Mention { .. } => {} _ => {} } diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 56d0bce102..1774921288 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -718,6 +718,9 @@ pub enum ContentItem { #[ts(optional)] detail: Option, }, + InputAudio { + audio_url: String, + }, OutputText { text: String, }, @@ -1334,16 +1337,37 @@ fn should_serialize_reasoning_content(content: &Option } } -fn local_image_error_placeholder( +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LocalMediaKind { + Audio, + Image, +} + +impl LocalMediaKind { + fn name(self) -> &'static str { + match self { + Self::Audio => "audio", + Self::Image => "image", + } + } +} + +fn local_media_error_placeholder( path: &std::path::Path, error: impl std::fmt::Display, + media_kind: LocalMediaKind, ) -> ContentItem { + let media_name = media_kind.name(); + let path = path.display(); ContentItem::InputText { - text: format!( - "Codex could not read the local image at `{}`: {}", - path.display(), - error - ), + text: format!("Codex could not read the local {media_name} at `{path}`: {error}"), + } +} + +fn local_media_kind_unsupported(media_kind: LocalMediaKind) -> ContentItem { + let media_name = media_kind.name(); + ContentItem::InputText { + text: format!("Codex does not support local {media_name} input yet."), } } @@ -1431,13 +1455,21 @@ pub fn local_image_content_items_with_label_number( | ImageProcessingError::Encode { .. } | ImageProcessingError::InvalidDataUrl { .. } | ImageProcessingError::ImageTooLarge { .. } => { - vec![local_image_error_placeholder(path, &err)] + vec![local_media_error_placeholder( + path, + &err, + LocalMediaKind::Image, + )] } ImageProcessingError::Decode { .. } if err.is_invalid_image() => { vec![invalid_image_error_placeholder(path, &err)] } ImageProcessingError::Decode { .. } => { - vec![local_image_error_placeholder(path, &err)] + vec![local_media_error_placeholder( + path, + &err, + LocalMediaKind::Image, + )] } ImageProcessingError::UnsupportedImageFormat { mime } => { vec![unsupported_image_error_placeholder(path, mime)] @@ -1647,9 +1679,20 @@ impl ResponseInputItem { detail, ), }, - Err(err) => vec![local_image_error_placeholder(&path, err)], + Err(err) => vec![local_media_error_placeholder( + &path, + err, + LocalMediaKind::Image, + )], } } + UserInput::Audio { .. } => vec![ContentItem::InputText { + text: "Codex does not support audio input yet.".to_string(), + }], + // TODO: Load local audio inputs once they are supported. + UserInput::LocalAudio { .. } => { + vec![local_media_kind_unsupported(LocalMediaKind::Audio)] + } UserInput::Skill { .. } | UserInput::Mention { .. } => Vec::new(), // Tool bodies are injected later in core }) .collect::>(), @@ -1707,6 +1750,10 @@ pub enum FunctionCallOutputContentItem { #[ts(optional)] detail: Option, }, + // Do not rename, these are serialized and used directly in the responses API. + InputAudio { + audio_url: String, + }, EncryptedContent { encrypted_content: String, }, @@ -1734,6 +1781,7 @@ pub fn function_call_output_content_items_to_text( } FunctionCallOutputContentItem::InputText { .. } | FunctionCallOutputContentItem::InputImage { .. } + | FunctionCallOutputContentItem::InputAudio { .. } | FunctionCallOutputContentItem::EncryptedContent { .. } => None, }) .collect::>(); @@ -3192,6 +3240,44 @@ mod tests { Ok(()) } + #[test] + fn replaces_unsupported_audio_user_input_with_placeholder() { + let item = ResponseInputItem::from(vec![UserInput::Audio { + audio_url: "data:audio/wav;base64,abc".to_string(), + }]); + + assert_eq!( + item, + ResponseInputItem::Message { + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "Codex does not support audio input yet.".to_string(), + }], + phase: None, + } + ); + } + + #[test] + fn replaces_unsupported_local_audio_user_input_with_placeholder() -> Result<()> { + let item = ResponseInputItem::from(vec![UserInput::LocalAudio { + path: "sample.mp3".into(), + }]); + + assert_eq!( + item, + ResponseInputItem::Message { + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "Codex does not support local audio input yet.".to_string(), + }], + phase: None, + } + ); + + Ok(()) + } + #[test] fn image_user_input_preserves_requested_detail() -> Result<()> { let image_url = "data:image/png;base64,abc".to_string(); diff --git a/codex-rs/protocol/src/user_input.rs b/codex-rs/protocol/src/user_input.rs index ce4cf99eba..e8252d6c37 100644 --- a/codex-rs/protocol/src/user_input.rs +++ b/codex-rs/protocol/src/user_input.rs @@ -29,7 +29,6 @@ pub enum UserInput { #[ts(optional)] detail: Option, }, - /// Local image path provided by the user. This will be converted to an /// `Image` variant (base64 data URL) during request serialization. LocalImage { @@ -38,6 +37,10 @@ pub enum UserInput { #[ts(optional)] detail: Option, }, + /// Pre-encoded audio data URI. Audio input is not yet forwarded to the Responses API. + Audio { audio_url: String }, + /// Local audio path provided by the user. Local audio is not converted in this layer. + LocalAudio { path: std::path::PathBuf }, /// Skill selected by the user (name + path to SKILL.md). Skill { diff --git a/codex-rs/rollout/src/search.rs b/codex-rs/rollout/src/search.rs index 805a6b274e..67a2723b49 100644 --- a/codex-rs/rollout/src/search.rs +++ b/codex-rs/rollout/src/search.rs @@ -293,7 +293,7 @@ fn conversation_text_from_item(item: &RolloutItem) -> Option { fn content_item_text(item: &ContentItem) -> Option<&str> { match item { ContentItem::InputText { text } | ContentItem::OutputText { text } => Some(text.as_str()), - ContentItem::InputImage { .. } => None, + ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => None, } } diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs index f2eed18fac..7d58b35e89 100644 --- a/codex-rs/tools/src/tool_output.rs +++ b/codex-rs/tools/src/tool_output.rs @@ -202,6 +202,9 @@ fn response_input_to_code_mode_result(response: ResponseInputItem) -> JsonValue detail: detail.or(Some(DEFAULT_IMAGE_DETAIL)), } } + codex_protocol::models::ContentItem::InputAudio { audio_url } => { + FunctionCallOutputContentItem::InputAudio { audio_url } + } }) .collect::>(), ), @@ -233,8 +236,14 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) -> { Some(image_url.clone()) } + FunctionCallOutputContentItem::InputAudio { audio_url } + if !audio_url.trim().is_empty() => + { + Some(audio_url.clone()) + } FunctionCallOutputContentItem::InputText { .. } | FunctionCallOutputContentItem::InputImage { .. } + | FunctionCallOutputContentItem::InputAudio { .. } | FunctionCallOutputContentItem::EncryptedContent { .. } => None, }) .collect::>() diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__live_app_server_user_message_omits_unsupported_media.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__live_app_server_user_message_omits_unsupported_media.snap new file mode 100644 index 0000000000..6f7384c1b3 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__live_app_server_user_message_omits_unsupported_media.snap @@ -0,0 +1,6 @@ +--- +source: tui/src/chatwidget/tests/app_server.rs +expression: "lines_to_single_string(&inserted[0])" +--- + +› Please inspect the attachments. diff --git a/codex-rs/tui/src/chatwidget/tests/app_server.rs b/codex-rs/tui/src/chatwidget/tests/app_server.rs index 4db7692ff3..1181120aaa 100644 --- a/codex-rs/tui/src/chatwidget/tests/app_server.rs +++ b/codex-rs/tui/src/chatwidget/tests/app_server.rs @@ -538,6 +538,43 @@ async fn live_app_server_user_message_item_completed_does_not_duplicate_rendered assert!(drain_insert_history(&mut rx).is_empty()); } +#[tokio::test] +async fn live_app_server_user_message_omits_unsupported_media() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + + chat.handle_server_notification( + ServerNotification::ItemCompleted(ItemCompletedNotification { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + completed_at_ms: 0, + item: AppServerThreadItem::UserMessage { + id: "user-1".to_string(), + client_id: None, + content: vec![ + AppServerUserInput::Text { + text: "Please inspect the attachments.".to_string(), + text_elements: Vec::new(), + }, + AppServerUserInput::Audio { + url: "https://example.com/one.wav".to_string(), + }, + AppServerUserInput::LocalAudio { + path: test_path_buf("/tmp/two.wav"), + }, + ], + }, + }), + /*replay_kind*/ None, + ); + + let inserted = drain_insert_history(&mut rx); + assert_eq!(inserted.len(), 1); + assert_chatwidget_snapshot!( + "live_app_server_user_message_omits_unsupported_media", + lines_to_single_string(&inserted[0]), + ); +} + #[tokio::test] async fn live_app_server_turn_completed_clears_working_status_after_answer_item() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; diff --git a/codex-rs/tui/src/chatwidget/user_messages.rs b/codex-rs/tui/src/chatwidget/user_messages.rs index 255ebb6e11..12f0207656 100644 --- a/codex-rs/tui/src/chatwidget/user_messages.rs +++ b/codex-rs/tui/src/chatwidget/user_messages.rs @@ -589,7 +589,11 @@ pub(crate) fn mention_bindings_from_user_inputs( path: path.clone(), }) } - UserInput::Text { .. } | UserInput::Image { .. } | UserInput::LocalImage { .. } => None, + UserInput::Text { .. } + | UserInput::Image { .. } + | UserInput::LocalImage { .. } + | UserInput::Audio { .. } + | UserInput::LocalAudio { .. } => None, }) .collect(); mention_bindings.sort_by_key(|binding| { @@ -655,7 +659,10 @@ impl ChatWidget { match item { UserInput::Text { text, .. } => message.push_str(text), UserInput::Image { .. } | UserInput::LocalImage { .. } => image_count += 1, - UserInput::Skill { .. } | UserInput::Mention { .. } => {} + UserInput::Audio { .. } + | UserInput::LocalAudio { .. } + | UserInput::Skill { .. } + | UserInput::Mention { .. } => {} } } @@ -666,6 +673,12 @@ impl ChatWidget { } pub(crate) fn user_message_display_from_inputs(items: &[UserInput]) -> UserMessageDisplay { + if items + .iter() + .any(|item| matches!(item, UserInput::Audio { .. } | UserInput::LocalAudio { .. })) + { + tracing::warn!("audio user inputs are not supported by the TUI and will be omitted"); + } let mut message = String::new(); let mut remote_image_urls = Vec::new(); let mut local_images = Vec::new(); @@ -694,7 +707,10 @@ impl ChatWidget { ), UserInput::Image { url, .. } => remote_image_urls.push(url.clone()), UserInput::LocalImage { path, .. } => local_images.push(path.clone()), - UserInput::Skill { .. } | UserInput::Mention { .. } => {} + UserInput::Audio { .. } + | UserInput::LocalAudio { .. } + | UserInput::Skill { .. } + | UserInput::Mention { .. } => {} } } diff --git a/codex-rs/tui/src/thread_transcript.rs b/codex-rs/tui/src/thread_transcript.rs index e9b54a7769..b051d3ed2b 100644 --- a/codex-rs/tui/src/thread_transcript.rs +++ b/codex-rs/tui/src/thread_transcript.rs @@ -13,6 +13,7 @@ use crate::history_cell::split_reasoning_summary_parts; use crate::multi_agents::sub_agent_activity_summary; use codex_app_server_protocol::Thread; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::UserInput; use codex_protocol::ThreadId; use codex_protocol::items::UserMessageItem; use ratatui::style::Stylize as _; @@ -54,6 +55,17 @@ pub(crate) fn thread_to_transcript_cells( client_id, content, } => { + if content.iter().any(|input| { + matches!( + input, + UserInput::Audio { .. } | UserInput::LocalAudio { .. } + ) + }) { + tracing::warn!( + user_message_id = id, + "audio user inputs are not supported by the TUI and will be omitted" + ); + } let item = UserMessageItem { id: id.clone(), client_id: client_id.clone(), diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/utils/output-truncation/src/lib.rs index 4fcff7dce4..e2b4a45c88 100644 --- a/codex-rs/utils/output-truncation/src/lib.rs +++ b/codex-rs/utils/output-truncation/src/lib.rs @@ -38,12 +38,20 @@ pub fn formatted_truncate_text_content_items_with_policy( .filter_map(|item| match item { FunctionCallOutputContentItem::InputText { text } => Some(text.as_str()), FunctionCallOutputContentItem::InputImage { .. } + | FunctionCallOutputContentItem::InputAudio { .. } | FunctionCallOutputContentItem::EncryptedContent { .. } => None, }) .collect::>(); + let without_audio = || { + items + .iter() + .filter(|item| !matches!(item, FunctionCallOutputContentItem::InputAudio { .. })) + .cloned() + .collect() + }; if text_segments.is_empty() { - return (items.to_vec(), None); + return (without_audio(), None); } let mut combined = String::new(); @@ -55,7 +63,7 @@ pub fn formatted_truncate_text_content_items_with_policy( } if combined.len() <= policy.byte_budget() { - return (items.to_vec(), None); + return (without_audio(), None); } let original_token_count = approx_token_count(&combined); @@ -69,6 +77,7 @@ pub fn formatted_truncate_text_content_items_with_policy( detail: *detail, }) } + FunctionCallOutputContentItem::InputAudio { .. } => None, FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => { Some(FunctionCallOutputContentItem::EncryptedContent { encrypted_content: encrypted_content.clone(), @@ -127,6 +136,7 @@ pub fn truncate_function_output_items_with_policy( detail: *detail, }); } + FunctionCallOutputContentItem::InputAudio { .. } => {} FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => { out.push(FunctionCallOutputContentItem::EncryptedContent { encrypted_content: encrypted_content.clone(), diff --git a/codex-rs/utils/output-truncation/src/truncate_tests.rs b/codex-rs/utils/output-truncation/src/truncate_tests.rs index e8907cf8f8..7f8a7cdf60 100644 --- a/codex-rs/utils/output-truncation/src/truncate_tests.rs +++ b/codex-rs/utils/output-truncation/src/truncate_tests.rs @@ -208,7 +208,7 @@ fn formatted_truncate_text_content_items_with_policy_preserves_empty_leading_tex } #[test] -fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_images() { +fn formatted_truncate_text_content_items_with_policy_merges_text_and_drops_audio() { let items = vec![ FunctionCallOutputContentItem::InputText { text: "abcd".to_string(), @@ -220,6 +220,9 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_ima FunctionCallOutputContentItem::InputText { text: "efgh".to_string(), }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "audio:one".to_string(), + }, FunctionCallOutputContentItem::InputText { text: "ijkl".to_string(), }, @@ -280,7 +283,7 @@ fn formatted_truncate_text_content_items_with_policy_preserves_encrypted_content } #[test] -fn truncate_function_output_items_with_policy_preserves_encrypted_content() { +fn truncate_function_output_items_with_policy_preserves_encrypted_content_and_drops_audio() { let items = vec![ FunctionCallOutputContentItem::InputText { text: "abcdefgh".to_string(), @@ -288,6 +291,9 @@ fn truncate_function_output_items_with_policy_preserves_encrypted_content() { FunctionCallOutputContentItem::EncryptedContent { encrypted_content: "enc_opaque".to_string(), }, + FunctionCallOutputContentItem::InputAudio { + audio_url: "audio:one".to_string(), + }, ]; let output = truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(2));