mirror of
https://github.com/openai/codex.git
synced 2026-10-31 19:28:49 +00:00
Add audio file references with explicit runtime rejection (#58430)
## What changed - Allow audio inputs to carry uploaded file references via `fileId` in the app-server API and `file_id` in model content, preserving existing inline audio shapes and Python class names. - Preserve mixed inline and file-backed audio ordering through history reconstruction and migration, and budget file-backed audio conservatively when duration is unknown. - Keep uploaded audio disabled at runtime: reject it before inference or raw history injection, preserve accompanying text, and prevent rejected attachments from reaching follow-up or resumed requests. ## Testing Add coverage for wire-format round trips, SDK compatibility, mixed audio history ordering, and rejection during turn start, steering, task completion, and raw injection. GitOrigin-RevId: 70895c47a2494c0c9936348ad1250eb1b7b892a6
This commit is contained in:
Hung
+54
-9
@@ -719,10 +719,34 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url"
|
||||
],
|
||||
"title": "AudioUrlContentItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"file_id": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"file_id"
|
||||
],
|
||||
"title": "FileIdContentItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
"input_audio"
|
||||
@@ -732,7 +756,6 @@
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url",
|
||||
"type"
|
||||
],
|
||||
"title": "InputAudioContentItem",
|
||||
@@ -6193,6 +6216,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -6200,14 +6249,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -6943,6 +6943,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -6950,14 +6976,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
+54
-9
@@ -10960,10 +10960,34 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url"
|
||||
],
|
||||
"title": "AudioUrlContentItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"file_id": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"file_id"
|
||||
],
|
||||
"title": "FileIdContentItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
"input_audio"
|
||||
@@ -10973,7 +10997,6 @@
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url",
|
||||
"type"
|
||||
],
|
||||
"title": "InputAudioContentItem",
|
||||
@@ -27301,6 +27324,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -27308,14 +27357,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
+54
-9
@@ -6741,10 +6741,34 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url"
|
||||
],
|
||||
"title": "AudioUrlContentItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"file_id": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"file_id"
|
||||
],
|
||||
"title": "FileIdContentItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
"input_audio"
|
||||
@@ -6754,7 +6778,6 @@
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url",
|
||||
"type"
|
||||
],
|
||||
"title": "InputAudioContentItem",
|
||||
@@ -25004,6 +25027,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -25011,14 +25060,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
+27
-5
@@ -1846,6 +1846,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -1853,14 +1879,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -1846,6 +1846,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -1853,14 +1879,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
+27
-4
@@ -132,10 +132,34 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url"
|
||||
],
|
||||
"title": "AudioUrlContentItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"file_id": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"file_id"
|
||||
],
|
||||
"title": "FileIdContentItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
"input_audio"
|
||||
@@ -145,7 +169,6 @@
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url",
|
||||
"type"
|
||||
],
|
||||
"title": "InputAudioContentItem",
|
||||
|
||||
@@ -2200,6 +2200,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2207,14 +2233,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -2940,6 +2940,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2947,14 +2973,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -1878,6 +1878,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -1885,14 +1911,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -2795,6 +2795,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2802,14 +2828,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
+27
-5
@@ -2731,6 +2731,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2738,14 +2764,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -2795,6 +2795,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2802,14 +2828,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -194,10 +194,34 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"audio_url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url"
|
||||
],
|
||||
"title": "AudioUrlContentItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"file_id": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"file_id"
|
||||
],
|
||||
"title": "FileIdContentItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"description": "Inline audio data or an uploaded file loaded by the Responses API.",
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
"input_audio"
|
||||
@@ -207,7 +231,6 @@
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"audio_url",
|
||||
"type"
|
||||
],
|
||||
"title": "InputAudioContentItem",
|
||||
|
||||
@@ -3018,6 +3018,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -3025,14 +3051,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -2731,6 +2731,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2738,14 +2764,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -2940,6 +2940,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2947,14 +2973,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
+27
-5
@@ -2731,6 +2731,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2738,14 +2764,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -2200,6 +2200,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2207,14 +2233,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -2731,6 +2731,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2738,14 +2764,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
+27
-5
@@ -2200,6 +2200,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2207,14 +2233,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -667,6 +667,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -674,14 +700,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -2200,6 +2200,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2207,14 +2233,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -2200,6 +2200,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -2207,14 +2233,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
@@ -196,6 +196,32 @@
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"url"
|
||||
],
|
||||
"title": "UrlUserInput",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"fileId": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"fileId"
|
||||
],
|
||||
"title": "FileIdUserInput",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
@@ -203,14 +229,10 @@
|
||||
],
|
||||
"title": "AudioUserInputType",
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"url"
|
||||
"type"
|
||||
],
|
||||
"title": "AudioUserInput",
|
||||
"type": "object"
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -3,4 +3,4 @@
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { ImageDetail } from "./ImageDetail";
|
||||
|
||||
export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", audio_url: string, } | { "type": "output_text", text: string, annotations?: unknown[], logprobs?: unknown[], };
|
||||
export type ContentItem = { "type": "input_text", text: string, } | { "type": "input_image", detail?: ImageDetail, } & ({ image_url: string, } | { file_id: string, }) | { "type": "input_audio", } & ({ audio_url: string, } | { file_id: string, }) | { "type": "output_text", text: string, annotations?: unknown[], logprobs?: unknown[], };
|
||||
|
||||
@@ -8,4 +8,4 @@ export type UserInput = { "type": "text", text: string,
|
||||
/**
|
||||
* UI-defined spans within `text` used to render or persist special elements.
|
||||
*/
|
||||
text_elements: Array<TextElement>, } | { "type": "image", detail?: ImageDetail, } & ({ url: string, } | { fileId: string, }) | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", url: string, } | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, };
|
||||
text_elements: Array<TextElement>, } | { "type": "image", detail?: ImageDetail, } & ({ url: string, } | { fileId: string, }) | { "type": "localImage", detail?: ImageDetail, path: string, } | { "type": "audio", } & ({ url: string, } | { fileId: string, }) | { "type": "localAudio", path: string, } | { "type": "skill", name: string, path: string, } | { "type": "mention", name: string, path: string, };
|
||||
|
||||
@@ -59,6 +59,7 @@ use codex_protocol::protocol::ThreadRolledBackEvent;
|
||||
use codex_protocol::protocol::TurnAbortedEvent;
|
||||
use codex_protocol::protocol::TurnCompleteEvent;
|
||||
use codex_protocol::protocol::TurnStartedEvent;
|
||||
use codex_protocol::protocol::UserMessageAudioKind;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_protocol::protocol::UserMessageImageKind;
|
||||
use codex_protocol::protocol::ViewImageToolCallEvent;
|
||||
@@ -1749,8 +1750,48 @@ impl ThreadHistoryBuilder {
|
||||
detail: payload.local_image_details.get(idx).copied().flatten(),
|
||||
});
|
||||
}
|
||||
if let Some(audio) = &payload.audio {
|
||||
content.extend(audio.iter().cloned().map(|url| UserInput::Audio { url }));
|
||||
if payload.has_complete_audio_order() {
|
||||
let mut inline_index = 0;
|
||||
let mut file_index = 0;
|
||||
for audio_kind in &payload.audio_order {
|
||||
let audio = match audio_kind {
|
||||
UserMessageAudioKind::Inline => {
|
||||
let Some(url) = payload
|
||||
.audio
|
||||
.as_deref()
|
||||
.and_then(|audio| audio.get(inline_index))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
inline_index += 1;
|
||||
codex_protocol::models::AudioReference::Inline {
|
||||
audio_url: url.clone(),
|
||||
}
|
||||
}
|
||||
UserMessageAudioKind::File => {
|
||||
let Some(audio) = payload
|
||||
.audio_references
|
||||
.as_deref()
|
||||
.and_then(|audio| audio.get(file_index))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
file_index += 1;
|
||||
audio.clone()
|
||||
}
|
||||
};
|
||||
content.push(UserInput::from(
|
||||
codex_protocol::user_input::UserInput::Audio { audio },
|
||||
));
|
||||
}
|
||||
} else if let Some(audio) = &payload.audio_references {
|
||||
content.extend(audio.iter().cloned().map(|audio| {
|
||||
UserInput::from(codex_protocol::user_input::UserInput::Audio { audio })
|
||||
}));
|
||||
} else if let Some(audio) = &payload.audio {
|
||||
content.extend(audio.iter().cloned().map(|url| UserInput::Audio {
|
||||
audio: crate::AudioReference::Inline { url },
|
||||
}));
|
||||
}
|
||||
content.extend(
|
||||
payload
|
||||
@@ -1948,6 +1989,7 @@ mod tests {
|
||||
use codex_protocol::items::UserMessageItem as CoreUserMessageItem;
|
||||
use codex_protocol::items::build_hook_prompt_message;
|
||||
use codex_protocol::mcp::CallToolResult;
|
||||
use codex_protocol::models::AudioReference as CoreAudioReference;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::MessagePhase as CoreMessagePhase;
|
||||
use codex_protocol::models::WebSearchAction as CoreWebSearchAction;
|
||||
@@ -1976,6 +2018,7 @@ mod tests {
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_protocol::protocol::WebSearchBeginEvent;
|
||||
use codex_protocol::protocol::WebSearchEndEvent;
|
||||
use codex_protocol::user_input::UserInput as CoreUserInput;
|
||||
use codex_rollout::CompactedItem;
|
||||
use codex_utils_absolute_path::test_support::PathBufExt;
|
||||
use codex_utils_absolute_path::test_support::test_path_buf;
|
||||
@@ -2323,6 +2366,8 @@ mod tests {
|
||||
local_images: vec![local_image_path.clone()],
|
||||
local_image_details: vec![Some(ImageDetail::Original)],
|
||||
audio: Some(vec!["https://example.com/audio.mp3".into()]),
|
||||
audio_references: None,
|
||||
audio_order: Vec::new(),
|
||||
local_audio: vec![local_audio_path.clone()],
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
@@ -2358,7 +2403,9 @@ mod tests {
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::Audio {
|
||||
url: "https://example.com/audio.mp3".into(),
|
||||
audio: crate::AudioReference::Inline {
|
||||
url: "https://example.com/audio.mp3".into()
|
||||
}
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: local_audio_path,
|
||||
@@ -2405,6 +2452,32 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebuilds_uploaded_and_inline_audio_in_order_from_legacy_events() {
|
||||
let item = CoreUserMessageItem::new(&[
|
||||
CoreUserInput::Audio {
|
||||
audio: CoreAudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
},
|
||||
},
|
||||
CoreUserInput::Audio {
|
||||
audio: CoreAudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".into(),
|
||||
},
|
||||
},
|
||||
]);
|
||||
let turns =
|
||||
build_turns_from_rollout_items(&[RolloutItem::EventMsg(item.as_legacy_event())]);
|
||||
|
||||
assert_eq!(
|
||||
serde_json::to_value(&turns[0].items[0]).unwrap()["content"],
|
||||
serde_json::json!([
|
||||
{"type": "audio", "fileId": "file_audio"},
|
||||
{"type": "audio", "url": "data:audio/wav;base64,YXVkaW8="},
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_user_message_item_lifecycle_events() {
|
||||
let turn_id = "turn-1";
|
||||
|
||||
@@ -26,6 +26,7 @@ use codex_protocol::mcp::McpServerInfo;
|
||||
use codex_protocol::memory_citation::MemoryCitation as CoreMemoryCitation;
|
||||
use codex_protocol::memory_citation::MemoryCitationEntry as CoreMemoryCitationEntry;
|
||||
use codex_protocol::models::AdditionalPermissionProfile as CoreAdditionalPermissionProfile;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE;
|
||||
use codex_protocol::models::FileSystemPermissions as CoreFileSystemPermissions;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
@@ -3142,6 +3143,28 @@ fn network_requirements_serializes_canonical_and_legacy_fields() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_references_round_trip_through_core_and_json() {
|
||||
for wire in [
|
||||
json!({"type": "audio", "url": "data:audio/wav;base64,AAA"}),
|
||||
json!({"type": "audio", "fileId": "file_audio"}),
|
||||
] {
|
||||
let input: UserInput = serde_json::from_value(wire.clone()).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(UserInput::from(input.into_core())).unwrap(),
|
||||
wire
|
||||
);
|
||||
}
|
||||
let old_input: UserInput = serde_json::from_value(
|
||||
json!({"type": "audio", "fileId": "file_audio", "durationMs": 1_000}),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(UserInput::from(old_input.into_core())).unwrap(),
|
||||
json!({"type": "audio", "fileId": "file_audio"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_turn_item_into_thread_item_converts_supported_variants() {
|
||||
let user_item = TurnItem::UserMessage(UserMessageItem {
|
||||
@@ -3163,7 +3186,14 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
CoreUserInput::Audio {
|
||||
audio_url: "data:audio/wav;base64,AAA".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,AAA".to_string(),
|
||||
},
|
||||
},
|
||||
CoreUserInput::Audio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
},
|
||||
},
|
||||
CoreUserInput::LocalAudio {
|
||||
path: PathBuf::from("local/audio.mp3"),
|
||||
@@ -3200,7 +3230,14 @@ fn core_turn_item_into_thread_item_converts_supported_variants() {
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::Audio {
|
||||
url: "data:audio/wav;base64,AAA".to_string(),
|
||||
audio: crate::AudioReference::Inline {
|
||||
url: "data:audio/wav;base64,AAA".to_string()
|
||||
}
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio: crate::AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
}
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: PathBuf::from("local/audio.mp3"),
|
||||
@@ -3854,11 +3891,15 @@ fn user_input_into_core_preserves_media_fields() {
|
||||
|
||||
assert_eq!(
|
||||
UserInput::Audio {
|
||||
url: "data:audio/wav;base64,AAA".to_string(),
|
||||
audio: crate::AudioReference::Inline {
|
||||
url: "data:audio/wav;base64,AAA".to_string()
|
||||
}
|
||||
}
|
||||
.into_core(),
|
||||
CoreUserInput::Audio {
|
||||
audio_url: "data:audio/wav;base64,AAA".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,AAA".to_string()
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ use codex_protocol::config_types::CollaborationMode;
|
||||
use codex_protocol::config_types::MultiAgentMode;
|
||||
use codex_protocol::config_types::Personality;
|
||||
use codex_protocol::config_types::ReasoningSummary;
|
||||
use codex_protocol::models::AudioReference as CoreAudioReference;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ImageReference as CoreImageReference;
|
||||
@@ -451,6 +452,21 @@ pub enum ImageReference {
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(untagged)]
|
||||
#[ts(untagged)]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub enum AudioReference {
|
||||
Inline {
|
||||
url: String,
|
||||
},
|
||||
File {
|
||||
#[serde(rename = "fileId")]
|
||||
#[ts(rename = "fileId")]
|
||||
file_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(tag = "type", rename_all = "camelCase")]
|
||||
#[ts(tag = "type")]
|
||||
@@ -476,7 +492,8 @@ pub enum UserInput {
|
||||
path: PathBuf,
|
||||
},
|
||||
Audio {
|
||||
url: String,
|
||||
#[serde(flatten)]
|
||||
audio: AudioReference,
|
||||
},
|
||||
LocalAudio {
|
||||
path: PathBuf,
|
||||
@@ -509,7 +526,12 @@ impl UserInput {
|
||||
detail,
|
||||
},
|
||||
UserInput::LocalImage { path, detail } => CoreUserInput::LocalImage { path, detail },
|
||||
UserInput::Audio { url } => CoreUserInput::Audio { audio_url: url },
|
||||
UserInput::Audio { audio } => CoreUserInput::Audio {
|
||||
audio: match audio {
|
||||
AudioReference::Inline { url } => CoreAudioReference::Inline { audio_url: url },
|
||||
AudioReference::File { file_id } => CoreAudioReference::File { file_id },
|
||||
},
|
||||
},
|
||||
UserInput::LocalAudio { path } => CoreUserInput::LocalAudio { path },
|
||||
UserInput::Skill { name, path } => CoreUserInput::Skill { name, path },
|
||||
UserInput::Mention { name, path } => CoreUserInput::Mention { name, path },
|
||||
@@ -537,7 +559,14 @@ impl From<CoreUserInput> for UserInput {
|
||||
detail,
|
||||
},
|
||||
CoreUserInput::LocalImage { path, detail } => UserInput::LocalImage { path, detail },
|
||||
CoreUserInput::Audio { audio_url } => UserInput::Audio { url: audio_url },
|
||||
CoreUserInput::Audio { audio } => UserInput::Audio {
|
||||
audio: match audio {
|
||||
CoreAudioReference::Inline { audio_url } => {
|
||||
AudioReference::Inline { url: audio_url }
|
||||
}
|
||||
CoreAudioReference::File { file_id } => AudioReference::File { file_id },
|
||||
},
|
||||
},
|
||||
CoreUserInput::LocalAudio { path } => UserInput::LocalAudio { path },
|
||||
CoreUserInput::Skill { name, path } => UserInput::Skill { name, path },
|
||||
CoreUserInput::Mention { name, path } => UserInput::Mention { name, path },
|
||||
|
||||
@@ -1083,7 +1083,9 @@ async fn thread_read_loaded_include_turns_reads_store_history_without_rollout_pa
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
UserInput::Audio {
|
||||
url: "https://example.com/recording.mp3".to_string(),
|
||||
audio: codex_app_server_protocol::AudioReference::Inline {
|
||||
url: "https://example.com/recording.mp3".to_string()
|
||||
}
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: "recording.wav".into(),
|
||||
|
||||
@@ -9,6 +9,7 @@ use crate::session::SessionIo;
|
||||
use crate::session::SessionSettingsUpdate;
|
||||
use crate::session::Submission;
|
||||
use crate::session::emit_subagent_session_started;
|
||||
use crate::session::inject::validate_injected_response_item;
|
||||
use crate::session::new_submission_id;
|
||||
use crate::session::session::AppServerClientMetadata;
|
||||
use crate::session::session::Session;
|
||||
@@ -673,11 +674,18 @@ impl CodexThread {
|
||||
///
|
||||
/// This is the thread-level bridge to `Session::inject_if_running` for
|
||||
/// callers that only hold a `CodexThread`.
|
||||
/// It returns the unchanged items when this thread has no active turn.
|
||||
/// Returns the unchanged batch when there is no active turn or an item requires typed turn input.
|
||||
pub async fn inject_if_running(
|
||||
&self,
|
||||
items: Vec<ResponseItem>,
|
||||
) -> Result<(), Vec<ResponseItem>> {
|
||||
if items
|
||||
.iter()
|
||||
.try_for_each(validate_injected_response_item)
|
||||
.is_err()
|
||||
{
|
||||
return Err(items);
|
||||
}
|
||||
self.session.inject_if_running(items).await
|
||||
}
|
||||
|
||||
@@ -952,6 +960,7 @@ impl CodexThread {
|
||||
"items must not be empty".to_string(),
|
||||
));
|
||||
}
|
||||
items.iter().try_for_each(validate_injected_response_item)?;
|
||||
|
||||
let had_reference_context = self.session.reference_context_item().await.is_some();
|
||||
let mut turn_context = if had_reference_context {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::*;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItemKind;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::InternalChatMessageMetadataPassthrough;
|
||||
@@ -88,7 +89,9 @@ fn later_image_parts_preserve_labels_audio_and_annotations() {
|
||||
text(&image_close_tag_text()),
|
||||
text("keep"),
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,abc".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,abc".to_string(),
|
||||
},
|
||||
},
|
||||
];
|
||||
let kinds = (0..parts.len())
|
||||
@@ -137,7 +140,9 @@ fn image_treatment_preserves_text_only_boundary_behavior() {
|
||||
text(&"middle".repeat(20)),
|
||||
text("later text"),
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,abc".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,abc".to_string(),
|
||||
},
|
||||
},
|
||||
]));
|
||||
for max_tokens in [0, 1, 4, 16, 100] {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use super::*;
|
||||
use codex_context_fragments::AnnotatedContent;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ContentItemKind;
|
||||
use codex_protocol::models::input_text;
|
||||
@@ -81,7 +82,9 @@ fn world_state_hash_tracks_typed_content_and_role() {
|
||||
role,
|
||||
("", ""),
|
||||
ContentItem::InputAudio {
|
||||
audio_url: audio_url.to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: audio_url.to_string(),
|
||||
},
|
||||
},
|
||||
),
|
||||
content_kind: ContentItemKind("generic.test".to_string()),
|
||||
|
||||
@@ -50,6 +50,7 @@ use codex_prompts::render_model_instructions;
|
||||
use codex_protocol::DEFAULT_FUNCTION_NAMESPACE;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::models::AgentMessageInputContent;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
@@ -87,6 +88,11 @@ use codex_features::Feature;
|
||||
|
||||
static GUARDIAN_REVIEW_CONTEXT_REVISION: AtomicU64 = AtomicU64::new(/*v*/ 1);
|
||||
|
||||
// Responses validates uploaded audio against its 20-minute duration limit.
|
||||
// File durations are unavailable locally, so budget that maximum at the existing
|
||||
// inline estimate of 10 audio tokens per second.
|
||||
pub(crate) const FILE_AUDIO_TOKENS_ESTIMATE: usize = 20 * 60 * 10;
|
||||
|
||||
fn next_guardian_review_context_revision() -> u64 {
|
||||
GUARDIAN_REVIEW_CONTEXT_REVISION.fetch_add(/*val*/ 1, Ordering::Relaxed)
|
||||
}
|
||||
@@ -1190,7 +1196,7 @@ fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> i64 {
|
||||
ContentItem::InputImage { image, detail } => {
|
||||
estimate_image_reference_bytes(image, *detail)
|
||||
}
|
||||
ContentItem::InputAudio { audio_url } => estimate_audio_bytes(audio_url),
|
||||
ContentItem::InputAudio { audio } => estimate_audio_reference_bytes(audio),
|
||||
})
|
||||
.fold(0i64, i64::saturating_add),
|
||||
ResponseItem::AgentMessage {
|
||||
@@ -1403,6 +1409,17 @@ fn estimate_audio_bytes(audio_url: &str) -> i64 {
|
||||
.unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Audio estimate for callers that only have the reference. Uploaded file durations are not
|
||||
/// available locally, so file references use the maximum duration accepted by Responses.
|
||||
fn estimate_audio_reference_bytes(audio: &AudioReference) -> i64 {
|
||||
match audio {
|
||||
AudioReference::Inline { audio_url } => estimate_audio_bytes(audio_url),
|
||||
AudioReference::File { .. } => {
|
||||
i64::try_from(approx_bytes_for_tokens(FILE_AUDIO_TOKENS_ESTIMATE)).unwrap_or(i64::MAX)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn estimate_function_output_bytes(output: &FunctionCallOutputBody) -> i64 {
|
||||
match output {
|
||||
FunctionCallOutputBody::Text(text) => text_bytes(text),
|
||||
|
||||
@@ -15,6 +15,7 @@ use codex_history::CodexHarnessMetadata;
|
||||
use codex_history::ResponseItemEnvelope;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::ResponseItemId;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ConfigurationReasoning;
|
||||
use codex_protocol::models::ContentItem;
|
||||
@@ -1381,7 +1382,9 @@ async fn for_prompt_strips_media_when_model_does_not_support_it() {
|
||||
detail: Some(DEFAULT_IMAGE_DETAIL),
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: "caption".to_string(),
|
||||
@@ -1623,7 +1626,9 @@ async fn for_prompt_strips_media_when_model_does_not_support_it() {
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
}],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: Some(unknown_content_metadata()),
|
||||
@@ -3344,7 +3349,9 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() {
|
||||
encrypted_content: None,
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputAudio { audio_url }],
|
||||
content: vec![ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline { audio_url },
|
||||
}],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
};
|
||||
@@ -3357,6 +3364,35 @@ fn audio_data_url_payload_does_not_dominate_message_estimate() {
|
||||
assert!(estimated < raw_len);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uploaded_audio_estimate_accounts_for_each_recording_instead_of_its_id() {
|
||||
let item = ResponseItem::Message {
|
||||
status: None,
|
||||
encrypted_content: None,
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_a".into(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_a_longer_reference".into(),
|
||||
},
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
estimate_response_item_model_visible_bytes(&item),
|
||||
approx_bytes_for_tokens(/*tokens*/ 24_000) as i64,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_data_url_payload_does_not_dominate_function_call_output_estimate() {
|
||||
let (audio_url, _) = pcm_wav_data_url(/*sample_count*/ 800);
|
||||
@@ -3411,7 +3447,9 @@ fn malformed_audio_data_url_falls_back_to_whole_url_size_cost() {
|
||||
encrypted_content: None,
|
||||
id: None,
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputAudio { audio_url }],
|
||||
content: vec![ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline { audio_url },
|
||||
}],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
};
|
||||
|
||||
@@ -145,9 +145,9 @@ fn parse_user_message(message: &[ContentItem]) -> Option<UserMessageItem> {
|
||||
detail: *detail,
|
||||
});
|
||||
}
|
||||
ContentItem::InputAudio { audio_url } => {
|
||||
ContentItem::InputAudio { audio } => {
|
||||
content.push(UserInput::Audio {
|
||||
audio_url: audio_url.clone(),
|
||||
audio: audio.clone(),
|
||||
});
|
||||
}
|
||||
ContentItem::OutputText { text, .. } => {
|
||||
|
||||
@@ -9,6 +9,7 @@ use codex_protocol::items::HookPromptFragment;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::items::WebSearchItem;
|
||||
use codex_protocol::items::build_hook_prompt_message;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::ImageReference;
|
||||
@@ -245,7 +246,9 @@ fn skips_local_audio_label_text() {
|
||||
content: vec![
|
||||
ContentItem::InputText { text: label },
|
||||
ContentItem::InputAudio {
|
||||
audio_url: audio_url.clone(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: audio_url.clone(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: "</audio>".to_string(),
|
||||
@@ -265,7 +268,9 @@ fn skips_local_audio_label_text() {
|
||||
assert_eq!(
|
||||
user.content,
|
||||
vec![
|
||||
UserInput::Audio { audio_url },
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::Inline { audio_url }
|
||||
},
|
||||
UserInput::Text {
|
||||
text: user_text,
|
||||
text_elements: Vec::new(),
|
||||
|
||||
@@ -5,9 +5,32 @@ use codex_analytics::ImagePreparationMetadata;
|
||||
use codex_features::Feature;
|
||||
use codex_history::CodexHarnessMetadata;
|
||||
use codex_history::ResponseItemEnvelope;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::ModelInfo;
|
||||
|
||||
/// Uploaded files need the admission checks on typed turn input before entering history.
|
||||
pub(crate) fn validate_injected_response_item(item: &ResponseItem) -> CodexResult<()> {
|
||||
if let ResponseItem::Message { content, .. } = item
|
||||
&& content.iter().any(|part| {
|
||||
matches!(
|
||||
part,
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::File { .. }
|
||||
}
|
||||
)
|
||||
})
|
||||
{
|
||||
return Err(CodexErr::InvalidRequest(
|
||||
"Uploaded audio files must be submitted as turn input.".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// Returns the input if there is no active turn to inject into.
|
||||
#[expect(
|
||||
|
||||
@@ -253,7 +253,7 @@ mod extension_interruption;
|
||||
pub(crate) mod extension_metrics;
|
||||
mod guardian_checkpoint;
|
||||
mod handlers;
|
||||
mod inject;
|
||||
pub(crate) mod inject;
|
||||
mod reasoning_effort;
|
||||
mod submission;
|
||||
pub(crate) use reasoning_effort::RequestEffortUsage;
|
||||
|
||||
@@ -99,6 +99,7 @@ use codex_protocol::error::Result as CodexResult;
|
||||
use codex_protocol::items::PlanItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::items::build_hook_prompt_message;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::BaseInstructions;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::MessagePhase;
|
||||
@@ -181,6 +182,20 @@ pub(crate) async fn run_turn(
|
||||
prewarmed_client_session: Option<ModelClientSession>,
|
||||
cancellation_token: CancellationToken,
|
||||
) -> CodexResult<Option<String>> {
|
||||
if input.iter().any(turn_input_has_uploaded_audio_file) {
|
||||
let result = run_hooks_and_record_inputs(
|
||||
&sess,
|
||||
&turn_context,
|
||||
&turn_context.capture_current_model_info(),
|
||||
&input,
|
||||
PersistContext::TurnStart,
|
||||
)
|
||||
.await;
|
||||
return match result {
|
||||
Err(err) => Err(err),
|
||||
Ok(_) => Err(uploaded_audio_disabled_error()),
|
||||
};
|
||||
}
|
||||
let auto_compaction_enabled = sess
|
||||
.services
|
||||
.thread_extension_data
|
||||
@@ -210,7 +225,7 @@ pub(crate) async fn run_turn(
|
||||
.await
|
||||
{
|
||||
// Compaction runs before the new input is recorded, so preserve it on every failure.
|
||||
run_hooks_and_record_inputs(
|
||||
let _ = run_hooks_and_record_inputs(
|
||||
&sess,
|
||||
&turn_context,
|
||||
&turn_context.capture_current_model_info(),
|
||||
@@ -259,7 +274,7 @@ pub(crate) async fn run_turn(
|
||||
{
|
||||
Ok(requirements) => requirements,
|
||||
Err(err) => {
|
||||
run_hooks_and_record_inputs(
|
||||
let _ = run_hooks_and_record_inputs(
|
||||
&sess,
|
||||
&turn_context,
|
||||
&turn_context.capture_current_model_info(),
|
||||
@@ -287,7 +302,7 @@ pub(crate) async fn run_turn(
|
||||
{
|
||||
Ok(step_context) => step_context,
|
||||
Err(err) if matches!(err.details(), CodexErrorDetails::TurnAborted) => {
|
||||
run_hooks_and_record_inputs(
|
||||
let _ = run_hooks_and_record_inputs(
|
||||
&sess,
|
||||
&turn_context,
|
||||
&turn_context.capture_current_model_info(),
|
||||
@@ -393,7 +408,7 @@ pub(crate) async fn run_turn(
|
||||
&input,
|
||||
PersistContext::TurnStart,
|
||||
)
|
||||
.await
|
||||
.await?
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -492,7 +507,7 @@ pub(crate) async fn run_turn(
|
||||
&pending_input,
|
||||
PersistContext::SteeredUserInput,
|
||||
)
|
||||
.await
|
||||
.await?
|
||||
{
|
||||
break;
|
||||
}
|
||||
@@ -953,7 +968,8 @@ pub(crate) async fn run_hooks_and_record_inputs(
|
||||
model_info: &ModelInfo,
|
||||
input: &[TurnInput],
|
||||
persist_context: PersistContext,
|
||||
) -> bool {
|
||||
) -> CodexResult<bool> {
|
||||
let rejected_uploaded_audio = input.iter().any(turn_input_has_uploaded_audio_file);
|
||||
// Cancellation can reach this path before Guardian's tools and context are
|
||||
// resolved. Only finalized evidence may enter reusable reviewer history.
|
||||
if sess
|
||||
@@ -962,11 +978,31 @@ pub(crate) async fn run_hooks_and_record_inputs(
|
||||
.get::<crate::guardian::PendingReviewContext>()
|
||||
.is_some()
|
||||
{
|
||||
return false;
|
||||
return if rejected_uploaded_audio {
|
||||
Err(uploaded_audio_disabled_error())
|
||||
} else {
|
||||
Ok(false)
|
||||
};
|
||||
}
|
||||
let mut blocked_input = false;
|
||||
let mut accepted_user_input = false;
|
||||
for input_item in input {
|
||||
let filtered_input;
|
||||
let input_item = match input_item {
|
||||
TurnInput::UserInput { content, .. } if content.iter().any(is_uploaded_audio_file) => {
|
||||
let mut filtered = input_item.clone();
|
||||
let TurnInput::UserInput { content, .. } = &mut filtered else {
|
||||
unreachable!("cloned user input must remain user input");
|
||||
};
|
||||
content.retain(|item| !is_uploaded_audio_file(item));
|
||||
if content.is_empty() {
|
||||
continue;
|
||||
}
|
||||
filtered_input = filtered;
|
||||
&filtered_input
|
||||
}
|
||||
_ => input_item,
|
||||
};
|
||||
let hook_outcome = inspect_pending_input(sess, turn_context, input_item).await;
|
||||
if hook_outcome.should_stop {
|
||||
blocked_input = true;
|
||||
@@ -998,7 +1034,29 @@ pub(crate) async fn run_hooks_and_record_inputs(
|
||||
.await;
|
||||
}
|
||||
}
|
||||
blocked_input && !accepted_user_input
|
||||
if rejected_uploaded_audio {
|
||||
return Err(uploaded_audio_disabled_error());
|
||||
}
|
||||
Ok(blocked_input && !accepted_user_input)
|
||||
}
|
||||
|
||||
fn turn_input_has_uploaded_audio_file(input: &TurnInput) -> bool {
|
||||
matches!(input, TurnInput::UserInput { content, .. } if content.iter().any(is_uploaded_audio_file))
|
||||
}
|
||||
|
||||
fn uploaded_audio_disabled_error() -> CodexErr {
|
||||
CodexErr::UnsupportedOperation(
|
||||
"Uploaded audio file inputs are not enabled in this build.".to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_uploaded_audio_file(input: &UserInput) -> bool {
|
||||
matches!(
|
||||
input,
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File { .. }
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
fn turn_user_input(input: &[TurnInput]) -> Vec<UserInput> {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use super::TurnInput;
|
||||
use super::UserInputMetadata;
|
||||
use super::inject::validate_injected_response_item;
|
||||
use super::session::Session;
|
||||
use super::session::SessionConfiguration;
|
||||
use super::session::SessionSettingsUpdate;
|
||||
@@ -237,6 +238,12 @@ pub(super) async fn handle(
|
||||
submission_id: String,
|
||||
) -> CodexResult<TurnInputSubmission> {
|
||||
let request = request.into();
|
||||
if let SubmittedTurnInput::ResponseItem(item)
|
||||
| SubmittedTurnInput::AnnotatedResponseItem(AnnotatedResponseItem { item, .. }) =
|
||||
&request.request.input
|
||||
{
|
||||
validate_injected_response_item(item)?;
|
||||
}
|
||||
request
|
||||
.request
|
||||
.additional_context
|
||||
|
||||
@@ -715,14 +715,28 @@ impl Session {
|
||||
std::mem::take(&mut ts.token_usage_by_model),
|
||||
)
|
||||
};
|
||||
run_hooks_and_record_inputs(
|
||||
if let Err(err) = run_hooks_and_record_inputs(
|
||||
self,
|
||||
&turn_context,
|
||||
&turn_context.capture_current_model_info(),
|
||||
&pending_input,
|
||||
PersistContext::Standard,
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
self.emit_turn_error_lifecycle(
|
||||
turn_context.as_ref(),
|
||||
err.to_codex_protocol_error(),
|
||||
err.details(),
|
||||
)
|
||||
.await;
|
||||
self.track_turn_codex_error(turn_context.as_ref(), &err);
|
||||
self.send_event(
|
||||
turn_context.as_ref(),
|
||||
EventMsg::Error(err.to_error_event(/*message_prefix*/ None)),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
drop(finish_recording);
|
||||
let turn_telemetry = &turn_context.session_telemetry;
|
||||
// Emit token usage metrics.
|
||||
|
||||
@@ -19,8 +19,14 @@ use codex_otel::TURN_TOOL_CALL_METRIC;
|
||||
use codex_otel::TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::CodexErrorInfo;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use opentelemetry::KeyValue;
|
||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
|
||||
@@ -54,6 +60,88 @@ impl SessionTask for PendingTask {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn task_finish_rejects_uploaded_audio_without_losing_other_pending_input() {
|
||||
let (session, turn_context, receiver) = make_session_and_context_with_rx().await;
|
||||
session
|
||||
.spawn_task(Arc::clone(&turn_context), Vec::new(), PendingTask)
|
||||
.await;
|
||||
while receiver.try_recv().is_ok() {}
|
||||
|
||||
let turn_state = {
|
||||
let active_turn = session.active_turn.lock().await;
|
||||
Arc::clone(
|
||||
&active_turn
|
||||
.as_ref()
|
||||
.expect("active pending task")
|
||||
.turn_state,
|
||||
)
|
||||
};
|
||||
session
|
||||
.input_queue
|
||||
.extend_pending_input_for_turn_state(
|
||||
turn_state.as_ref(),
|
||||
vec![TurnInput::UserInput {
|
||||
content: vec![
|
||||
UserInput::Text {
|
||||
text: "preserve pending text".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_disabled_at_finish".to_string(),
|
||||
},
|
||||
},
|
||||
],
|
||||
client_id: None,
|
||||
metadata: Default::default(),
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
|
||||
session
|
||||
.on_task_finished(Arc::clone(&turn_context), /*task_result*/ Ok(None))
|
||||
.await;
|
||||
|
||||
let history = session.clone_history().await;
|
||||
assert!(history.raw_items().any(|item| {
|
||||
matches!(item, ResponseItem::Message { content, .. } if content.iter().any(|part| {
|
||||
matches!(part, ContentItem::InputText { text } if text == "preserve pending text")
|
||||
}))
|
||||
}));
|
||||
assert!(history.raw_items().all(|item| {
|
||||
!matches!(item, ResponseItem::Message { content, .. } if content.iter().any(|part| {
|
||||
matches!(part, ContentItem::InputAudio {
|
||||
audio: AudioReference::File { file_id, .. },
|
||||
} if file_id == "file_disabled_at_finish")
|
||||
}))
|
||||
}));
|
||||
|
||||
let (saw_error, completed) = tokio::time::timeout(std::time::Duration::from_secs(2), async {
|
||||
let mut saw_error = false;
|
||||
loop {
|
||||
match receiver.recv().await.expect("event channel open").msg {
|
||||
EventMsg::Error(error) => {
|
||||
saw_error = true;
|
||||
assert_eq!(error.codex_error_info, Some(CodexErrorInfo::BadRequest));
|
||||
}
|
||||
EventMsg::TurnComplete(completed) => break (saw_error, completed),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("task finish events");
|
||||
assert!(saw_error, "the rejected drain should emit an error event");
|
||||
assert_eq!(
|
||||
completed
|
||||
.error
|
||||
.as_ref()
|
||||
.and_then(|error| error.codex_error_info.clone()),
|
||||
Some(CodexErrorInfo::BadRequest)
|
||||
);
|
||||
}
|
||||
|
||||
fn test_session_telemetry() -> SessionTelemetry {
|
||||
let exporter = InMemoryMetricExporter::default();
|
||||
let metrics = MetricsClient::new(
|
||||
|
||||
@@ -94,7 +94,7 @@ impl SessionTask for RegularTask {
|
||||
.await;
|
||||
let prewarmed_client_session = match prewarmed_client_session {
|
||||
SessionStartupPrewarmResolution::Cancelled => {
|
||||
run_hooks_and_record_inputs(
|
||||
let _ = run_hooks_and_record_inputs(
|
||||
&sess,
|
||||
&ctx,
|
||||
&ctx.capture_current_model_info(),
|
||||
|
||||
@@ -37,6 +37,7 @@ use codex_protocol::config_types::ReasoningSummary;
|
||||
use codex_protocol::config_types::Settings;
|
||||
use codex_protocol::config_types::Verbosity;
|
||||
use codex_protocol::error::CodexErr;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::DEFAULT_IMAGE_DETAIL;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
@@ -528,7 +529,9 @@ async fn sends_audio_urls_to_responses() {
|
||||
|
||||
codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Audio {
|
||||
audio_url: audio_url.to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: audio_url.to_string(),
|
||||
},
|
||||
}]))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -4,6 +4,7 @@ use codex_core::config::CurrentTimeReminderConfig;
|
||||
use codex_core::config::RolloutBudgetConfig;
|
||||
use codex_core::config::TokenBudgetConfig;
|
||||
use codex_features::Feature;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::protocol::AdditionalContextEntry;
|
||||
@@ -119,9 +120,7 @@ async fn first_request_item_types_roles_and_content_annotations() -> Result<()>
|
||||
},
|
||||
detail: None,
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio_url: "data:audio/wav;base64,AAAA".to_string(),
|
||||
},
|
||||
UserInput::Audio { audio: AudioReference::Inline { audio_url: "data:audio/wav;base64,AAAA".to_string() } },
|
||||
])
|
||||
.with_additional_context(BTreeMap::from([
|
||||
(
|
||||
|
||||
@@ -18,6 +18,7 @@ use codex_protocol::config_types::SERVICE_TIER_DEFAULT_REQUEST_VALUE;
|
||||
use codex_protocol::config_types::ServiceTier;
|
||||
use codex_protocol::config_types::Settings;
|
||||
use codex_protocol::mcp::ClientMcpExtensions;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::BaseInstructionsProvenance;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::PermissionProfile;
|
||||
@@ -908,7 +909,9 @@ async fn model_change_projects_media_without_changing_live_or_replayed_history(
|
||||
detail: None,
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
},
|
||||
UserInput::Text {
|
||||
text: "first turn".to_string(),
|
||||
|
||||
@@ -1530,6 +1530,18 @@ async fn injected_response_item_reopens_turn_after_final_answer() {
|
||||
submit_user_input(&codex, INITIAL_PROMPT).await;
|
||||
wait_for_reasoning_item_started(&codex).await;
|
||||
|
||||
let rejected = vec![
|
||||
responses::user_message_item("rejected injection prefix"),
|
||||
serde_json::from_value(json!({
|
||||
"type": "message", "role": "user",
|
||||
"content": [{"type": "input_audio", "file_id": "file_live_injection"}],
|
||||
}))
|
||||
.expect("raw audio message"),
|
||||
];
|
||||
assert_eq!(
|
||||
codex.inject_if_running(rejected.clone()).await,
|
||||
Err(rejected)
|
||||
);
|
||||
assert!(
|
||||
codex
|
||||
.inject_if_running(vec![responses::user_message_item(INJECTED_CONTEXT)])
|
||||
@@ -1555,6 +1567,11 @@ async fn injected_response_item_reopens_turn_after_final_answer() {
|
||||
let second: Value = from_slice(&requests[1]).expect("parse second request");
|
||||
responses::assert_root_turn(&second, Some(first_turn_id))
|
||||
.expect("external injection should preserve the active turn root");
|
||||
assert!(
|
||||
!message_input_texts(&second, "user")
|
||||
.iter()
|
||||
.any(|text| text == "rejected injection prefix")
|
||||
);
|
||||
let relevant_user_input = message_input_texts(&second, "user")
|
||||
.into_iter()
|
||||
.filter(|text| {
|
||||
|
||||
@@ -109,6 +109,9 @@ const ONE_PIXEL_PNG_BASE64: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ
|
||||
#[path = "scenarios_dynamic_tools.rs"]
|
||||
mod dynamic_tools;
|
||||
|
||||
#[path = "scenarios_audio_file_tests.rs"]
|
||||
mod audio_file;
|
||||
|
||||
#[path = "scenarios_retained_tool_outputs.rs"]
|
||||
mod retained_tool_outputs;
|
||||
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
//! New uploaded-audio inputs stay inactive until the runtime integration is installed.
|
||||
|
||||
use super::*;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn unsupported_steered_audio_is_rejected_before_continuation_and_resume() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let (release, gate) = tokio::sync::oneshot::channel();
|
||||
let (streaming, _completions) = start_streaming_sse_server(vec![
|
||||
vec![
|
||||
StreamingSseChunk {
|
||||
gate: None,
|
||||
body: sse(vec![
|
||||
ev_response_created("active"),
|
||||
ev_assistant_message("working", "I am working on your request."),
|
||||
]),
|
||||
},
|
||||
StreamingSseChunk {
|
||||
gate: Some(gate),
|
||||
body: sse(vec![ev_completed("active")]),
|
||||
},
|
||||
],
|
||||
vec![StreamingSseChunk {
|
||||
gate: None,
|
||||
body: sse(vec![ev_completed("follow_up")]),
|
||||
}],
|
||||
vec![StreamingSseChunk {
|
||||
gate: None,
|
||||
body: sse(vec![ev_completed("resumed")]),
|
||||
}],
|
||||
])
|
||||
.await;
|
||||
let config_server = start_mock_server().await;
|
||||
let base_url = format!("{}/v1", streaming.uri());
|
||||
let builder = || {
|
||||
let base_url = base_url.clone();
|
||||
test_codex()
|
||||
.with_model_info_override("gpt-5.5", |model| {
|
||||
model.input_modalities.push(InputModality::Audio);
|
||||
})
|
||||
.with_config(move |config| {
|
||||
config.model_provider.base_url = Some(base_url);
|
||||
config.model_provider.supports_websockets = false;
|
||||
config
|
||||
.features
|
||||
.disable(Feature::InstantInterrupt)
|
||||
.expect("queue steering");
|
||||
config
|
||||
.features
|
||||
.disable(Feature::EnableRequestCompression)
|
||||
.expect("record request bodies");
|
||||
})
|
||||
};
|
||||
let test = builder().build_with_auto_env(&config_server).await?;
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![text("Start working.")]))
|
||||
.await?;
|
||||
let turn_id = wait_for_event_match(&test.codex, |event| match event {
|
||||
EventMsg::TurnStarted(event) => Some(event.turn_id.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::ItemCompleted(event)
|
||||
if matches!(&event.item, TurnItem::AgentMessage(message) if message.id == "working"))
|
||||
})
|
||||
.await;
|
||||
test.codex
|
||||
.steer_turn(
|
||||
TurnInputRequest::user_input(vec![
|
||||
text("Keep this queued text."),
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_unsupported_steering".into(),
|
||||
},
|
||||
},
|
||||
]),
|
||||
turn_id,
|
||||
)
|
||||
.await?;
|
||||
release.send(()).expect("release the active response");
|
||||
let completed = wait_for_event_match(&test.codex, |event| match event {
|
||||
EventMsg::TurnComplete(completed) => Some(completed.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.await;
|
||||
assert!(completed.error.is_some());
|
||||
assert_eq!(streaming.requests().await.len(), 1);
|
||||
test.submit_text_turn("Continue without audio.").await?;
|
||||
let resumed = builder()
|
||||
.restart_with_auto_env(&config_server, &test)
|
||||
.await?;
|
||||
resumed.submit_text_turn("Continue after restart.").await?;
|
||||
let requests = streaming
|
||||
.requests()
|
||||
.await
|
||||
.iter()
|
||||
.map(|body| serde_json::from_slice::<serde_json::Value>(body))
|
||||
.collect::<serde_json::Result<Vec<_>>>()?;
|
||||
assert_eq!(requests.len(), 3);
|
||||
for request in &requests {
|
||||
assert!(!request["input"].as_array().unwrap().iter().any(|item| {
|
||||
item["content"]
|
||||
.as_array()
|
||||
.is_some_and(|content| content.iter().any(|part| part["type"] == "input_audio"))
|
||||
}));
|
||||
}
|
||||
for request in &requests[1..] {
|
||||
assert!(
|
||||
request.to_string().contains("Keep this queued text."),
|
||||
"rejecting the attachment must preserve unrelated queued input"
|
||||
);
|
||||
}
|
||||
let entries = requests.iter().map(SnapshotEntry::body).collect::<Vec<_>>();
|
||||
insta::assert_snapshot!(
|
||||
"unsupported_steered_audio_follow_up_and_resume",
|
||||
context_snapshot::format_context_snapshot(
|
||||
"Unsupported audio queued during an active turn is never forwarded or persisted; the next turn and resumed thread continue without it.",
|
||||
&entries,
|
||||
&ContextSnapshotOptions::default().rewrite_known_segments(),
|
||||
)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn unsupported_uploaded_audio_fails_before_inference_without_poisoning_follow_up()
|
||||
-> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let mock = mount_sse_sequence(&server, vec![sse(vec![ev_completed("follow_up")])]).await;
|
||||
let test = test_codex()
|
||||
.with_model_info_override("gpt-5.5", |model| {
|
||||
model.input_modalities.push(InputModality::Audio);
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
test.codex
|
||||
.start_or_steer_turn(TurnInputRequest::user_input(vec![
|
||||
text("Listen to this recording."),
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_unsupported".into(),
|
||||
},
|
||||
},
|
||||
]))
|
||||
.await?;
|
||||
let EventMsg::TurnComplete(completed) = wait_for_event(&test.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
assert!(completed.error.is_some());
|
||||
assert!(
|
||||
mock.requests().is_empty(),
|
||||
"unsupported audio must not reach inference"
|
||||
);
|
||||
test.submit_text_turn("Continue without the attachment.")
|
||||
.await?;
|
||||
let requests = mock.requests();
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(!requests[0].input().iter().any(|item| {
|
||||
item["content"]
|
||||
.as_array()
|
||||
.is_some_and(|parts| parts.iter().any(|part| part["type"] == "input_audio"))
|
||||
}));
|
||||
assert!(
|
||||
serde_json::to_string(&requests[0].input())
|
||||
.unwrap()
|
||||
.contains("Listen to this recording."),
|
||||
"rejecting the attachment must preserve surrounding text"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,10 +1,70 @@
|
||||
//! Verifies prepared context reaches both the first model request and a follow-up.
|
||||
|
||||
use super::*;
|
||||
use codex_protocol::error::CodexErrorDetails;
|
||||
use codex_protocol::protocol::ThreadHistoryMode;
|
||||
use codex_protocol::turn_input::AnnotatedResponseItem;
|
||||
use pretty_assertions::assert_eq;
|
||||
use test_case::test_case;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn raw_injection_rejects_uploaded_audio_before_modifying_history() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let test = test_codex().build_with_auto_env(&server).await?;
|
||||
let before = test.codex.conversation_history_snapshot().await;
|
||||
let audio: ResponseItem = serde_json::from_value(json!({
|
||||
"type": "message", "role": "developer",
|
||||
"content": [{"type": "input_audio", "file_id": "file_injected"}]
|
||||
}))?;
|
||||
let batch = vec![
|
||||
core_test_support::responses::user_message_item("must not be partially written"),
|
||||
audio.clone(),
|
||||
];
|
||||
|
||||
let history_error = test
|
||||
.codex
|
||||
.inject_response_items(batch.clone())
|
||||
.await
|
||||
.unwrap_err();
|
||||
let preparation_error = test
|
||||
.codex
|
||||
.inject_response_items_for_turn(batch)
|
||||
.await
|
||||
.unwrap_err();
|
||||
for error in [history_error, preparation_error] {
|
||||
assert!(matches!(
|
||||
error.details(),
|
||||
CodexErrorDetails::InvalidRequest(_)
|
||||
));
|
||||
}
|
||||
for input in [
|
||||
TurnInput::ResponseItem(audio.clone()),
|
||||
TurnInput::AnnotatedResponseItem(AnnotatedResponseItem {
|
||||
item: audio,
|
||||
annotations: Default::default(),
|
||||
}),
|
||||
] {
|
||||
let error = test
|
||||
.codex
|
||||
.start_or_steer_turn(TurnInputRequest::new(input))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error.details(),
|
||||
CodexErrorDetails::InvalidRequest(_)
|
||||
));
|
||||
}
|
||||
let after = test.codex.conversation_history_snapshot().await;
|
||||
assert_eq!(after.history_version(), before.history_version());
|
||||
assert_eq!(
|
||||
after.items().collect::<Vec<_>>(),
|
||||
before.items().collect::<Vec<_>>()
|
||||
);
|
||||
test.codex.shutdown_and_wait().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test_case(ThreadHistoryMode::Legacy; "legacy")]
|
||||
#[test_case(ThreadHistoryMode::Paginated; "paginated")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_audio_file_tests.rs
|
||||
expression: "context_snapshot::format_context_snapshot(\"Unsupported audio queued during an active turn is never forwarded or persisted; the next turn and resumed thread continue without it.\",\n&entries, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
|
||||
---
|
||||
Scenario: Unsupported audio queued during an active turn is never forwarded or persisted; the next turn and resumed thread continue without it.
|
||||
|
||||
## Window 1
|
||||
-- request 1 (request) --
|
||||
00:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
01:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
02:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
03:message/user:
|
||||
Start working.
|
||||
-- request 2 (request) --
|
||||
04:message/assistant:
|
||||
I am working on your request.
|
||||
05:message/user:
|
||||
Keep this queued text.
|
||||
06:message/user:
|
||||
Continue without audio.
|
||||
-- request 3 (request) --
|
||||
07:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
08:message/user:
|
||||
Continue after restart.
|
||||
+24
-12
@@ -16,7 +16,7 @@ Settings:
|
||||
text: {"verbosity":"low"}
|
||||
tool_choice: "auto"
|
||||
-- request 1 (turn) --
|
||||
00:additional_tools/developer (2; hash=49f981f03ae5c5ce):
|
||||
00:additional_tools/developer (2; hash=b07492b5514a41d0):
|
||||
- namespace/functions; hash=cc6e8cf0e5405f3a
|
||||
- custom/exec
|
||||
- function/wait
|
||||
@@ -25,11 +25,15 @@ Settings:
|
||||
- function/request_user_input
|
||||
- custom/apply_patch
|
||||
- function/view_image
|
||||
- tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=400cd6ac61d524a9
|
||||
- tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=b573ed69c0b0e6c7
|
||||
01:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
02:message/developer:
|
||||
<PERMISSIONS_INSTRUCTIONS>
|
||||
02:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
03:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
04:message/user:
|
||||
@@ -42,7 +46,7 @@ Settings:
|
||||
Settings: relative to window 1
|
||||
model: "updated-tools-model"
|
||||
-- request 3 (turn) --
|
||||
00:additional_tools/developer (2; hash=49f981f03ae5c5ce):
|
||||
00:additional_tools/developer (2; hash=b07492b5514a41d0):
|
||||
- namespace/functions; hash=cc6e8cf0e5405f3a
|
||||
- custom/exec
|
||||
- function/wait
|
||||
@@ -51,11 +55,15 @@ Settings: relative to window 1
|
||||
- function/request_user_input
|
||||
- custom/apply_patch
|
||||
- function/view_image
|
||||
- tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=400cd6ac61d524a9
|
||||
- tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=b573ed69c0b0e6c7
|
||||
01:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
02:message/developer:
|
||||
<PERMISSIONS_INSTRUCTIONS>
|
||||
02:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
03:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
04:message/user:
|
||||
@@ -84,7 +92,7 @@ Settings: relative to window 1
|
||||
## Window 3 (after request 5: input diverged at item 00)
|
||||
Settings: same as window 2
|
||||
-- request 6 (turn) --
|
||||
00:additional_tools/developer (2; hash=1322c3a339378659):
|
||||
00:additional_tools/developer (2; hash=f9f0840b2e42f7f5):
|
||||
- namespace/functions; hash=9d5d7a0adab05fef
|
||||
- custom/exec
|
||||
- function/wait
|
||||
@@ -93,7 +101,7 @@ Settings: same as window 2
|
||||
- function/request_user_input
|
||||
- custom/apply_patch
|
||||
- function/view_image
|
||||
- tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=400cd6ac61d524a9
|
||||
- tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=b573ed69c0b0e6c7
|
||||
01:message/developer:
|
||||
<MODEL_INSTRUCTIONS>
|
||||
02:message/user:
|
||||
@@ -102,8 +110,12 @@ Settings: same as window 2
|
||||
Continue with the same tools.
|
||||
04:message/user:
|
||||
Continue with the updated execution instructions.
|
||||
05:message/developer:
|
||||
<PERMISSIONS_INSTRUCTIONS>
|
||||
05:message/developer[2]:
|
||||
[01] <PERMISSIONS_INSTRUCTIONS>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
06:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
07:message/user:
|
||||
|
||||
+20
-16
@@ -161,7 +161,7 @@ Scenario: A host updates and clears shared instructions for surviving descendant
|
||||
08:message/user:
|
||||
continue with the current instructions
|
||||
|
||||
## Window 3 (after request 8: input diverged at item 03, settings changed (prompt_cache_key, tools))
|
||||
## Window 3 (after request 8: input diverged at item 01, settings changed (prompt_cache_key, tools))
|
||||
-- request 9 (turn) --
|
||||
00:message/developer:
|
||||
You are Codex, a coding agent based on GPT-5. You and the user share one workspace, and your job is to c...rate with them until their goal is genuinely handled. [hash=2a3efec66e7f8963]
|
||||
@@ -181,24 +181,28 @@ Scenario: A host updates and clears shared instructions for surviving descendant
|
||||
- Before performing file edits of any kind, you provide updates explaining what edits you are making.
|
||||
- Tone of your updates must match your personality.
|
||||
|
||||
01:message/developer:
|
||||
<permissions instructions>
|
||||
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c]
|
||||
# Escalation Requests
|
||||
01:message/developer[2]:
|
||||
[01] <permissions instructions>
|
||||
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c]
|
||||
# Escalation Requests
|
||||
|
||||
Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb]
|
||||
Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb]
|
||||
|
||||
- Pipes: |
|
||||
- Logical operators: &&, ||
|
||||
<OMITTED 44 LINES; ~790 TOKENS; hash=3a8900343e968d61>
|
||||
NEVER provide a prefix_rule if your command uses a heredoc or herestring.
|
||||
- Pipes: |
|
||||
- Logical operators: &&, ||
|
||||
<OMITTED 44 LINES; ~790 TOKENS; hash=3a8900343e968d61>
|
||||
NEVER provide a prefix_rule if your command uses a heredoc or herestring.
|
||||
|
||||
### Examples
|
||||
Good examples of prefixes:
|
||||
- ["npm", "run", "dev"]
|
||||
- ["gh", "pr", "check"]
|
||||
- ["cargo", "test"]
|
||||
</permissions instructions>
|
||||
### Examples
|
||||
Good examples of prefixes:
|
||||
- ["npm", "run", "dev"]
|
||||
- ["gh", "pr", "check"]
|
||||
- ["cargo", "test"]
|
||||
</permissions instructions>
|
||||
[02] <tools>
|
||||
Deferred tool namespaces:
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
02:message/user[2]:
|
||||
[01] # AGENTS.md instructions
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use codex_core::TurnInputRequest;
|
||||
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::openai_models::InputModality;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
@@ -72,7 +73,9 @@ mv "${tmp_path}" "${payload_path}""#,
|
||||
detail: None,
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio_url: audio_url.into(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: audio_url.into(),
|
||||
},
|
||||
},
|
||||
]))
|
||||
.await?;
|
||||
|
||||
@@ -5,6 +5,7 @@ use codex_guardian_context::ContextTarget;
|
||||
use codex_protocol::AgentPath;
|
||||
use codex_protocol::ResponseItemId;
|
||||
use codex_protocol::models::AgentMessageInputContent;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
@@ -1192,7 +1193,9 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
|
||||
detail: None,
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,audio-payload".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,audio-payload".to_string(),
|
||||
},
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use codex_protocol::models::AudioReference;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
@@ -854,7 +855,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
@@ -896,7 +899,9 @@ async fn local_attachments_are_snapshotted_before_enqueue_and_update() -> anyhow
|
||||
detail: Some(ImageDetail::High),
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio_url: "data:audio/mp4;base64,ZWRpdGVkIGF1ZGlv".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/mp4;base64,ZWRpdGVkIGF1ZGlv".to_string(),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
@@ -935,7 +940,9 @@ async fn invalid_local_attachments_do_not_mutate_queue() -> anyhow::Result<()> {
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ use codex_protocol::openai_models::CatalogModel;
|
||||
use codex_protocol::openai_models::ModelOrigin;
|
||||
use std::io::Write;
|
||||
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
@@ -295,11 +296,16 @@ impl SectionCost {
|
||||
} => {
|
||||
self.image_count = self.image_count.saturating_add(1);
|
||||
}
|
||||
ContentItem::InputAudio { audio_url } => {
|
||||
// Guardian currently has no audio contributor. Count a future opaque
|
||||
// payload conservatively until its consumer supplies modality costs.
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline { audio_url },
|
||||
} => {
|
||||
// Guardian has no audio contributor. Keep the existing inline
|
||||
// transport cost, but do not treat a file handle as evidence.
|
||||
self.text_bytes = self.text_bytes.saturating_add(audio_url.len());
|
||||
}
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::File { .. },
|
||||
} => {}
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ use super::*;
|
||||
use crate::Budgeted;
|
||||
use crate::composition::SectionOutput;
|
||||
use crate::composition::user_message as message;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
@@ -62,6 +63,16 @@ fn section_costs_keep_multimodal_payloads_separate() {
|
||||
},
|
||||
detail: None,
|
||||
}),
|
||||
Budgeted::required(ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,AAAA".to_owned(),
|
||||
},
|
||||
}),
|
||||
Budgeted::required(ContentItem::InputAudio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_audio".to_owned(),
|
||||
},
|
||||
}),
|
||||
]),
|
||||
},
|
||||
SectionOutput {
|
||||
@@ -82,7 +93,7 @@ fn section_costs_keep_multimodal_payloads_separate() {
|
||||
(
|
||||
"transcript",
|
||||
SectionCost {
|
||||
text_bytes: "évidence".len(),
|
||||
text_bytes: "évidence".len() + "data:audio/wav;base64,AAAA".len(),
|
||||
image_bytes: "data:image/png;base64,AAAA".len(),
|
||||
image_count: 2
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
|
||||
use codex_history::ResponseItemEnvelope;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
use codex_protocol::models::ImageReference;
|
||||
@@ -422,7 +423,12 @@ pub(crate) fn retain_content(
|
||||
image: ImageReference::File { .. },
|
||||
..
|
||||
}) => 0,
|
||||
SectionContent::Other(ContentItem::InputAudio { audio_url }) => audio_url.len(),
|
||||
SectionContent::Other(ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline { audio_url },
|
||||
}) => audio_url.len(),
|
||||
SectionContent::Other(ContentItem::InputAudio {
|
||||
audio: AudioReference::File { .. },
|
||||
}) => 0,
|
||||
};
|
||||
truncations.push(TruncationObservation {
|
||||
component: section.id,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use anyhow::Result;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ConfigurationReasoning;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::ThreadSettingsSnapshot;
|
||||
@@ -718,6 +719,79 @@ fn compacted_metadata_remains_compatible_with_legacy_response_item_readers() ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uploaded_audio_round_trips_in_canonical_message_content() -> Result<()> {
|
||||
let mut message = response_message("user");
|
||||
let ResponseItem::Message { content, .. } = &mut message else {
|
||||
unreachable!()
|
||||
};
|
||||
content.push(ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,aGVsbG8=".to_string(),
|
||||
},
|
||||
});
|
||||
content.push(ContentItem::InputAudio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_audio".to_string(),
|
||||
},
|
||||
});
|
||||
content.push(ContentItem::InputText {
|
||||
text: "after audio".to_string(),
|
||||
});
|
||||
let envelope = ResponseItemEnvelope::new(message);
|
||||
let response = RolloutItem::ResponseItem(envelope.clone());
|
||||
let compacted = RolloutItem::Compacted(Box::new(CompactedItem {
|
||||
message: String::new(),
|
||||
replacement_history: Some(vec![envelope.clone()]),
|
||||
guardian_history: Some(GuardianHistoryCheckpoint(vec![envelope])),
|
||||
retained_context: None,
|
||||
mcp_resource_origins: None,
|
||||
window_number: None,
|
||||
first_window_id: None,
|
||||
previous_window_id: None,
|
||||
window_id: None,
|
||||
compaction_response_id: None,
|
||||
latest_token_usage_record: None,
|
||||
resume_metadata: None,
|
||||
}));
|
||||
for (item, paths) in [
|
||||
(response, vec!["/payload/content"]),
|
||||
(
|
||||
compacted,
|
||||
vec![
|
||||
"/payload/replacement_history/0/content",
|
||||
"/payload/guardian_history/0/content",
|
||||
],
|
||||
),
|
||||
] {
|
||||
let mut serialized = serde_json::to_value(&item)?;
|
||||
for path in paths {
|
||||
assert_eq!(
|
||||
serialized.pointer(path),
|
||||
Some(&json!([
|
||||
{"type": "input_text", "text": "hello"},
|
||||
{"type": "input_audio", "audio_url": "data:audio/wav;base64,aGVsbG8="},
|
||||
{"type": "input_audio", "file_id": "file_audio"},
|
||||
{"type": "input_text", "text": "after audio"},
|
||||
]))
|
||||
);
|
||||
// Rollouts from the prototype can contain this extra field. Reading
|
||||
// them must preserve the same file reference in every history shape.
|
||||
serialized.pointer_mut(path).unwrap()[2]["duration_ms"] = json!(1_000);
|
||||
}
|
||||
match (serde_json::from_value::<RolloutItem>(serialized)?, item) {
|
||||
(RolloutItem::ResponseItem(restored), RolloutItem::ResponseItem(original)) => {
|
||||
assert_eq!(restored, original)
|
||||
}
|
||||
(RolloutItem::Compacted(restored), RolloutItem::Compacted(original)) => {
|
||||
assert_eq!(restored, original)
|
||||
}
|
||||
_ => panic!("unexpected rollout variant"),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Preserves the established tagged payload representation for every rollout variant.
|
||||
fn rollout_item_variants_preserve_existing_payload_shapes() -> Result<()> {
|
||||
|
||||
@@ -4,6 +4,7 @@ use crate::ThreadId;
|
||||
use crate::dynamic_tools::DynamicToolCallOutputContentItem;
|
||||
use crate::mcp::CallToolResult;
|
||||
use crate::memory_citation::MemoryCitation;
|
||||
use crate::models::AudioReference;
|
||||
use crate::models::ContentItem;
|
||||
use crate::models::ContentItemMetadata;
|
||||
use crate::models::FunctionCallOutputBody;
|
||||
@@ -24,6 +25,7 @@ use crate::protocol::PatchApplyStatus;
|
||||
use crate::protocol::ReviewOutputEvent;
|
||||
use crate::protocol::ReviewTarget;
|
||||
use crate::protocol::SubAgentActivityKind;
|
||||
use crate::protocol::UserMessageAudioKind;
|
||||
use crate::user_input::ByteRange;
|
||||
use crate::user_input::TextElement;
|
||||
use crate::user_input::UserInput;
|
||||
@@ -635,7 +637,52 @@ impl UserMessageItem {
|
||||
self.content
|
||||
.iter()
|
||||
.filter_map(|c| match c {
|
||||
UserInput::Audio { audio_url } => Some(audio_url.clone()),
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::Inline { audio_url },
|
||||
} => Some(audio_url.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns file-backed audio references when the message includes an upload.
|
||||
/// Inline payloads remain exclusively in the legacy `audio` field.
|
||||
pub fn audio_references(&self) -> Option<Vec<AudioReference>> {
|
||||
let audio = self
|
||||
.content
|
||||
.iter()
|
||||
.filter_map(|content| match content {
|
||||
UserInput::Audio {
|
||||
audio: audio @ AudioReference::File { .. },
|
||||
} => Some(audio.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
(!audio.is_empty()).then_some(audio)
|
||||
}
|
||||
|
||||
/// Returns the original inline/file ordering when uploaded audio is present.
|
||||
pub fn audio_order(&self) -> Vec<UserMessageAudioKind> {
|
||||
if !self.content.iter().any(|content| {
|
||||
matches!(
|
||||
content,
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File { .. }
|
||||
}
|
||||
)
|
||||
}) {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
self.content
|
||||
.iter()
|
||||
.filter_map(|content| match content {
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::Inline { .. },
|
||||
} => Some(UserMessageAudioKind::Inline),
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File { .. },
|
||||
} => Some(UserMessageAudioKind::File),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
@@ -812,7 +859,14 @@ mod tests {
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio_url: "https://example.com/remote.mp3".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "https://example.com/remote.mp3".to_string(),
|
||||
},
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_audio".to_string(),
|
||||
},
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: std::path::PathBuf::from("local.wav"),
|
||||
@@ -826,6 +880,16 @@ mod tests {
|
||||
vec![std::path::PathBuf::from("local.wav")],
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
item.audio_references(),
|
||||
Some(vec![AudioReference::File {
|
||||
file_id: "file_audio".to_string(),
|
||||
}])
|
||||
);
|
||||
assert_eq!(
|
||||
item.audio_order(),
|
||||
vec![UserMessageAudioKind::Inline, UserMessageAudioKind::File]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -112,6 +112,8 @@ impl UserMessageItem {
|
||||
local_images: self.local_image_paths(),
|
||||
local_image_details: self.local_image_details(),
|
||||
audio: Some(self.audio_urls()),
|
||||
audio_references: self.audio_references(),
|
||||
audio_order: self.audio_order(),
|
||||
local_audio: self.local_audio_paths(),
|
||||
text_elements: self.text_elements(),
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ use codex_utils_image::PromptImageMode;
|
||||
use codex_utils_image::data_url_from_bytes;
|
||||
use codex_utils_image::load_for_prompt_bytes;
|
||||
|
||||
use crate::models::AudioReference;
|
||||
use crate::models::ImageDetail;
|
||||
use crate::models::ImageReference;
|
||||
use crate::user_input::UserInput;
|
||||
@@ -42,7 +43,9 @@ pub fn snapshot_local_user_input(input: &mut UserInput) -> io::Result<()> {
|
||||
})?;
|
||||
let file_bytes = read_bounded_local_media(path, MAX_PROMPT_AUDIO_INPUT_BYTES, "audio")?;
|
||||
*input = UserInput::Audio {
|
||||
audio_url: data_url_from_bytes(mime, &file_bytes),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: data_url_from_bytes(mime, &file_bytes),
|
||||
},
|
||||
};
|
||||
}
|
||||
UserInput::Text { .. }
|
||||
|
||||
@@ -63,7 +63,9 @@ fn snapshots_local_audio_user_input_with_supported_mime_types() -> Result<()> {
|
||||
|
||||
assert_eq!(
|
||||
UserInput::Audio {
|
||||
audio_url: format!("data:{mime};base64,YXVkaW8="),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: format!("data:{mime};base64,YXVkaW8=")
|
||||
},
|
||||
},
|
||||
input
|
||||
);
|
||||
|
||||
@@ -893,7 +893,8 @@ pub enum ContentItem {
|
||||
detail: Option<ImageDetail>,
|
||||
},
|
||||
InputAudio {
|
||||
audio_url: String,
|
||||
#[serde(flatten)]
|
||||
audio: AudioReference,
|
||||
},
|
||||
OutputText {
|
||||
text: String,
|
||||
@@ -934,6 +935,15 @@ pub enum ImageReference {
|
||||
File { file_id: String },
|
||||
}
|
||||
|
||||
/// Inline audio data or an uploaded file loaded by the Responses API.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
|
||||
#[serde(untagged)]
|
||||
#[ts(untagged)]
|
||||
pub enum AudioReference {
|
||||
Inline { audio_url: String },
|
||||
File { file_id: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema, TS)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum AgentMessageInputContent {
|
||||
@@ -1982,7 +1992,9 @@ fn local_audio_content_items(
|
||||
text: local_audio_open_tag_text_with_path(label_number, path),
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: data_url_from_bytes(mime, file_bytes),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: data_url_from_bytes(mime, file_bytes),
|
||||
},
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: LOCAL_AUDIO_CLOSE_TAG.to_string(),
|
||||
@@ -2199,9 +2211,9 @@ impl ResponseInputItem {
|
||||
)],
|
||||
}
|
||||
}
|
||||
UserInput::Audio { audio_url } => {
|
||||
UserInput::Audio { audio } => {
|
||||
audio_index += 1;
|
||||
vec![ContentItem::InputAudio { audio_url }]
|
||||
vec![ContentItem::InputAudio { audio }]
|
||||
}
|
||||
UserInput::LocalAudio { path } => {
|
||||
audio_index += 1;
|
||||
@@ -2277,7 +2289,12 @@ impl From<ContentItem> for FunctionCallOutputContentItem {
|
||||
Self::InputText { text }
|
||||
}
|
||||
ContentItem::InputImage { image, detail } => Self::InputImage { image, detail },
|
||||
ContentItem::InputAudio { audio_url } => Self::InputAudio { audio_url },
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline { audio_url },
|
||||
} => Self::InputAudio { audio_url },
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::File { file_id, .. },
|
||||
} => Self::InputText { text: file_id },
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4147,34 +4164,47 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn serializes_audio_user_input_without_tags() -> Result<()> {
|
||||
let audio_url = "data:audio/wav;base64,abc".to_string();
|
||||
|
||||
let item = ResponseInputItem::from(vec![UserInput::Audio {
|
||||
audio_url: audio_url.clone(),
|
||||
}]);
|
||||
|
||||
assert_eq!(
|
||||
item,
|
||||
ResponseInputItem::Message {
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputAudio { audio_url }],
|
||||
phase: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(item)?,
|
||||
serde_json::json!({
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"audio_url": "data:audio/wav;base64,abc",
|
||||
},
|
||||
],
|
||||
})
|
||||
);
|
||||
|
||||
for (audio, wire) in [
|
||||
(
|
||||
AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,abc".into(),
|
||||
},
|
||||
serde_json::json!({"type": "input_audio", "audio_url": "data:audio/wav;base64,abc"}),
|
||||
),
|
||||
(
|
||||
AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
},
|
||||
serde_json::json!({"type": "input_audio", "file_id": "file_audio"}),
|
||||
),
|
||||
] {
|
||||
let input = UserInput::Audio {
|
||||
audio: audio.clone(),
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::from_value::<UserInput>(serde_json::to_value(&input)?)?,
|
||||
input
|
||||
);
|
||||
let item = ResponseInputItem::from(vec![input]);
|
||||
assert_eq!(
|
||||
item,
|
||||
ResponseInputItem::Message {
|
||||
role: "user".into(),
|
||||
content: vec![ContentItem::InputAudio { audio }],
|
||||
phase: None,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&item)?,
|
||||
serde_json::json!({
|
||||
"type": "message", "role": "user", "content": [wire],
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ResponseInputItem>(serde_json::to_value(&item)?)?,
|
||||
item
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -4207,7 +4237,9 @@ mod tests {
|
||||
),
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: format!("data:{mime};base64,YXVkaW8="),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: format!("data:{mime};base64,YXVkaW8=")
|
||||
},
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: audio_close_tag_text(),
|
||||
@@ -4583,7 +4615,9 @@ mod tests {
|
||||
|
||||
let item = ResponseInputItem::from(vec![
|
||||
UserInput::Audio {
|
||||
audio_url: audio_url.clone(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: audio_url.clone(),
|
||||
},
|
||||
},
|
||||
UserInput::LocalAudio {
|
||||
path: local_path.clone(),
|
||||
@@ -4595,7 +4629,9 @@ mod tests {
|
||||
ResponseInputItem::Message {
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputAudio { audio_url },
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline { audio_url }
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: local_audio_open_tag_text_with_path(
|
||||
/*label_number*/ 2,
|
||||
@@ -4603,7 +4639,9 @@ mod tests {
|
||||
),
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/mpeg;base64,YXVkaW8=".to_string()
|
||||
}
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: audio_close_tag_text(),
|
||||
|
||||
@@ -41,6 +41,7 @@ use crate::mcp::RequestId;
|
||||
use crate::memory_citation::MemoryCitation;
|
||||
use crate::models::ActivePermissionProfile;
|
||||
use crate::models::AgentMessageInputContent;
|
||||
use crate::models::AudioReference;
|
||||
use crate::models::BaseInstructions;
|
||||
use crate::models::ContentItem;
|
||||
use crate::models::ImageDetail;
|
||||
@@ -2777,6 +2778,14 @@ pub enum UserMessageImageKind {
|
||||
File,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
#[ts(rename_all = "snake_case")]
|
||||
pub enum UserMessageAudioKind {
|
||||
Inline,
|
||||
File,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, JsonSchema, TS)]
|
||||
pub struct UserMessageEvent {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -2818,6 +2827,16 @@ pub struct UserMessageEvent {
|
||||
/// legacy UI history events and correspond to audio sent to the model.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub audio: Option<Vec<String>>,
|
||||
/// File-backed audio references. When `audio_order` is complete, consumers
|
||||
/// combine this with `audio` without duplicating inline payloads. Older
|
||||
/// producers may instead store the complete ordered reference list here.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub audio_references: Option<Vec<AudioReference>>,
|
||||
/// Inline and file-backed audio kinds in their original input order.
|
||||
/// New mixed-input producers populate this alongside the split `audio` and
|
||||
/// `audio_references` fields; when absent, consumers retain the prior shape.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub audio_order: Vec<UserMessageAudioKind>,
|
||||
/// Local file paths sourced from `UserInput::LocalAudio`. These are kept so
|
||||
/// clients can reattach audio when editing history and should not be
|
||||
/// treated as API-ready URLs.
|
||||
@@ -2847,6 +2866,30 @@ impl UserMessageEvent {
|
||||
inline_count == self.images.as_ref().map_or(0, Vec::len)
|
||||
&& file_count == self.file_ids.as_ref().map_or(0, Vec::len)
|
||||
}
|
||||
|
||||
/// Returns whether `audio_order` accounts for every split audio reference exactly once.
|
||||
pub fn has_complete_audio_order(&self) -> bool {
|
||||
if self.audio_order.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut inline_count = 0;
|
||||
let mut file_count = 0;
|
||||
for audio_kind in &self.audio_order {
|
||||
match audio_kind {
|
||||
UserMessageAudioKind::Inline => inline_count += 1,
|
||||
UserMessageAudioKind::File => file_count += 1,
|
||||
}
|
||||
}
|
||||
|
||||
inline_count == self.audio.as_ref().map_or(0, Vec::len)
|
||||
&& file_count == self.audio_references.as_ref().map_or(0, Vec::len)
|
||||
&& self.audio_references.as_ref().is_none_or(|references| {
|
||||
references
|
||||
.iter()
|
||||
.all(|reference| matches!(reference, AudioReference::File { .. }))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the user-facing preview text for a user message.
|
||||
@@ -2867,7 +2910,13 @@ pub fn user_message_preview(user: &UserMessageEvent) -> Option<String> {
|
||||
{
|
||||
return Some("[Image]".to_string());
|
||||
}
|
||||
if user.audio.as_ref().is_some_and(|audio| !audio.is_empty()) || !user.local_audio.is_empty() {
|
||||
if user.audio.as_ref().is_some_and(|audio| !audio.is_empty())
|
||||
|| user
|
||||
.audio_references
|
||||
.as_ref()
|
||||
.is_some_and(|audio| !audio.is_empty())
|
||||
|| !user.local_audio.is_empty()
|
||||
{
|
||||
return Some("[Audio]".to_string());
|
||||
}
|
||||
None
|
||||
@@ -4762,6 +4811,7 @@ mod tests {
|
||||
use crate::items::UserMessageItem;
|
||||
use crate::items::WebSearchItem;
|
||||
use crate::mcp::CallToolResult;
|
||||
use crate::models::AudioReference;
|
||||
use crate::permissions::FileSystemAccessMode;
|
||||
use crate::permissions::FileSystemPath;
|
||||
use crate::permissions::FileSystemSandboxEntry;
|
||||
@@ -6297,7 +6347,14 @@ mod tests {
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
crate::user_input::UserInput::Audio {
|
||||
audio_url: "https://example.com/remote.mp3".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "https://example.com/remote.mp3".to_string(),
|
||||
},
|
||||
},
|
||||
crate::user_input::UserInput::Audio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_audio".to_string(),
|
||||
},
|
||||
},
|
||||
crate::user_input::UserInput::LocalAudio {
|
||||
path: local_audio_path.clone(),
|
||||
@@ -6342,16 +6399,41 @@ mod tests {
|
||||
Some(vec!["https://example.com/remote.mp3".to_string()])
|
||||
);
|
||||
assert_eq!(event.local_audio, vec![local_audio_path]);
|
||||
assert_eq!(
|
||||
event.audio_references,
|
||||
Some(vec![AudioReference::File {
|
||||
file_id: "file_audio".to_string(),
|
||||
}])
|
||||
);
|
||||
assert_eq!(
|
||||
event.audio_order,
|
||||
vec![UserMessageAudioKind::Inline, UserMessageAudioKind::File]
|
||||
);
|
||||
// Earlier readers ignore the new ordered field and retain inline recordings.
|
||||
#[derive(Deserialize)]
|
||||
struct EarlierUserMessageEvent {
|
||||
audio: Option<Vec<String>>,
|
||||
}
|
||||
let earlier: EarlierUserMessageEvent = serde_json::from_value(event_json).unwrap();
|
||||
assert_eq!(earlier.audio, event.audio);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_only_user_message_has_placeholder_preview() {
|
||||
let event = UserMessageEvent {
|
||||
audio: Some(vec!["https://example.com/remote.mp3".to_string()]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(user_message_preview(&event), Some("[Audio]".to_string()));
|
||||
for event in [
|
||||
UserMessageEvent {
|
||||
audio: Some(vec!["https://example.com/remote.mp3".to_string()]),
|
||||
..Default::default()
|
||||
},
|
||||
UserMessageEvent {
|
||||
audio_references: Some(vec![AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert_eq!(user_message_preview(&event), Some("[Audio]".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -3,6 +3,7 @@ use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use ts_rs::TS;
|
||||
|
||||
use crate::models::AudioReference;
|
||||
use crate::models::ImageDetail;
|
||||
use crate::models::ImageReference;
|
||||
|
||||
@@ -40,8 +41,11 @@ pub enum UserInput {
|
||||
#[ts(optional)]
|
||||
detail: Option<ImageDetail>,
|
||||
},
|
||||
/// Pre-encoded audio data URI forwarded to the Responses API.
|
||||
Audio { audio_url: String },
|
||||
/// Audio reference forwarded to the Responses API.
|
||||
Audio {
|
||||
#[serde(flatten)]
|
||||
audio: AudioReference,
|
||||
},
|
||||
/// Local audio path converted to an `Audio` data URI during request serialization.
|
||||
LocalAudio { path: std::path::PathBuf },
|
||||
|
||||
|
||||
@@ -26,8 +26,10 @@ use codex_protocol::items::SubAgentActivityItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::items::UserMessageItem;
|
||||
use codex_protocol::items::WebSearchItem;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::UserMessageAudioKind;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_protocol::protocol::UserMessageImageKind;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
@@ -40,6 +42,7 @@ pub(super) fn user_message_item(
|
||||
) -> ThreadStoreResult<TurnItem> {
|
||||
let mut content = Vec::new();
|
||||
let has_image_order = event.has_complete_image_order();
|
||||
let has_audio_order = event.has_complete_audio_order();
|
||||
if !event.message.trim().is_empty() {
|
||||
content.push(UserInput::Text {
|
||||
text: event.message,
|
||||
@@ -110,12 +113,26 @@ pub(super) fn user_message_item(
|
||||
detail: event.local_image_details.get(index).copied().flatten(),
|
||||
});
|
||||
}
|
||||
if let Some(audio) = event.audio {
|
||||
content.extend(
|
||||
audio
|
||||
.into_iter()
|
||||
.map(|audio_url| UserInput::Audio { audio_url }),
|
||||
);
|
||||
if has_audio_order {
|
||||
let mut inline_audio = event.audio.unwrap_or_default().into_iter();
|
||||
let mut file_audio = event.audio_references.unwrap_or_default().into_iter();
|
||||
for audio_kind in event.audio_order {
|
||||
let audio = match audio_kind {
|
||||
UserMessageAudioKind::Inline => inline_audio
|
||||
.next()
|
||||
.map(|audio_url| AudioReference::Inline { audio_url }),
|
||||
UserMessageAudioKind::File => file_audio.next(),
|
||||
};
|
||||
if let Some(audio) = audio {
|
||||
content.push(UserInput::Audio { audio });
|
||||
}
|
||||
}
|
||||
} else if let Some(audio) = event.audio_references {
|
||||
content.extend(audio.into_iter().map(|audio| UserInput::Audio { audio }));
|
||||
} else if let Some(audio) = event.audio {
|
||||
content.extend(audio.into_iter().map(|audio_url| UserInput::Audio {
|
||||
audio: AudioReference::Inline { audio_url },
|
||||
}));
|
||||
}
|
||||
content.extend(
|
||||
event
|
||||
|
||||
@@ -11,10 +11,12 @@ use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
|
||||
use codex_protocol::ResponseItemId;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::UserMessageAudioKind;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_protocol::protocol::UserMessageImageKind;
|
||||
use codex_rollout::CompactedItem;
|
||||
@@ -580,7 +582,7 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent
|
||||
file_ids.push(file_id.as_str());
|
||||
}
|
||||
},
|
||||
ContentItem::InputAudio { audio_url } => audio.push(audio_url.as_str()),
|
||||
ContentItem::InputAudio { audio: reference } => audio.push(reference),
|
||||
ContentItem::OutputText { .. } => return false,
|
||||
}
|
||||
}
|
||||
@@ -602,14 +604,48 @@ fn user_response_matches_event(content: &[ContentItem], event: &UserMessageEvent
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.collect::<Vec<_>>()
|
||||
&& audio
|
||||
== event
|
||||
.audio
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.collect::<Vec<_>>()
|
||||
&& {
|
||||
if event.has_complete_audio_order() {
|
||||
let mut actual = audio.into_iter();
|
||||
let mut inline = event.audio.as_deref().unwrap_or_default().iter();
|
||||
let mut file = event.audio_references.as_deref().unwrap_or_default().iter();
|
||||
event.audio_order.iter().all(|kind| {
|
||||
let Some(actual) = actual.next() else {
|
||||
return false;
|
||||
};
|
||||
match kind {
|
||||
UserMessageAudioKind::Inline => matches!(
|
||||
(actual, inline.next()),
|
||||
(
|
||||
AudioReference::Inline { audio_url },
|
||||
Some(expected),
|
||||
) if audio_url == expected
|
||||
),
|
||||
UserMessageAudioKind::File => {
|
||||
file.next().is_some_and(|expected| actual == expected)
|
||||
}
|
||||
}
|
||||
}) && actual.next().is_none()
|
||||
&& inline.next().is_none()
|
||||
&& file.next().is_none()
|
||||
} else {
|
||||
match &event.audio_references {
|
||||
Some(references) => audio.into_iter().eq(references),
|
||||
None => audio
|
||||
.into_iter()
|
||||
.map(|reference| match reference {
|
||||
AudioReference::Inline { audio_url } => Some(audio_url.as_str()),
|
||||
AudioReference::File { .. } => None,
|
||||
})
|
||||
.eq(event
|
||||
.audio
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|url| Some(url.as_str()))),
|
||||
}
|
||||
}
|
||||
}
|
||||
&& event.local_images.is_empty()
|
||||
&& event.local_audio.is_empty()
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ use codex_protocol::items::ReasoningItem;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::mcp::McpResourceOrigin;
|
||||
use codex_protocol::mcp::McpResourceOriginCheckpoint;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ContentItemKind;
|
||||
use codex_protocol::models::ImageDetail;
|
||||
@@ -43,6 +44,7 @@ use codex_protocol::protocol::ThreadRolledBackEvent;
|
||||
use codex_protocol::protocol::TurnCompleteEvent;
|
||||
use codex_protocol::protocol::TurnContextItem;
|
||||
use codex_protocol::protocol::TurnStartedEvent;
|
||||
use codex_protocol::protocol::UserMessageAudioKind;
|
||||
use codex_protocol::protocol::UserMessageEvent;
|
||||
use codex_protocol::protocol::UserMessageImageKind;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
@@ -340,6 +342,11 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
|
||||
file_ids: Some(vec!["file_123".to_string()]),
|
||||
file_id_details: vec![Some(ImageDetail::High)],
|
||||
image_order: vec![UserMessageImageKind::File, UserMessageImageKind::Inline],
|
||||
audio: Some(vec!["data:audio/wav;base64,YXVkaW8=".into()]),
|
||||
audio_references: Some(vec![AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
}]),
|
||||
audio_order: vec![UserMessageAudioKind::File, UserMessageAudioKind::Inline],
|
||||
..Default::default()
|
||||
};
|
||||
let path = write_rollout(
|
||||
@@ -403,6 +410,16 @@ async fn migration_publishes_canonical_projected_history_and_is_idempotent() {
|
||||
},
|
||||
detail: Some(ImageDetail::Original),
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
},
|
||||
},
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".into(),
|
||||
},
|
||||
},
|
||||
];
|
||||
assert_eq!(user_item.map(|item| &item.content), Some(&expected_content));
|
||||
|
||||
@@ -1010,6 +1027,9 @@ async fn migration_coalesces_response_first_user_message_rollback_boundary() {
|
||||
let home = TempDir::new().expect("create Codex home");
|
||||
let thread_id = ThreadId::new();
|
||||
let file_id = "file_123".to_string();
|
||||
let audio = AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
};
|
||||
let response = ResponseItem::Message {
|
||||
status: None,
|
||||
encrypted_content: None,
|
||||
@@ -1025,6 +1045,9 @@ async fn migration_coalesces_response_first_user_message_rollback_boundary() {
|
||||
},
|
||||
detail: None,
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: audio.clone(),
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
@@ -1032,6 +1055,8 @@ async fn migration_coalesces_response_first_user_message_rollback_boundary() {
|
||||
let event = UserMessageEvent {
|
||||
message: "remove question".to_string(),
|
||||
file_ids: Some(vec![file_id]),
|
||||
audio_references: Some(vec![audio]),
|
||||
audio_order: vec![UserMessageAudioKind::File],
|
||||
..Default::default()
|
||||
};
|
||||
let path = write_rollout(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::ImageReference;
|
||||
use codex_protocol::models::ResponseInputItem;
|
||||
@@ -31,7 +32,14 @@ fn code_mode_text_preserves_message_content() {
|
||||
detail: None,
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,def".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,def".to_string(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_audio".to_string(),
|
||||
},
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
@@ -40,7 +48,7 @@ fn code_mode_text_preserves_message_content() {
|
||||
assert_eq!(
|
||||
response_input_to_code_mode_result(response),
|
||||
serde_json::json!(
|
||||
"Input text\nOutput text\nfile_123\ndata:image/png;base64,abc\ndata:audio/wav;base64,def"
|
||||
"Input text\nOutput text\nfile_123\ndata:image/png;base64,abc\ndata:audio/wav;base64,def\nfile_audio"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
+10
-2
@@ -9,10 +9,14 @@ Scenario: The fork excludes the blocked turn and submits the selected rewrite.
|
||||
-- request 1 (turn) --
|
||||
00:message/developer:
|
||||
Help the user.
|
||||
01:message/developer[3]:
|
||||
01:message/developer[4]:
|
||||
[01] <SKILLS_INSTRUCTIONS>
|
||||
[02] <PERMISSIONS_INSTRUCTIONS>
|
||||
[03] <COLLABORATION_MODE>
|
||||
[04] <tools>
|
||||
Deferred tool namespaces:
|
||||
- codex_tui: Manage Codex tasks available through the connected app server.
|
||||
</tools>
|
||||
02:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
03:message/user:
|
||||
@@ -22,10 +26,14 @@ Scenario: The fork excludes the blocked turn and submits the selected rewrite.
|
||||
-- request 2 (turn) --
|
||||
00:message/developer:
|
||||
Help the user.
|
||||
01:message/developer[3]:
|
||||
01:message/developer[4]:
|
||||
[01] <SKILLS_INSTRUCTIONS>
|
||||
[02] <PERMISSIONS_INSTRUCTIONS>
|
||||
[03] <COLLABORATION_MODE>
|
||||
[04] <tools>
|
||||
Deferred tool namespaces:
|
||||
- codex_tui: Manage Codex tasks available through the connected app server.
|
||||
</tools>
|
||||
02:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
03:message/user:
|
||||
|
||||
@@ -911,7 +911,9 @@ async fn live_app_server_user_message_omits_unsupported_media() {
|
||||
text_elements: Vec::new(),
|
||||
},
|
||||
AppServerUserInput::Audio {
|
||||
url: "https://example.com/one.wav".to_string(),
|
||||
audio: codex_app_server_protocol::AudioReference::Inline {
|
||||
url: "https://example.com/one.wav".to_string(),
|
||||
},
|
||||
},
|
||||
AppServerUserInput::LocalAudio {
|
||||
path: test_path_buf("/tmp/two.wav"),
|
||||
|
||||
@@ -5,6 +5,7 @@ use crate::app_event_sender::AppEventSender;
|
||||
use codex_app_server_client::AppServerRequestHandle;
|
||||
use codex_app_server_client::TypedRequestError;
|
||||
use codex_app_server_protocol::Account;
|
||||
use codex_app_server_protocol::AudioReference;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
||||
use codex_app_server_protocol::DynamicToolCallParams;
|
||||
@@ -1385,7 +1386,10 @@ fn turn_summary(turn: &Turn, include_outputs: bool, output_chars: usize) -> Valu
|
||||
..
|
||||
} => json!({"type": "image", "fileId": file_id}),
|
||||
UserInput::LocalImage { path, .. } => json!({"type": "localImage", "path": path}),
|
||||
UserInput::Audio { url } => json!({"type": "audio", "url": url}),
|
||||
UserInput::Audio { audio: AudioReference::Inline { url } } => json!({"type": "audio", "url": url}),
|
||||
UserInput::Audio {
|
||||
audio: AudioReference::File { file_id, .. },
|
||||
} => json!({"type": "audio", "fileId": file_id}),
|
||||
UserInput::LocalAudio { path } => json!({"type": "localAudio", "path": path}),
|
||||
UserInput::Skill { name, path } => json!({"type": "skill", "name": name, "path": path}),
|
||||
UserInput::Mention { name, path } => json!({"type": "mention", "name": name, "path": path}),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::FunctionCallOutputBody;
|
||||
use codex_protocol::models::FunctionCallOutputPayload;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -13,13 +14,24 @@ fn preparation_canonicalizes_data_urls_and_rejects_remote_urls() {
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/x-wav;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/x-wav;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "https://example.com/audio.mp3".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "https://example.com/audio.mp3".to_string(),
|
||||
},
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
@@ -37,10 +49,19 @@ fn preparation_canonicalizes_data_urls_and_rejects_remote_urls() {
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::File {
|
||||
file_id: "file_audio".into(),
|
||||
}
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string()
|
||||
}
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/ogg;base64,YXVkaW8=".to_string()
|
||||
}
|
||||
},
|
||||
ContentItem::InputText {
|
||||
text: "audio content omitted because it could not be processed".to_string(),
|
||||
@@ -61,10 +82,14 @@ fn preparation_replaces_invalid_message_audio_with_placeholders() {
|
||||
role: "user".to_string(),
|
||||
content: vec![
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/wav;base64,%%%".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/wav;base64,%%%".to_string(),
|
||||
},
|
||||
},
|
||||
ContentItem::InputAudio {
|
||||
audio_url: "data:audio/flac;base64,YXVkaW8=".to_string(),
|
||||
audio: AudioReference::Inline {
|
||||
audio_url: "data:audio/flac;base64,YXVkaW8=".to_string(),
|
||||
},
|
||||
},
|
||||
],
|
||||
phase: None,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_protocol::models::AudioReference;
|
||||
use codex_protocol::models::ContentItem;
|
||||
use codex_protocol::models::FunctionCallOutputContentItem;
|
||||
use codex_protocol::models::MAX_PROMPT_AUDIO_INPUT_BYTES;
|
||||
@@ -91,7 +92,9 @@ pub fn prepare_response_items(items: &mut [ResponseItem]) {
|
||||
|
||||
fn prepare_message_content(items: &mut [ContentItem]) {
|
||||
for item in items {
|
||||
if let ContentItem::InputAudio { audio_url } = item
|
||||
if let ContentItem::InputAudio {
|
||||
audio: AudioReference::Inline { audio_url },
|
||||
} = item
|
||||
&& let Err(error) = prepare_audio(audio_url)
|
||||
{
|
||||
warn!(%error, "failed to prepare message audio");
|
||||
|
||||
@@ -605,7 +605,7 @@ def generate_v2_all(schema_dir: Path, *, format_output: bool = True) -> None:
|
||||
+ (["--formatters", "ruff-format"] if format_output else []),
|
||||
cwd=sdk_root() if format_output else Path(td),
|
||||
)
|
||||
_preserve_inline_image_class_names(out_path)
|
||||
_preserve_inline_media_class_names(out_path)
|
||||
_add_legacy_image_generation_thread_item_alias(out_path)
|
||||
_preserve_open_enum(out_path, "CodexErrorInfoValue", allow_empty=True)
|
||||
_preserve_reasoning_effort_enum(out_path)
|
||||
@@ -614,13 +614,15 @@ def generate_v2_all(schema_dir: Path, *, format_output: bool = True) -> None:
|
||||
_normalize_generated_timestamps(out_path)
|
||||
|
||||
|
||||
def _preserve_inline_image_class_names(out_path: Path) -> None:
|
||||
"""Keep the public class names used before ImageReference was introduced."""
|
||||
def _preserve_inline_media_class_names(out_path: Path) -> None:
|
||||
"""Keep the public inline class names used before file references were introduced."""
|
||||
source = out_path.read_text()
|
||||
stable_names = {
|
||||
"UrlUserInput": "ImageUserInput",
|
||||
"ImageUrlContentItem": "InputImageContentItem",
|
||||
"ImageUrlFunctionCallOutputContentItem": "InputImageFunctionCallOutputContentItem",
|
||||
"UrlUserInput1": "AudioUserInput",
|
||||
"AudioUrlContentItem": "InputAudioContentItem",
|
||||
}
|
||||
for generated_name, stable_name in stable_names.items():
|
||||
if source.count(f"class {generated_name}(") != 1:
|
||||
|
||||
@@ -1101,8 +1101,16 @@ class InputAudioContentItem(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
populate_by_name=True,
|
||||
)
|
||||
audio_url: str
|
||||
type: Annotated[Literal["input_audio"], Field(title="InputAudioContentItemType")]
|
||||
audio_url: str
|
||||
|
||||
|
||||
class FileIdContentItem1(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
populate_by_name=True,
|
||||
)
|
||||
type: Annotated[Literal["input_audio"], Field(title="InputAudioContentItemType")]
|
||||
file_id: str
|
||||
|
||||
|
||||
class OutputTextContentItem(BaseModel):
|
||||
@@ -6762,6 +6770,14 @@ class AudioUserInput(BaseModel):
|
||||
url: str
|
||||
|
||||
|
||||
class FileIdUserInput1(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
populate_by_name=True,
|
||||
)
|
||||
type: Annotated[Literal["audio"], Field(title="AudioUserInputType")]
|
||||
file_id: Annotated[str, Field(alias="fileId")]
|
||||
|
||||
|
||||
class LocalAudioUserInput(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
populate_by_name=True,
|
||||
@@ -6795,6 +6811,7 @@ class UserInput(
|
||||
| FileIdUserInput
|
||||
| LocalImageUserInput
|
||||
| AudioUserInput
|
||||
| FileIdUserInput1
|
||||
| LocalAudioUserInput
|
||||
| SkillUserInput
|
||||
| MentionUserInput
|
||||
@@ -6806,6 +6823,7 @@ class UserInput(
|
||||
| FileIdUserInput
|
||||
| LocalImageUserInput
|
||||
| AudioUserInput
|
||||
| FileIdUserInput1
|
||||
| LocalAudioUserInput
|
||||
| SkillUserInput
|
||||
| MentionUserInput
|
||||
@@ -8375,6 +8393,7 @@ class ContentItem(
|
||||
| InputImageContentItem
|
||||
| FileIdContentItem
|
||||
| InputAudioContentItem
|
||||
| FileIdContentItem1
|
||||
| OutputTextContentItem
|
||||
]
|
||||
):
|
||||
@@ -8383,6 +8402,7 @@ class ContentItem(
|
||||
| InputImageContentItem
|
||||
| FileIdContentItem
|
||||
| InputAudioContentItem
|
||||
| FileIdContentItem1
|
||||
| OutputTextContentItem
|
||||
)
|
||||
|
||||
|
||||
@@ -555,6 +555,28 @@ def test_generated_inline_image_class_names_remain_stable() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_generated_audio_inputs_preserve_inline_and_file_shapes() -> None:
|
||||
"""Existing audio imports still work, and file references round-trip through the unions."""
|
||||
from openai_codex.generated.v2_all import (
|
||||
AudioUserInput,
|
||||
ContentItem,
|
||||
FileIdUserInput1,
|
||||
InputAudioContentItem,
|
||||
UserInput,
|
||||
)
|
||||
|
||||
audio_url = "data:audio/wav;base64,YXVkaW8="
|
||||
for model, wire in [
|
||||
(AudioUserInput, {"type": "audio", "url": audio_url}),
|
||||
(InputAudioContentItem, {"type": "input_audio", "audio_url": audio_url}),
|
||||
(UserInput, {"type": "audio", "fileId": "file_audio"}),
|
||||
(FileIdUserInput1, {"type": "audio", "fileId": "file_audio"}),
|
||||
(ContentItem, {"type": "input_audio", "file_id": "file_audio"}),
|
||||
]:
|
||||
audio = model.model_validate(wire)
|
||||
assert audio.model_dump(by_alias=True, exclude_unset=True, mode="json") == wire
|
||||
|
||||
|
||||
def test_generated_image_generation_thread_item_alias_remains_available() -> None:
|
||||
"""Existing SDK callers can still construct and validate inline image-generation items."""
|
||||
from openai_codex.generated.v2_all import (
|
||||
|
||||
Reference in New Issue
Block a user