Add audio output support to dynamic tools and code mode (#34080)

## What changed

- Add `inputAudio` content items to dynamic tool responses, app-server events, thread history, and generated protocol schemas.
- Add an `audio()` code-mode helper that accepts inline data URLs, audio URL objects, and MCP audio blocks.
- Convert MCP audio blocks into model input when audio is supported, and replace unsupported audio with an explanatory text item.
- Reject non-data audio URLs and track audio item counts in dynamic tool analytics.

## Testing

- Cover audio serialization, protocol round trips, thread-history conversion, MCP modality filtering, code-mode helper inputs, and invalid URL handling.

GitOrigin-RevId: 1ed52a8f9c62d4840fb71c5ec736b4a3566243d6
This commit is contained in:
nhamidi-oai
2026-07-18 23:22:13 +00:00
committed by copyberry
parent 312caf176a
commit 643de86a19
46 changed files with 987 additions and 72 deletions
@@ -130,6 +130,9 @@ fn content_items() -> Vec<WireContentItem> {
image_url: "data:image/png;base64,original".to_string(),
detail: Some(WireImageDetail::Original),
},
WireContentItem::InputAudio {
audio_url: "data:audio/wav;base64,YXVkaW8=".to_string(),
},
]
}
@@ -157,6 +160,10 @@ fn content_items_json() -> Value {
"image_url": "data:image/png;base64,original",
"detail": "original",
},
{
"type": "input_audio",
"audio_url": "data:audio/wav;base64,YXVkaW8=",
},
])
}
@@ -245,6 +245,9 @@ pub enum WireContentItem {
#[serde(default, skip_serializing_if = "Option::is_none")]
detail: Option<WireImageDetail>,
},
InputAudio {
audio_url: String,
},
}
impl From<FunctionCallOutputContentItem> for WireContentItem {
@@ -255,6 +258,9 @@ impl From<FunctionCallOutputContentItem> for WireContentItem {
image_url,
detail: detail.map(Into::into),
},
FunctionCallOutputContentItem::InputAudio { audio_url } => {
Self::InputAudio { audio_url }
}
}
}
}
@@ -267,6 +273,7 @@ impl From<WireContentItem> for FunctionCallOutputContentItem {
image_url,
detail: detail.map(Into::into),
},
WireContentItem::InputAudio { audio_url } => Self::InputAudio { audio_url },
}
}
}