Serialize Responses routing fields before large inputs (#49675)

## Why

Gateways that inspect request bodies incrementally need routing fields before potentially multi-megabyte prompts so they can route requests without buffering the input first.

## What changed

Serialize `model`, `stream`, and optional `service_tier` before `input` in both HTTP Responses requests and WebSocket `response.create` messages. Preserve field values and omission of unset `service_tier`.

## Testing

Add an HTTP regression test with a 2 MiB input that checks routing field order and unchanged JSON contents. Extend WebSocket serialization coverage to assert the routing fields appear first, and check that HTTP requests omit unset `service_tier`.

GitOrigin-RevId: dbc174e1fe7ca30d0c00f075bc37dd5f97b27b37
This commit is contained in:
Justin Beckwith
2026-09-30 16:57:01 +00:00
committed by copyberry
parent 67727e7cf1
commit ed0cc1a4ab
3 changed files with 71 additions and 6 deletions
+58
View File
@@ -376,6 +376,7 @@ async fn responses_client_stream_request_preserves_item_ids() -> Result<()> {
serde_json::from_slice(prepared.body.as_deref().expect("body should be JSON"))?;
assert_eq!(body, expected);
assert_eq!(body["input"][0]["id"], "msg_1");
assert_eq!(body.get("service_tier"), None);
assert_eq!(
prepared.headers.get(http::header::CONTENT_TYPE),
Some(&HeaderValue::from_static("application/json"))
@@ -383,6 +384,63 @@ async fn responses_client_stream_request_preserves_item_ids() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn responses_client_stream_request_sends_routing_fields_ahead_of_large_input() -> Result<()> {
let state = RecordingState::default();
let transport = RecordingTransport::new(state.clone());
let client = ResponsesClient::new(transport, provider("openai"), Arc::new(NoAuth));
// Exercise the customer gateway case where routing fields must be available without
// buffering a potentially multi-megabyte prompt first.
let large_input = "x".repeat(2 * 1024 * 1024);
let request = ResponsesApiRequest {
model: "gpt-test".into(),
instructions: "Say hi".into(),
input: vec![ResponseItem::Message {
id: None,
role: "user".into(),
content: vec![ContentItem::InputText { text: large_input }],
phase: None,
internal_chat_message_metadata_passthrough: None,
}],
tools: None,
tool_choice: "auto".into(),
parallel_tool_calls: false,
reasoning: None,
store: false,
stream: true,
stream_options: None,
include: Vec::new(),
service_tier: Some("priority".into()),
prompt_cache_key: None,
text: None,
client_metadata: None,
access_programs: None,
};
let expected = serde_json::to_value(&request)?;
let _stream = client
.stream_request(request, ResponsesOptions::default())
.await?;
let requests = state.take_stream_requests();
assert_eq!(requests.len(), 1);
let body = std::str::from_utf8(request_body_bytes(&requests[0]))?;
let input_position = body.find(r#""input":"#).expect("input should be present");
for routing_field in [r#""model":"#, r#""stream":"#, r#""service_tier":"#] {
assert!(
body.find(routing_field)
.is_some_and(|position| position < input_position),
"{routing_field} should precede input"
);
}
assert!(body.starts_with(
r#"{"model":"gpt-test","stream":true,"service_tier":"priority","instructions":"Say hi","input":[{"type":"message""#
));
assert!(body.len() > 2 * 1024 * 1024);
assert_eq!(serde_json::from_str::<serde_json::Value>(body)?, expected);
Ok(())
}
#[tokio::test]
async fn streaming_client_adds_auth_headers() -> Result<()> {
let state = RecordingState::default();