Allow per-sample model overrides in thread/sample (#62419)

## Why

Clients sampling a thread could only use the source thread's model.

## What changed

Add an optional `model` field to the experimental `thread/sample`
request. Omitting it or passing `null` inherits the source thread's
model. When an override is supplied, the server uses the selected
model's capabilities to decide whether to apply the requested
reasoning effort.

## Testing

Add an app-server integration test that checks model selection and
reasoning effort for the overridden sample, then verifies that a
subsequent sample and a follow-up turn use the source model.

GitOrigin-RevId: f5c54ca3d693d31e1f3e0628a520583330a5d56d
This commit is contained in:
Keyan Zhang
2026-11-19 03:37:31 +00:00
committed by copyberry
parent cf881a627d
commit 0bc1e7db9a
4 changed files with 101 additions and 2 deletions
@@ -55,6 +55,7 @@ use tokio::time::timeout;
use wiremock::ResponseTemplate;
const WAIT: Duration = Duration::from_secs(40);
const SAMPLE_MODEL: &str = "gpt-5.4-mini";
async fn client(
home: &TempDir,
@@ -89,10 +90,13 @@ async fn client_with_effort_updates(
name: "Fast".to_string(),
description: "Priority processing.".to_string(),
}];
let mut sample_model = model.clone();
sample_model.slug = SAMPLE_MODEL.to_string();
sample_model.supports_reasoning_effort_updates = false;
let catalog = home.path().join("catalog.json");
std::fs::write(
&catalog,
serde_json::to_string(&json!({"models": [model]}))?,
serde_json::to_string(&json!({"models": [model, sample_model]}))?,
)?;
let mut config = MockResponsesConfig::new(&server.uri()).with_model("gpt-5.4").with_provider_name("OpenAI")
.with_root_config(&format!("model_catalog_json = {}\nmodel_reasoning_effort = \"high\"\nmodel_post_turn_compact_threshold_percent = 0", serde_json::to_string(&catalog)?));
@@ -173,6 +177,92 @@ async fn prediction_fork(client: &mut TestAppServer, parent: &str) -> Result<Str
Ok(fork.thread.id)
}
#[tokio::test]
async fn sample_model_override_is_local_to_the_sample() -> Result<()> {
let server = responses::start_mock_server().await;
let mock = responses::mount_sse_sequence(
&server,
["parent", "override", "inherited", "follow-up"]
.map(|id| {
responses::sse(vec![
responses::ev_assistant_message(id, "Ready."),
responses::ev_completed(id),
])
})
.to_vec(),
)
.await;
let home = TempDir::new()?;
let mut client = client(&home, &server, /*overrides*/ true).await?;
let (thread_id, turn_id) = ready_parent(&mut client, ThreadStartParams::default()).await?;
let result = sample(
&mut client,
ThreadSampleParams {
thread_id: thread_id.clone(),
expected_turn_id: turn_id.clone(),
prompt: "Predict the next message.".to_string(),
model: Some(SAMPLE_MODEL.to_string()),
effort: Some(ReasoningEffort::Medium),
..Default::default()
},
)
.await?;
assert_eq!(result.text.as_deref(), Some("Ready."));
let request_id = client
.send_raw_request(
"thread/sample",
Some(json!({
"threadId": thread_id,
"expectedTurnId": turn_id,
"prompt": "Predict with inherited settings.",
})),
)
.await?;
let _: ThreadSampleResponse = timeout(WAIT, client.read_response(request_id)).await??;
client
.start_turn_and_wait_for_completion(TurnStartParams {
thread_id,
input: input("Follow-up"),
..Default::default()
})
.await?;
let requests = mock.requests();
let models: Vec<_> = requests
.iter()
.map(|request| request.body_json()["model"].clone())
.collect();
assert_eq!(
models,
["gpt-5.4", SAMPLE_MODEL, "gpt-5.4", "gpt-5.4"].map(|model| json!(model)),
);
assert_eq!(
requests[1].body_json()["reasoning"]["effort"],
json!("high")
);
let announcement = requests[1]
.message_input_texts("developer")
.into_iter()
.find(|text| text.starts_with("<model_instructions>"))
.expect("sample model announcement");
insta::assert_snapshot!(announcement, @"
<model_instructions>
The current model is gpt-5.4-mini.
</model_instructions>
");
assert!(
!requests[2].body_contains_text("<model_instructions>")
&& !requests[2].body_contains_text("<model_switch>"),
"sample without a model override should not announce a model switch"
);
assert!(requests[1].body_contains_text("Initial turn"));
assert!(requests[1].body_contains_text("Predict the next message."));
assert!(!requests[3].body_contains_text("Predict the next message."));
Ok(())
}
#[test_matrix(
[ThreadHistoryMode::Legacy, ThreadHistoryMode::Paginated],
[TurnStatus::Completed, TurnStatus::Failed, TurnStatus::Interrupted],