Make Ultra reasoning fallback model-aware (#41206)

## What changed

- Add an optional `multi_agent_reasoning_effort` field to model metadata.
- When `Ultra` is selected, use a supported catalog override when present. Otherwise, prefer `max`, then the highest supported non-`ultra` effort, and finally `medium` when the model has no reasoning levels.
- Preserve existing request behavior for other reasoning efforts, including mapping `persistent` to `disabled`.

## Testing

- Cover configured, missing, unsupported, and empty-catalog fallbacks for both parent and spawned-agent requests.

GitOrigin-RevId: b7ef576f32e0ea95ef38bf95c25789ed16aae94e
This commit is contained in:
Shijie Rao
2026-08-27 21:03:09 +00:00
committed by copyberry
parent e931d07b88
commit 7f135e1314
16 changed files with 267 additions and 18 deletions
@@ -74,6 +74,7 @@ fn preset_to_info(preset: &ModelPreset, priority: i32) -> ModelInfo {
model_specialty: None,
tool_mode: None,
multi_agent_version: preset.multi_agent_version,
multi_agent_reasoning_effort: None,
}
}
@@ -108,6 +108,7 @@ async fn models_client_hits_models_endpoint() {
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
}],
};
+33 -5
View File
@@ -179,9 +179,36 @@ pub(crate) struct CompactConversationRequestSettings {
pub(crate) service_tier: Option<String>,
}
fn reasoning_effort_for_request(effort: ReasoningEffortConfig) -> ReasoningEffortConfig {
fn reasoning_effort_for_request(
model_info: &ModelInfo,
effort: ReasoningEffortConfig,
) -> ReasoningEffortConfig {
match effort {
ReasoningEffortConfig::Ultra => ReasoningEffortConfig::Max,
ReasoningEffortConfig::Ultra => model_info
.multi_agent_reasoning_effort
.as_ref()
.filter(|effort| {
*effort != &ReasoningEffortConfig::Ultra
&& model_info
.supported_reasoning_levels
.iter()
.any(|preset| &preset.effort == *effort)
})
.cloned()
.or_else(|| {
let supported_reasoning_levels = &model_info.supported_reasoning_levels;
supported_reasoning_levels
.iter()
.find(|preset| preset.effort == ReasoningEffortConfig::Max)
.or_else(|| {
supported_reasoning_levels
.iter()
.rev()
.find(|preset| preset.effort != ReasoningEffortConfig::Ultra)
})
.map(|preset| preset.effort.clone())
})
.unwrap_or(ReasoningEffortConfig::Medium),
// Keep "persistent" in local settings; the Responses API calls it "disabled".
ReasoningEffortConfig::Persistent => ReasoningEffortConfig::Custom("disabled".to_string()),
effort => effort,
@@ -771,7 +798,7 @@ impl ModelClient {
model: model_info.slug.clone(),
raw_memories,
reasoning: effort
.map(reasoning_effort_for_request)
.map(|effort| reasoning_effort_for_request(model_info, effort))
.map(|effort| Reasoning {
effort: Some(effort),
summary: None,
@@ -869,6 +896,7 @@ impl ModelClient {
}
fn build_reasoning(
&self,
model_info: &ModelInfo,
effort: Option<ReasoningEffortConfig>,
summary: ReasoningSummaryConfig,
@@ -876,7 +904,7 @@ impl ModelClient {
Reasoning {
effort: effort
.or_else(|| model_info.default_reasoning_level.clone())
.map(reasoning_effort_for_request),
.map(|effort| reasoning_effort_for_request(model_info, effort)),
summary: (model_info.supports_reasoning_summary_parameter
&& summary != ReasoningSummaryConfig::None)
.then_some(summary),
@@ -949,7 +977,7 @@ impl ModelClient {
}
}
}
let reasoning = Self::build_reasoning(model_info, effort, summary);
let reasoning = self.build_reasoning(model_info, effort, summary);
let stream_options = (self.state.concurrent_reasoning_summaries_enabled
&& is_openai
&& reasoning.summary.is_some())
+124 -5
View File
@@ -50,6 +50,7 @@ use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::openai_models::ModelsResponse;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::openai_models::ReasoningEffortPreset;
use codex_protocol::protocol::InternalSessionSource;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
@@ -364,16 +365,134 @@ fn test_session_telemetry() -> SessionTelemetry {
)
}
fn spawned_session_source() -> SessionSource {
SessionSource::SubAgent(SubAgentSource::ThreadSpawn {
parent_thread_id: ThreadId::new(),
depth: 1,
agent_path: None,
agent_nickname: None,
agent_role: None,
})
}
fn reasoning_effort_in_request(
model_info: &ModelInfo,
session_source: SessionSource,
effort: ReasoningEffort,
) -> ReasoningEffort {
let client = test_model_client(session_source);
client
.build_responses_request(
&Prompt::default(),
model_info,
Some(effort),
codex_protocol::config_types::ReasoningSummary::None,
/*service_tier*/ None,
&test_responses_metadata_for_client(
&client,
/*turn_id*/ None,
format!("{}:0", client.state.thread_id),
/*parent_thread_id*/ None,
TestCodexResponsesRequestKind::Turn,
),
)
.expect("build responses request")
.reasoning
.expect("request should include reasoning")
.effort
.expect("request should include reasoning effort")
}
#[test]
fn reasoning_effort_for_requests_maps_ultra_and_persistent() {
fn reasoning_effort_for_requests_uses_multi_agent_override_for_ultra() {
let mut model_info = test_model_info();
model_info.multi_agent_reasoning_effort = Some(ReasoningEffort::High);
model_info
.supported_reasoning_levels
.push(ReasoningEffortPreset {
effort: ReasoningEffort::High,
description: "high".to_string(),
});
let actual = [SessionSource::Cli, spawned_session_source()].map(|session_source| {
reasoning_effort_in_request(&model_info, session_source, ReasoningEffort::Ultra)
});
assert_eq!(actual, [ReasoningEffort::High, ReasoningEffort::High]);
}
#[test]
fn reasoning_effort_for_requests_falls_back_for_missing_or_invalid_override() {
let mut model_info = test_model_info();
model_info.supported_reasoning_levels = vec![
ReasoningEffortPreset {
effort: ReasoningEffort::Low,
description: "low".to_string(),
},
ReasoningEffortPreset {
effort: ReasoningEffort::XHigh,
description: "xhigh".to_string(),
},
ReasoningEffortPreset {
effort: ReasoningEffort::Ultra,
description: "ultra".to_string(),
},
];
let actual = [
None,
Some(ReasoningEffort::Ultra),
Some(ReasoningEffort::High),
]
.map(|multi_agent_reasoning_effort| {
model_info.multi_agent_reasoning_effort = multi_agent_reasoning_effort;
reasoning_effort_in_request(&model_info, SessionSource::Cli, ReasoningEffort::Ultra)
});
assert_eq!(
actual,
[
ReasoningEffort::XHigh,
ReasoningEffort::XHigh,
ReasoningEffort::XHigh,
]
);
model_info.multi_agent_reasoning_effort = None;
model_info.supported_reasoning_levels.insert(
1,
ReasoningEffortPreset {
effort: ReasoningEffort::Max,
description: "max".to_string(),
},
);
assert_eq!(
reasoning_effort_in_request(&model_info, SessionSource::Cli, ReasoningEffort::Ultra),
ReasoningEffort::Max
);
model_info.supported_reasoning_levels.clear();
assert_eq!(
reasoning_effort_in_request(&model_info, SessionSource::Cli, ReasoningEffort::Ultra),
ReasoningEffort::Medium
);
}
#[test]
fn reasoning_effort_for_requests_preserves_non_ultra_and_persistent_behavior() {
let mut model_info = test_model_info();
model_info.multi_agent_reasoning_effort = Some(ReasoningEffort::Low);
assert_eq!(
(
super::reasoning_effort_for_request(ReasoningEffort::Ultra),
super::reasoning_effort_for_request(ReasoningEffort::High),
super::reasoning_effort_for_request(ReasoningEffort::Persistent),
reasoning_effort_in_request(&model_info, SessionSource::Cli, ReasoningEffort::High,),
reasoning_effort_in_request(
&model_info,
SessionSource::Cli,
ReasoningEffort::Persistent,
),
),
(
ReasoningEffort::Max,
ReasoningEffort::High,
ReasoningEffort::Custom("disabled".to_string()),
)
+1
View File
@@ -629,6 +629,7 @@ fn remote_model_with_auto_review_override(slug: &str, review_model: &str) -> Mod
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
priority: 1,
additional_speed_tiers: Vec::new(),
service_tiers: Vec::new(),
@@ -128,6 +128,7 @@ fn test_model_info(
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
priority: 1,
additional_speed_tiers: Vec::new(),
service_tiers: Vec::new(),
@@ -1364,6 +1365,7 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result<
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
priority: 1,
additional_speed_tiers: Vec::new(),
service_tiers: Vec::new(),
@@ -544,5 +544,6 @@ fn test_remote_model(slug: &str, priority: i32) -> ModelInfo {
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
}
}
@@ -125,7 +125,7 @@ async fn submit_turn(
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn ultra_reasoning_uses_max_and_proactive_mode() -> Result<()> {
async fn ultra_reasoning_uses_highest_non_ultra_and_proactive_mode() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
@@ -145,7 +145,7 @@ async fn ultra_reasoning_uses_max_and_proactive_mode() -> Result<()> {
let request = response.single_request();
assert_eq!(
request.body_json()["reasoning"]["effort"].as_str(),
Some("max")
Some("xhigh")
);
let input = request.input();
let texts = developer_texts(&input);
@@ -522,7 +522,7 @@ async fn leaving_ultra_after_cold_resume_emits_explicit_mode() -> Result<()> {
.as_str()
.map(str::to_string),
),
(Some("max".to_string()), Some("high".to_string()))
(Some("xhigh".to_string()), Some("high".to_string()))
);
let resumed_input = requests[1].input();
let texts = developer_texts(&resumed_input);
@@ -539,7 +539,7 @@ async fn leaving_ultra_after_cold_resume_emits_explicit_mode() -> Result<()> {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn ultra_on_multi_agent_v1_uses_max_without_mode_instructions() -> Result<()> {
async fn ultra_on_multi_agent_v1_uses_highest_non_ultra_without_mode_instructions() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
@@ -561,7 +561,7 @@ async fn ultra_on_multi_agent_v1_uses_max_without_mode_instructions() -> Result<
let request = response.single_request();
assert_eq!(
request.body_json()["reasoning"]["effort"].as_str(),
Some("max")
Some("xhigh")
);
let input = request.input();
let texts = developer_texts(&input);
+2
View File
@@ -677,6 +677,7 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
};
let _models_mock = mount_models_once(
@@ -805,6 +806,7 @@ async fn user_turn_personality_remote_model_template_includes_update_message() -
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
};
let _models_mock = mount_models_once(
@@ -574,6 +574,7 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> {
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
priority: 1,
additional_speed_tiers: Vec::new(),
service_tiers: Vec::new(),
@@ -842,6 +843,7 @@ async fn remote_models_apply_legacy_instructions() -> Result<()> {
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
priority: 1,
additional_speed_tiers: Vec::new(),
service_tiers: Vec::new(),
@@ -1426,6 +1428,7 @@ fn test_remote_model_with_policy(
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
priority,
additional_speed_tiers: Vec::new(),
service_tiers: Vec::new(),
+1
View File
@@ -2762,6 +2762,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
}],
},
)
@@ -98,6 +98,7 @@ fn test_model_info(
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
priority: 1,
additional_speed_tiers: Vec::new(),
service_tiers,
@@ -408,6 +408,7 @@ async fn setup_turn_one_with_spawned_child(
}),
child_response_delay,
/*wait_for_parent_notification*/ true,
INHERITED_REASONING_EFFORT,
|builder| builder,
)
.await?;
@@ -419,6 +420,7 @@ async fn setup_turn_one_with_custom_spawned_child(
spawn_args: serde_json::Value,
child_response_delay: Option<Duration>,
wait_for_parent_notification: bool,
turn_reasoning_effort: ReasoningEffort,
configure_test: impl FnOnce(
core_test_support::test_codex::TestCodexBuilder,
) -> core_test_support::test_codex::TestCodexBuilder,
@@ -481,16 +483,32 @@ async fn setup_turn_one_with_custom_spawned_child(
)
.await;
let mut builder = configure_test(test_codex().with_config(|config| {
let configured_reasoning_effort = turn_reasoning_effort.clone();
let mut builder = configure_test(test_codex().with_config(move |config| {
config
.features
.enable(Feature::Collab)
.expect("test config should allow feature update");
config.model = Some(INHERITED_MODEL.to_string());
config.model_reasoning_effort = Some(INHERITED_REASONING_EFFORT);
config.model_reasoning_effort = Some(configured_reasoning_effort);
}));
let test = builder.build_with_auto_env(server).await?;
test.submit_turn(TURN_1_PROMPT).await?;
test.codex
.start_or_steer_turn(
TurnInputRequest::user_input(vec![UserInput::Text {
text: TURN_1_PROMPT.to_string(),
text_elements: Vec::new(),
}])
.with_thread_settings(ThreadSettingsOverrides {
effort: Some(Some(turn_reasoning_effort)),
..Default::default()
}),
)
.await?;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
if child_response_delay.is_none() && wait_for_parent_notification {
let _ = wait_for_requests(&child_request_log).await?;
let rollout_path = test
@@ -530,6 +548,7 @@ async fn spawn_child_and_capture_snapshot(
spawn_args,
/*child_response_delay*/ None,
/*wait_for_parent_notification*/ false,
INHERITED_REASONING_EFFORT,
configure_test,
)
.await?;
@@ -542,6 +561,47 @@ async fn spawn_child_and_capture_snapshot(
.await)
}
#[test_case(
ReasoningEffort::Ultra,
ReasoningEffort::XHigh;
"absent catalog override uses highest non-ultra"
)]
#[test_case(
ReasoningEffort::High,
ReasoningEffort::High;
"non-ultra selection is unchanged"
)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn spawned_agent_uses_multi_agent_reasoning_effort_for_requests(
selected_reasoning_effort: ReasoningEffort,
expected_request_reasoning_effort: ReasoningEffort,
) -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let (_test, _spawned_id, child_request_log) = setup_turn_one_with_custom_spawned_child(
&server,
json!({ "message": CHILD_PROMPT }),
/*child_response_delay*/ None,
/*wait_for_parent_notification*/ false,
selected_reasoning_effort,
std::convert::identity,
)
.await?;
let child_request = wait_for_requests(&child_request_log)
.await?
.into_iter()
.next()
.expect("wait_for_requests should return a request");
assert_eq!(
child_request.body_json()["reasoning"]["effort"],
expected_request_reasoning_effort.to_string()
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn subagent_start_replaces_session_start_and_injects_context() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -1662,6 +1722,7 @@ async fn spawned_agent_uses_summary_support_for_final_model(
}),
/*child_response_delay*/ Some(Duration::from_secs(1)),
/*wait_for_parent_notification*/ false,
INHERITED_REASONING_EFFORT,
move |builder| {
builder.with_config(move |config| {
config.model_catalog = Some(model_catalog);
+1
View File
@@ -1629,6 +1629,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
priority: 1,
additional_speed_tiers: Vec::new(),
service_tiers: Vec::new(),
@@ -184,6 +184,7 @@ pub fn model_info_from_slug(slug: &str) -> ModelInfo {
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
}
}
+26
View File
@@ -479,6 +479,9 @@ pub struct ModelInfo {
deserialize_with = "deserialize_optional_model_selector"
)]
pub multi_agent_version: Option<MultiAgentVersion>,
/// Reasoning effort used for multi-agent work when the user selects Ultra.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub multi_agent_reasoning_effort: Option<ReasoningEffort>,
}
impl ModelInfo {
@@ -973,6 +976,7 @@ mod tests {
model_specialty: None,
tool_mode: None,
multi_agent_version: None,
multi_agent_reasoning_effort: None,
}
}
@@ -1654,6 +1658,28 @@ mod tests {
assert_eq!(model.comp_hash, None);
assert_eq!(model.auto_review_model_override, None);
assert_eq!(model.tool_mode, None);
assert_eq!(model.multi_agent_reasoning_effort, None);
}
#[test]
fn model_info_deserializes_multi_agent_reasoning_effort() {
let mut value =
serde_json::to_value(test_model(/*spec*/ None)).expect("serialize test model");
value
.as_object_mut()
.expect("model info should be an object")
.insert(
"multi_agent_reasoning_effort".to_string(),
serde_json::Value::String("high".to_string()),
);
let model = serde_json::from_value::<ModelInfo>(value)
.expect("deserialize multi-agent reasoning effort");
assert_eq!(
model.multi_agent_reasoning_effort,
Some(ReasoningEffort::High)
);
}
#[test]