#![cfg(not(target_os = "windows"))] // unified exec is not supported on Windows OS use std::sync::Arc; use anyhow::Result; use codex_core::CodexAuth; use codex_core::CodexConversation; use codex_core::ConversationManager; use codex_core::ModelProviderInfo; use codex_core::built_in_model_providers; use codex_core::config::Config; use codex_core::features::Feature; use codex_core::openai_models::models_manager::ModelsManager; use codex_core::protocol::AskForApproval; use codex_core::protocol::EventMsg; use codex_core::protocol::ExecCommandSource; use codex_core::protocol::Op; use codex_core::protocol::SandboxPolicy; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::openai_models::ClientVersion; use codex_protocol::openai_models::ConfigShellToolType; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelPreset; use codex_protocol::openai_models::ModelVisibility; use codex_protocol::openai_models::ModelsResponse; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::openai_models::ReasoningEffortPreset; use codex_protocol::openai_models::ReasoningSummaryFormat; use codex_protocol::openai_models::TruncationPolicyConfig; use codex_protocol::user_input::UserInput; use core_test_support::load_default_config_for_test; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call; use core_test_support::responses::ev_response_created; use core_test_support::responses::ev_shell_command_call; use core_test_support::responses::mount_models_once; use core_test_support::responses::mount_models_once_with_etag; use core_test_support::responses::mount_response_once_match; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_once_match; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::skip_if_no_network; use core_test_support::skip_if_sandbox; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; use pretty_assertions::assert_eq; use serde_json::Value; use serde_json::json; use tempfile::TempDir; use tokio::time::Duration; use tokio::time::Instant; use tokio::time::sleep; use wiremock::BodyPrintLimit; use wiremock::MockServer; use wiremock::ResponseTemplate; const REMOTE_MODEL_SLUG: &str = "codex-test"; #[derive(Clone, Default)] struct ResponsesMatch { etag: Option, user_text: Option, call_id: Option, } impl ResponsesMatch { fn with_etag(mut self, etag: &str) -> Self { self.etag = Some(etag.to_string()); self } fn with_user_text(mut self, text: &str) -> Self { self.user_text = Some(text.to_string()); self } fn with_function_call_output(mut self, call_id: &str) -> Self { self.call_id = Some(call_id.to_string()); self } } impl wiremock::Match for ResponsesMatch { fn matches(&self, request: &wiremock::Request) -> bool { if let Some(expected_etag) = &self.etag { let header = request .headers .get("X-If-Models-Match") .and_then(|value| value.to_str().ok()); if header != Some(expected_etag.as_str()) { return false; } } let Ok(body): Result = request.body_json() else { return false; }; let Some(items) = body.get("input").and_then(Value::as_array) else { return false; }; if let Some(expected_text) = &self.user_text && !input_has_user_text(items, expected_text) { return false; } if let Some(expected_call_id) = &self.call_id && !input_has_function_call_output(items, expected_call_id) { return false; } true } } fn input_has_user_text(items: &[Value], expected: &str) -> bool { items.iter().any(|item| { item.get("type").and_then(Value::as_str) == Some("message") && item.get("role").and_then(Value::as_str) == Some("user") && item .get("content") .and_then(Value::as_array) .is_some_and(|content| { content.iter().any(|span| { span.get("type").and_then(Value::as_str) == Some("input_text") && span.get("text").and_then(Value::as_str) == Some(expected) }) }) }) } fn input_has_function_call_output(items: &[Value], call_id: &str) -> bool { items.iter().any(|item| { item.get("type").and_then(Value::as_str) == Some("function_call_output") && item.get("call_id").and_then(Value::as_str) == Some(call_id) }) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { skip_if_no_network!(Ok(())); skip_if_sandbox!(Ok(())); let server = MockServer::builder() .body_print_limit(BodyPrintLimit::Limited(80_000)) .start() .await; let remote_model = ModelInfo { slug: REMOTE_MODEL_SLUG.to_string(), display_name: "Remote Test".to_string(), description: Some("A remote model that requires the test shell".to_string()), default_reasoning_level: ReasoningEffort::Medium, supported_reasoning_levels: vec![ReasoningEffortPreset { effort: ReasoningEffort::Medium, description: ReasoningEffort::Medium.to_string(), }], shell_type: ConfigShellToolType::UnifiedExec, visibility: ModelVisibility::List, minimal_client_version: ClientVersion(0, 1, 0), supported_in_api: true, priority: 1, upgrade: None, base_instructions: None, supports_reasoning_summaries: false, support_verbosity: false, default_verbosity: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, context_window: None, reasoning_summary_format: ReasoningSummaryFormat::None, experimental_supported_tools: Vec::new(), }; let models_mock = mount_models_once( &server, ModelsResponse { models: vec![remote_model], }, ) .await; let harness = build_remote_models_harness(&server, |config| { config.features.enable(Feature::RemoteModels); config.model = Some("gpt-5.1".to_string()); }) .await?; let RemoteModelsHarness { codex, cwd, config, conversation_manager, .. } = harness; let models_manager = conversation_manager.get_models_manager(); let available_model = wait_for_model_available(&models_manager, REMOTE_MODEL_SLUG, &config).await; assert_eq!(available_model.model, REMOTE_MODEL_SLUG); let requests = models_mock.requests(); assert_eq!( requests.len(), 1, "expected a single /models refresh request for the remote models feature" ); assert_eq!(requests[0].url.path(), "/v1/models"); let family = models_manager .construct_model_family(REMOTE_MODEL_SLUG, &config) .await; assert_eq!(family.shell_type, ConfigShellToolType::UnifiedExec); codex .submit(Op::OverrideTurnContext { cwd: None, approval_policy: None, sandbox_policy: None, model: Some(REMOTE_MODEL_SLUG.to_string()), effort: None, summary: None, }) .await?; let call_id = "call"; let args = json!({ "cmd": "/bin/echo call", "yield_time_ms": 250, }); let responses = vec![ sse(vec![ ev_response_created("resp-1"), ev_function_call(call_id, "exec_command", &serde_json::to_string(&args)?), ev_completed("resp-1"), ]), sse(vec![ ev_response_created("resp-2"), ev_assistant_message("msg-1", "done"), ev_completed("resp-2"), ]), ]; mount_sse_sequence(&server, responses).await; codex .submit(Op::UserTurn { items: vec![UserInput::Text { text: "run call".into(), }], final_output_json_schema: None, cwd: cwd.path().to_path_buf(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: REMOTE_MODEL_SLUG.to_string(), effort: None, summary: ReasoningSummary::Auto, }) .await?; let begin_event = wait_for_event_match(&codex, |msg| match msg { EventMsg::ExecCommandBegin(event) if event.call_id == call_id => Some(event.clone()), _ => None, }) .await; assert_eq!(begin_event.source, ExecCommandSource::UnifiedExecStartup); wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; Ok(()) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_models_apply_remote_base_instructions() -> Result<()> { skip_if_no_network!(Ok(())); skip_if_sandbox!(Ok(())); let server = MockServer::builder() .body_print_limit(BodyPrintLimit::Limited(80_000)) .start() .await; let model = "test-gpt-5-remote"; let remote_base = "Use the remote base instructions only."; let remote_model = ModelInfo { slug: model.to_string(), display_name: "Parallel Remote".to_string(), description: Some("A remote model with custom instructions".to_string()), default_reasoning_level: ReasoningEffort::Medium, supported_reasoning_levels: vec![ReasoningEffortPreset { effort: ReasoningEffort::Medium, description: ReasoningEffort::Medium.to_string(), }], shell_type: ConfigShellToolType::ShellCommand, visibility: ModelVisibility::List, minimal_client_version: ClientVersion(0, 1, 0), supported_in_api: true, priority: 1, upgrade: None, base_instructions: Some(remote_base.to_string()), supports_reasoning_summaries: false, support_verbosity: false, default_verbosity: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, context_window: None, reasoning_summary_format: ReasoningSummaryFormat::None, experimental_supported_tools: Vec::new(), }; mount_models_once( &server, ModelsResponse { models: vec![remote_model], }, ) .await; let response_mock = mount_sse_once( &server, sse(vec![ ev_response_created("resp-1"), ev_assistant_message("msg-1", "done"), ev_completed("resp-1"), ]), ) .await; let harness = build_remote_models_harness(&server, |config| { config.features.enable(Feature::RemoteModels); config.model = Some("gpt-5.1".to_string()); }) .await?; let RemoteModelsHarness { codex, cwd, config, conversation_manager, .. } = harness; let models_manager = conversation_manager.get_models_manager(); wait_for_model_available(&models_manager, model, &config).await; codex .submit(Op::OverrideTurnContext { cwd: None, approval_policy: None, sandbox_policy: None, model: Some(model.to_string()), effort: None, summary: None, }) .await?; codex .submit(Op::UserTurn { items: vec![UserInput::Text { text: "hello remote".into(), }], final_output_json_schema: None, cwd: cwd.path().to_path_buf(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: model.to_string(), effort: None, summary: ReasoningSummary::Auto, }) .await?; wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; let body = response_mock.single_request().body_json(); let instructions = body["instructions"].as_str().unwrap(); assert_eq!(instructions, remote_base); Ok(()) } /// Exercises the remote-models retry flow: /// 1) initial `/models` fetch stores an ETag, /// 2) `/responses` uses that ETag for a tool call, /// 3) the tool-output turn receives a 412 (stale models), /// 4) Codex refreshes `/models` to get a new ETag and retries, /// 5) subsequent user turns keep sending the refreshed ETag. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_models_refresh_etag_after_outdated_models() -> Result<()> { skip_if_no_network!(Ok(())); skip_if_sandbox!(Ok(())); let server = MockServer::builder() .body_print_limit(BodyPrintLimit::Limited(80_000)) .start() .await; let remote_model = test_remote_model("remote-etag", ModelVisibility::List, 1); let initial_etag = "models-etag-initial"; let refreshed_etag = "models-etag-refreshed"; // Phase 1a: seed the initial `/models` response with an ETag. let models_mock = mount_models_once_with_etag( &server, ModelsResponse { models: vec![remote_model.clone()], }, initial_etag, ) .await; // Phase 1b: boot a Codex session configured for remote models. let harness = build_remote_models_harness(&server, |config| { config.features.enable(Feature::RemoteModels); config.model = Some("gpt-5.1".to_string()); }) .await?; let RemoteModelsHarness { codex, cwd, config, conversation_manager, .. } = harness; let models_manager = conversation_manager.get_models_manager(); wait_for_model_available(&models_manager, "remote-etag", &config).await; // Phase 1c: confirm the ETag is stored and `/models` was called. assert_eq!( models_manager.get_models_etag().await.as_deref(), Some(initial_etag), ); assert_eq!( models_mock.requests().len(), 1, "expected an initial /models request", ); assert_eq!(models_mock.requests()[0].url.path(), "/v1/models"); // Phase 2a: reset mocks so the next `/models` call must be explicit. server.reset().await; // Phase 2b: mount a refreshed `/models` response with a new ETag. let refreshed_models_mock = mount_models_once_with_etag( &server, ModelsResponse { models: vec![remote_model], }, refreshed_etag, ) .await; let call_id = "shell-command-call"; let first_prompt = "run a shell command"; let followup_prompt = "send another message"; // Phase 2c: first `/responses` turn uses the initial ETag and emits a tool call. let first_response = mount_sse_once_match( &server, ResponsesMatch::default() .with_etag(initial_etag) .with_user_text(first_prompt), sse(vec![ ev_response_created("resp-1"), ev_shell_command_call(call_id, "echo refreshed"), ev_completed("resp-1"), ]), ) .await; // Phase 2d: the tool-output follow-up returns 412 (stale models). let stale_response = mount_response_once_match( &server, ResponsesMatch::default() .with_etag(initial_etag) .with_function_call_output(call_id), ResponseTemplate::new(412) .set_body_string("Models catalog has changed. Please refresh your models list."), ) .await; // Phase 2e: retry tool-output follow-up should use the refreshed ETag. let refreshed_response = mount_sse_once_match( &server, ResponsesMatch::default() .with_etag(refreshed_etag) .with_function_call_output(call_id), sse(vec![ ev_response_created("resp-2"), ev_assistant_message("msg-1", "done"), ev_completed("resp-2"), ]), ) .await; // Phase 3a: next user turn should also use the refreshed ETag. let next_turn_response = mount_sse_once_match( &server, ResponsesMatch::default() .with_etag(refreshed_etag) .with_user_text(followup_prompt), sse(vec![ ev_response_created("resp-3"), ev_assistant_message("msg-2", "ok"), ev_completed("resp-3"), ]), ) .await; // Phase 3b: run the first user turn and let retries complete. codex .submit(Op::UserTurn { items: vec![UserInput::Text { text: first_prompt.into(), }], final_output_json_schema: None, cwd: cwd.path().to_path_buf(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: "gpt-5.1".to_string(), effort: None, summary: ReasoningSummary::Auto, }) .await?; wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; // Phase 3c: assert the refresh happened and the ETag was updated. assert_eq!( refreshed_models_mock.requests().len(), 1, "expected a refreshed /models request", ); assert_eq!( models_manager.get_models_etag().await.as_deref(), Some(refreshed_etag), ); // Phase 3d: assert the ETag header progression across the retry sequence. assert_eq!( first_response.single_request().header("X-If-Models-Match"), Some(initial_etag.to_string()), ); assert_eq!( stale_response.single_request().header("X-If-Models-Match"), Some(initial_etag.to_string()), ); assert_eq!( refreshed_response .single_request() .header("X-If-Models-Match"), Some(refreshed_etag.to_string()), ); // Phase 3e: execute a new user turn and ensure the refreshed ETag persists. codex .submit(Op::UserTurn { items: vec![UserInput::Text { text: followup_prompt.into(), }], final_output_json_schema: None, cwd: cwd.path().to_path_buf(), approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, model: "gpt-5.1".to_string(), effort: None, summary: ReasoningSummary::Auto, }) .await?; wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; assert_eq!( next_turn_response .single_request() .header("X-If-Models-Match"), Some(refreshed_etag.to_string()), ); Ok(()) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_models_preserve_builtin_presets() -> Result<()> { skip_if_no_network!(Ok(())); skip_if_sandbox!(Ok(())); let server = MockServer::start().await; let remote_model = test_remote_model("remote-alpha", ModelVisibility::List, 0); let models_mock = mount_models_once( &server, ModelsResponse { models: vec![remote_model.clone()], }, ) .await; let codex_home = TempDir::new()?; let mut config = load_default_config_for_test(&codex_home).await; config.features.enable(Feature::RemoteModels); let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), ..built_in_model_providers()["openai"].clone() }; let manager = ModelsManager::with_provider( codex_core::auth::AuthManager::from_auth_for_testing(auth), provider, ); manager .try_refresh_available_models(&config) .await .expect("refresh succeeds"); let available = manager.list_models(&config).await; let remote = available .iter() .find(|model| model.model == "remote-alpha") .expect("remote model should be listed"); let mut expected_remote: ModelPreset = remote_model.into(); expected_remote.is_default = true; assert_eq!(*remote, expected_remote); assert!( available .iter() .any(|model| model.model == "gpt-5.1-codex-max"), "builtin presets should remain available after refresh" ); assert_eq!( models_mock.requests().len(), 1, "expected a single /models request" ); Ok(()) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_models_hide_picker_only_models() -> Result<()> { skip_if_no_network!(Ok(())); skip_if_sandbox!(Ok(())); let server = MockServer::start().await; let remote_model = test_remote_model("codex-auto-balanced", ModelVisibility::Hide, 0); mount_models_once( &server, ModelsResponse { models: vec![remote_model], }, ) .await; let codex_home = TempDir::new()?; let mut config = load_default_config_for_test(&codex_home).await; config.features.enable(Feature::RemoteModels); let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), ..built_in_model_providers()["openai"].clone() }; let manager = ModelsManager::with_provider( codex_core::auth::AuthManager::from_auth_for_testing(auth), provider, ); let selected = manager.get_model(&None, &config).await; assert_eq!(selected, "gpt-5.2-codex"); let available = manager.list_models(&config).await; assert!( available .iter() .all(|model| model.model != "codex-auto-balanced"), "hidden models should not appear in the picker list" ); Ok(()) } async fn wait_for_model_available( manager: &Arc, slug: &str, config: &Config, ) -> ModelPreset { let deadline = Instant::now() + Duration::from_secs(2); loop { if let Some(model) = { let guard = manager.list_models(config).await; guard.iter().find(|model| model.model == slug).cloned() } { return model; } if Instant::now() >= deadline { panic!("timed out waiting for the remote model {slug} to appear"); } sleep(Duration::from_millis(25)).await; } } struct RemoteModelsHarness { codex: Arc, cwd: Arc, config: Config, conversation_manager: Arc, } // todo(aibrahim): move this to with_model_provier in test_codex async fn build_remote_models_harness( server: &MockServer, mutate_config: F, ) -> Result where F: FnOnce(&mut Config), { let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); let home = Arc::new(TempDir::new()?); let cwd = Arc::new(TempDir::new()?); let mut config = load_default_config_for_test(&home).await; config.cwd = cwd.path().to_path_buf(); config.features.enable(Feature::RemoteModels); let provider = ModelProviderInfo { base_url: Some(format!("{}/v1", server.uri())), ..built_in_model_providers()["openai"].clone() }; config.model_provider = provider.clone(); mutate_config(&mut config); let conversation_manager = Arc::new(ConversationManager::with_models_provider(auth, provider)); let new_conversation = conversation_manager .new_conversation(config.clone()) .await?; Ok(RemoteModelsHarness { codex: new_conversation.conversation, cwd, config, conversation_manager, }) } fn test_remote_model(slug: &str, visibility: ModelVisibility, priority: i32) -> ModelInfo { ModelInfo { slug: slug.to_string(), display_name: format!("{slug} display"), description: Some(format!("{slug} description")), default_reasoning_level: ReasoningEffort::Medium, supported_reasoning_levels: vec![ReasoningEffortPreset { effort: ReasoningEffort::Medium, description: ReasoningEffort::Medium.to_string(), }], shell_type: ConfigShellToolType::ShellCommand, visibility, minimal_client_version: ClientVersion(0, 1, 0), supported_in_api: true, priority, upgrade: None, base_instructions: None, supports_reasoning_summaries: false, support_verbosity: false, default_verbosity: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, context_window: None, reasoning_summary_format: ReasoningSummaryFormat::None, experimental_supported_tools: Vec::new(), } }