//! Compare model-visible prefixes from two app-server versions using the same client and inputs. //! The contract runner supplies a released CLI; ordinary runs use the current binary twice. //! Resume coverage checks that changed tool declarations leave the retained prefix intact. #[path = "prompt_prefix/resume_tests.rs"] mod resume_tests; use anyhow::Context; use anyhow::Result; use app_test_support::MockResponsesConfig; use app_test_support::TestAppServer; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput; use core_test_support::responses; use pretty_assertions::assert_eq; use serde_json::Value; use serde_json::json; use std::time::Duration; use tempfile::TempDir; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn prompt_prefix_matches_release() -> Result<()> { let release = std::env::var_os("CONTRACT_EXECUTOR_BIN").map(std::path::PathBuf::from); let mut prefixes = Vec::new(); for program in [release.as_deref(), None] { let server = responses::start_mock_server().await; let response = responses::mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("prefix-response"), responses::ev_assistant_message("prefix-message", "Done"), responses::ev_completed("prefix-response"), ]), ) .await; let home = TempDir::new()?; // Use each binary's bundled model catalog and feature defaults. Supplying a // catalog or base instructions from the test would hide changes to them. MockResponsesConfig::new(&server.uri()) .with_model("gpt-6-astra") // CCA disables bundled skills in its orchestrator. .with_root_config("skills.bundled.enabled = false") .write(home.path())?; let mut builder = TestAppServer::builder() .with_codex_home(home.path()) .with_plugin_startup_tasks() .without_managed_config(); if release.is_some() { // Released CLIs ignore the debug-only managed-config overrides. // Give both versions the same host configuration in version-skew runs. builder = builder.with_env_overrides(&[ ("CODEX_APP_SERVER_MANAGED_CONFIG_PATH", None), ("CODEX_APP_SERVER_DISABLE_MANAGED_CONFIG", None), ]); } if let Some(program) = program { // Reuse the exact package downloaded for executor version-skew tests. // Keep it in place so its own bundled resources remain discoverable. builder = builder.with_program(program).with_args(&["app-server"]); } // Cold startup includes plugin startup tasks for both binaries. let mut app = builder .build_initialized_with_timeout(Duration::from_secs(60)) .await .context("initializing app-server for the prompt-prefix comparison")?; let thread = app.start_thread(ThreadStartParams::default()).await?.thread; let completed = app .start_turn_and_wait_for_completion(TurnStartParams { thread_id: thread.id.clone(), input: vec![UserInput::Text { text: "prefix-contract-user-message".to_string(), text_elements: Vec::new(), }], ..Default::default() }) .await?; assert_eq!(completed.turn.status, TurnStatus::Completed); let request = response.single_request(); let body = request.body_json(); let input = responses::strip_response_item_ids_from_json(json!(request.input())); let input = input.as_array().context("request input must be an array")?; let user_index = input .iter() .position(|item| { item["role"] == "user" && item["content"][0]["text"] == "prefix-contract-user-message" }) .context("request must contain the submitted user message")?; let prefix = responses::strip_metadata_from_json(json!({ "instructions": body.get("instructions"), "tools": body.get("tools"), "input": &input[..user_index], })); // Normalize only values created by this fixture, retaining all prompt // text, whitespace, tool descriptions, schemas, and array ordering. let mut prefix = serde_json::to_string_pretty(&prefix)?; let environment = app.auto_env_params()?; for (value, replacement) in [ (environment.cwd.as_str(), ""), (thread.cwd.to_string_lossy().as_ref(), ""), (home.path().to_string_lossy().as_ref(), ""), (thread.id.as_str(), ""), (thread.session_id.as_str(), ""), ] { // Windows paths can be rendered with either separator, notably in // skill roots. Normalize both spellings of these fixture values only. for spelling in [value.to_string(), value.replace('\\', "/")] { let escaped = serde_json::to_string(&spelling)?; prefix = prefix.replace(&escaped[1..escaped.len() - 1], replacement); } } prefixes.push(prefix); } if release.is_some() && prefixes[0] != prefixes[1] { let mut released: Value = serde_json::from_str(&prefixes[0])?; let current: Value = serde_json::from_str(&prefixes[1])?; accept_spawn_agent_shortlist_rollout(&mut released, ¤t); accept_default_tool_description_rollouts(&mut released, ¤t); prefixes[0] = serde_json::to_string_pretty(&released)?; } assert_eq!( prefixes[0], prefixes[1], "the default inference prefix differs from the released version; gate the change behind a default-off feature key" ); Ok(()) } // Accept the exact default-on shortlist expansion from #18296 while comparing with // releases that still advertise five models. All other description bytes stay checked. fn accept_spawn_agent_shortlist_rollout(released: &mut Value, current: &Value) { const TOOL: &str = "/input/0/tools/2/tools/4"; for prefix in [&*released, current] { if prefix.pointer("/input/0/role").and_then(Value::as_str) != Some("developer") || prefix .pointer("/input/0/tools/2/name") .and_then(Value::as_str) != Some("collaboration") || prefix.pointer(TOOL).and_then(|tool| tool["name"].as_str()) != Some("spawn_agent") { return; } } let Some(description) = released .pointer_mut(TOOL) .and_then(|tool| tool.get_mut("description")) else { return; }; let Some(old) = description.as_str() else { return; }; const OLD_MODELS: &str = r#"Available model overrides (optional; inherited parent model is preferred): - `gpt-6.1-sol`: Latest workhorse model for coding and everyday work. Reasoning efforts: low (default), medium, high, xhigh, max, ultra. Service tiers: priority. - `gpt-6-astra`: Frontier intelligence for the most demanding work. Reasoning efforts: low (default), medium, high, xhigh, max, ultra. Service tiers: priority. - `gpt-6-sol`: Previous generation workhorse model. Reasoning efforts: low, medium (default), high, xhigh, max, ultra. Service tiers: priority. - `gpt-6-luna`: Fast and affordable model for easier tasks. Reasoning efforts: low, medium (default), high, xhigh, max. Service tiers: priority. - `gpt-5.6-sol`: Older generation workhorse model. Reasoning efforts: low (default), medium, high, xhigh, max, ultra. Service tiers: priority. "#; const EXTRA_MODELS: &str = r#"- `gpt-6.1-terra`: GPT-6.1 Terra Codex model. Reasoning efforts: low (default), medium, high, xhigh, max, ultra. Service tiers: priority. - `gpt-5.6-terra`: Older balanced model for straightforward work. Reasoning efforts: low, medium (default), high, xhigh, max, ultra. Service tiers: priority. - `gpt-5.6-luna`: Older fast and efficient model. Reasoning efforts: low, medium (default), high, xhigh, max. Service tiers: priority. - `gpt-5.5`: Legacy coding model. Reasoning efforts: low, medium (default), high, xhigh. Service tiers: priority. "#; if old.matches(OLD_MODELS).count() != 1 { return; } let updated = old.replace(OLD_MODELS, &format!("{OLD_MODELS}{EXTRA_MODELS}")); if current .pointer(TOOL) .and_then(|tool| tool["description"].as_str()) == Some(updated.as_str()) { *description = Value::String(updated); } } // Accept only the exact default-on changes to the released functions.exec description. // Once the release includes them, the ordinary byte-for-byte comparison succeeds. fn accept_default_tool_description_rollouts(released: &mut Value, current: &Value) { const TOOL: &str = "/input/0/tools/0/tools/0"; for prefix in [&*released, current] { if prefix.pointer("/input/0/role").and_then(Value::as_str) != Some("developer") || prefix .pointer("/input/0/tools/0/name") .and_then(Value::as_str) != Some("functions") || prefix.pointer(TOOL).and_then(|tool| tool["name"].as_str()) != Some("exec") { return; } } let Some(description) = released .pointer_mut(TOOL) .and_then(|tool| tool.get_mut("description")) else { return; }; let Some(old) = description.as_str() else { return; }; const OLD_IMAGE: &str = r#"- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument."#; const NEW_IMAGE: &str = r#"- `image(imageUrlOrItem: string | { image_url: string } | ImageContent)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`."#; const OLD_VIEW_IMAGE: &str = r#"declare const tools: { view_image(args: { // Image detail level. Defaults to `high`; use `original` to preserve exact resolution. detail?: "high" | "original"; // Local filesystem path to an image file. path: string; }): Promise<{ // Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved. detail: "high" | "original"; // Data URL for the loaded image. image_url: string; }>; };"#; const NEW_VIEW_IMAGE: &str = r#"declare const tools: { view_image(args: { // Local filesystem path to an image file. path: string; }): Promise<{ // Data URL for the loaded image. image_url: string; }>; };"#; const OLD_LOGIN: &str = " // True runs the shell with -l/-i semantics; false disables them. Defaults to true.\n login?: boolean;"; const UNIX_LOGIN: &str = " // Request login-shell startup. Omit to inherit.\n login?: boolean;"; const POWERSHELL_LOGIN: &str = " // Login-shell startup. In PowerShell, false skips profiles. Keep them enabled for profile-defined commands, even unattended. Omit to inherit.\n login?: boolean;"; let current_description = current .pointer(TOOL) .and_then(|tool| tool["description"].as_str()) .unwrap_or_default(); let new_login = if current_description.contains(POWERSHELL_LOGIN) { POWERSHELL_LOGIN } else { UNIX_LOGIN }; let mut updated = old.to_owned(); if old.matches(OLD_IMAGE).count() == 1 && old.matches(OLD_VIEW_IMAGE).count() == 1 { updated = updated .replace(OLD_IMAGE, NEW_IMAGE) .replace(OLD_VIEW_IMAGE, NEW_VIEW_IMAGE); } if old.matches(OLD_LOGIN).count() == 1 { updated = updated.replace(OLD_LOGIN, new_login); } if current .pointer(TOOL) .and_then(|tool| tool["description"].as_str()) == Some(updated.as_str()) { *description = Value::String(updated); } }