//! Compare model-visible prefixes from two app-server versions using the same client and inputs. //! The contract runner supplies a released CLI; ordinary runs check same-version determinism. use anyhow::Context; use anyhow::Result; use app_test_support::MockResponsesConfig; use app_test_support::TestAppServer; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput; use core_test_support::responses; use pretty_assertions::assert_eq; use serde_json::Value; use serde_json::json; use std::time::Duration; use tempfile::TempDir; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn prompt_prefix_matches_release() -> Result<()> { let release = std::env::var_os("CONTRACT_EXECUTOR_BIN").map(std::path::PathBuf::from); let mut prefixes = Vec::new(); for program in [release.as_deref(), None] { let server = responses::start_mock_server().await; let response = responses::mount_sse_once( &server, responses::sse(vec![ responses::ev_response_created("prefix-response"), responses::ev_assistant_message("prefix-message", "Done"), responses::ev_completed("prefix-response"), ]), ) .await; let home = TempDir::new()?; // Use each binary's bundled model catalog and feature defaults. Supplying a // catalog or base instructions from the test would hide changes to them. MockResponsesConfig::new(&server.uri()) .with_model("gpt-6-astra") // CCA disables bundled skills in its orchestrator. .with_root_config("skills.bundled.enabled = false") .write(home.path())?; let mut builder = TestAppServer::builder() .with_codex_home(home.path()) .with_plugin_startup_tasks() .without_managed_config(); if release.is_some() { // Released CLIs ignore the debug-only managed-config overrides. // Give both versions the same host configuration in version-skew runs. builder = builder.with_env_overrides(&[ ("CODEX_APP_SERVER_MANAGED_CONFIG_PATH", None), ("CODEX_APP_SERVER_DISABLE_MANAGED_CONFIG", None), ]); } if let Some(program) = program { // Reuse the exact package downloaded for executor version-skew tests. // Keep it in place so its own bundled resources remain discoverable. builder = builder.with_program(program).with_args(&["app-server"]); } // Cold startup includes plugin startup tasks for both binaries. let mut app = builder .build_initialized_with_timeout(Duration::from_secs(60)) .await .context("initializing app-server for the prompt-prefix comparison")?; let thread = app.start_thread(ThreadStartParams::default()).await?.thread; let completed = app .start_turn_and_wait_for_completion(TurnStartParams { thread_id: thread.id.clone(), input: vec![UserInput::Text { text: "prefix-contract-user-message".to_string(), text_elements: Vec::new(), }], ..Default::default() }) .await?; assert_eq!(completed.turn.status, TurnStatus::Completed); let request = response.single_request(); let body = request.body_json(); let input = responses::strip_response_item_ids_from_json(json!(request.input())); let input = input.as_array().context("request input must be an array")?; let user_index = input .iter() .position(|item| { item["role"] == "user" && item["content"][0]["text"] == "prefix-contract-user-message" }) .context("request must contain the submitted user message")?; let prefix = responses::strip_metadata_from_json(json!({ "instructions": body.get("instructions"), "tools": body.get("tools"), "input": &input[..user_index], })); // Normalize only values created by this fixture, retaining all prompt // text, whitespace, tool descriptions, schemas, and array ordering. let mut prefix = serde_json::to_string_pretty(&prefix)?; let environment = app.auto_env_params()?; for (value, replacement) in [ (environment.cwd.as_str(), ""), (thread.cwd.to_string_lossy().as_ref(), ""), (home.path().to_string_lossy().as_ref(), ""), (thread.id.as_str(), ""), (thread.session_id.as_str(), ""), ] { // Windows paths can be rendered with either separator, notably in // skill roots. Normalize both spellings of these fixture values only. for spelling in [value.to_string(), value.replace('\\', "/")] { let escaped = serde_json::to_string(&spelling)?; prefix = prefix.replace(&escaped[1..escaped.len() - 1], replacement); } } prefixes.push(prefix); } if release.is_some() && prefixes[0] != prefixes[1] { let mut released: Value = serde_json::from_str(&prefixes[0])?; let current: Value = serde_json::from_str(&prefixes[1])?; accept_unified_image_budget_rollout(&mut released, ¤t); prefixes[0] = serde_json::to_string_pretty(&released)?; } assert_eq!( prefixes[0], prefixes[1], "the default inference prefix differs from the released version; gate the change behind a default-off feature key" ); Ok(()) } // Compare the released functions.exec description against the exact default-on image budget // rollout. Once the release includes it, the ordinary byte-for-byte comparison succeeds. fn accept_unified_image_budget_rollout(released: &mut Value, current: &Value) { const TOOL: &str = "/input/0/tools/0/tools/0"; for prefix in [&*released, current] { if prefix.pointer("/input/0/role").and_then(Value::as_str) != Some("developer") || prefix .pointer("/input/0/tools/0/name") .and_then(Value::as_str) != Some("functions") || prefix.pointer(TOOL).and_then(|tool| tool["name"].as_str()) != Some("exec") { return; } } let Some(description) = released .pointer_mut(TOOL) .and_then(|tool| tool.get_mut("description")) else { return; }; let Some(old) = description.as_str() else { return; }; const OLD_IMAGE: &str = r#"- `image(imageUrlOrItem: string | { image_url: string; detail?: "auto" | "low" | "high" | "original" | null } | ImageContent, detail?: "auto" | "low" | "high" | "original" | null)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`. MCP image blocks may request detail with `_meta: { "codex/imageDetail": "original" }`. When provided, the second `detail` argument overrides any detail embedded in the first argument."#; const NEW_IMAGE: &str = r#"- `image(imageUrlOrItem: string | { image_url: string } | ImageContent)`: Appends an image item. `image_url` should be a base64-encoded `data:` URL. To forward an MCP tool image, pass an individual `ImageContent` block from `result.content`, for example `image(result.content[0])`."#; const OLD_VIEW_IMAGE: &str = r#"declare const tools: { view_image(args: { // Image detail level. Defaults to `high`; use `original` to preserve exact resolution. detail?: "high" | "original"; // Local filesystem path to an image file. path: string; }): Promise<{ // Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved. detail: "high" | "original"; // Data URL for the loaded image. image_url: string; }>; };"#; const NEW_VIEW_IMAGE: &str = r#"declare const tools: { view_image(args: { // Local filesystem path to an image file. path: string; }): Promise<{ // Data URL for the loaded image. image_url: string; }>; };"#; if old.matches(OLD_IMAGE).count() != 1 || old.matches(OLD_VIEW_IMAGE).count() != 1 { return; } let updated = old .replace(OLD_IMAGE, NEW_IMAGE) .replace(OLD_VIEW_IMAGE, NEW_VIEW_IMAGE); if current .pointer(TOOL) .and_then(|tool| tool["description"].as_str()) == Some(updated.as_str()) { *description = Value::String(updated); } }