From cac9284d24cc01f85df2d470717749514d06f02e Mon Sep 17 00:00:00 2001 From: jif Date: Tue, 17 Nov 2026 14:02:04 +0000 Subject: [PATCH] Test prompt-prefix preservation across app-server resume (#62071) Add a persisted-thread resume test with incremental tools enabled. Change the model description between app-server runs to force an updated `spawn_agent` declaration, then verify that resume preserves the original instructions, tools, and input prefix and appends the updated declaration. Use the released binary from `CONTRACT_EXECUTOR_BIN` for the initial run when available; otherwise, use the current binary for both runs. GitOrigin-RevId: 4a53d54ca0e8638277b27d9d6bec6ac46bdef5dd --- codex-rs/app-server/tests/prompt_prefix.rs | 6 +- .../tests/prompt_prefix/resume_tests.rs | 175 ++++++++++++++++++ 2 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 codex-rs/app-server/tests/prompt_prefix/resume_tests.rs diff --git a/codex-rs/app-server/tests/prompt_prefix.rs b/codex-rs/app-server/tests/prompt_prefix.rs index c0d2b5e602..3ac1d50391 100644 --- a/codex-rs/app-server/tests/prompt_prefix.rs +++ b/codex-rs/app-server/tests/prompt_prefix.rs @@ -1,5 +1,9 @@ //! Compare model-visible prefixes from two app-server versions using the same client and inputs. -//! The contract runner supplies a released CLI; ordinary runs check same-version determinism. +//! The contract runner supplies a released CLI; ordinary runs use the current binary twice. +//! Resume coverage checks that changed tool declarations leave the retained prefix intact. + +#[path = "prompt_prefix/resume_tests.rs"] +mod resume_tests; use anyhow::Context; use anyhow::Result; diff --git a/codex-rs/app-server/tests/prompt_prefix/resume_tests.rs b/codex-rs/app-server/tests/prompt_prefix/resume_tests.rs new file mode 100644 index 0000000000..8a966cde13 --- /dev/null +++ b/codex-rs/app-server/tests/prompt_prefix/resume_tests.rs @@ -0,0 +1,175 @@ +//! Exercise incremental tool updates across a persisted old-to-new app-server resume. + +use anyhow::Context; +use anyhow::Result; +use app_test_support::MockResponsesConfig; +use app_test_support::TestAppServer; +use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::ThreadResumeParams; +use codex_app_server_protocol::ThreadResumeResponse; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStatus; +use codex_app_server_protocol::UserInput; +use codex_features::Feature; +use core_test_support::responses; +use pretty_assertions::assert_eq; +use serde_json::Value; +use serde_json::json; +use tempfile::TempDir; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resume_preserves_prefix_and_appends_changed_tool_declaration() -> Result<()> { + let release = std::env::var_os("CONTRACT_EXECUTOR_BIN").map(std::path::PathBuf::from); + let server = responses::start_mock_server().await; + let home = TempDir::new()?; + let catalog_path = home.path().join("models.json"); + let mut catalog = codex_models_manager::bundled_models_response()?; + let model_index = catalog + .models + .iter() + .position(|model| model.slug == "gpt-6-astra") + .context("bundled catalog must contain the test model")?; + // Force a real declaration change even when the released catalog catches up. + // The separate fresh-start test still compares each binary's bundled defaults. + catalog.models[model_index].description = Some("Before the app-server upgrade".into()); + std::fs::write(&catalog_path, serde_json::to_vec(&catalog)?)?; + MockResponsesConfig::new(&server.uri()) + .with_model("gpt-6-astra") + .enable_feature(Feature::IncrementalTools) + .disable_feature(Feature::ModelCatalogInContext) + .with_root_config(&format!( + "skills.bundled.enabled = false\nmodel_catalog_json = {}\nmulti_agent_v2.expose_spawn_agent_model_overrides = true", + serde_json::to_string(&catalog_path)? + )) + .write(home.path())?; + let builder = || { + TestAppServer::builder() + .with_codex_home(home.path()) + .with_plugin_startup_tasks() + .without_managed_config() + .with_env_overrides(&[ + ("CODEX_APP_SERVER_MANAGED_CONFIG_PATH", None), + ("CODEX_APP_SERVER_DISABLE_MANAGED_CONFIG", None), + ]) + }; + let mut old_builder = builder(); + if let Some(program) = release.as_deref() { + old_builder = old_builder.with_program(program).with_args(&["app-server"]); + } + let initial_response = responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("before-upgrade"), + responses::ev_assistant_message("before-message", "Before upgrade"), + responses::ev_completed("before-upgrade"), + ]), + ) + .await; + let mut old_app = old_builder.build_initialized().await?; + let thread = old_app + .start_thread(ThreadStartParams::default()) + .await? + .thread; + let completed = old_app + .start_turn_and_wait_for_completion(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![UserInput::Text { + text: "Before upgrade".into(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + assert_eq!(completed.turn.status, TurnStatus::Completed); + let initial_request = initial_response.single_request().body_json(); + let initial_input = initial_request["input"] + .as_array() + .context("initial request input")?; + let initial_declarations = spawn_agent_declarations(initial_input); + let [initial_tool] = initial_declarations.as_slice() else { + anyhow::bail!("expected one initial incremental spawn_agent declaration"); + }; + assert!( + initial_tool["description"] + .as_str() + .context("initial spawn_agent description")? + .contains("Before the app-server upgrade") + ); + // Finish persistence and retire the old process before the new binary resumes. + assert!(old_app.shutdown_gracefully().await?.success()); + + catalog.models[model_index].description = Some("After the app-server upgrade".into()); + std::fs::write(&catalog_path, serde_json::to_vec(&catalog)?)?; + server.reset().await; + let resumed_response = responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("after-upgrade"), + responses::ev_assistant_message("after-message", "After upgrade"), + responses::ev_completed("after-upgrade"), + ]), + ) + .await; + let mut app = builder().build_initialized().await?; + let resumed: ThreadResumeResponse = app + .request(|request_id| ClientRequest::ThreadResume { + request_id, + params: ThreadResumeParams { + thread_id: thread.id.clone(), + ..Default::default() + }, + }) + .await?; + assert_eq!(resumed.thread.id, thread.id); + let completed = app + .start_turn_and_wait_for_completion(TurnStartParams { + thread_id: thread.id, + input: vec![UserInput::Text { + text: "After upgrade".into(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + assert_eq!(completed.turn.status, TurnStatus::Completed); + let resumed_request = resumed_response.single_request().body_json(); + let resumed_input = resumed_request["input"] + .as_array() + .context("resumed request input")?; + let retained = resumed_input + .get(..initial_input.len()) + .context("resume must retain the entire previous request prefix")?; + assert_eq!( + json!({ + "instructions": initial_request.get("instructions"), + "tools": initial_request.get("tools"), + "input": initial_input, + }), + json!({ + "instructions": resumed_request.get("instructions"), + "tools": resumed_request.get("tools"), + "input": retained, + }), + "resuming on the new binary must preserve the previous inference prefix" + ); + let appended_declarations = spawn_agent_declarations(&resumed_input[initial_input.len()..]); + let [updated_tool] = appended_declarations.as_slice() else { + anyhow::bail!("expected one appended spawn_agent declaration after the retained prefix"); + }; + let updated_description = updated_tool["description"] + .as_str() + .context("updated spawn_agent description")?; + assert!(updated_description.contains("After the app-server upgrade")); + assert!(!updated_description.contains("Before the app-server upgrade")); + assert!(app.shutdown_gracefully().await?.success()); + Ok(()) +} + +fn spawn_agent_declarations(input: &[Value]) -> Vec<&Value> { + input + .iter() + .filter(|item| item["type"] == "additional_tools") + .filter_map(|item| responses::namespace_child_tool(item, "collaboration", "spawn_agent")) + .collect() +}