mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Test prompt-prefix preservation across app-server resume (#62071)
Add a persisted-thread resume test with incremental tools enabled. Change the model description between app-server runs to force an updated `spawn_agent` declaration, then verify that resume preserves the original instructions, tools, and input prefix and appends the updated declaration. Use the released binary from `CONTRACT_EXECUTOR_BIN` for the initial run when available; otherwise, use the current binary for both runs. GitOrigin-RevId: 4a53d54ca0e8638277b27d9d6bec6ac46bdef5dd
This commit is contained in:
@@ -1,5 +1,9 @@
|
||||
//! Compare model-visible prefixes from two app-server versions using the same client and inputs.
|
||||
//! The contract runner supplies a released CLI; ordinary runs check same-version determinism.
|
||||
//! The contract runner supplies a released CLI; ordinary runs use the current binary twice.
|
||||
//! Resume coverage checks that changed tool declarations leave the retained prefix intact.
|
||||
|
||||
#[path = "prompt_prefix/resume_tests.rs"]
|
||||
mod resume_tests;
|
||||
|
||||
use anyhow::Context;
|
||||
use anyhow::Result;
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
//! Exercise incremental tool updates across a persisted old-to-new app-server resume.
|
||||
|
||||
use anyhow::Context;
|
||||
use anyhow::Result;
|
||||
use app_test_support::MockResponsesConfig;
|
||||
use app_test_support::TestAppServer;
|
||||
use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::ThreadResumeParams;
|
||||
use codex_app_server_protocol::ThreadResumeResponse;
|
||||
use codex_app_server_protocol::ThreadStartParams;
|
||||
use codex_app_server_protocol::TurnStartParams;
|
||||
use codex_app_server_protocol::TurnStatus;
|
||||
use codex_app_server_protocol::UserInput;
|
||||
use codex_features::Feature;
|
||||
use core_test_support::responses;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn resume_preserves_prefix_and_appends_changed_tool_declaration() -> Result<()> {
|
||||
let release = std::env::var_os("CONTRACT_EXECUTOR_BIN").map(std::path::PathBuf::from);
|
||||
let server = responses::start_mock_server().await;
|
||||
let home = TempDir::new()?;
|
||||
let catalog_path = home.path().join("models.json");
|
||||
let mut catalog = codex_models_manager::bundled_models_response()?;
|
||||
let model_index = catalog
|
||||
.models
|
||||
.iter()
|
||||
.position(|model| model.slug == "gpt-6-astra")
|
||||
.context("bundled catalog must contain the test model")?;
|
||||
// Force a real declaration change even when the released catalog catches up.
|
||||
// The separate fresh-start test still compares each binary's bundled defaults.
|
||||
catalog.models[model_index].description = Some("Before the app-server upgrade".into());
|
||||
std::fs::write(&catalog_path, serde_json::to_vec(&catalog)?)?;
|
||||
MockResponsesConfig::new(&server.uri())
|
||||
.with_model("gpt-6-astra")
|
||||
.enable_feature(Feature::IncrementalTools)
|
||||
.disable_feature(Feature::ModelCatalogInContext)
|
||||
.with_root_config(&format!(
|
||||
"skills.bundled.enabled = false\nmodel_catalog_json = {}\nmulti_agent_v2.expose_spawn_agent_model_overrides = true",
|
||||
serde_json::to_string(&catalog_path)?
|
||||
))
|
||||
.write(home.path())?;
|
||||
let builder = || {
|
||||
TestAppServer::builder()
|
||||
.with_codex_home(home.path())
|
||||
.with_plugin_startup_tasks()
|
||||
.without_managed_config()
|
||||
.with_env_overrides(&[
|
||||
("CODEX_APP_SERVER_MANAGED_CONFIG_PATH", None),
|
||||
("CODEX_APP_SERVER_DISABLE_MANAGED_CONFIG", None),
|
||||
])
|
||||
};
|
||||
let mut old_builder = builder();
|
||||
if let Some(program) = release.as_deref() {
|
||||
old_builder = old_builder.with_program(program).with_args(&["app-server"]);
|
||||
}
|
||||
let initial_response = responses::mount_sse_once(
|
||||
&server,
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("before-upgrade"),
|
||||
responses::ev_assistant_message("before-message", "Before upgrade"),
|
||||
responses::ev_completed("before-upgrade"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let mut old_app = old_builder.build_initialized().await?;
|
||||
let thread = old_app
|
||||
.start_thread(ThreadStartParams::default())
|
||||
.await?
|
||||
.thread;
|
||||
let completed = old_app
|
||||
.start_turn_and_wait_for_completion(TurnStartParams {
|
||||
thread_id: thread.id.clone(),
|
||||
input: vec![UserInput::Text {
|
||||
text: "Before upgrade".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(completed.turn.status, TurnStatus::Completed);
|
||||
let initial_request = initial_response.single_request().body_json();
|
||||
let initial_input = initial_request["input"]
|
||||
.as_array()
|
||||
.context("initial request input")?;
|
||||
let initial_declarations = spawn_agent_declarations(initial_input);
|
||||
let [initial_tool] = initial_declarations.as_slice() else {
|
||||
anyhow::bail!("expected one initial incremental spawn_agent declaration");
|
||||
};
|
||||
assert!(
|
||||
initial_tool["description"]
|
||||
.as_str()
|
||||
.context("initial spawn_agent description")?
|
||||
.contains("Before the app-server upgrade")
|
||||
);
|
||||
// Finish persistence and retire the old process before the new binary resumes.
|
||||
assert!(old_app.shutdown_gracefully().await?.success());
|
||||
|
||||
catalog.models[model_index].description = Some("After the app-server upgrade".into());
|
||||
std::fs::write(&catalog_path, serde_json::to_vec(&catalog)?)?;
|
||||
server.reset().await;
|
||||
let resumed_response = responses::mount_sse_once(
|
||||
&server,
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("after-upgrade"),
|
||||
responses::ev_assistant_message("after-message", "After upgrade"),
|
||||
responses::ev_completed("after-upgrade"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let mut app = builder().build_initialized().await?;
|
||||
let resumed: ThreadResumeResponse = app
|
||||
.request(|request_id| ClientRequest::ThreadResume {
|
||||
request_id,
|
||||
params: ThreadResumeParams {
|
||||
thread_id: thread.id.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(resumed.thread.id, thread.id);
|
||||
let completed = app
|
||||
.start_turn_and_wait_for_completion(TurnStartParams {
|
||||
thread_id: thread.id,
|
||||
input: vec![UserInput::Text {
|
||||
text: "After upgrade".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(completed.turn.status, TurnStatus::Completed);
|
||||
let resumed_request = resumed_response.single_request().body_json();
|
||||
let resumed_input = resumed_request["input"]
|
||||
.as_array()
|
||||
.context("resumed request input")?;
|
||||
let retained = resumed_input
|
||||
.get(..initial_input.len())
|
||||
.context("resume must retain the entire previous request prefix")?;
|
||||
assert_eq!(
|
||||
json!({
|
||||
"instructions": initial_request.get("instructions"),
|
||||
"tools": initial_request.get("tools"),
|
||||
"input": initial_input,
|
||||
}),
|
||||
json!({
|
||||
"instructions": resumed_request.get("instructions"),
|
||||
"tools": resumed_request.get("tools"),
|
||||
"input": retained,
|
||||
}),
|
||||
"resuming on the new binary must preserve the previous inference prefix"
|
||||
);
|
||||
let appended_declarations = spawn_agent_declarations(&resumed_input[initial_input.len()..]);
|
||||
let [updated_tool] = appended_declarations.as_slice() else {
|
||||
anyhow::bail!("expected one appended spawn_agent declaration after the retained prefix");
|
||||
};
|
||||
let updated_description = updated_tool["description"]
|
||||
.as_str()
|
||||
.context("updated spawn_agent description")?;
|
||||
assert!(updated_description.contains("After the app-server upgrade"));
|
||||
assert!(!updated_description.contains("Before the app-server upgrade"));
|
||||
assert!(app.shutdown_gracefully().await?.success());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn spawn_agent_declarations(input: &[Value]) -> Vec<&Value> {
|
||||
input
|
||||
.iter()
|
||||
.filter(|item| item["type"] == "additional_tools")
|
||||
.filter_map(|item| responses::namespace_child_tool(item, "collaboration", "spawn_agent"))
|
||||
.collect()
|
||||
}
|
||||
Reference in New Issue
Block a user