tighten inline instruction tests

This commit is contained in:
pakrym-oai
2026-06-20 23:51:22 -07:00
parent f6f0b8efca
commit ea0d3f7a3b
15 changed files with 118 additions and 135 deletions
+11 -14
View File
@@ -1907,25 +1907,21 @@ async fn build_initial_context_inlines_base_instructions_for_new_turn_contexts()
assert!(turn_context.inline_instructions);
let initial_context = session.build_initial_context(&turn_context).await;
assert!(
developer_input_texts(&initial_context)
.iter()
.any(|text| text.contains(&base_instructions))
);
let developer_texts = developer_input_texts(&initial_context);
assert_eq!(developer_texts.first(), Some(&base_instructions.as_str()));
turn_context.inline_instructions = false;
let legacy_initial_context = session.build_initial_context(&turn_context).await;
assert!(
developer_input_texts(&legacy_initial_context)
.iter()
.all(|text| !text.contains(&base_instructions))
assert_eq!(
developer_input_texts(&legacy_initial_context),
developer_texts[1..]
);
}
#[tokio::test]
async fn new_turn_context_inherits_inline_instructions_from_reference_context() {
let (session, turn_context) = make_session_and_context().await;
assert!(turn_context.inline_instructions);
assert_eq!(turn_context.inline_instructions, true);
let mut legacy_context_item = turn_context.to_turn_context_item();
legacy_context_item.inline_instructions = false;
@@ -1936,11 +1932,12 @@ async fn new_turn_context_inherits_inline_instructions_from_reference_context()
.set_reference_context_item(Some(legacy_context_item));
let resumed_turn_context = session.new_default_turn().await;
assert!(!resumed_turn_context.inline_instructions);
assert!(
!resumed_turn_context
assert_eq!(resumed_turn_context.inline_instructions, false);
assert_eq!(
resumed_turn_context
.to_turn_context_item()
.inline_instructions
.inline_instructions,
false
);
}
+5 -3
View File
@@ -201,9 +201,11 @@ async fn remote_model_override_uses_catalog_model_for_strict_auto_review() -> Re
.into_iter()
.find(|request| {
request.body_contains_text("auto-review-model-override.txt")
&& request.message_input_texts("developer").iter().any(|text| {
text.starts_with("You are judging one planned coding-agent action.")
})
&& request
.message_input_texts("developer")
.first()
.and_then(|text| text.lines().next())
== Some("You are judging one planned coding-agent action.")
})
.expect("expected Guardian request for apply_patch");
assert_eq!(
+4 -10
View File
@@ -346,11 +346,8 @@ async fn exec_cli_applies_model_instructions_file() {
// Inspect the captured request and verify our custom base instructions were
// included in developer input.
let request = resp_mock.single_request();
let instructions = request.message_input_texts("developer").join("\n\n");
assert!(
instructions.contains(marker),
"instructions did not contain custom marker; got: {instructions}"
);
let developer_texts = request.message_input_texts("developer");
assert_eq!(developer_texts.first().map(String::as_str), Some(marker));
}
/// Verify that `codex exec --profile ...` preserves the active user config
@@ -409,11 +406,8 @@ async fn exec_cli_profile_applies_model_instructions_file() {
assert!(output.status.success());
let request = resp_mock.single_request();
let instructions = request.message_input_texts("developer").join("\n\n");
assert!(
instructions.contains(marker),
"instructions did not contain profile marker; got: {instructions}"
);
let developer_texts = request.message_input_texts("developer");
assert_eq!(developer_texts.first().map(String::as_str), Some(marker));
}
/// Tests streaming responses through the CLI using a local Responses API server.
+10 -16
View File
@@ -1240,12 +1240,10 @@ async fn inlines_base_instructions_override_in_request() {
let request_body = request.body_json();
let developer_messages = request.message_input_texts("developer");
assert!(request_body.get("instructions").is_none());
assert!(
developer_messages
.iter()
.any(|text| text.contains("test instructions")),
"expected base instructions in developer input, got {developer_messages:?}"
assert_eq!(request_body.get("instructions"), None);
assert_eq!(
developer_messages.first().map(String::as_str),
Some("test instructions")
);
}
@@ -1466,12 +1464,10 @@ async fn includes_user_instructions_message_in_request() {
let request = resp_mock.single_request();
let request_body = request.body_json();
assert!(request_body.get("instructions").is_none());
assert_eq!(request_body.get("instructions"), None);
assert_message_role(&request_body["input"][0], "developer");
let developer_texts = request.message_input_texts("developer");
let permissions_text = developer_texts
.iter()
.find(|text| text.contains("`sandbox_mode`"))
let permissions_text = request_body["input"][0]["content"][1]["text"]
.as_str()
.expect("permissions message content");
assert!(
permissions_text.contains("`sandbox_mode`"),
@@ -2649,12 +2645,10 @@ async fn includes_developer_instructions_message_in_request() {
let request = resp_mock.single_request();
let request_body = request.body_json();
assert!(request_body.get("instructions").is_none());
assert_eq!(request_body.get("instructions"), None);
assert_message_role(&request_body["input"][0], "developer");
let developer_texts = request.message_input_texts("developer");
let permissions_text = developer_texts
.iter()
.find(|text| text.contains("`sandbox_mode`"))
let permissions_text = request_body["input"][0]["content"][1]["text"]
.as_str()
.expect("permissions message content");
assert!(
permissions_text.contains("`sandbox_mode`"),
+20 -17
View File
@@ -547,13 +547,15 @@ async fn summarize_context_three_requests_and_instructions() {
let body3 = requests[2].body_json();
// Manual compact should keep the baseline developer instructions.
assert!(body1.get("instructions").is_none());
assert!(body2.get("instructions").is_none());
assert_eq!(body1.get("instructions"), None);
assert_eq!(body2.get("instructions"), None);
let developer_texts1 = requests[0].message_input_texts("developer");
let developer_texts2 = requests[1].message_input_texts("developer");
assert!(
developer_texts2.starts_with(&developer_texts1),
"manual compact should keep the standard developer instructions: {developer_texts2:?}"
let baseline_instructions = developer_texts1.first().expect("developer instructions");
assert_eq!(
developer_texts2.first(),
Some(baseline_instructions),
"manual compact should keep the standard developer instructions"
);
// The summarization request should include the injected user input marker.
@@ -1215,13 +1217,10 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() {
&& value
.get("content")
.and_then(|content| content.as_array())
.is_some_and(|content| {
content.iter().any(|item| {
item.get("text")
.and_then(|text| text.as_str())
.is_some_and(|text| text.contains("`sandbox_mode`"))
})
})
.and_then(|content| content.get(1))
.and_then(|item| item.get("text"))
.and_then(|text| text.as_str())
.is_some_and(|text| text.contains("`sandbox_mode`"))
{
return None;
}
@@ -1717,13 +1716,17 @@ async fn auto_compact_runs_after_token_limit_hit() {
let body_first = requests[0].body_json();
let body_auto = requests[auto_compact_index].body_json();
let body_follow_up = requests[follow_up_index].body_json();
assert!(body_first.get("instructions").is_none());
assert!(body_auto.get("instructions").is_none());
assert_eq!(body_first.get("instructions"), None);
assert_eq!(body_auto.get("instructions"), None);
let baseline_developer_texts = requests[0].message_input_texts("developer");
let auto_developer_texts = requests[auto_compact_index].message_input_texts("developer");
assert!(
auto_developer_texts.starts_with(&baseline_developer_texts),
"auto compact should keep the standard developer instructions: {auto_developer_texts:?}",
let baseline_instructions = baseline_developer_texts
.first()
.expect("developer instructions");
assert_eq!(
auto_developer_texts.first(),
Some(baseline_instructions),
"auto compact should keep the standard developer instructions",
);
let input_auto = body_auto.get("input").and_then(|v| v.as_array()).unwrap();
+6 -10
View File
@@ -2180,18 +2180,14 @@ async fn remote_compact_trim_estimate_uses_session_base_instructions() -> Result
.await;
let override_compact_request = override_compact_mock.single_request();
assert!(
override_compact_request
.body_json()
.get("instructions")
.is_none()
assert_eq!(
override_compact_request.body_json().get("instructions"),
None
);
let override_developer_texts = override_compact_request.message_input_texts("developer");
assert!(
override_developer_texts
.iter()
.any(|text| text.contains(&override_base_instructions)),
"expected override instructions in developer input, got {override_developer_texts:?}"
assert_eq!(
override_developer_texts.first().map(String::as_str),
Some(override_base_instructions.as_str())
);
assert!(
override_compact_request.has_function_call(override_retained_call_id),
@@ -318,8 +318,9 @@ async fn apps_default_prompt_with_auto_review_routes_actual_mcp_approval_to_guar
.find(|request| {
request
.message_input_texts("developer")
.iter()
.any(|text| text.starts_with("You are judging one planned coding-agent action."))
.first()
.and_then(|text| text.lines().next())
== Some("You are judging one planned coding-agent action.")
})
.expect("expected a Guardian request for the app MCP approval");
assert!(guardian_request.body_contains_text("calendar_create_event"));
@@ -5,6 +5,7 @@ use std::sync::Arc;
use anyhow::Result;
use codex_config::types::Personality;
use codex_features::Feature;
use codex_protocol::models::ContentItem;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::AskForApproval;
@@ -29,6 +30,7 @@ use core_test_support::skip_if_no_network;
use core_test_support::test_codex::test_codex;
use core_test_support::test_codex::turn_permission_fields;
use core_test_support::wait_for_event;
use pretty_assertions::assert_eq;
use serde_json::json;
const PRETURN_CONTEXT_DIFF_CWD: &str = "PRETURN_CONTEXT_DIFF_CWD";
@@ -376,6 +378,10 @@ async fn snapshot_model_visible_layout_resume_with_personality_change() -> Resul
.await?;
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
let initial_request = initial_mock.single_request();
let initial_developer_texts = initial_request.message_input_texts("developer");
let initial_model_instructions = initial_developer_texts
.first()
.expect("initial model instructions");
// Rewrite the seed turn into the legacy representation: no persisted compatibility flag and
// no model instructions in history. The resumed request should therefore use the top-level
@@ -394,8 +400,10 @@ async fn snapshot_model_visible_layout_resume_with_personality_change() -> Resul
RolloutItem::ResponseItem(ResponseItem::Message { role, content, .. })
if role == "developer" && !removed_inline_instructions =>
{
assert!(!content.is_empty(), "developer message should have content");
content.remove(0);
let ContentItem::InputText { text } = content.remove(0) else {
panic!("first developer content should be model instructions");
};
assert_eq!(text, initial_model_instructions.as_str());
removed_inline_instructions = true;
}
_ => {}
+8 -6
View File
@@ -194,14 +194,14 @@ async fn config_personality_some_sets_instructions_template() -> anyhow::Result<
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let request = resp_mock.single_request();
let instructions_text = request.message_input_texts("developer").join("\n\n");
let developer_texts = request.message_input_texts("developer");
let instructions_text = developer_texts.first().expect("developer instructions");
assert!(
instructions_text.contains(LOCAL_FRIENDLY_TEMPLATE),
"expected personality update to include the local friendly template, got: {instructions_text:?}"
);
let developer_texts = request.message_input_texts("developer");
for text in developer_texts {
assert!(
!text.contains("<personality_spec>"),
@@ -241,7 +241,8 @@ async fn config_personality_none_sends_no_personality() -> anyhow::Result<()> {
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let request = resp_mock.single_request();
let instructions_text = request.message_input_texts("developer").join("\n\n");
let developer_texts = request.message_input_texts("developer");
let instructions_text = developer_texts.first().expect("developer instructions");
assert!(
!instructions_text.contains(LOCAL_FRIENDLY_TEMPLATE),
"expected no friendly personality template, got: {instructions_text:?}"
@@ -255,7 +256,6 @@ async fn config_personality_none_sends_no_personality() -> anyhow::Result<()> {
"expected personality placeholder to be removed, got: {instructions_text:?}"
);
let developer_texts = request.message_input_texts("developer");
assert!(
!developer_texts
.iter()
@@ -294,7 +294,8 @@ async fn default_personality_is_pragmatic_without_config_toml() -> anyhow::Resul
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let request = resp_mock.single_request();
let instructions_text = request.message_input_texts("developer").join("\n\n");
let developer_texts = request.message_input_texts("developer");
let instructions_text = developer_texts.first().expect("developer instructions");
assert!(
instructions_text.contains(LOCAL_PRAGMATIC_TEMPLATE),
"expected default friendly template, got: {instructions_text:?}"
@@ -636,7 +637,8 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let request = resp_mock.single_request();
let instructions_text = request.message_input_texts("developer").join("\n\n");
let developer_texts = request.message_input_texts("developer");
let instructions_text = developer_texts.first().expect("developer instructions");
assert!(
instructions_text.contains(friendly_personality_message),
+21 -31
View File
@@ -206,25 +206,21 @@ async fn prompt_tools_are_consistent_across_requests() -> anyhow::Result<()> {
[base_instructions, APPLY_PATCH_TOOL_INSTRUCTIONS.to_string()].join("\n")
};
assert!(body0.get("instructions").is_none());
assert_eq!(body0.get("instructions"), None);
let developer_texts0 = request0.message_input_texts("developer");
assert!(
developer_texts0
.iter()
.any(|text| text.starts_with(&expected_instructions)),
"expected model instructions in developer input, got {developer_texts0:?}"
assert_eq!(
developer_texts0.first().map(String::as_str),
Some(expected_instructions.as_str())
);
assert_tool_names(&body0, &expected_tools_names);
let request1 = req2.single_request();
let body1 = request1.body_json();
assert!(body1.get("instructions").is_none());
assert_eq!(body1.get("instructions"), None);
let developer_texts1 = request1.message_input_texts("developer");
assert!(
developer_texts1
.iter()
.any(|text| text.starts_with(&expected_instructions)),
"expected model instructions in developer input, got {developer_texts1:?}"
assert_eq!(
developer_texts1.first().map(String::as_str),
Some(expected_instructions.as_str())
);
assert_tool_names(&body1, &expected_tools_names);
@@ -290,7 +286,7 @@ async fn gpt_5_tools_without_apply_patch_append_apply_patch_instructions() -> an
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let request0 = req1.single_request();
assert!(request0.body_json().get("instructions").is_none());
assert_eq!(request0.body_json().get("instructions"), None);
let developer_texts0 = request0.message_input_texts("developer");
let instructions0 = developer_texts0.first().expect("developer instructions");
assert!(
@@ -299,7 +295,7 @@ async fn gpt_5_tools_without_apply_patch_append_apply_patch_instructions() -> an
);
let request1 = req2.single_request();
assert!(request1.body_json().get("instructions").is_none());
assert_eq!(request1.body_json().get("instructions"), None);
let developer_texts1 = request1.message_input_texts("developer");
let instructions1 = developer_texts1.first().expect("developer instructions");
assert_eq!(
@@ -654,24 +650,18 @@ async fn override_before_first_turn_emits_environment_context() -> anyhow::Resul
"environment context should appear at least once, found {env_count}"
);
let permissions_texts: Vec<&str> = input
.iter()
.filter(|msg| msg["role"].as_str() == Some("developer"))
.flat_map(|msg| {
msg["content"]
.as_array()
.into_iter()
.flatten()
.filter_map(|item| item["text"].as_str())
})
.collect();
let permissions_text = input
.first()
.and_then(|message| message["content"].as_array())
.and_then(|content| content.get(1))
.and_then(|item| item["text"].as_str())
.expect("permissions instructions");
let normalized_permissions = permissions_text.to_ascii_lowercase();
assert!(
permissions_texts.iter().any(|text| {
let lower = text.to_ascii_lowercase();
(lower.contains("approval policy") || lower.contains("approval_policy"))
&& lower.contains("never")
}),
"permissions message should reflect overridden approval policy: {permissions_texts:?}"
(normalized_permissions.contains("approval policy")
|| normalized_permissions.contains("approval_policy"))
&& normalized_permissions.contains("never"),
"permissions message should reflect overridden approval policy: {permissions_text:?}"
);
let user_texts: Vec<&str> = input
+4 -6
View File
@@ -827,13 +827,11 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> {
.get_model_info("gpt-5.2", &config.to_models_manager_config())
.await;
let request = response_mock.single_request();
assert!(request.body_json().get("instructions").is_none());
assert_eq!(request.body_json().get("instructions"), None);
let developer_texts = request.message_input_texts("developer");
assert!(
developer_texts
.iter()
.any(|text| text.contains(&base_model_info.base_instructions)),
"expected model instructions in developer input, got {developer_texts:?}"
assert_eq!(
developer_texts.first().map(String::as_str),
Some(base_model_info.base_instructions.as_str())
);
Ok(())
+9 -9
View File
@@ -275,8 +275,11 @@ async fn resume_switches_models_preserves_base_instructions() -> Result<()> {
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
let initial_request = initial_mock.single_request();
assert!(initial_request.body_json().get("instructions").is_none());
assert_eq!(initial_request.body_json().get("instructions"), None);
let initial_developer_texts = initial_request.message_input_texts("developer");
let initial_instructions = initial_developer_texts
.first()
.expect("initial developer instructions");
let resumed_mock = mount_sse_sequence(
&server,
@@ -339,22 +342,19 @@ async fn resume_switches_models_preserves_base_instructions() -> Result<()> {
assert_eq!(requests.len(), 2, "expected two resumed requests");
let first_resumed = &requests[0];
assert!(first_resumed.body_json().get("instructions").is_none());
assert_eq!(first_resumed.body_json().get("instructions"), None);
let first_developer_texts = first_resumed.message_input_texts("developer");
assert!(first_developer_texts.starts_with(&initial_developer_texts));
assert_eq!(first_developer_texts.first(), Some(initial_instructions));
let first_model_switch_count = first_developer_texts
.iter()
.filter(|text| text.contains("<model_switch>"))
.count();
assert!(
first_model_switch_count >= 1,
"expected model switch message on first post-resume turn"
);
assert_eq!(first_model_switch_count, 1);
let second_resumed = &requests[1];
assert!(second_resumed.body_json().get("instructions").is_none());
assert_eq!(second_resumed.body_json().get("instructions"), None);
let second_developer_texts = second_resumed.message_input_texts("developer");
assert!(second_developer_texts.starts_with(&initial_developer_texts));
assert_eq!(second_developer_texts.first(), Some(initial_instructions));
let second_model_switch_count = second_developer_texts
.iter()
.filter(|text| text.contains("<model_switch>"))
+4 -6
View File
@@ -628,13 +628,11 @@ async fn review_input_isolated_from_parent_history() {
);
// Ensure the REVIEW_PROMPT rubric is sent via developer input.
assert!(body.get("instructions").is_none());
assert_eq!(body.get("instructions"), None);
let developer_texts = request.message_input_texts("developer");
assert!(
developer_texts
.iter()
.any(|text| text.contains(REVIEW_PROMPT)),
"expected review instructions in developer input, got {developer_texts:?}"
assert_eq!(
developer_texts.first().map(String::as_str),
Some(REVIEW_PROMPT)
);
// Also verify that a user interruption note was recorded in the rollout.
+2 -2
View File
@@ -305,9 +305,9 @@ mod job {
phase: None,
metadata: None,
}];
prompt.base_instructions = BaseInstructions {
prompt.base_instructions = Some(BaseInstructions {
text: crate::stage_one::PROMPT.to_string(),
};
});
prompt.output_schema = Some(output_schema());
prompt.output_schema_strict = true;
+1 -1
View File
@@ -5376,7 +5376,7 @@ mod tests {
assert_eq!(item.network, None);
assert_eq!(item.file_system_sandbox_policy, None);
assert_eq!(item.comp_hash, None);
assert!(!item.inline_instructions);
assert_eq!(item.inline_instructions, false);
Ok(())
}