mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Support model catalog Apps usage instructions (#61213)
## What changed - Add the disabled-by-default `catalog_apps_instructions` feature to use the selected model's `model_messages.apps.usage_instructions`, subject to existing Apps availability and model gates. - Preserve valid catalog text literally, allow an empty string to clear guidance, and fall back to bundled instructions for missing, whitespace-only, or oversized values. Overrides are limited to 800 UTF-8 bytes. - Reconcile guidance with the latest retained Apps fragment, emitting replacement or removal notices when needed and avoiding repeated updates across resumes. Preserve compatibility with bundled guidance and legacy checkpoints. - Recognize catalog Apps fragments as contextual developer messages in event mapping and rollout rollback. ## Testing Add resolver, world-state, client, and scenario coverage for literal text and byte limits, feature and availability gates, model switches, clearing guidance, resumes, and legacy history recovery. GitOrigin-RevId: 6240d407ada8f22fa101753ff3ddb338632ae283
This commit is contained in:
@@ -2188,8 +2188,33 @@ async fn includes_user_instructions_message_in_request() {
|
||||
);
|
||||
}
|
||||
|
||||
const CATALOG_APPS_GUIDANCE: &str = "Distinctive catalog Apps guidance.";
|
||||
|
||||
fn assert_no_apps_guidance(request: &ResponsesRequest) {
|
||||
for text in [
|
||||
"<apps_instructions>",
|
||||
"<catalog_apps_instructions>",
|
||||
CATALOG_APPS_GUIDANCE,
|
||||
] {
|
||||
assert!(
|
||||
!request.body_contains_text(text),
|
||||
"unexpected Apps guidance: {text:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test_case::test_case(false, true, true, true; "catalog_missing")]
|
||||
#[test_case::test_case(true, false, true, true; "catalog_feature_off_by_default")]
|
||||
#[test_case::test_case(true, true, true, true; "catalog_override")]
|
||||
#[test_case::test_case(true, true, false, true; "model_excludes")]
|
||||
#[test_case::test_case(true, true, true, false; "apps_feature_off")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn includes_apps_guidance_as_developer_message_for_chatgpt_auth() {
|
||||
async fn includes_apps_guidance_as_developer_message_for_chatgpt_auth(
|
||||
catalog_override: bool,
|
||||
enable_catalog: bool,
|
||||
model_includes: bool,
|
||||
apps_enabled: bool,
|
||||
) {
|
||||
skip_if_no_network!();
|
||||
let server = MockServer::start().await;
|
||||
let apps_server = AppsTestServer::mount(&server)
|
||||
@@ -2208,12 +2233,27 @@ async fn includes_apps_guidance_as_developer_message_for_chatgpt_auth() {
|
||||
.with_config(move |config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::Apps)
|
||||
.set_enabled(Feature::Apps, apps_enabled)
|
||||
.expect("test config should allow feature update");
|
||||
if enable_catalog {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::CatalogAppsInstructions)
|
||||
.expect("enable catalog Apps instructions");
|
||||
}
|
||||
config.chatgpt_base_url = apps_base_url;
|
||||
})
|
||||
.with_model_info_override("gpt-5.5", move |model| {
|
||||
model.include_apps_usage_instructions = model_includes;
|
||||
model.model_messages = catalog_override.then(|| {
|
||||
serde_json::from_value(
|
||||
json!({"apps": {"usage_instructions": CATALOG_APPS_GUIDANCE}}),
|
||||
)
|
||||
.expect("catalog Apps messages")
|
||||
});
|
||||
});
|
||||
let codex = builder
|
||||
.build(&server)
|
||||
.build_with_auto_env(&server)
|
||||
.await
|
||||
.expect("create new conversation")
|
||||
.codex;
|
||||
@@ -2224,19 +2264,32 @@ async fn includes_apps_guidance_as_developer_message_for_chatgpt_auth() {
|
||||
text_elements: Vec::new(),
|
||||
}]))
|
||||
.await
|
||||
.unwrap();
|
||||
.expect("start Apps guidance turn");
|
||||
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let request = resp_mock.single_request();
|
||||
let apps_snippet =
|
||||
"Apps (Connectors) can be explicitly triggered in user messages in the format";
|
||||
|
||||
assert!(
|
||||
message_input_text_contains(&request, "developer", apps_snippet),
|
||||
"expected apps guidance in a developer message, got {:?}",
|
||||
request.body_json()["input"]
|
||||
let use_catalog = catalog_override && enable_catalog;
|
||||
assert_eq!(
|
||||
request.body_contains_text(CATALOG_APPS_GUIDANCE),
|
||||
use_catalog && model_includes && apps_enabled
|
||||
);
|
||||
let apps_snippet = if use_catalog {
|
||||
CATALOG_APPS_GUIDANCE
|
||||
} else {
|
||||
"Apps (Connectors) can be explicitly triggered in user messages in the format"
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
message_input_text_contains(&request, "developer", apps_snippet),
|
||||
model_includes && apps_enabled
|
||||
);
|
||||
if !model_includes || !apps_enabled {
|
||||
assert_no_apps_guidance(&request);
|
||||
}
|
||||
if !apps_enabled {
|
||||
assert!(!request.body_contains_text("mcp__codex_apps"));
|
||||
}
|
||||
|
||||
assert!(
|
||||
!message_input_text_contains(&request, "user", apps_snippet),
|
||||
@@ -2245,6 +2298,121 @@ async fn includes_apps_guidance_as_developer_message_for_chatgpt_auth() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn resume_without_apps_checkpoint_restores_bundled_guidance_once() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = MockServer::start().await;
|
||||
let apps = AppsTestServer::mount(&server).await?;
|
||||
let builder = || {
|
||||
let base_url = apps.chatgpt_base_url.clone();
|
||||
let mut builder = test_codex()
|
||||
.with_auth(create_dummy_codex_auth())
|
||||
.with_config(move |config| {
|
||||
config.chatgpt_base_url = base_url;
|
||||
config.features.enable(Feature::Apps).unwrap();
|
||||
});
|
||||
for (model, wording) in [
|
||||
("apps-bundled", None),
|
||||
("apps-custom", Some(CATALOG_APPS_GUIDANCE)),
|
||||
("apps-clear", Some("")),
|
||||
] {
|
||||
builder = builder.with_model_info_override(model, move |info| {
|
||||
info.include_apps_usage_instructions = true;
|
||||
info.model_messages = Some(
|
||||
serde_json::from_value(json!({"apps": {"usage_instructions": wording}}))
|
||||
.unwrap(),
|
||||
);
|
||||
});
|
||||
}
|
||||
builder
|
||||
};
|
||||
let writer = builder()
|
||||
.with_model("apps-bundled")
|
||||
.with_config(|config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::CatalogAppsInstructions)
|
||||
.unwrap();
|
||||
})
|
||||
.build(&server)
|
||||
.await?;
|
||||
let mock = mount_sse_sequence(
|
||||
&server,
|
||||
(0..5)
|
||||
.map(|i| sse(vec![ev_completed(&format!("apps-{i}"))]))
|
||||
.collect(),
|
||||
)
|
||||
.await;
|
||||
writer.submit_text_turn("Use bundled guidance").await?;
|
||||
for model in ["apps-custom", "apps-clear"] {
|
||||
core_test_support::submit_thread_settings(
|
||||
&writer.codex,
|
||||
ThreadSettingsOverrides {
|
||||
model: Some(model.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
writer.submit_text_turn(model).await?;
|
||||
}
|
||||
writer.codex.shutdown_and_wait().await?;
|
||||
|
||||
// Preserve the recorded conversation while simulating loss of the Apps checkpoint.
|
||||
let path = writer.codex.rollout_path().expect("rollout path");
|
||||
let mut rollout = String::new();
|
||||
let mut removed_checkpoint = false;
|
||||
for line in std::fs::read_to_string(&path)?.lines() {
|
||||
let mut record = codex_rollout::parse_rollout_line(line)?;
|
||||
if let RolloutItem::WorldState(state) = &mut record.item
|
||||
&& state.state.remove("apps_instructions").is_some()
|
||||
{
|
||||
removed_checkpoint = true;
|
||||
rollout.push_str(&serde_json::to_string(&record)?);
|
||||
} else {
|
||||
rollout.push_str(line);
|
||||
}
|
||||
rollout.push('\n');
|
||||
}
|
||||
assert!(removed_checkpoint);
|
||||
std::fs::write(&path, rollout)?;
|
||||
|
||||
// Resume on the updated reader with catalog wording still off.
|
||||
let reader = builder()
|
||||
.with_model("apps-clear")
|
||||
.resume(&server, Arc::clone(&writer.home), path)
|
||||
.await?;
|
||||
reader
|
||||
.submit_text_turn("Resume with bundled guidance")
|
||||
.await?;
|
||||
reader.submit_text_turn("Continue").await?;
|
||||
let guidance = |request: &ResponsesRequest| {
|
||||
request
|
||||
.message_input_texts("developer")
|
||||
.into_iter()
|
||||
.filter(|text| {
|
||||
text.starts_with("<apps_instructions>")
|
||||
|| text.starts_with("<catalog_apps_instructions>")
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let requests = mock.requests();
|
||||
assert_eq!(requests.len(), 5);
|
||||
let [bundled, replacement, retirement]: [String; 3] = guidance(&requests[2])
|
||||
.try_into()
|
||||
.expect("bundled, replaced, then retired");
|
||||
assert!(bundled.starts_with("<apps_instructions>"));
|
||||
assert!(replacement.contains(CATALOG_APPS_GUIDANCE));
|
||||
assert!(
|
||||
retirement.contains("The previously provided Apps usage instructions no longer apply.")
|
||||
);
|
||||
let expected = vec![bundled.clone(), replacement, retirement, bundled];
|
||||
assert_eq!(guidance(&requests[0]), expected[..1]);
|
||||
assert_eq!(guidance(&requests[1]), expected[..2]);
|
||||
assert_eq!(guidance(&requests[3]), expected);
|
||||
assert_eq!(guidance(&requests[4]), expected);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn omits_apps_guidance_for_api_key_auth_even_when_feature_enabled() {
|
||||
skip_if_no_network!();
|
||||
@@ -2267,10 +2435,22 @@ async fn omits_apps_guidance_for_api_key_auth_even_when_feature_enabled() {
|
||||
.features
|
||||
.enable(Feature::Apps)
|
||||
.expect("test config should allow feature update");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::CatalogAppsInstructions)
|
||||
.expect("enable catalog Apps instructions");
|
||||
config.chatgpt_base_url = apps_base_url;
|
||||
})
|
||||
.with_model_info_override("gpt-5.5", |model| {
|
||||
model.model_messages = Some(
|
||||
serde_json::from_value(
|
||||
json!({"apps": {"usage_instructions": CATALOG_APPS_GUIDANCE}}),
|
||||
)
|
||||
.expect("catalog Apps messages"),
|
||||
);
|
||||
});
|
||||
let codex = builder
|
||||
.build(&server)
|
||||
.build_with_auto_env(&server)
|
||||
.await
|
||||
.expect("create new conversation")
|
||||
.codex;
|
||||
@@ -2281,19 +2461,16 @@ async fn omits_apps_guidance_for_api_key_auth_even_when_feature_enabled() {
|
||||
text_elements: Vec::new(),
|
||||
}]))
|
||||
.await
|
||||
.unwrap();
|
||||
.expect("start API-key Apps guidance turn");
|
||||
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let request = resp_mock.single_request();
|
||||
let apps_snippet =
|
||||
"Apps (Connectors) can be explicitly triggered in user messages in the format";
|
||||
|
||||
assert_no_apps_guidance(&request);
|
||||
assert!(
|
||||
!message_input_text_contains(&request, "developer", apps_snippet)
|
||||
&& !message_input_text_contains(&request, "user", apps_snippet),
|
||||
"did not expect apps guidance for API key auth, got {:?}",
|
||||
request.body_json()["input"]
|
||||
!request.body_contains_text("mcp__codex_apps"),
|
||||
"did not expect Apps tools for API key auth, got {:?}",
|
||||
request.body_json()["tools"]
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2319,11 +2496,23 @@ async fn omits_apps_guidance_when_configured_off() {
|
||||
.features
|
||||
.enable(Feature::Apps)
|
||||
.expect("test config should allow feature update");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::CatalogAppsInstructions)
|
||||
.expect("enable catalog Apps instructions");
|
||||
config.chatgpt_base_url = apps_base_url;
|
||||
config.include_apps_instructions = false;
|
||||
})
|
||||
.with_model_info_override("gpt-5.5", |model| {
|
||||
model.model_messages = Some(
|
||||
serde_json::from_value(
|
||||
json!({"apps": {"usage_instructions": CATALOG_APPS_GUIDANCE}}),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
});
|
||||
let codex = builder
|
||||
.build(&server)
|
||||
.build_with_auto_env(&server)
|
||||
.await
|
||||
.expect("create new conversation")
|
||||
.codex;
|
||||
@@ -2339,11 +2528,7 @@ async fn omits_apps_guidance_when_configured_off() {
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let request = resp_mock.single_request();
|
||||
assert!(
|
||||
!message_input_text_contains(&request, "developer", "<apps_instructions>"),
|
||||
"did not expect apps instructions when include_apps_instructions = false, got {:?}",
|
||||
request.body_json()["input"]
|
||||
);
|
||||
assert_no_apps_guidance(&request);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -2390,11 +2575,23 @@ async fn omits_apps_guidance_when_orchestrator_mcp_is_disabled() {
|
||||
.features
|
||||
.enable(Feature::Apps)
|
||||
.expect("test config should allow feature update");
|
||||
config
|
||||
.features
|
||||
.enable(Feature::CatalogAppsInstructions)
|
||||
.expect("enable catalog Apps instructions");
|
||||
config.chatgpt_base_url = apps_base_url;
|
||||
config.orchestrator_mcp_enabled = false;
|
||||
})
|
||||
.with_model_info_override("gpt-5.5", |model| {
|
||||
model.model_messages = Some(
|
||||
serde_json::from_value(
|
||||
json!({"apps": {"usage_instructions": CATALOG_APPS_GUIDANCE}}),
|
||||
)
|
||||
.expect("catalog Apps messages"),
|
||||
);
|
||||
});
|
||||
let codex = builder
|
||||
.build(&server)
|
||||
.build_with_auto_env(&server)
|
||||
.await
|
||||
.expect("create new conversation")
|
||||
.codex;
|
||||
@@ -2405,18 +2602,16 @@ async fn omits_apps_guidance_when_orchestrator_mcp_is_disabled() {
|
||||
text_elements: Vec::new(),
|
||||
}]))
|
||||
.await
|
||||
.unwrap();
|
||||
.expect("start disabled-orchestrator Apps guidance turn");
|
||||
|
||||
wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let requests = resp_mock.requests();
|
||||
assert_eq!(requests.len(), 3);
|
||||
for request in &requests {
|
||||
assert_no_apps_guidance(request);
|
||||
}
|
||||
let request = &requests[0];
|
||||
assert!(
|
||||
!message_input_text_contains(request, "developer", "<apps_instructions>"),
|
||||
"did not expect apps instructions when orchestrator MCP is disabled, got {:?}",
|
||||
request.body_json()["input"]
|
||||
);
|
||||
assert!(
|
||||
!request.body_contains_text("mcp__codex_apps"),
|
||||
"did not expect codex_apps MCP tools when orchestrator MCP is disabled, got {:?}",
|
||||
|
||||
@@ -144,6 +144,9 @@ mod catalog_settings;
|
||||
#[path = "scenarios_catalog_approval_instructions_tests.rs"]
|
||||
mod catalog_approval_instructions;
|
||||
|
||||
#[path = "scenarios_catalog_apps_instructions_tests.rs"]
|
||||
mod catalog_apps_instructions;
|
||||
|
||||
#[path = "scenarios_managed_human_approval_tests.rs"]
|
||||
mod managed_human_approval;
|
||||
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
#![cfg(test)]
|
||||
|
||||
//! Complete request history for catalog Apps wording across model switches and resumes.
|
||||
|
||||
use super::*;
|
||||
use codex_protocol::protocol::ThreadSettingsOverrides;
|
||||
use core_test_support::apps_test_server::AppsTestServer;
|
||||
use core_test_support::submit_thread_settings;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn catalog_apps_instructions_follow_selected_model() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
let server = start_mock_server().await;
|
||||
let apps = AppsTestServer::mount_searchable(&server).await?;
|
||||
let configure = move |config: &mut Config| {
|
||||
configure_scenario_catalog(config);
|
||||
config.workspace_roots = vec![config.cwd.clone()];
|
||||
config.chatgpt_base_url = apps.chatgpt_base_url.clone();
|
||||
for feature in [Feature::Apps, Feature::CatalogAppsInstructions] {
|
||||
config.features.enable(feature).unwrap();
|
||||
}
|
||||
let catalog = config.model_catalog.as_mut().unwrap();
|
||||
let base = catalog
|
||||
.models
|
||||
.iter()
|
||||
.find(|model| model.slug == "gpt-5.5")
|
||||
.unwrap()
|
||||
.clone();
|
||||
for (slug, wording, include_guidance) in [
|
||||
("apps-bundled", None, true),
|
||||
("apps-no-guidance", Some("Calendar guidance B."), false),
|
||||
(
|
||||
"apps-a",
|
||||
Some(" Calendar guidance A: {{literal}}.\n"),
|
||||
true,
|
||||
),
|
||||
("apps-b", Some("Calendar guidance B."), true),
|
||||
("apps-clear", Some(""), true),
|
||||
] {
|
||||
let mut model = base.clone();
|
||||
model.slug = slug.to_string();
|
||||
model.include_apps_usage_instructions = include_guidance;
|
||||
model.model_messages = Some(
|
||||
serde_json::from_value(json!({"apps": {"usage_instructions": wording}})).unwrap(),
|
||||
);
|
||||
catalog.models.push(model);
|
||||
}
|
||||
};
|
||||
let mut builder = test_codex()
|
||||
.with_model("apps-bundled")
|
||||
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||
.with_config(configure.clone());
|
||||
let test = builder.build_with_auto_env(&server).await?;
|
||||
let mock = mount_sse_sequence(
|
||||
&server,
|
||||
(0..8)
|
||||
.map(|index| sse(vec![ev_completed(&format!("apps-{index}"))]))
|
||||
.collect(),
|
||||
)
|
||||
.await;
|
||||
|
||||
test.submit_text_turn("Check with bundled guidance.")
|
||||
.await?;
|
||||
// Selecting catalog wording cannot override the receiving model's availability gate.
|
||||
for (model, prompt) in [
|
||||
("apps-no-guidance", "Check while guidance is unavailable."),
|
||||
("apps-a", "Check the calendar."),
|
||||
("apps-b", "Check again with the new model."),
|
||||
] {
|
||||
submit_thread_settings(
|
||||
&test.codex,
|
||||
ThreadSettingsOverrides {
|
||||
model: Some(model.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
test.submit_text_turn(prompt).await?;
|
||||
}
|
||||
builder = builder.with_config(configure.clone()).with_model("apps-b");
|
||||
let test = builder.restart_with_auto_env(&server, &test).await?;
|
||||
test.submit_text_turn("Check after restarting with the same guidance.")
|
||||
.await?;
|
||||
submit_thread_settings(
|
||||
&test.codex,
|
||||
ThreadSettingsOverrides {
|
||||
model: Some("apps-clear".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
test.submit_text_turn("Clear the guidance.").await?;
|
||||
test.submit_text_turn("Continue with cleared guidance.")
|
||||
.await?;
|
||||
builder = builder.with_config(configure).with_model("apps-clear");
|
||||
let resumed = builder.restart_with_auto_env(&server, &test).await?;
|
||||
resumed
|
||||
.submit_text_turn("Check after restarting with cleared guidance.")
|
||||
.await?;
|
||||
|
||||
let requests = mock.requests();
|
||||
assert_eq!(requests.len(), 8);
|
||||
assert_eq!(
|
||||
requests
|
||||
.iter()
|
||||
.map(|request| request.body_json()["model"].clone())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
json!("apps-bundled"),
|
||||
json!("apps-no-guidance"),
|
||||
json!("apps-a"),
|
||||
json!("apps-b"),
|
||||
json!("apps-b"),
|
||||
json!("apps-clear"),
|
||||
json!("apps-clear"),
|
||||
json!("apps-clear"),
|
||||
]
|
||||
);
|
||||
let guidance = requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
request
|
||||
.message_input_texts("developer")
|
||||
.into_iter()
|
||||
.filter(|text| {
|
||||
text.starts_with("<apps_instructions>")
|
||||
|| text.starts_with("<catalog_apps_instructions>")
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
guidance.iter().map(Vec::len).collect::<Vec<_>>(),
|
||||
vec![1, 2, 3, 4, 4, 5, 5, 5]
|
||||
);
|
||||
assert!(guidance[0][0].starts_with("<apps_instructions>"));
|
||||
assert!(guidance[2][2].contains(" Calendar guidance A: {{literal}}.\n"));
|
||||
assert!(guidance[3][3].contains("Calendar guidance B."));
|
||||
assert!(guidance[3][3].contains("replace all previously provided Apps usage instructions"));
|
||||
assert!(
|
||||
guidance[1][1].contains("The previously provided Apps usage instructions no longer apply.")
|
||||
);
|
||||
assert_eq!(guidance[1][1], guidance[5][4]);
|
||||
for pair in guidance.windows(2) {
|
||||
assert!(
|
||||
pair[1].starts_with(&pair[0]),
|
||||
"history must stay append-only"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
guidance[4], guidance[3],
|
||||
"custom resume must not repeat guidance"
|
||||
);
|
||||
assert_eq!(
|
||||
guidance[6], guidance[5],
|
||||
"clearing twice must not repeat a notice"
|
||||
);
|
||||
assert_eq!(
|
||||
guidance[7], guidance[6],
|
||||
"cleared resume must not repeat a notice"
|
||||
);
|
||||
insta::assert_snapshot!(
|
||||
"catalog_apps_instructions_follow_selected_model",
|
||||
context_snapshot::format_request_history_snapshot(
|
||||
"Apps wording follows the selected model, retires bundled guidance when an unavailable model selects an override, preserves literal bytes, and survives both custom and cleared restarts without duplication.",
|
||||
&requests,
|
||||
&ContextSnapshotOptions::default().include_request_settings(),
|
||||
)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_catalog_apps_instructions_tests.rs
|
||||
assertion_line: 144
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Apps wording follows the selected model, retires bundled guidance when an unavailable model selects an override, preserves literal bytes, and survives both custom and cleared restarts without duplication.\",\n&requests, &ContextSnapshotOptions::default().include_request_settings(),)"
|
||||
---
|
||||
Scenario: Apps wording follows the selected model, retires bundled guidance when an unavailable model selects an override, preserves literal bytes, and survives both custom and cleared restarts without duplication.
|
||||
|
||||
## Window 1
|
||||
Settings:
|
||||
include: ["reasoning.encrypted_content"]
|
||||
model: "apps-bundled"
|
||||
parallel_tool_calls: true
|
||||
prompt_cache_key: "<PROMPT_CACHE_KEY 1>"
|
||||
reasoning: {"effort":"medium"}
|
||||
store: false
|
||||
stream: true
|
||||
text: {"verbosity":"low"}
|
||||
tool_choice: "auto"
|
||||
tools (10; hash=9f8074d045f3bd93):
|
||||
- function/exec_command: Runs a command in a PTY, returning output or a session ID for ongoing interaction.; args=[cmd, justification, login, max_output_tokens, prefix_rule, sandbox_permissions, shell, tty, workdir, yield_time_ms]; hash=c04108e51a8548c5
|
||||
- function/write_stdin: Writes characters to an existing unified exec session and returns recent output.; args=[chars, max_output_tokens, session_id, yield_time_ms]; hash=7fb62d8a3e2df4e2
|
||||
- function/list_mcp_resources: Lists resources provided by MCP servers. Resources allow servers to share data that provides cont...; args=[cursor, server]; hash=6792edaff13561bd
|
||||
- function/list_mcp_resource_templates: Lists resource templates provided by MCP servers. Parameterized resource templates allow servers ...; args=[cursor, server]; hash=60f17c90ef7a4ebb
|
||||
- function/read_mcp_resource: Read a specific resource from an MCP server given the server name and resource URI.; args=[server, uri]; hash=24246896b39008ae
|
||||
- function/request_user_input: Request user input for one to three short questions and wait for the response. This tool is only ...; args=[questions]; hash=64a729b25c424fc6
|
||||
- custom/apply_patch: The `apply_patch` tool can be used to edit files. This is a FREEFORM tool, so do not wrap the pat...; hash=dea9f12b0111a4ba
|
||||
- function/view_image: View a local image file from the filesystem when visual inspection is needed. Use this for images...; args=[path]; hash=1908de9c0be7118a
|
||||
- tool_search: # Tool discovery Searches over deferred tool metadata with BM25 and exposes matching tools for t...; args=[limit, query]; hash=b573ed69c0b0e6c7
|
||||
- web_search; hash=f103e327c6870a46
|
||||
-- request 1 (turn) --
|
||||
00:message/developer[3]:
|
||||
[01] <permissions instructions>
|
||||
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c]
|
||||
# Escalation Requests
|
||||
|
||||
Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb]
|
||||
|
||||
- Pipes: |
|
||||
- Logical operators: &&, ||
|
||||
<OMITTED 44 LINES; ~790 TOKENS; hash=3a8900343e968d61>
|
||||
NEVER provide a prefix_rule if your command uses a heredoc or herestring.
|
||||
|
||||
### Examples
|
||||
Good examples of prefixes:
|
||||
- ["npm", "run", "dev"]
|
||||
- ["gh", "pr", "check"]
|
||||
- ["cargo", "test"]
|
||||
</permissions instructions>
|
||||
[02] <apps_instructions>
|
||||
## Apps (Connectors)
|
||||
Apps (Connectors) can be explicitly triggered in user messages in the format `[$app-name](app://{connect...long as the context suggests usage of available apps. [hash=8c2ddeb1adaf185d]
|
||||
An app is equivalent to a set of MCP tools within the `codex_apps` MCP.
|
||||
An installed app's MCP tools are either provided to you already, or can be lazy-loaded through the `tool...re searchable by `tools_search` will be listed by it. [hash=4f9c0d35da3968a5]
|
||||
Do not additionally call list_mcp_resources or list_mcp_resource_templates for apps.
|
||||
</apps_instructions>
|
||||
[03] <tools>
|
||||
Deferred tool namespaces:
|
||||
- mcp__codex_apps__calendar: Plan events and manage your calendar.
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
01:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD></cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
<current_date><CURRENT_DATE></current_date>
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
<filesystem><workspace_roots><root><CWD></root></workspace_roots><permission_profile type="managed"><f...ntry></file_system></permission_profile></filesystem> [hash=e8c1de60307d0a3d]
|
||||
</environment_context>
|
||||
02:message/user:
|
||||
Check with bundled guidance.
|
||||
|
||||
## Window 2 (after request 1)
|
||||
Settings: relative to window 1
|
||||
model: "apps-no-guidance"
|
||||
-- request 2 (turn) --
|
||||
00:message/developer[3]:
|
||||
[01] <permissions instructions>
|
||||
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c]
|
||||
# Escalation Requests
|
||||
|
||||
Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb]
|
||||
|
||||
- Pipes: |
|
||||
- Logical operators: &&, ||
|
||||
<OMITTED 44 LINES; ~790 TOKENS; hash=3a8900343e968d61>
|
||||
NEVER provide a prefix_rule if your command uses a heredoc or herestring.
|
||||
|
||||
### Examples
|
||||
Good examples of prefixes:
|
||||
- ["npm", "run", "dev"]
|
||||
- ["gh", "pr", "check"]
|
||||
- ["cargo", "test"]
|
||||
</permissions instructions>
|
||||
[02] <apps_instructions>
|
||||
## Apps (Connectors)
|
||||
Apps (Connectors) can be explicitly triggered in user messages in the format `[$app-name](app://{connect...long as the context suggests usage of available apps. [hash=8c2ddeb1adaf185d]
|
||||
An app is equivalent to a set of MCP tools within the `codex_apps` MCP.
|
||||
An installed app's MCP tools are either provided to you already, or can be lazy-loaded through the `tool...re searchable by `tools_search` will be listed by it. [hash=4f9c0d35da3968a5]
|
||||
Do not additionally call list_mcp_resources or list_mcp_resource_templates for apps.
|
||||
</apps_instructions>
|
||||
[03] <tools>
|
||||
Deferred tool namespaces:
|
||||
- mcp__codex_apps__calendar: Plan events and manage your calendar.
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
01:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD></cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
<current_date><CURRENT_DATE></current_date>
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
<filesystem><workspace_roots><root><CWD></root></workspace_roots><permission_profile type="managed"><f...ntry></file_system></permission_profile></filesystem> [hash=e8c1de60307d0a3d]
|
||||
</environment_context>
|
||||
02:message/user:
|
||||
Check with bundled guidance.
|
||||
03:message/developer:
|
||||
<catalog_apps_instructions>
|
||||
The previously provided Apps usage instructions no longer apply.
|
||||
</catalog_apps_instructions>
|
||||
04:message/user:
|
||||
Check while guidance is unavailable.
|
||||
|
||||
## Window 3 (after request 2)
|
||||
Settings: relative to window 2
|
||||
model: "apps-a"
|
||||
-- request 3 (turn) --
|
||||
00:message/developer[3]:
|
||||
[01] <permissions instructions>
|
||||
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c]
|
||||
# Escalation Requests
|
||||
|
||||
Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb]
|
||||
|
||||
- Pipes: |
|
||||
- Logical operators: &&, ||
|
||||
<OMITTED 44 LINES; ~790 TOKENS; hash=3a8900343e968d61>
|
||||
NEVER provide a prefix_rule if your command uses a heredoc or herestring.
|
||||
|
||||
### Examples
|
||||
Good examples of prefixes:
|
||||
- ["npm", "run", "dev"]
|
||||
- ["gh", "pr", "check"]
|
||||
- ["cargo", "test"]
|
||||
</permissions instructions>
|
||||
[02] <apps_instructions>
|
||||
## Apps (Connectors)
|
||||
Apps (Connectors) can be explicitly triggered in user messages in the format `[$app-name](app://{connect...long as the context suggests usage of available apps. [hash=8c2ddeb1adaf185d]
|
||||
An app is equivalent to a set of MCP tools within the `codex_apps` MCP.
|
||||
An installed app's MCP tools are either provided to you already, or can be lazy-loaded through the `tool...re searchable by `tools_search` will be listed by it. [hash=4f9c0d35da3968a5]
|
||||
Do not additionally call list_mcp_resources or list_mcp_resource_templates for apps.
|
||||
</apps_instructions>
|
||||
[03] <tools>
|
||||
Deferred tool namespaces:
|
||||
- mcp__codex_apps__calendar: Plan events and manage your calendar.
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
01:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD></cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
<current_date><CURRENT_DATE></current_date>
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
<filesystem><workspace_roots><root><CWD></root></workspace_roots><permission_profile type="managed"><f...ntry></file_system></permission_profile></filesystem> [hash=e8c1de60307d0a3d]
|
||||
</environment_context>
|
||||
02:message/user:
|
||||
Check with bundled guidance.
|
||||
03:message/developer:
|
||||
<catalog_apps_instructions>
|
||||
The previously provided Apps usage instructions no longer apply.
|
||||
</catalog_apps_instructions>
|
||||
04:message/user:
|
||||
Check while guidance is unavailable.
|
||||
05:message/developer:
|
||||
<catalog_apps_instructions> Calendar guidance A: {{literal}}.
|
||||
</catalog_apps_instructions>
|
||||
06:message/user:
|
||||
Check the calendar.
|
||||
|
||||
## Window 4 (after request 3)
|
||||
Settings: relative to window 3
|
||||
model: "apps-b"
|
||||
-- request 4 (turn) --
|
||||
00:message/developer[3]:
|
||||
[01] <permissions instructions>
|
||||
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c]
|
||||
# Escalation Requests
|
||||
|
||||
Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb]
|
||||
|
||||
- Pipes: |
|
||||
- Logical operators: &&, ||
|
||||
<OMITTED 44 LINES; ~790 TOKENS; hash=3a8900343e968d61>
|
||||
NEVER provide a prefix_rule if your command uses a heredoc or herestring.
|
||||
|
||||
### Examples
|
||||
Good examples of prefixes:
|
||||
- ["npm", "run", "dev"]
|
||||
- ["gh", "pr", "check"]
|
||||
- ["cargo", "test"]
|
||||
</permissions instructions>
|
||||
[02] <apps_instructions>
|
||||
## Apps (Connectors)
|
||||
Apps (Connectors) can be explicitly triggered in user messages in the format `[$app-name](app://{connect...long as the context suggests usage of available apps. [hash=8c2ddeb1adaf185d]
|
||||
An app is equivalent to a set of MCP tools within the `codex_apps` MCP.
|
||||
An installed app's MCP tools are either provided to you already, or can be lazy-loaded through the `tool...re searchable by `tools_search` will be listed by it. [hash=4f9c0d35da3968a5]
|
||||
Do not additionally call list_mcp_resources or list_mcp_resource_templates for apps.
|
||||
</apps_instructions>
|
||||
[03] <tools>
|
||||
Deferred tool namespaces:
|
||||
- mcp__codex_apps__calendar: Plan events and manage your calendar.
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
01:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD></cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
<current_date><CURRENT_DATE></current_date>
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
<filesystem><workspace_roots><root><CWD></root></workspace_roots><permission_profile type="managed"><f...ntry></file_system></permission_profile></filesystem> [hash=e8c1de60307d0a3d]
|
||||
</environment_context>
|
||||
02:message/user:
|
||||
Check with bundled guidance.
|
||||
03:message/developer:
|
||||
<catalog_apps_instructions>
|
||||
The previously provided Apps usage instructions no longer apply.
|
||||
</catalog_apps_instructions>
|
||||
04:message/user:
|
||||
Check while guidance is unavailable.
|
||||
05:message/developer:
|
||||
<catalog_apps_instructions> Calendar guidance A: {{literal}}.
|
||||
</catalog_apps_instructions>
|
||||
06:message/user:
|
||||
Check the calendar.
|
||||
07:message/developer:
|
||||
<catalog_apps_instructions>
|
||||
These Apps usage instructions replace all previously provided Apps usage instructions.
|
||||
|
||||
Calendar guidance B.</catalog_apps_instructions>
|
||||
08:message/user:
|
||||
Check again with the new model.
|
||||
-- request 5 (turn) --
|
||||
09:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD 2></cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
<current_date><CURRENT_DATE></current_date>
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
<filesystem><workspace_roots><root><CWD 2></root></workspace_roots><permission_profile type="managed">...ntry></file_system></permission_profile></filesystem> [hash=8fc9d186c1133657]
|
||||
</environment_context>
|
||||
10:message/user:
|
||||
Check after restarting with the same guidance.
|
||||
|
||||
## Window 5 (after request 5)
|
||||
Settings: relative to window 4
|
||||
model: "apps-clear"
|
||||
-- request 6 (turn) --
|
||||
00:message/developer[3]:
|
||||
[01] <permissions instructions>
|
||||
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `read-only`: The san... permits reading files. Network access is restricted. [hash=be7da4425f89053c]
|
||||
# Escalation Requests
|
||||
|
||||
Commands are run outside the sandbox if they are approved by the user, or match an existing rule that al...hell control operators, including but not limited to: [hash=d24ebf5fa4e4b9bb]
|
||||
|
||||
- Pipes: |
|
||||
- Logical operators: &&, ||
|
||||
<OMITTED 44 LINES; ~790 TOKENS; hash=3a8900343e968d61>
|
||||
NEVER provide a prefix_rule if your command uses a heredoc or herestring.
|
||||
|
||||
### Examples
|
||||
Good examples of prefixes:
|
||||
- ["npm", "run", "dev"]
|
||||
- ["gh", "pr", "check"]
|
||||
- ["cargo", "test"]
|
||||
</permissions instructions>
|
||||
[02] <apps_instructions>
|
||||
## Apps (Connectors)
|
||||
Apps (Connectors) can be explicitly triggered in user messages in the format `[$app-name](app://{connect...long as the context suggests usage of available apps. [hash=8c2ddeb1adaf185d]
|
||||
An app is equivalent to a set of MCP tools within the `codex_apps` MCP.
|
||||
An installed app's MCP tools are either provided to you already, or can be lazy-loaded through the `tool...re searchable by `tools_search` will be listed by it. [hash=4f9c0d35da3968a5]
|
||||
Do not additionally call list_mcp_resources or list_mcp_resource_templates for apps.
|
||||
</apps_instructions>
|
||||
[03] <tools>
|
||||
Deferred tool namespaces:
|
||||
- mcp__codex_apps__calendar: Plan events and manage your calendar.
|
||||
- multi_agent_v1: Tools for spawning and managing sub-agents.
|
||||
</tools>
|
||||
01:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD></cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
<current_date><CURRENT_DATE></current_date>
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
<filesystem><workspace_roots><root><CWD></root></workspace_roots><permission_profile type="managed"><f...ntry></file_system></permission_profile></filesystem> [hash=e8c1de60307d0a3d]
|
||||
</environment_context>
|
||||
02:message/user:
|
||||
Check with bundled guidance.
|
||||
03:message/developer:
|
||||
<catalog_apps_instructions>
|
||||
The previously provided Apps usage instructions no longer apply.
|
||||
</catalog_apps_instructions>
|
||||
04:message/user:
|
||||
Check while guidance is unavailable.
|
||||
05:message/developer:
|
||||
<catalog_apps_instructions> Calendar guidance A: {{literal}}.
|
||||
</catalog_apps_instructions>
|
||||
06:message/user:
|
||||
Check the calendar.
|
||||
07:message/developer:
|
||||
<catalog_apps_instructions>
|
||||
These Apps usage instructions replace all previously provided Apps usage instructions.
|
||||
|
||||
Calendar guidance B.</catalog_apps_instructions>
|
||||
08:message/user:
|
||||
Check again with the new model.
|
||||
09:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD 2></cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
<current_date><CURRENT_DATE></current_date>
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
<filesystem><workspace_roots><root><CWD 2></root></workspace_roots><permission_profile type="managed">...ntry></file_system></permission_profile></filesystem> [hash=8fc9d186c1133657]
|
||||
</environment_context>
|
||||
10:message/user:
|
||||
Check after restarting with the same guidance.
|
||||
11:message/developer:
|
||||
<catalog_apps_instructions>
|
||||
The previously provided Apps usage instructions no longer apply.
|
||||
</catalog_apps_instructions>
|
||||
12:message/user:
|
||||
Clear the guidance.
|
||||
-- request 7 (turn) --
|
||||
13:message/user:
|
||||
Continue with cleared guidance.
|
||||
-- request 8 (turn) --
|
||||
14:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD 3></cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
<current_date><CURRENT_DATE></current_date>
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
<filesystem><workspace_roots><root><CWD 3></root></workspace_roots><permission_profile type="managed">...ntry></file_system></permission_profile></filesystem> [hash=1d8e57fdb1985164]
|
||||
</environment_context>
|
||||
15:message/user:
|
||||
Check after restarting with cleared guidance.
|
||||
Reference in New Issue
Block a user