Enforce strict third-party tool deferral in Code Mode Only (#59071)

## What changed

Restore `code_mode_only_strict_3p_tools` as an opt-in feature under development. When the effective tool mode is Code Mode Only, keep eligible MCP/app and client-supplied dynamic tools deferred and callable through `exec`, regardless of `deferLoading`, `omit_tools_from`, or namespace exclusion and direct-only settings.

Preserve hidden tools, tool policy restrictions, and built-in tool classification. Log conflicting exposure settings without adding warnings to the model prompt. Leave other effective tool modes unaffected.

## Testing

Add coverage for exposure overrides, model mode overrides, built-in name collisions, and hidden or denied tools. Update integration coverage for app access restrictions and MCP catalog refreshes that preserve the cached prompt prefix while `exec` uses the current tools.

GitOrigin-RevId: 0cb085ba9d23bebc48dbc3fa298b982de0a66c47
This commit is contained in:
rishab
2026-11-03 06:05:55 +00:00
committed by copyberry
parent 9e23a4708a
commit 29eed28082
13 changed files with 356 additions and 102 deletions
@@ -177,6 +177,10 @@ impl CoreToolRuntime for DynamicToolHandler {
fn finishes_on_cancellation(&self) -> bool {
true
}
fn is_third_party_tool(&self) -> bool {
true
}
}
#[expect(
+4
View File
@@ -352,6 +352,10 @@ impl CoreToolRuntime for McpHandler {
}))
}
fn is_third_party_tool(&self) -> bool {
true
}
fn mcp_server_name(&self) -> Option<&str> {
Some(&self.definition.tool_info.server_name)
}
+6
View File
@@ -92,6 +92,12 @@ pub(crate) trait CoreToolRuntime: ToolExecutor<ToolInvocation> {
None
}
/// True for MCP/app and client-supplied dynamic tools.
/// A built-in stays false even if a client's tool has the same name.
fn is_third_party_tool(&self) -> bool {
false
}
/// Identify the environments selected by the original input.
fn environments_for_input(
&self,
+51 -1
View File
@@ -217,6 +217,15 @@ pub(crate) fn build_tool_router(
)
}
/// Use the effective mode because the model can override the thread's configured mode.
fn code_mode_only_strict_3p_tools(turn_context: &TurnContext, model_info: &ModelInfo) -> bool {
effective_tool_mode(turn_context, model_info) == ToolMode::CodeModeOnly
&& turn_context
.config
.features
.enabled(Feature::CodeModeOnlyStrictThirdPartyTools)
}
fn apply_mcp_tool_exposure_policy(
turn_context: &TurnContext,
model_info: &ModelInfo,
@@ -261,6 +270,17 @@ fn apply_mcp_tool_exposure_policy(
let Some(omitted_exposures) = omitted_exposures_by_tool.get(&tool_name) else {
continue;
};
if code_mode_only_strict_3p_tools(turn_context, model_info) {
// Ignore MCP/app omit_tools_from; eligible tools must stay deferred in exec.
// Log conflicts without adding a warning to the model's prompt.
if omitted_exposures.intersects(ToolExposures::DEFERRED | ToolExposures::CODE_MODE) {
tracing::warn!(
tool_name = %tool_name,
"Ignoring MCP/app omit_tools_from because code_mode_only_strict_3p_tools is enabled"
);
}
continue;
}
let tool_name = tool_name.with_default_namespace();
let mut exposures = ToolExposures::ALL.difference(*omitted_exposures);
@@ -388,6 +408,7 @@ pub(crate) fn finalize_tool_router(
let model_messages = ResolvedModelMessages::from_model(model_info);
hosted_specs.retain(|spec| registry.tool_policy.allows(&ToolName::plain(spec.name())));
apply_direct_model_only_namespace_overrides(turn_context, &mut registry);
enforce_strict_3p_tools(turn_context, model_info, &mut registry);
let tool_mode = effective_tool_mode(turn_context, model_info);
let code_mode_enabled = matches!(tool_mode, ToolMode::CodeMode | ToolMode::CodeModeOnly);
if code_mode_enabled {
@@ -610,6 +631,31 @@ fn tool_exposure_with_namespace_override(
}
}
/// Run after namespace overrides so eligible third-party tools always remain deferred.
fn enforce_strict_3p_tools(
turn_context: &TurnContext,
model_info: &ModelInfo,
registry: &mut ToolRegistry,
) {
if !code_mode_only_strict_3p_tools(turn_context, model_info) {
return;
}
for tool in registry.entries_mut() {
// Remaining Hidden entries failed eligibility or budget checks; keep them hidden.
if !tool.runtime.is_third_party_tool() || tool.exposure == ToolExposure::Hidden {
continue;
}
if tool.exposure != ToolExposure::Deferred {
tracing::warn!(
tool_name = %tool.runtime.tool_name().with_default_namespace(),
"Deferring third-party tool because code_mode_only_strict_3p_tools is enabled"
);
tool.exposure = ToolExposure::Deferred;
}
}
}
#[instrument(level = "trace", skip_all)]
fn build_model_visible_specs(
turn_context: &TurnContext,
@@ -956,7 +1002,11 @@ fn code_mode_tools<'a>(
}
let tool_name = tool.runtime.tool_name();
if is_excluded_from_code_mode(turn_context, &tool_name) {
// Deferred third-party tools still need an exec route to be discoverable.
if is_excluded_from_code_mode(turn_context, &tool_name)
&& !(code_mode_only_strict_3p_tools(turn_context, model_info)
&& tool.runtime.is_third_party_tool())
{
return None;
}
@@ -1,80 +0,0 @@
//! Code Mode finalization preserves tool policy and hidden-tool eligibility.
use super::*;
use crate::tools::registry::ToolRegistry;
use codex_extension_api::ToolPolicy;
use pretty_assertions::assert_eq;
#[tokio::test]
async fn code_mode_finalization_preserves_hidden_and_policy_denied_tools() {
let (_, mut turn) = make_session_and_context().await;
set_features(&mut turn, &[Feature::CodeMode, Feature::CodeModeOnly]);
update_config(&mut turn, |config| {
config.code_mode.direct_only_tool_namespaces = vec!["mcp__allowed".to_string()];
});
let allowed_mcp = ToolName::namespaced("mcp__allowed", "lookup");
let hidden_mcp = ToolName::namespaced("mcp__allowed", "over_budget");
let mut registry = ToolRegistry::with_tool_policy(Arc::new(ToolPolicy {
allowed_tools: Some(vec![
ToolName::plain("exec"),
ToolName::plain("wait"),
allowed_mcp.clone(),
hidden_mcp.clone(),
ToolName::plain("allowed_dynamic"),
]),
..Default::default()
}));
let hosted = append_source_tools(
&turn,
turn.model_info(),
&mut registry,
vec![
mcp_runtime("allowed", "mcp__allowed", "lookup", ToolExposure::Direct),
// MCP registration keeps budget-overflow tools in the registry as Hidden.
// This is different from a caller setting omit_tools_from.
mcp_runtime(
"allowed",
"mcp__allowed",
"over_budget",
ToolExposure::Hidden,
),
mcp_runtime("denied", "mcp__denied", "lookup", ToolExposure::Direct),
],
std::iter::empty::<Arc<dyn for<'call> ToolExecutor<ExtensionToolCall<'call>>>>(),
&[
dynamic_tool(
/*namespace*/ None,
"allowed_dynamic",
/*defer_loading*/ false,
),
dynamic_tool(
/*namespace*/ None,
"denied_dynamic",
/*defer_loading*/ false,
),
],
);
let plan = ToolPlanProbe::from_router(ToolRouter::from_registry(
&turn,
turn.model_info(),
registry,
hosted,
&Default::default(),
));
plan.assert_registered_lacks(&[
&ToolName::namespaced("mcp__denied", "lookup").to_string(),
"denied_dynamic",
]);
assert_eq!(
plan.exposure(&allowed_mcp.to_string()),
ToolExposure::DirectModelOnly
);
assert_eq!(plan.exposure(&hidden_mcp.to_string()), ToolExposure::Hidden);
assert_eq!(plan.exposure("allowed_dynamic"), ToolExposure::Direct);
assert_eq!(
plan.code_mode_tool_names
.values()
.cloned()
.collect::<std::collections::HashSet<_>>(),
[ToolName::plain("allowed_dynamic")].into_iter().collect(),
);
}
@@ -0,0 +1,248 @@
//! Strict Code Mode Only: deferral, mode overrides, and hidden/denied tools.
use super::*;
use crate::tools::registry::ToolRegistry;
use codex_extension_api::ToolPolicy;
use pretty_assertions::assert_eq;
use tracing_test::internal::MockWriter;
#[tokio::test(flavor = "current_thread")]
async fn strict_defers_third_party_tools_despite_caller_settings_and_logs_overrides() {
let buffer: &'static std::sync::Mutex<Vec<u8>> =
Box::leak(Box::new(std::sync::Mutex::new(Vec::new())));
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.without_time()
.with_max_level(tracing::Level::WARN)
.with_writer(MockWriter::new(buffer))
.finish();
// Keep callsite interest independent of untraced parallel test threads.
let _untraced = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());
// The test runner may already own the global subscriber.
let _subscriber = tracing::subscriber::set_default(subscriber);
for supports_search_tool in [false, true] {
let plan = probe_with(
|turn| {
set_features(
turn,
&[
Feature::CodeMode,
Feature::CodeModeOnly,
Feature::CodeModeOnlyStrictThirdPartyTools,
],
);
update_turn_settings_for_test(turn, |settings| {
Arc::make_mut(&mut settings.model_info).supports_search_tool =
supports_search_tool;
});
update_config(turn, |config| {
config.update_plan_enabled = true;
config.code_mode.direct_only_tool_namespaces =
vec!["direct_client".to_string()];
config.code_mode.excluded_tool_namespaces = vec!["excluded_client".to_string()];
});
},
ToolPlanInputs {
dynamic_tools: vec![
dynamic_tool(Some("client"), "eager", /*defer_loading*/ false),
dynamic_tool(Some("client"), "deferred", /*defer_loading*/ true),
dynamic_tool(Some("direct_client"), "eager", /*defer_loading*/ false),
dynamic_tool(
Some("excluded_client"),
"eager",
/*defer_loading*/ false,
),
// Legal legacy input: app-server forbids deferLoading=true on a plain tool.
dynamic_tool(
/*namespace*/ None,
"client_echo",
/*defer_loading*/ false,
),
// A collision must not make the built-in tool third-party.
dynamic_tool(
/*namespace*/ None,
"update_plan",
/*defer_loading*/ false,
),
],
..ToolPlanInputs::default()
},
)
.await;
plan.assert_visible_contains(&["exec", "wait"]);
plan.assert_visible_lacks(&[
"tool_search",
"client",
"direct_client",
"excluded_client",
"client_echo",
]);
assert_eq!(plan.exposure("update_plan"), ToolExposure::Direct);
let ToolSpec::Freeform(exec) = plan.visible_spec("exec") else {
panic!("expected exec");
};
for name in [
ToolName::namespaced("client", "eager"),
ToolName::namespaced("client", "deferred"),
ToolName::namespaced("direct_client", "eager"),
ToolName::namespaced("excluded_client", "eager"),
ToolName::plain("client_echo"),
] {
assert_eq!(plan.exposure(&name.to_string()), ToolExposure::Deferred);
assert!(
plan.code_mode_tool_names
.values()
.any(|nested| nested == &name),
"{name} must be callable via exec"
);
assert!(
!exec.description.contains(&format!("{}(args:", name.name)),
"{name} schema must not be eager"
);
}
assert!(
!exec.description.contains("Deferring third-party tool"),
"warnings must not enter the model prompt"
);
}
let logs = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();
assert!(
logs.contains(
"Deferring third-party tool because code_mode_only_strict_3p_tools is enabled"
)
);
assert!(logs.contains("client_echo"));
}
#[tokio::test]
async fn strict_flag_does_nothing_outside_effective_code_mode_only() {
let (_, mut turn) = make_session_and_context().await;
set_features(&mut turn, &[Feature::CodeMode, Feature::CodeModeOnly]);
turn.code_mode_available = true;
update_config(&mut turn, |config| {
config.code_mode.direct_only_tool_namespaces = vec!["client".to_string()];
config.code_mode.excluded_tool_namespaces = vec!["client".to_string()];
});
for mode in [ToolMode::Direct, ToolMode::CodeMode] {
update_turn_settings_for_test(&mut turn, |settings| {
let model = Arc::make_mut(&mut settings.model_info);
model.tool_mode = Some(mode);
model.supports_search_tool = true;
});
let inputs = || ToolPlanInputs {
dynamic_tools: vec![
dynamic_tool(Some("client"), "eager", /*defer_loading*/ false),
dynamic_tool(Some("client"), "deferred", /*defer_loading*/ true),
dynamic_tool(
/*namespace*/ None,
"client_echo",
/*defer_loading*/ false,
),
],
tool_runtimes: vec![mcp_runtime(
"sample",
"mcp__sample",
"lookup",
ToolExposure::Direct,
)],
..ToolPlanInputs::default()
};
set_feature(
&mut turn,
Feature::CodeModeOnlyStrictThirdPartyTools,
/*enabled*/ false,
);
let off = ToolPlanProbe::from_router(plan_with_model(&turn, turn.model_info(), inputs()));
assert_eq!(off.tool_mode, mode);
set_feature(
&mut turn,
Feature::CodeModeOnlyStrictThirdPartyTools,
/*enabled*/ true,
);
let on = ToolPlanProbe::from_router(plan_with_model(&turn, turn.model_info(), inputs()));
assert_eq!(on, off, "strict flag changed effective {mode:?}");
}
}
#[tokio::test]
async fn strict_finalization_cannot_resurrect_hidden_or_policy_denied_tools() {
let (_, mut turn) = make_session_and_context().await;
set_features(
&mut turn,
&[
Feature::CodeMode,
Feature::CodeModeOnly,
Feature::CodeModeOnlyStrictThirdPartyTools,
],
);
update_config(&mut turn, |config| {
config.code_mode.direct_only_tool_namespaces = vec!["mcp__allowed".to_string()];
});
let allowed_mcp = ToolName::namespaced("mcp__allowed", "lookup");
let hidden_mcp = ToolName::namespaced("mcp__allowed", "over_budget");
let mut registry = ToolRegistry::with_tool_policy(Arc::new(ToolPolicy {
allowed_tools: Some(vec![
ToolName::plain("exec"),
ToolName::plain("wait"),
allowed_mcp.clone(),
hidden_mcp.clone(),
ToolName::plain("allowed_dynamic"),
]),
..Default::default()
}));
let hosted = append_source_tools(
&turn,
turn.model_info(),
&mut registry,
vec![
mcp_runtime("allowed", "mcp__allowed", "lookup", ToolExposure::Direct),
// MCP registration keeps budget-overflow tools in the registry as Hidden.
// This is different from a caller setting omit_tools_from.
mcp_runtime(
"allowed",
"mcp__allowed",
"over_budget",
ToolExposure::Hidden,
),
mcp_runtime("denied", "mcp__denied", "lookup", ToolExposure::Direct),
],
std::iter::empty::<Arc<dyn for<'call> ToolExecutor<ExtensionToolCall<'call>>>>(),
&[
dynamic_tool(
/*namespace*/ None,
"allowed_dynamic",
/*defer_loading*/ false,
),
dynamic_tool(
/*namespace*/ None,
"denied_dynamic",
/*defer_loading*/ false,
),
],
);
let plan = ToolPlanProbe::from_router(ToolRouter::from_registry(
&turn,
turn.model_info(),
registry,
hosted,
&Default::default(),
));
plan.assert_registered_lacks(&[
&ToolName::namespaced("mcp__denied", "lookup").to_string(),
"denied_dynamic",
]);
assert_eq!(
plan.exposure(&allowed_mcp.to_string()),
ToolExposure::Deferred
);
assert_eq!(plan.exposure(&hidden_mcp.to_string()), ToolExposure::Hidden);
assert_eq!(plan.exposure("allowed_dynamic"), ToolExposure::Deferred);
assert_eq!(
plan.code_mode_tool_names
.values()
.cloned()
.collect::<std::collections::HashSet<_>>(),
[allowed_mcp, ToolName::plain("allowed_dynamic")]
.into_iter()
.collect(),
);
}
+2 -2
View File
@@ -3428,5 +3428,5 @@ async fn hosted_web_search_and_standalone_image_generation_follow_runtime_gates(
bedrock_with_standalone_web_search.assert_visible_lacks(&["web"]);
}
#[path = "spec_plan_code_mode_policy_tests.rs"]
mod code_mode_policy;
#[path = "spec_plan_strict_third_party_tests.rs"]
mod strict_third_party;