Add startup tool allowlists for threads (#45711)

## What changed

- Add `AllowedTools` to the extension API, captured once at thread startup to restrict which tools can be advertised or executed. An empty list disables all tools; an absent value preserves ordinary tool setup. Callers must supply the list again when resuming a thread.
- Filter tool registration and hosted tool specifications before Code Mode and discovery. Match names with their namespaces and require generated tools such as `exec` and `wait` to be explicitly allowed.
- Move Guardian reviewer tool selection into the allowlist while retaining feature and sandbox restrictions, with a fallback for older reviewer sessions.

## Testing

Add coverage for selected, empty, and absent allowlists across tool sources and Code Mode. Add managed-thread tests verifying advertised tools, rejection of excluded calls, and that replacing extension state after startup cannot change the captured allowlist.

GitOrigin-RevId: 33c1722c4241f368963d8385fa0415112e16a72d
This commit is contained in:
jif
2026-09-15 14:45:35 +00:00
committed by copyberry
parent 7f01a84eff
commit 2fdcdeaf0e
12 changed files with 328 additions and 93 deletions
+3
View File
@@ -64,6 +64,9 @@ pub(crate) fn install(session: &Session, config: &Config) {
options
.thread_extension_init
.insert(SessionIsolation::Isolated);
options
.thread_extension_init
.insert(codex_guardian_reviewer::reviewer_allowed_tools());
let session_cancel = cancel.clone();
let until = async move {
let _cancel_on_exit = cancel.clone().drop_guard();
+9
View File
@@ -69,6 +69,7 @@ pub(crate) struct Session {
pub(super) features: ManagedFeatures,
pub(crate) guardian_context_mode: GuardianContextMode,
pub(super) isolation: codex_extension_api::SessionIsolation,
pub(crate) allowed_tools: Option<Arc<codex_extension_api::AllowedTools>>,
pub(crate) windows_sandbox_proxy_settings_mode:
codex_sandboxing::WindowsSandboxProxySettingsMode,
pub(super) multi_agent_version: OnceLock<MultiAgentVersion>,
@@ -946,6 +947,13 @@ impl Session {
.get::<codex_extension_api::SessionIsolation>()
.map(|policy| *policy)
.unwrap_or_default();
let allowed_tools = thread_extension_init
.get::<codex_extension_api::AllowedTools>()
.or_else(|| {
// Older reviewer rollouts predate the explicit startup setting.
crate::guardian::is_basic_session_source(&session_configuration.session_source)
.then(|| Arc::new(codex_guardian_reviewer::reviewer_allowed_tools()))
});
let mcp_thread_init = thread_extension_init.clone();
let thread_extension_data = codex_extension_api::ExtensionData::new_with_init(
thread_id.to_string(),
@@ -1695,6 +1703,7 @@ impl Session {
features: config.features.clone(),
guardian_context_mode,
isolation,
allowed_tools,
windows_sandbox_proxy_settings_mode,
multi_agent_version,
mcp_refresh: McpRefresh::new(),
+2
View File
@@ -6110,6 +6110,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
features: config.features.clone(),
guardian_context_mode: GuardianContextMode::from_features(&config.features),
isolation: codex_extension_api::SessionIsolation::Inherit,
allowed_tools: None,
windows_sandbox_proxy_settings_mode:
codex_sandboxing::WindowsSandboxProxySettingsMode::Reconcile,
multi_agent_version: OnceLock::from(config.multi_agent_version_from_features()),
@@ -8291,6 +8292,7 @@ where
features: config.features.clone(),
guardian_context_mode: GuardianContextMode::from_features(&config.features),
isolation: codex_extension_api::SessionIsolation::Inherit,
allowed_tools: None,
windows_sandbox_proxy_settings_mode:
codex_sandboxing::WindowsSandboxProxySettingsMode::Reconcile,
multi_agent_version: OnceLock::from(config.multi_agent_version_from_features()),
+30
View File
@@ -27,6 +27,7 @@ use crate::tools::router::tool_log_payload;
use crate::tools::tool_dispatch_trace::ToolDispatchTrace;
use crate::util::error_or_panic;
use codex_analytics::ControlToolCallStatus;
use codex_extension_api::AllowedTools;
use codex_extension_api::ToolCallOutcome;
use codex_history::CodexHarnessMetadata;
use codex_history::ResponseItemEnvelope;
@@ -287,9 +288,17 @@ pub(crate) struct RegisteredTool {
pub struct ToolRegistry {
tools: IndexMap<ToolName, RegisteredTool>,
first_collision: Option<ToolName>,
pub(crate) allowed_tools: Option<Arc<AllowedTools>>,
}
impl ToolRegistry {
pub(crate) fn with_allowed_tools(allowed_tools: Option<Arc<AllowedTools>>) -> Self {
Self {
allowed_tools,
..Self::default()
}
}
#[cfg(test)]
pub(crate) fn from_tools(tools: impl IntoIterator<Item = Arc<dyn CoreToolRuntime>>) -> Self {
let mut registry = Self::default();
@@ -326,6 +335,13 @@ impl ToolRegistry {
exposure: ToolExposure,
) {
let tool_name = runtime.tool_name().with_default_namespace();
if self
.allowed_tools
.as_ref()
.is_some_and(|allowed| !allowed.contains(&tool_name))
{
return;
}
match self.tools.entry(tool_name) {
Entry::Vacant(entry) => {
entry.insert(RegisteredTool { runtime, exposure });
@@ -339,6 +355,13 @@ impl ToolRegistry {
pub(crate) fn prepend_trusted(&mut self, runtime: Arc<dyn CoreToolRuntime>) {
let tool_name = runtime.tool_name().with_default_namespace();
if self
.allowed_tools
.as_ref()
.is_some_and(|allowed| !allowed.contains(&tool_name))
{
return;
}
if self.tools.contains_key(&tool_name) {
error_or_panic(format!("tool {tool_name} already registered"));
return;
@@ -360,6 +383,13 @@ impl ToolRegistry {
exposure: ToolExposure,
) -> bool {
let tool_name = runtime.tool_name().with_default_namespace();
if self
.allowed_tools
.as_ref()
.is_some_and(|allowed| !allowed.contains(&tool_name))
{
return false;
}
if tool_name.is_default_namespace()
&& matches!(tool_name.name.as_str(), "exec_command" | "shell_command")
{
+46 -89
View File
@@ -150,40 +150,36 @@ pub(crate) fn build_tool_router(
default_agent_type_description: &default_agent_type_description,
wait_agent_timeouts: wait_agent_timeout_options(turn_context),
};
let mut registry = ToolRegistry::default();
let mut registry = ToolRegistry::with_allowed_tools(session.allowed_tools.clone());
add_core_tool_sources(&context, &mut registry);
let hosted_specs = if crate::guardian::is_basic_session_source(&turn_context.session_source) {
Vec::new()
} else {
let registered_mcp_tools = session.services.mcp_handler_cache.append_mcp_tools(
mcp,
&turn_context.config,
apps_enabled,
&mcp.config().mcp_server_catalog,
search_tool_enabled(turn_context, model_info),
&mut registry,
);
apply_mcp_tool_exposure_policy(
turn_context,
model_info,
mcp,
&registered_mcp_tools,
&mut registry,
);
let standalone_web_search_tool = append_extension_tool_executors(
turn_context,
model_info,
extension_tool_executors(session, step_store),
&mut registry,
);
append_dynamic_tool_runtimes(&turn_context.dynamic_tools, &mut registry);
hosted_model_tool_specs(
turn_context,
model_info,
standalone_web_search_tool.as_slice(),
)
};
let registered_mcp_tools = session.services.mcp_handler_cache.append_mcp_tools(
mcp,
&turn_context.config,
apps_enabled,
&mcp.config().mcp_server_catalog,
search_tool_enabled(turn_context, model_info),
&mut registry,
);
apply_mcp_tool_exposure_policy(
turn_context,
model_info,
mcp,
&registered_mcp_tools,
&mut registry,
);
let standalone_web_search_tool = append_extension_tool_executors(
turn_context,
model_info,
extension_tool_executors(session, step_store),
&mut registry,
);
append_dynamic_tool_runtimes(&turn_context.dynamic_tools, &mut registry);
let hosted_specs = hosted_model_tool_specs(
turn_context,
model_info,
standalone_web_search_tool.as_slice(),
);
finalize_tool_router(
turn_context,
@@ -308,10 +304,6 @@ pub(crate) fn append_source_tools(
>,
dynamic_tools: &[DynamicToolSpec],
) -> Vec<ToolSpec> {
if crate::guardian::is_basic_session_source(&turn_context.session_source) {
return Vec::new();
}
for tool in mcp_tools {
registry.register_external_with_exposure(tool.runtime, tool.exposure);
}
@@ -353,9 +345,12 @@ pub(crate) fn finalize_tool_router(
turn_context: &TurnContext,
model_info: &ModelInfo,
mut registry: ToolRegistry,
hosted_specs: Vec<ToolSpec>,
mut hosted_specs: Vec<ToolSpec>,
tool_search_handler_cache: &ToolSearchHandlerCache,
) -> CodexResult<ToolRouter> {
if let Some(allowed) = &registry.allowed_tools {
hosted_specs.retain(|spec| allowed.contains(&ToolName::plain(spec.name())));
}
apply_direct_model_only_namespace_overrides(turn_context, &mut registry);
let tool_mode = effective_tool_mode(turn_context, model_info);
let code_mode_enabled = matches!(tool_mode, ToolMode::CodeMode | ToolMode::CodeModeOnly);
@@ -599,9 +594,7 @@ fn hosted_model_tool_specs(
registered_extension_tool_names: &[ToolName],
) -> Vec<ToolSpec> {
// Responses Lite accepts schemas for client-executed tools, not hosted Responses tools.
if model_info.use_responses_lite
|| crate::guardian::is_basic_session_source(&turn_context.session_source)
{
if model_info.use_responses_lite {
return Vec::new();
}
@@ -972,60 +965,19 @@ fn code_mode_namespace_descriptions(
#[instrument(level = "trace", skip_all)]
fn add_core_tool_sources(context: &CoreToolPlanContext<'_>, registry: &mut ToolRegistry) {
// Guardian reviewers receive only `exec_command`, `write_stdin`, and `view_image`
// when a managed sandbox can enforce the parent's filesystem restrictions;
// all general tool sources stay excluded.
if crate::guardian::is_basic_session_source(&context.turn_context.session_source) {
let turn_context = context.turn_context;
if !matches!(
turn_context.permission_profile(),
// Preserve the reviewer's existing sandbox requirements. Tool selection is
// supplied separately by the extension through AllowedTools.
if crate::guardian::is_basic_session_source(&context.turn_context.session_source)
&& (!matches!(
context.turn_context.permission_profile(),
PermissionProfile::Managed { .. }
) || context.environments.turn_environments().any(|environment| {
!matches!(
environment.permission_profile(),
PermissionProfile::Managed { .. }
)
}) {
return;
}
let environment_mode = tool_environment_mode(context.environments);
if environment_mode.has_environment() {
let include_environment_id = matches!(environment_mode, ToolEnvironmentMode::Multiple);
if turn_context.config.features.enabled(Feature::ShellTool)
&& turn_context.config.features.enabled(Feature::UnifiedExec)
&& !matches!(context.model_info.shell_type, ConfigShellToolType::Disabled)
{
registry.add(ExecCommandHandler::new(ExecCommandHandlerOptions {
allow_login_shell: any_environment_allows_login_shell(context.environments),
allow_tty: turn_context
.config
.features
.enabled(Feature::UnifiedExecTty),
exec_permission_approvals_enabled: false,
include_environment_id,
include_shell_parameter: unified_exec_should_include_shell_parameter(
turn_context,
context.environments,
),
include_windows_shell_guidance: should_include_windows_shell_guidance(
context.environments,
),
}));
registry.add(WriteStdinHandler);
}
if turn_context.config.features.enabled(Feature::ViewImage) {
registry.add(ViewImageHandler::new(ViewImageToolOptions {
can_request_original_image_detail: can_request_original_image_detail(
context.model_info,
),
unified_image_budget: unified_image_budget_enabled(
&turn_context.config.features,
context.model_info,
),
include_environment_id,
}));
}
}
}))
{
return;
}
@@ -1088,7 +1040,12 @@ fn add_shell_tools(context: &CoreToolPlanContext<'_>, registry: &mut ToolRegistr
}
let allow_login_shell = any_environment_allows_login_shell(context.environments);
let exec_permission_approvals_enabled = features.enabled(Feature::ExecPermissionApprovals);
let is_guardian = crate::guardian::is_basic_session_source(&turn_context.session_source);
if is_guardian && !features.enabled(Feature::UnifiedExec) {
return;
}
let exec_permission_approvals_enabled =
features.enabled(Feature::ExecPermissionApprovals) && !is_guardian;
let include_environment_id = matches!(environment_mode, ToolEnvironmentMode::Multiple);
let options = ExecCommandHandlerOptions {
allow_login_shell,
+90 -4
View File
@@ -514,10 +514,93 @@ fn apply_patch_accepts_environment_id(spec: &ToolSpec) -> bool {
}
}
#[tokio::test]
async fn allowed_tools_filter_sources_before_code_mode_and_discovery() {
use crate::tools::registry::ToolRegistry;
use codex_extension_api::AllowedTools;
for allowed in [
None,
Some(AllowedTools(vec![
ToolName::namespaced("kept", "lookup"),
ToolName::plain("exec"),
ToolName::plain("wait"),
])),
Some(AllowedTools::default()),
] {
let (_, mut turn) = make_session_and_context().await;
set_feature(&mut turn, Feature::CodeMode, /*enabled*/ true);
set_web_search_mode(&mut turn, WebSearchMode::Live);
update_turn_settings_for_test(&mut turn, |settings| {
let model = Arc::make_mut(&mut settings.model_info);
model.supports_search_tool = true;
model.use_responses_lite = false;
});
let mut registry = ToolRegistry::with_allowed_tools(allowed.clone().map(Arc::new));
registry.add(crate::tools::handlers::PlanHandler);
let hosted = append_source_tools(
&turn,
turn.model_info(),
&mut registry,
vec![
mcp_runtime("kept", "kept", "lookup", ToolExposure::Deferred),
mcp_runtime("excluded", "excluded", "lookup", ToolExposure::Deferred),
],
[Arc::new(DeferredExtensionTool)
as Arc<
dyn for<'call> ToolExecutor<ExtensionToolCall<'call>>,
>],
&[dynamic_tool(
/*namespace*/ None,
"dynamic_echo",
/*defer_loading*/ false,
)],
);
assert!(!hosted.is_empty(), "exercise hosted tool filtering too");
let router = ToolRouter::from_registry(
&turn,
turn.model_info(),
registry,
hosted,
&Default::default(),
);
let plan = ToolPlanProbe::from_router(router);
match allowed {
None => {
plan.assert_registered_contains(&["update_plan", "extension_echo", "dynamic_echo"]);
plan.assert_visible_contains(&["web_search", "tool_search", "exec", "wait"]);
}
Some(allowed) if allowed.0.is_empty() => {
assert_eq!(plan.registered_names, Vec::<String>::new());
assert_eq!(plan.visible_specs, Vec::<ToolSpec>::new());
assert_eq!(plan.code_mode_tool_names, BTreeMap::new());
}
Some(_) => {
assert_eq!(plan.registered_names, vec!["exec", "wait", "keptlookup"]);
assert_eq!(
plan.code_mode_tool_names
.values()
.cloned()
.collect::<Vec<_>>(),
vec![ToolName::namespaced("kept", "lookup")]
);
plan.assert_visible_lacks(&[
"web_search",
"tool_search",
"update_plan",
"dynamic_echo",
"extension_echo",
]);
}
}
}
}
#[tokio::test]
async fn internal_guardian_sessions_exclude_optional_core_tools() {
let (session, mut turn) = make_session_and_context().await;
let (mut session, mut turn) = make_session_and_context().await;
turn.session_source = SessionSource::Internal(InternalSessionSource::Guardian);
session.allowed_tools = Some(Arc::new(codex_guardian_reviewer::reviewer_allowed_tools()));
set_feature(&mut turn, Feature::ViewImage, /*enabled*/ true);
Arc::make_mut(&mut turn.config).update_plan_enabled = true;
turn.multi_agent_version = MultiAgentVersion::V2;
@@ -554,8 +637,9 @@ async fn internal_guardian_sessions_respect_managed_shell_restrictions() {
(Some(Feature::UnifiedExec), ConfigShellToolType::UnifiedExec),
(None, ConfigShellToolType::Disabled),
] {
let (session, mut turn) = make_session_and_context().await;
let (mut session, mut turn) = make_session_and_context().await;
turn.session_source = SessionSource::Internal(InternalSessionSource::Guardian);
session.allowed_tools = Some(Arc::new(codex_guardian_reviewer::reviewer_allowed_tools()));
set_feature(&mut turn, Feature::ViewImage, /*enabled*/ true);
set_feature(&mut turn, Feature::CodeMode, /*enabled*/ true);
if let Some(feature) = disabled_feature {
@@ -608,8 +692,9 @@ async fn internal_guardian_sessions_respect_managed_shell_restrictions() {
#[tokio::test]
async fn internal_guardian_sessions_preserve_code_mode() {
let (session, mut turn) = make_session_and_context().await;
let (mut session, mut turn) = make_session_and_context().await;
turn.session_source = SessionSource::Internal(InternalSessionSource::Guardian);
session.allowed_tools = Some(Arc::new(codex_guardian_reviewer::reviewer_allowed_tools()));
set_feature(&mut turn, Feature::CodeMode, /*enabled*/ true);
let turn = Arc::new(turn);
let step_context = StepContext::for_test(Arc::clone(&turn));
@@ -653,8 +738,9 @@ async fn internal_guardian_sessions_require_managed_secondary_environments() {
Vec::new(),
),
] {
let (session, mut turn) = make_session_and_context().await;
let (mut session, mut turn) = make_session_and_context().await;
turn.session_source = SessionSource::Internal(InternalSessionSource::Guardian);
session.allowed_tools = Some(Arc::new(codex_guardian_reviewer::reviewer_allowed_tools()));
set_feature(&mut turn, Feature::ViewImage, /*enabled*/ true);
let TurnEnvironmentState::Ready(primary) = turn
.environments
@@ -9,7 +9,9 @@ use std::time::Duration;
use anyhow::Context;
use codex_core::StartThreadOptions;
use codex_core::TurnInputRequest;
use codex_extension_api::AllowedTools;
use codex_extension_api::SessionIsolation;
use codex_extension_api::ToolName;
use codex_protocol::protocol::EventMsg;
use codex_protocol::user_input::UserInput;
use codex_thread_store::ThreadStoreError;
@@ -20,6 +22,7 @@ use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use pretty_assertions::assert_eq;
use serde_json::json;
use test_case::test_case;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
@@ -242,3 +245,105 @@ async fn owner_cancellation_closes_agent_and_preserves_history_and_parent() -> a
fixture.codex.shutdown_and_wait().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[test_case(vec![ToolName::namespaced("functions", "update_plan")]; "selected_tool")]
#[test_case(vec![]; "no_tools")]
async fn startup_allowlist_controls_advertising_and_execution(
tools: Vec<ToolName>,
) -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let fixture = test_codex()
.with_config(|config| {
config.update_plan_enabled = true;
config
.features
.disable(codex_features::Feature::CodeMode)
.expect("disable Code Mode for direct-tool assertions");
})
.build_with_auto_env(&server)
.await?;
let mut options = StartThreadOptions::new(fixture.config.clone());
options.environments = Some(fixture.codex.environment_selections().await);
options
.thread_extension_init
.insert(SessionIsolation::Isolated);
options
.thread_extension_init
.insert(AllowedTools(tools.clone()));
let cancelled = CancellationToken::new();
let tasks = TaskTracker::new();
let agent = fixture
.thread_manager
.start_thread_until(options, cancelled.clone().cancelled_owned(), &tasks)
.await?;
// Replacing extension state after startup must not change the captured ceiling.
agent
.thread
.thread_extension_data()
.insert(AllowedTools(vec![ToolName::plain("exec_command")]));
let response = responses::mount_sse_sequence(
&server,
vec![
responses::sse(vec![
responses::ev_function_call(
"plan",
"update_plan",
r#"{"plan":[{"step":"inspect","status":"completed"}]}"#,
),
responses::ev_function_call(
"excluded",
"exec_command",
r#"{"cmd":"echo should-not-run"}"#,
),
responses::ev_completed("tools"),
]),
responses::sse(vec![responses::ev_completed("done")]),
],
)
.await;
agent
.thread
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
text: "Try both tools".to_string(),
text_elements: Vec::new(),
}]))
.await?;
wait_for_event(&agent.thread, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
let requests = response.requests();
assert_eq!(requests.len(), 2);
assert_eq!(
requests[0].body_json()["tools"]
.as_array()
.expect("request tools array")
.iter()
.map(|tool| tool["name"].as_str().expect("plain tool name"))
.collect::<Vec<_>>(),
tools
.iter()
.map(|tool| tool.name.as_str())
.collect::<Vec<_>>(),
);
assert_eq!(
requests[1].function_call_output("excluded")["output"],
"unsupported call: exec_command"
);
assert_eq!(
requests[1].function_call_output("plan")["output"],
if tools.is_empty() {
"unsupported call: update_plan"
} else {
"Plan updated"
}
);
cancelled.cancel();
tasks.close();
tasks.wait().await;
fixture.codex.shutdown_and_wait().await?;
Ok(())
}
@@ -0,0 +1,24 @@
//! A startup ceiling on the tools a thread may advertise or execute.
use crate::ToolName;
/// Supply through `ExtensionDataInit` before starting a thread. The host captures
/// this value once; changing extension state later cannot widen the tool set.
/// Callers must supply it again when resuming a thread.
///
/// An absent value keeps ordinary tool setup. An empty list permits no tools.
/// Names include their namespace; a plain name uses the default namespace.
/// This only removes tools: configuration and permission checks still apply.
/// Generated tools such as Code Mode's `exec` and `wait` must also be listed.
#[derive(Clone, Debug, Default)]
pub struct AllowedTools(pub Vec<ToolName>);
impl AllowedTools {
pub fn contains(&self, tool: &ToolName) -> bool {
self.0.iter().any(|allowed| {
allowed.name == tool.name
&& (allowed.namespace == tool.namespace
|| (allowed.is_default_namespace() && tool.is_default_namespace()))
})
}
}
+2
View File
@@ -1,3 +1,4 @@
mod allowed_tools;
mod capabilities;
mod contributors;
mod registry;
@@ -6,6 +7,7 @@ mod state;
mod turn_admission;
mod user_instructions;
pub use allowed_tools::AllowedTools;
pub use session_isolation::SessionIsolation;
pub use capabilities::ConversationHistorySnapshot;
@@ -58,6 +58,7 @@ pub use execution::start_review_turn;
pub use execution::wait_for_guardian_review;
pub use settings::ReviewerConfigOverrides;
pub use settings::ReviewerTurn;
pub use settings::reviewer_allowed_tools;
pub use settings::reviewer_config_overrides;
pub use feedback::FailedReviewFeedback;
@@ -3,6 +3,8 @@
use std::collections::HashMap;
use codex_extension_api::AllowedTools;
use codex_extension_api::ToolName;
use codex_features::Feature;
use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::ModeKind;
@@ -21,6 +23,17 @@ use codex_protocol::turn_input::TurnStartOptions;
use codex_protocol::user_input::UserInput;
use serde_json::Value;
/// Reviewer tools, including the existing Code Mode dispatchers when enabled.
/// The host still applies feature flags and sandbox restrictions to these tools.
pub fn reviewer_allowed_tools() -> AllowedTools {
AllowedTools(
["exec_command", "write_stdin", "view_image", "exec", "wait"]
.into_iter()
.map(ToolName::plain)
.collect(),
)
}
/// Configuration policy for a reviewer. The host retains managed constraints and
/// live network rules while applying these values to its concrete runtime config.
pub struct ReviewerConfigOverrides {
@@ -82,6 +82,9 @@ impl ThreadLifecycleContributor<Config> for GuardianExtension {
options
.thread_extension_init
.insert(SessionIsolation::Isolated);
options
.thread_extension_init
.insert(codex_guardian_reviewer::reviewer_allowed_tools());
let session_cancel = cancel.clone();
let until = async move {
let _cancel_on_exit = cancel.clone().drop_guard();