From 641b4cac88d08e03d3d72ed91ea7e0dfd8b12299 Mon Sep 17 00:00:00 2001 From: viyatb-oai Date: Mon, 16 Nov 2026 21:46:52 +0000 Subject: [PATCH] Tie Linux sandbox proxy bridges to the launcher lifetime (#61898) ## Why Host proxy bridges must close active connections when a sandbox exits or is cancelled. Proxy descriptor exhaustion must also leave synthetic mount cleanup able to run. ## What changed - Run host proxy bridges as threads in an outer supervisor that forwards signals and preserves the sandbox command's exit status. Use parent-death signals to terminate supervised children when their parent exits. - Give bridge routes private descriptor tables where supported, retry accepts after descriptor exhaustion, and keep mount cleanup in a separate supervisor. - Create the transparent TCP namespace in the bubblewrap child after re-executing the sandbox helper, clearing inherited credentials from memory and restoring dumpability for user namespace ID mapping. ## Testing Add regression tests for bridge teardown after normal exit, process-group cancellation, launcher termination, and caller-thread exit; exit status propagation with inherited `SIGCHLD` ignored; and SOCKS connectivity and synthetic mount cleanup under descriptor pressure. GitOrigin-RevId: dd297af98e9383942098fab5683ef2474c5ae554 --- codex-rs/linux-sandbox/src/launcher.rs | 21 +- codex-rs/linux-sandbox/src/linux_run_main.rs | 143 ++++++--- codex-rs/linux-sandbox/src/proxy_lifecycle.rs | 65 +++-- codex-rs/linux-sandbox/src/proxy_routing.rs | 187 ++++++------ codex-rs/linux-sandbox/src/tcp_net_setup.rs | 70 ++++- .../tests/suite/managed_proxy.rs | 114 ++------ .../managed_proxy_bridge_lifecycle_tests.rs | 274 ++++++++++++++++++ 7 files changed, 613 insertions(+), 261 deletions(-) create mode 100644 codex-rs/linux-sandbox/tests/suite/managed_proxy_bridge_lifecycle_tests.rs diff --git a/codex-rs/linux-sandbox/src/launcher.rs b/codex-rs/linux-sandbox/src/launcher.rs index 7398bc80f2..49256a9755 100644 --- a/codex-rs/linux-sandbox/src/launcher.rs +++ b/codex-rs/linux-sandbox/src/launcher.rs @@ -38,7 +38,11 @@ struct SystemBwrapCapabilities { static LAUNCHER: OnceLock = OnceLock::new(); -pub(crate) fn exec_bwrap(mut argv: Vec, preserved_files: Vec) -> ! { +pub(crate) fn exec_bwrap( + mut argv: Vec, + preserved_files: Vec, + transparent_tcp: bool, +) -> ! { if argv .iter() .take_while(|arg| arg.as_str() != "--") @@ -53,7 +57,11 @@ pub(crate) fn exec_bwrap(mut argv: Vec, preserved_files: Vec) -> ! translate_legacy_bwrap_fd_mounts(&mut argv) .unwrap_or_else(|error| panic!("invalid legacy bubblewrap fd mount: {error}")); } - exec_system_bwrap(&launcher.program, argv, preserved_files) + make_files_inheritable(&preserved_files); + if transparent_tcp { + crate::tcp_net_setup::exec_setup(launcher.program.as_path(), &argv[1..]); + } + exec_system_bwrap(&launcher.program, argv) } BubblewrapLauncher::Bundled(launcher) => launcher.exec(argv, preserved_files), BubblewrapLauncher::Unavailable => { @@ -223,14 +231,7 @@ fn system_bwrap_capabilities(system_bwrap_path: &Path) -> Option, - preserved_files: Vec, -) -> ! { - // System bwrap runs across an exec boundary, so preserved fds must survive exec. - make_files_inheritable(&preserved_files); - +fn exec_system_bwrap(program: &AbsolutePathBuf, argv: Vec) -> ! { let program_path = program.as_path().display().to_string(); let program = CString::new(program.as_path().as_os_str().as_bytes()) .unwrap_or_else(|err| panic!("invalid system bubblewrap path: {err}")); diff --git a/codex-rs/linux-sandbox/src/linux_run_main.rs b/codex-rs/linux-sandbox/src/linux_run_main.rs index 1849aeedd2..b6873663ac 100644 --- a/codex-rs/linux-sandbox/src/linux_run_main.rs +++ b/codex-rs/linux-sandbox/src/linux_run_main.rs @@ -23,8 +23,10 @@ use crate::launcher::exec_bwrap; use crate::launcher::initialize_bwrap_launcher; use crate::launcher::preferred_bwrap_supports_argv0; use crate::managed_tcp::Relay; +use crate::proxy_routing::HostProxyBridge; use crate::proxy_routing::activate_proxy_routes_in_netns; use crate::proxy_routing::prepare_host_proxy_route_spec; +use crate::proxy_routing::start_host_proxy_bridges; use crate::seccomp::ProxyRouting; use crate::seccomp::apply_sandbox_policies_to_current_thread; use codex_network_proxy::ManagedNetworkSandboxContext; @@ -142,6 +144,8 @@ pub struct LinuxSandboxCommand { /// 2. Apply in-process restrictions (no_new_privs + seccomp). /// 3. `execvp` into the final command. pub fn run_main() -> ! { + let caller_pid = unsafe { libc::getppid() }; + crate::tcp_net_setup::dispatch(); let LinuxSandboxCommand { sandbox_policy_cwd, command_cwd, @@ -290,21 +294,25 @@ pub fn run_main() -> ! { // Outer stage: bubblewrap first, then re-enter this binary in the // sandboxed environment to apply seccomp. This path never falls back // to legacy Landlock on failure. - let (proxy_route_spec, proxy_controls) = if let Some(context) = managed_network.as_ref() { + let (proxy_route_spec, proxy_bridges) = if let Some(context) = managed_network.as_ref() { // Older explicit-proxy callers may omit ports. The new transparent // switch always requires the executor's trusted ingress port list. let ports = (transparent_tcp || !context.loopback_ports.is_empty()) .then_some(context.loopback_ports.as_slice()); - let (proxy_route_spec, controls) = prepare_host_proxy_route_spec(ports, routing) + let (proxy_route_spec, bridges) = prepare_host_proxy_route_spec(ports, routing) .unwrap_or_else(|err| panic!("failed to prepare host proxy routing bridge: {err}")); - (Some(proxy_route_spec), controls) + (Some(proxy_route_spec), bridges) } else { (None, Vec::new()) }; let options = BwrapOptions { mount_proc: !no_proc && !inherit_pid_namespace, inherit_pid_namespace, - network_mode: bwrap_network_mode(network_sandbox_policy, allow_network_for_proxy), + network_mode: if transparent_tcp { + BwrapNetworkMode::PreparedProxy + } else { + bwrap_network_mode(network_sandbox_policy, allow_network_for_proxy) + }, mask_wsl_interop: !file_system_sandbox_policy.has_full_disk_write_access() && Path::new(WSL_INTEROP_DIR).is_dir(), mask_wslg_distro: (!file_system_sandbox_policy.has_full_disk_write_access() @@ -336,13 +344,13 @@ pub fn run_main() -> ! { command, }); run_bwrap_with_proc_fallback( + caller_pid, &sandbox_policy_cwd, command_cwd.as_deref(), &file_system_sandbox_policy, options, inner, - proxy_controls, - transparent_tcp, + proxy_bridges, ); } @@ -419,16 +427,26 @@ fn ensure_legacy_landlock_mode_supports_policy( } fn run_bwrap_with_proc_fallback( + caller_pid: libc::pid_t, sandbox_policy_cwd: &Path, command_cwd: Option<&Path>, file_system_sandbox_policy: &FileSystemSandboxPolicy, mut options: BwrapOptions, inner: Vec, - proxy_controls: Vec, - transparent_tcp: bool, + proxy_bridges: Vec, ) -> ! { + let transparent_tcp = options.network_mode == BwrapNetworkMode::PreparedProxy; let command_cwd = command_cwd.unwrap_or(sandbox_policy_cwd); + // An inherited SIG_IGN would auto-reap bubblewrap before we can collect + // its exit status. The preflight also waits for its own bubblewrap child. + if unsafe { libc::signal(libc::SIGCHLD, libc::SIG_DFL) } == libc::SIG_ERR { + panic!( + "failed to reset SIGCHLD: {}", + std::io::Error::last_os_error() + ); + } + // Select before the preflight, whose synthetic policy omits writable roots. initialize_bwrap_launcher( file_system_sandbox_policy, @@ -448,13 +466,7 @@ fn run_bwrap_with_proc_fallback( options.mount_proc = false; } - let tcp_files = transparent_tcp.then(|| { - let files = crate::tcp_namespace::create() - .unwrap_or_else(|err| panic!("failed to create private TCP namespace: {err}")); - options.network_mode = BwrapNetworkMode::PreparedProxy; - files - }); - let mut bwrap_args = build_bwrap_argv( + let bwrap_args = build_bwrap_argv( inner, file_system_sandbox_policy, sandbox_policy_cwd, @@ -462,18 +474,24 @@ fn run_bwrap_with_proc_fallback( options, ) .unwrap_or_else(|err| exit_with_bwrap_build_error(err)); - if let Some(files) = tcp_files { - crate::tcp_net_setup::prepare( - files, + if !proxy_bridges.is_empty() && bwrap_args.synthetic_mount_targets.is_empty() { + // This launcher would otherwise exec bubblewrap directly. Preserve + // --die-with-parent now that it stays alive to serve the proxy. + kill_on_parent_death(caller_pid); + } + // Host bridge threads must stay in the host network namespace. Only the + // bubblewrap child enters the private namespace and receives its TUN and resolver. + let mut bwrap_args = start_bwrap_supervisors(bwrap_args, proxy_bridges); + if transparent_tcp { + crate::tcp_net_setup::mount_resolver( &mut bwrap_args, file_system_sandbox_policy, sandbox_policy_cwd, ) .unwrap_or_else(|err| panic!("failed to prepare transparent TCP namespace: {err}")); } - bwrap_args.preserved_files.extend(proxy_controls); apply_inner_command_argv0(&mut bwrap_args.args); - run_or_exec_bwrap(bwrap_args); + exec_bwrap(bwrap_args.args, bwrap_args.preserved_files, transparent_tcp); } fn bwrap_network_mode( @@ -596,47 +614,81 @@ fn resolve_true_command() -> String { "true".to_string() } -fn run_or_exec_bwrap(bwrap_args: crate::bwrap::BwrapArgs) -> ! { - if bwrap_args.synthetic_mount_targets.is_empty() { - exec_bwrap(bwrap_args.args, bwrap_args.preserved_files); +/// With both supervisors, the launcher runs `proxy supervisor -> mount cleanup supervisor -> bubblewrap`. +/// Each optional supervisor waits for its child. Separate descriptor tables +/// let mount cleanup proceed even when proxy connections exhaust theirs. +/// Returns only in the process that will exec bubblewrap. +fn start_bwrap_supervisors( + mut bwrap_args: crate::bwrap::BwrapArgs, + proxy_bridges: Vec, +) -> crate::bwrap::BwrapArgs { + if bwrap_args.synthetic_mount_targets.is_empty() && proxy_bridges.is_empty() { + return bwrap_args; } - run_bwrap_in_child_with_synthetic_mount_cleanup(bwrap_args); + + let setup_signal_mask = ForwardedSignalMask::block(); + // Register before either fork so markers name the PID the caller will reap. + let registrations = register_synthetic_mount_targets(&bwrap_args.synthetic_mount_targets); + + if !proxy_bridges.is_empty() { + let child_pid = fork_supervised_child(); + if child_pid > 0 { + drop(bwrap_args.preserved_files); + // Bridge threads inherit the blocked signals. The supervising thread + // receives cancellation and forwards it to the child. + start_host_proxy_bridges(proxy_bridges); + supervise_child(child_pid, &setup_signal_mask, || {}); + } + bwrap_args.preserved_files.extend( + proxy_bridges + .into_iter() + .map(HostProxyBridge::into_sandbox_control), + ); + } + + if !registrations.is_empty() { + let child_pid = fork_supervised_child(); + if child_pid > 0 { + drop(bwrap_args.preserved_files); + supervise_child(child_pid, &setup_signal_mask, || { + cleanup_synthetic_mount_targets(®istrations); + }); + } + } + setup_signal_mask.restore(); + bwrap_args } -fn run_bwrap_in_child_with_synthetic_mount_cleanup(bwrap_args: crate::bwrap::BwrapArgs) -> ! { - let crate::bwrap::BwrapArgs { - args, - preserved_files, - synthetic_mount_targets, - } = bwrap_args; - let setup_signal_mask = ForwardedSignalMask::block(); - let synthetic_mount_registrations = register_synthetic_mount_targets(&synthetic_mount_targets); +fn fork_supervised_child() -> libc::pid_t { let parent_pid = unsafe { libc::getpid() }; let pid = unsafe { libc::fork() }; if pid < 0 { let err = std::io::Error::last_os_error(); panic!("failed to fork for bubblewrap: {err}"); } - if pid == 0 { reset_forwarded_signal_handlers_to_default(); - setup_signal_mask.restore(); let setpgid_res = unsafe { libc::setpgid(0, 0) }; if setpgid_res < 0 { let err = std::io::Error::last_os_error(); panic!("failed to place bubblewrap child in its own process group: {err}"); } - terminate_with_parent(parent_pid); - exec_bwrap(args, preserved_files); + kill_on_parent_death(parent_pid); } + pid +} - drop(preserved_files); +fn supervise_child( + pid: libc::pid_t, + setup_signal_mask: &ForwardedSignalMask, + cleanup: impl FnOnce(), +) -> ! { let signal_forwarders = install_bwrap_signal_forwarders(pid); setup_signal_mask.restore(); let status = wait_for_bwrap_child(pid); let cleanup_signal_mask = ForwardedSignalMask::block(); BWRAP_CHILD_PID.store(0, Ordering::SeqCst); - cleanup_synthetic_mount_targets(&synthetic_mount_registrations); + cleanup(); signal_forwarders.restore(); cleanup_signal_mask.restore(); exit_with_wait_status(status); @@ -681,15 +733,17 @@ impl ForwardedSignalMask { } } -fn terminate_with_parent(parent_pid: libc::pid_t) { - let res = unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) }; +fn kill_on_parent_death(parent_pid: libc::pid_t) { + // Match bubblewrap's --die-with-parent even while the child has TERM blocked + // for mount registration or cleanup. + let res = unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) }; if res < 0 { let err = std::io::Error::last_os_error(); - panic!("failed to set bubblewrap child parent-death signal: {err}"); + panic!("failed to set sandbox parent-death signal: {err}"); } if unsafe { libc::getppid() } != parent_pid { unsafe { - libc::raise(libc::SIGTERM); + libc::raise(libc::SIGKILL); } } } @@ -780,6 +834,9 @@ fn wait_for_bwrap_child(pid: libc::pid_t) -> libc::c_int { fn register_synthetic_mount_targets( targets: &[crate::bwrap::SyntheticMountTarget], ) -> Vec { + if targets.is_empty() { + return Vec::new(); + } with_synthetic_mount_registry_lock(|| { targets .iter() @@ -1117,7 +1174,7 @@ fn run_bwrap_in_child_capture_stderr(bwrap_args: crate::bwrap::BwrapArgs) -> Str close_fd_or_panic(write_fd, "close write end in bubblewrap child"); } - exec_bwrap(args, preserved_files); + exec_bwrap(args, preserved_files, /*transparent_tcp*/ false); } let signal_forwarders = install_bwrap_signal_forwarders(pid); diff --git a/codex-rs/linux-sandbox/src/proxy_lifecycle.rs b/codex-rs/linux-sandbox/src/proxy_lifecycle.rs index 1ee0148120..9e8401b668 100644 --- a/codex-rs/linux-sandbox/src/proxy_lifecycle.rs +++ b/codex-rs/linux-sandbox/src/proxy_lifecycle.rs @@ -1,4 +1,4 @@ -//! Manage privileged bridge processes and transfer their loopback TCP listeners. +//! Isolate host proxy descriptors and transfer loopback listeners from the sandbox. //! //! Received descriptors are CLOEXEC and owned even when validation rejects them. //! Shape validation does not authenticate the sender: the caller must retain exclusive @@ -26,10 +26,54 @@ use std::os::fd::AsFd; use std::os::fd::AsRawFd; use std::os::fd::FromRawFd; use std::os::fd::OwnedFd; +use std::os::fd::RawFd; use std::os::unix::net::UnixStream; const LISTENER_MESSAGE: u8 = 1; +/// Give one route its own descriptor budget. Its relay threads inherit this table. +/// +/// # Safety +/// Call only on a fresh bridge thread with no owned descriptors. The supervisor must +/// keep the bootstrap socket open at `control` until the caller signals readiness. +pub(crate) unsafe fn private_bridge_control(control: RawFd) -> io::Result { + // SAFETY: UNSHARE copies the calling thread's table before closing the upper range. + if unsafe { + libc::syscall( + libc::SYS_close_range, + control as u32 + 1, + u32::MAX, + libc::CLOSE_RANGE_UNSHARE, + ) + } == 0 + { + for fd in 3..control { + let _ = close_fd(fd); + } + } else { + // Older kernels support CLONE_FILES without close_range. + // SAFETY: unshare takes only flags and changes this thread's table. + if unsafe { libc::unshare(libc::CLONE_FILES) } == -1 { + return Err(io::Error::last_os_error()); + } + let mut inherited = Vec::new(); + for entry in std::fs::read_dir("/proc/thread-self/fd")? { + if let Ok(fd) = entry?.file_name().to_string_lossy().parse::() + && fd > libc::STDERR_FILENO + && fd != control + { + inherited.push(fd); + } + } + // The directory handle has dropped. No other thread can reuse these numbers. + for fd in inherited { + let _ = close_fd(fd); + } + } + // SAFETY: this thread owns the private copy of the supervisor's live socket. + Ok(unsafe { UnixStream::from_raw_fd(control) }) +} + pub(crate) fn send_listener(channel: &UnixStream, listener: &TcpListener) -> io::Result<()> { let descriptors = [listener.as_fd()]; let mut space = [MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(1))]; @@ -113,24 +157,7 @@ pub(crate) fn receive_listener(channel: &UnixStream) -> io::Result Ok(listener) } -pub(crate) fn harden_bridge_process(expected_parent_pid: libc::pid_t) -> io::Result<()> { - detach_bridge_stdio()?; - set_parent_death_signal(expected_parent_pid)?; - codex_process_hardening::disable_process_dumping() -} - -fn set_parent_death_signal(expected_parent_pid: libc::pid_t) -> io::Result<()> { - let res = unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) }; - if res != 0 { - Err(io::Error::last_os_error()) - } else if unsafe { libc::getppid() } != expected_parent_pid { - Err(io::Error::other("parent process already exited")) - } else { - Ok(()) - } -} - -fn detach_bridge_stdio() -> io::Result<()> { +pub(crate) fn detach_bridge_stdio() -> io::Result<()> { let null_read_fd = unsafe { libc::open(c"/dev/null".as_ptr(), libc::O_RDONLY) }; if null_read_fd < 0 { let err = io::Error::last_os_error(); diff --git a/codex-rs/linux-sandbox/src/proxy_routing.rs b/codex-rs/linux-sandbox/src/proxy_routing.rs index ff79b5bcf1..817169ee67 100644 --- a/codex-rs/linux-sandbox/src/proxy_routing.rs +++ b/codex-rs/linux-sandbox/src/proxy_routing.rs @@ -1,6 +1,7 @@ use crate::proxy_lifecycle::close_fd; -use crate::proxy_lifecycle::harden_bridge_process; +use crate::proxy_lifecycle::detach_bridge_stdio; use crate::proxy_lifecycle::move_fd_above_stdio; +use crate::proxy_lifecycle::private_bridge_control; use crate::proxy_lifecycle::receive_listener; use crate::proxy_lifecycle::send_listener; use crate::seccomp::ProxyRouting; @@ -11,6 +12,7 @@ use serde::Deserialize; use serde::Serialize; use std::collections::BTreeMap; use std::collections::HashMap; +use std::collections::btree_map::Entry; use std::fs::File; use std::io; use std::io::Read; @@ -21,9 +23,9 @@ use std::net::Shutdown; use std::net::SocketAddr; use std::net::TcpListener; use std::net::TcpStream; +use std::os::fd::AsFd; use std::os::fd::AsRawFd; use std::os::fd::FromRawFd; -use std::os::fd::OwnedFd; use std::os::unix::net::UnixStream; use std::sync::Arc; use std::sync::mpsc; @@ -57,13 +59,84 @@ struct ProxyRoutePlan { has_proxy_config: bool, } +/// One bootstrap socketpair per upstream endpoint. Only the sandbox end crosses exec. +pub(crate) struct HostProxyBridge { + endpoint: SocketAddr, + routing: ProxyRouting, + host_control: UnixStream, + sandbox_control: File, + attribution_token: Option, +} + +impl HostProxyBridge { + pub(crate) fn into_sandbox_control(self) -> File { + self.sandbox_control + } +} + +/// Start bridge threads only in the outer supervisor, after the child inherits +/// the command's stdio. If setup fails, closing a host socket fails the handoff. +pub(crate) fn start_host_proxy_bridges(bridges: Vec) { + let mut startup_stderr = None; + let result: io::Result<()> = bridges + .into_iter() + .enumerate() + .try_for_each(|(index, bridge)| { + drop(bridge.sandbox_control); + if index == 0 { + // Reuse the slot freed above for diagnostics, even at the FD limit. + // The child already inherited its stdio; detach ours before starting any worker. + startup_stderr = io::stderr() + .as_fd() + .try_clone_to_owned() + .ok() + .map(File::from); + detach_bridge_stdio()?; + } + let control_fd = bridge.host_control.as_raw_fd(); + let token = bridge.attribution_token.clone(); + let (ready, initialized) = mpsc::channel(); + let worker = std::thread::Builder::new() + .name("codex-proxy".to_string()) + .spawn(move || { + // SAFETY: this new thread owns no FDs. The supervisor holds the socket + // until ready and its own descriptor table is unaffected. + let control = unsafe { private_bridge_control(control_fd)? }; + ready.send(()).map_err(io::Error::other)?; + run_host_bridge(bridge.endpoint, bridge.routing, control, token) + })?; + if initialized.recv().is_ok() { + return Ok(()); + } + + // Reclaim the failed worker's task slot before falling back on hosts that + // prohibit private FD tables. + let _ = worker.join(); + std::thread::Builder::new() + .name("codex-proxy".to_string()) + .spawn(move || { + run_host_bridge( + bridge.endpoint, + bridge.routing, + bridge.host_control, + bridge.attribution_token, + ) + })?; + Ok(()) + }); + if let (Err(err), Some(mut stderr)) = (result, startup_stderr) { + let _ = writeln!(stderr, "failed to start host proxy bridges: {err}"); + } +} + pub(crate) fn prepare_host_proxy_route_spec( port_allowlist: Option<&[u16]>, routing: ProxyRouting, -) -> io::Result<(String, Vec)> { +) -> io::Result<(String, Vec)> { + // The outer supervisor holds the upstream credential and host sockets. + codex_process_hardening::disable_process_dumping()?; let (attribution_token, plan) = extract_attribution_token_and_plan(std::env::vars().collect()); - // SAFETY: the sandbox helper is single-threaded here, before it forks bridge workers or - // executes the user command. + // SAFETY: bubblewrap and the host bridge threads have not been started yet. unsafe { std::env::remove_var(PROXY_ATTRIBUTION_TOKEN_ENV_KEY); } @@ -77,7 +150,7 @@ pub(crate) fn prepare_host_proxy_route_spec( return Err(io::Error::new(io::ErrorKind::InvalidInput, message)); } - // Validate the entire plan before forking any host-side workers or sending + // Validate the entire plan before starting any host-side workers or sending // attribution. An empty enforced list grants no routes. if let Some(ports) = port_allowlist && plan @@ -91,36 +164,30 @@ pub(crate) fn prepare_host_proxy_route_spec( )); } - let mut control_by_endpoint = BTreeMap::new(); - for route in &plan.routes { - if control_by_endpoint.contains_key(&route.endpoint) { - continue; - } - let control = spawn_host_bridge( - route.endpoint, - routing, - attribution_token.as_deref(), - &mut control_by_endpoint, - )?; - control_by_endpoint.insert(route.endpoint, control); - } - + let mut bridges = BTreeMap::new(); let mut routes = Vec::with_capacity(plan.routes.len()); for route in plan.routes { - let Some(control) = control_by_endpoint.get(&route.endpoint) else { - return Err(io::Error::other(format!( - "missing bootstrap channel for endpoint {}", - route.endpoint - ))); + let bridge = match bridges.entry(route.endpoint) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let (host, sandbox) = UnixStream::pair()?; + entry.insert(HostProxyBridge { + endpoint: route.endpoint, + routing, + host_control: UnixStream::from(move_fd_above_stdio(host.into())?), + sandbox_control: File::from(move_fd_above_stdio(sandbox.into())?), + attribution_token: attribution_token.clone(), + }) + } }; routes.push(ProxyRouteEntry { env_key: route.env_key, - control_fd: control.as_raw_fd(), + control_fd: bridge.sandbox_control.as_raw_fd(), }); } let spec = serde_json::to_string(&ProxyRouteSpec { routes }).map_err(io::Error::other)?; - Ok((spec, control_by_endpoint.into_values().collect())) + Ok((spec, bridges.into_values().collect())) } fn extract_attribution_token_and_plan( @@ -329,51 +396,6 @@ fn rewrite_proxy_env_value(proxy_url: &str, local_port: u16) -> Option { Some(rewritten) } -fn spawn_host_bridge( - endpoint: SocketAddr, - routing: ProxyRouting, - attribution_token: Option<&str>, - inherited_controls: &mut BTreeMap, -) -> io::Result { - let (host_control, control) = UnixStream::pair()?; - let host_control = UnixStream::from(move_fd_above_stdio(host_control.into())?); - let mut control = UnixStream::from(move_fd_above_stdio(control.into())?); - control.set_read_timeout(Some(HANDOFF_TIMEOUT))?; - let parent_pid = unsafe { libc::getpid() }; - let pid = unsafe { libc::fork() }; - if pid < 0 { - return Err(io::Error::last_os_error()); - } - - if pid == 0 { - drop(control); - // A worker must not retain another route's inner bootstrap endpoint. - // Drop this child's inherited owners; the parent's copies are unaffected by fork. - inherited_controls.clear(); - let result = run_host_bridge( - endpoint, - routing, - host_control, - attribution_token, - parent_pid, - ); - if result.is_err() { - unsafe { libc::_exit(1) }; - } - unsafe { libc::_exit(0) }; - } - - drop(host_control); - let mut ready = [0_u8; 1]; - control.read_exact(&mut ready)?; - if ready != [HOST_BRIDGE_READY] { - return Err(io::Error::other( - "host bridge did not acknowledge readiness", - )); - } - Ok(File::from(OwnedFd::from(control))) -} - /// Each connection worker holds a permit until it exits. pub(super) struct ConnectionSlots(Arc); @@ -403,22 +425,25 @@ fn run_host_bridge( endpoint: SocketAddr, routing: ProxyRouting, mut control: UnixStream, - attribution_token: Option<&str>, - parent_pid: libc::pid_t, + attribution_token: Option, ) -> io::Result<()> { - harden_bridge_process(parent_pid)?; - control.write_all(&[HOST_BRIDGE_READY])?; - // The outer helper must launch bubblewrap before this listener can arrive. - // The socket keeps its isolated network namespace even though we accept in - // the host process; newly opened upstream sockets use our host namespace. + // The listener stays in the sandbox network namespace. Connections to the + // upstream proxy are opened here in the host network namespace. let listener = receive_listener(&control)?; control.write_all(&[HOST_BRIDGE_READY])?; drop(control); - let attribution_token = attribution_token.map(str::to_owned); let slots = ConnectionSlots::new(); loop { - let (client_stream, _) = listener.accept()?; + let client_stream = match listener.accept() { + Ok((stream, _)) => stream, + // Keep the listener available when descriptor pressure subsides. + Err(err) if matches!(err.raw_os_error(), Some(libc::EMFILE | libc::ENFILE)) => { + std::thread::sleep(Duration::from_millis(/*millis*/ 100)); + continue; + } + Err(err) => return Err(err), + }; let attribution_token = attribution_token.clone(); slots.spawn(routing, move || { let mut tcp_stream = match TcpStream::connect(endpoint) { diff --git a/codex-rs/linux-sandbox/src/tcp_net_setup.rs b/codex-rs/linux-sandbox/src/tcp_net_setup.rs index 8009457645..193b2e0c95 100644 --- a/codex-rs/linux-sandbox/src/tcp_net_setup.rs +++ b/codex-rs/linux-sandbox/src/tcp_net_setup.rs @@ -2,8 +2,12 @@ //! takes ownership of all three received descriptors before any workload exec. use crate::bwrap::BwrapArgs; +use crate::exec_util::make_files_inheritable; use crate::proxy_lifecycle::move_fd_above_stdio; use codex_protocol::protocol::FileSystemSandboxPolicy; +use codex_sandboxing::landlock::CODEX_LINUX_SANDBOX_ARG0; +use std::ffi::OsStr; +use std::ffi::OsString; use std::fs::File; use std::io; use std::io::Seek; @@ -11,10 +15,59 @@ use std::io::Write; use std::os::fd::AsRawFd; use std::os::fd::FromRawFd; use std::os::fd::OwnedFd; +use std::os::unix::process::CommandExt; use std::path::Path; +use std::process::Command; -pub(super) fn prepare( - files: [File; 3], +const SETUP_ARG: &str = "--setup-transparent-tcp"; + +/// Replace the forked supervisor image before creating user-namespace ID maps. +/// Exec drops the inherited attribution token from memory and restores normal +/// dumpability so an unprivileged caller can write its own UID/GID mappings. +pub(super) fn exec_setup(bwrap: &Path, args: &[String]) -> ! { + let err = Command::new("/proc/self/exe") + .arg0(CODEX_LINUX_SANDBOX_ARG0) + .arg(SETUP_ARG) + .arg(bwrap) + .args(args) + .exec(); + panic!("failed to exec transparent TCP setup: {err}") +} + +pub(super) fn dispatch() { + let mut args = std::env::args_os().skip(1); + if args.next().as_deref() != Some(OsStr::new(SETUP_ARG)) { + return; + } + let program = args + .next() + .unwrap_or_else(|| panic!("missing system bubblewrap path")); + let mut args: Vec = args.collect(); + let separator = args + .iter() + .position(|arg| arg == "--") + .filter(|separator| separator + 1 < args.len()) + .unwrap_or_else(|| panic!("missing inner sandbox command")); + let files = crate::tcp_namespace::create() + .unwrap_or_else(|err| panic!("failed to create private TCP namespace: {err}")); + let numbers = files + .each_ref() + .map(|fd| fd.as_raw_fd().to_string()) + .join(","); + args.splice( + separator + 2..separator + 2, + [ + "--transparent-tcp".into(), + "--transparent-tcp-fd".into(), + numbers.into(), + ], + ); + make_files_inheritable(&files); + let err = Command::new(program).arg0("bwrap").args(args).exec(); + panic!("failed to exec bubblewrap in transparent TCP namespace: {err}") +} + +pub(super) fn mount_resolver( bwrap: &mut BwrapArgs, policy: &FileSystemSandboxPolicy, cwd: &Path, @@ -33,23 +86,11 @@ pub(super) fn prepare( let mut resolv = File::from(move_fd_above_stdio(fd)?); resolv.write_all(b"nameserver 127.0.0.53\n")?; resolv.rewind()?; - let numbers = files - .each_ref() - .map(|f| f.as_raw_fd().to_string()) - .join(","); let separator = bwrap .args .iter() .position(|arg| arg == "--") .ok_or_else(|| io::Error::other("missing inner command"))?; - bwrap.args.splice( - separator + 2..separator + 2, - [ - "--transparent-tcp".into(), - "--transparent-tcp-fd".into(), - numbers, - ], - ); bwrap.args.splice( separator..separator, [ @@ -64,7 +105,6 @@ pub(super) fn prepare( ], ); bwrap.preserved_files.push(resolv); - bwrap.preserved_files.extend(files); Ok(()) } diff --git a/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs b/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs index 5e2c9b0c17..70efb1962e 100644 --- a/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs +++ b/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs @@ -37,6 +37,8 @@ use tempfile::NamedTempFile; use tokio::process::Command; use url::Url; +#[path = "managed_proxy_bridge_lifecycle_tests.rs"] +mod bridge_lifecycle; #[path = "managed_tcp.rs"] mod managed_tcp; #[path = "managed_proxy_unix_sockets_tests.rs"] @@ -189,32 +191,34 @@ fn linux_sandbox_command_with_options( options: &[&str], ) -> Command { let cwd = std::env::current_dir().expect("current directory should exist"); - let permission_profile_json = - serde_json::to_string(permission_profile).expect("permission profile should serialize"); - - let mut args = vec![ - "--sandbox-policy-cwd".to_string(), - cwd.to_string_lossy().to_string(), - "--permission-profile".to_string(), - permission_profile_json, - ]; - if let Some(context) = managed_network { - args.push("--managed-network".to_string()); - args.push(context.to_string()); - } - args.extend(options.iter().map(|arg| (*arg).to_string())); - args.push("--".to_string()); - args.extend(command.iter().map(|entry| (*entry).to_string())); + let mut cmd = linux_sandbox_launcher(&cwd, permission_profile, managed_network, env); + cmd.args(options).arg("--").args(command); + cmd +} +fn linux_sandbox_launcher( + cwd: &Path, + permission_profile: &PermissionProfile, + managed_network: Option<&str>, + env: HashMap, +) -> Command { let mut cmd = Command::new(cargo_bin("codex-linux-sandbox").expect("sandbox helper should exist")); - cmd.args(args) + cmd.arg("--sandbox-policy-cwd") + .arg(cwd) + .arg("--permission-profile") + .arg( + serde_json::to_string(permission_profile).expect("permission profile should serialize"), + ) .current_dir(cwd) .env_clear() .envs(env) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + if let Some(context) = managed_network { + cmd.args(["--managed-network", context]); + } cmd } @@ -1139,79 +1143,3 @@ async fn handoff_isolates_concurrent_endpoints_and_closes_privileged_descriptors server.join().expect("proxy responder should succeed"); } } - -#[tokio::test] -async fn cancelling_sandbox_closes_active_proxy_connection() { - if let Some(reason) = managed_proxy_skip_reason().await { - eprintln!("skipping managed proxy test: {reason}"); - return; - } - - for profile in [ - PermissionProfile::workspace_write(), - PermissionProfile::Disabled, - ] { - let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("local proxy listener"); - listener - .set_nonblocking(/*nonblocking*/ true) - .expect("bounded accept"); - let mut env = create_env_from_core_vars(); - strip_proxy_env(&mut env); - env.remove(PROXY_ATTRIBUTION_TOKEN_ENV_KEY); - env.insert( - "HTTP_PROXY".to_string(), - format!("http://{}", listener.local_addr().expect("proxy endpoint")), - ); - let mut command = linux_sandbox_command( - &[ - "bash", - "-c", - concat!( - "set -eu; proxy=${HTTP_PROXY#*://}; ", - "exec 3<>/dev/tcp/${proxy%:*}/${proxy##*:}; ", - "printf ready >&3; IFS= read -r response <&3", - ), - ], - &profile, - /*allow_network_for_proxy*/ true, - env, - ); - command.kill_on_drop(true); - let mut child = command.spawn().expect("launch sandbox"); - let (mut upstream, _) = tokio::time::timeout(OPERATION_TIMEOUT, async { - loop { - match listener.accept() { - Ok(connection) => break connection, - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - tokio::time::sleep(Duration::from_millis(/*millis*/ 10)).await; - } - Err(error) => panic!("accept proxy connection: {error}"), - } - } - }) - .await - .expect("sandbox should establish its proxy connection"); - upstream - .set_read_timeout(Some(OPERATION_TIMEOUT)) - .expect("bounded read"); - let mut ready = [0; 5]; - upstream - .read_exact(&mut ready) - .expect("read client readiness"); - assert_eq!(ready, *b"ready"); - - child.start_kill().expect("cancel sandbox"); - let output = tokio::time::timeout(OPERATION_TIMEOUT, child.wait_with_output()) - .await - .expect("cancelled sandbox should release output") - .expect("wait for cancelled sandbox"); - assert_eq!(output.status.success(), false); - assert_eq!( - upstream - .read(&mut [0]) - .expect("host bridge should close its active connection"), - 0, - "cancelling the sandbox must stop its host bridge" - ); - } -} diff --git a/codex-rs/linux-sandbox/tests/suite/managed_proxy_bridge_lifecycle_tests.rs b/codex-rs/linux-sandbox/tests/suite/managed_proxy_bridge_lifecycle_tests.rs new file mode 100644 index 0000000000..8505247127 --- /dev/null +++ b/codex-rs/linux-sandbox/tests/suite/managed_proxy_bridge_lifecycle_tests.rs @@ -0,0 +1,274 @@ +//! Verify bridge task lifetime and mount cleanup with live proxy connections. + +use super::*; +use pretty_assertions::assert_eq; +use std::fs; +use std::io; +use std::os::unix::process::ExitStatusExt; +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio::process::Child; +use tokio::time::timeout; + +#[derive(Clone, Copy, Debug)] +enum Finish { + Exit(u8), + KillGroup, + KillLauncher, +} + +struct LiveProxy { + child: Child, + upstreams: Vec, + socks_listener: TcpListener, +} + +fn managed_command(cwd: &Path, profile: &PermissionProfile) -> Command { + let mut env = create_env_from_core_vars(); + strip_proxy_env(&mut env); + env.remove(PROXY_ATTRIBUTION_TOKEN_ENV_KEY); + linux_sandbox_launcher(cwd, profile, Some("{}"), env) +} + +impl LiveProxy { + async fn start(mut command: Command, connections: usize) -> io::Result { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let socks_listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let script = r#" +set -eu +proxy=${HTTP_PROXY#*://} +while :; do + exec {fd}<>/dev/tcp/${proxy%:*}/${proxy##*:} + printf R >&"$fd" + IFS= read -r -u "$fd" reply + case $reply in + next) ;; + socks) proxy=${ALL_PROXY#*://} ;; + *) exit "$reply" ;; + esac +done +"#; + let child = command + .env("HTTP_PROXY", format!("http://{}", listener.local_addr()?)) + .env( + "ALL_PROXY", + format!("socks5://{}", socks_listener.local_addr()?), + ) + .process_group(/*pgroup*/ 0) + .kill_on_drop(true) + .args(["--", "bash", "-c", script]) + .spawn()?; + let mut upstreams = Vec::new(); + for n in 1..=connections { + let (mut stream, _) = listener.accept().await?; + assert_eq!(stream.read_u8().await?, b'R'); + if n < connections { + stream.write_all(b"next\n").await?; + } + upstreams.push(stream); + } + Ok(Self { + child, + upstreams, + socks_listener, + }) + } + + fn pid(&self) -> i32 { + self.child.id().unwrap().try_into().unwrap() + } + + async fn finish(mut self, action: Finish) -> io::Result<()> { + let pid = self.pid(); + let stream = self.upstreams.last_mut().unwrap(); + let expected = match action { + Finish::Exit(code) => { + stream.write_all(format!("{code}\n").as_bytes()).await?; + (Some(i32::from(code)), None) + } + Finish::KillGroup => { + assert_eq!(unsafe { libc::killpg(pid, libc::SIGKILL) }, 0); + (None, Some(libc::SIGKILL)) + } + Finish::KillLauncher => { + self.child.start_kill()?; + (None, Some(libc::SIGKILL)) + } + }; + let output = self.child.wait_with_output().await?; + assert_eq!( + (output.status.code(), output.status.signal()), + expected, + "{action:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + for stream in &mut self.upstreams { + assert_eq!(stream.read(&mut [0]).await?, 0, "proxy stayed open"); + } + Ok(()) + } +} + +fn host_bridge_tasks(launcher: i32) -> io::Result> { + let mut tasks = Vec::new(); + for thread in fs::read_dir(format!("/proc/{launcher}/task"))? { + let thread = thread?; + let tid = thread.file_name().to_str().unwrap().parse::().unwrap(); + if tid != launcher { + tasks.push(tid); + } + // Include forked bridges. Bubblewrap runs in a separate process group. + for child in fs::read_to_string(thread.path().join("children"))?.split_whitespace() { + let pid = child.parse().unwrap(); + if unsafe { libc::getpgid(pid) } == launcher { + tasks.push(pid); + } + } + } + Ok(tasks) +} + +#[tokio::test] +async fn reaps_bridges_after_exit_and_cancellation() -> io::Result<()> { + const OBSERVER: &str = "CODEX_TEST_BRIDGE_OBSERVER"; + if std::env::var_os(OBSERVER).is_some() { + timeout(SANDBOX_TIMEOUT, observe_bridge_teardown()).await??; + println!("verified bridge teardown"); + return Ok(()); + } + if let Some(reason) = managed_proxy_skip_reason().await { + eprintln!("skipping managed proxy test: {reason}"); + return Ok(()); + } + // Subreaper state is process-wide. Re-run this test in its own process so PID 1 + // cannot collect leaked children before the test inspects them. + let name = std::thread::current().name().unwrap().to_owned(); + let output = Command::new(std::env::current_exe()?) + .args(["--exact", &name, "--nocapture"]) + .env(OBSERVER, "1") + .kill_on_drop(true) + .output() + .await?; + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stderr}"); + assert!(String::from_utf8_lossy(&output.stdout).contains("verified bridge teardown")); + Ok(()) +} + +async fn observe_bridge_teardown() -> io::Result<()> { + assert_eq!( + unsafe { libc::prctl(libc::PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) }, + 0 + ); + let tmp = tempfile::tempdir()?; + let mut leaks = Vec::new(); + for profile in [ + PermissionProfile::workspace_write(), + PermissionProfile::Disabled, + ] { + for ending in [Finish::Exit(0), Finish::KillGroup, Finish::KillLauncher] { + let command = managed_command(tmp.path(), &profile); + let proxy = LiveProxy::start(command, /*connections*/ 1).await?; + let tasks = host_bridge_tasks(proxy.pid())?; + assert!(tasks.len() >= 2, "expected active and idle bridge tasks"); + proxy.finish(ending).await?; + for pid in tasks { + if Path::new(&format!("/proc/{pid}")).try_exists()? { + leaks.push((ending, pid)); + } + } + while unsafe { libc::waitpid(-1, std::ptr::null_mut(), libc::WNOHANG) } > 0 {} + } + } + assert!(leaks.is_empty(), "bridge tasks survived: {leaks:?}"); + + // This caller previously exec'd bwrap directly. Its inherited signal + // disposition must still allow the supervisor to recover the command's exit status. + let mut command = managed_command(tmp.path(), &PermissionProfile::Disabled); + command.arg("--inherit-pid-namespace"); + unsafe { + command.pre_exec(|| { + if libc::signal(libc::SIGCHLD, libc::SIG_IGN) == libc::SIG_ERR { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + let proxy = LiveProxy::start(command, /*connections*/ 1).await?; + proxy.finish(Finish::Exit(37)).await?; + + // Linux sends the parent-death signal when the thread that forked the launcher exits. + let mut command = managed_command(tmp.path(), &PermissionProfile::Disabled); + command.arg("--inherit-pid-namespace"); + let runtime = tokio::runtime::Handle::current(); + let (started, proxy) = tokio::sync::oneshot::channel(); + let caller = std::thread::spawn(move || { + let proxy = runtime.block_on(async { + timeout( + OPERATION_TIMEOUT, + LiveProxy::start(command, /*connections*/ 1), + ) + .await? + }); + let _ = started.send(proxy); + }); + let mut proxy = proxy.await.expect("caller returned the active proxy")?; + caller.join().expect("caller thread exited"); + let output = timeout(OPERATION_TIMEOUT, proxy.child.wait_with_output()) + .await + .expect("launcher survived its caller")?; + assert_eq!(output.status.signal(), Some(libc::SIGKILL)); + assert_eq!( + proxy.upstreams[0].read(&mut [0]).await?, + 0, + "proxy survived its caller" + ); + Ok(()) +} + +#[tokio::test] +async fn proxy_fd_pressure_preserves_socks_and_mount_cleanup() -> io::Result<()> { + if let Some(reason) = managed_proxy_skip_reason().await { + eprintln!("skipping managed proxy test: {reason}"); + return Ok(()); + } + let temp = tempfile::tempdir()?; + let protected = temp.path().join(".codex"); + timeout(SANDBOX_TIMEOUT, async { + for (http_connections, fd_limit, open_socks) in [(3, 20, true), (8, 16, false)] { + let command = managed_command(temp.path(), &PermissionProfile::workspace_write()); + let mut proxy = LiveProxy::start(command, http_connections).await?; + assert!( + protected.try_exists()?, + "expected a live synthetic mount target" + ); + let limit = libc::rlimit { + rlim_cur: fd_limit, + rlim_max: fd_limit, + }; + let pid = proxy.pid(); + let result = + unsafe { libc::prlimit(pid, libc::RLIMIT_NOFILE, &limit, std::ptr::null_mut()) }; + assert_eq!(result, 0, "prlimit: {}", io::Error::last_os_error()); + if open_socks { + proxy + .upstreams + .last_mut() + .unwrap() + .write_all(b"socks\n") + .await?; + let (mut socks, _) = timeout(OPERATION_TIMEOUT, proxy.socks_listener.accept()) + .await + .expect("SOCKS upstream should accept while HTTP connections are open")?; + assert_eq!(socks.read_u8().await?, b'R'); + proxy.upstreams.push(socks); + } + proxy.finish(Finish::Exit(37)).await?; + assert!(!protected.try_exists()?, "synthetic mount target survived"); + } + Ok(()) + }) + .await? +}