Tie Linux sandbox proxy bridges to the launcher lifetime (#61898)

## Why

Host proxy bridges must close active connections when a sandbox exits or is cancelled. Proxy descriptor exhaustion must also leave synthetic mount cleanup able to run.

## What changed

- Run host proxy bridges as threads in an outer supervisor that forwards signals and preserves the sandbox command's exit status. Use parent-death signals to terminate supervised children when their parent exits.
- Give bridge routes private descriptor tables where supported, retry accepts after descriptor exhaustion, and keep mount cleanup in a separate supervisor.
- Create the transparent TCP namespace in the bubblewrap child after re-executing the sandbox helper, clearing inherited credentials from memory and restoring dumpability for user namespace ID mapping.

## Testing

Add regression tests for bridge teardown after normal exit, process-group cancellation, launcher termination, and caller-thread exit; exit status propagation with inherited `SIGCHLD` ignored; and SOCKS connectivity and synthetic mount cleanup under descriptor pressure.

GitOrigin-RevId: dd297af98e9383942098fab5683ef2474c5ae554
This commit is contained in:
viyatb-oai
2026-11-16 22:27:42 +00:00
committed by copyberry
parent ce7079b506
commit 641b4cac88
7 changed files with 613 additions and 261 deletions
+11 -10
View File
@@ -38,7 +38,11 @@ struct SystemBwrapCapabilities {
static LAUNCHER: OnceLock<BubblewrapLauncher> = OnceLock::new();
pub(crate) fn exec_bwrap(mut argv: Vec<String>, preserved_files: Vec<File>) -> ! {
pub(crate) fn exec_bwrap(
mut argv: Vec<String>,
preserved_files: Vec<File>,
transparent_tcp: bool,
) -> ! {
if argv
.iter()
.take_while(|arg| arg.as_str() != "--")
@@ -53,7 +57,11 @@ pub(crate) fn exec_bwrap(mut argv: Vec<String>, preserved_files: Vec<File>) -> !
translate_legacy_bwrap_fd_mounts(&mut argv)
.unwrap_or_else(|error| panic!("invalid legacy bubblewrap fd mount: {error}"));
}
exec_system_bwrap(&launcher.program, argv, preserved_files)
make_files_inheritable(&preserved_files);
if transparent_tcp {
crate::tcp_net_setup::exec_setup(launcher.program.as_path(), &argv[1..]);
}
exec_system_bwrap(&launcher.program, argv)
}
BubblewrapLauncher::Bundled(launcher) => launcher.exec(argv, preserved_files),
BubblewrapLauncher::Unavailable => {
@@ -223,14 +231,7 @@ fn system_bwrap_capabilities(system_bwrap_path: &Path) -> Option<SystemBwrapCapa
})
}
fn exec_system_bwrap(
program: &AbsolutePathBuf,
argv: Vec<String>,
preserved_files: Vec<File>,
) -> ! {
// System bwrap runs across an exec boundary, so preserved fds must survive exec.
make_files_inheritable(&preserved_files);
fn exec_system_bwrap(program: &AbsolutePathBuf, argv: Vec<String>) -> ! {
let program_path = program.as_path().display().to_string();
let program = CString::new(program.as_path().as_os_str().as_bytes())
.unwrap_or_else(|err| panic!("invalid system bubblewrap path: {err}"));
+100 -43
View File
@@ -23,8 +23,10 @@ use crate::launcher::exec_bwrap;
use crate::launcher::initialize_bwrap_launcher;
use crate::launcher::preferred_bwrap_supports_argv0;
use crate::managed_tcp::Relay;
use crate::proxy_routing::HostProxyBridge;
use crate::proxy_routing::activate_proxy_routes_in_netns;
use crate::proxy_routing::prepare_host_proxy_route_spec;
use crate::proxy_routing::start_host_proxy_bridges;
use crate::seccomp::ProxyRouting;
use crate::seccomp::apply_sandbox_policies_to_current_thread;
use codex_network_proxy::ManagedNetworkSandboxContext;
@@ -142,6 +144,8 @@ pub struct LinuxSandboxCommand {
/// 2. Apply in-process restrictions (no_new_privs + seccomp).
/// 3. `execvp` into the final command.
pub fn run_main() -> ! {
let caller_pid = unsafe { libc::getppid() };
crate::tcp_net_setup::dispatch();
let LinuxSandboxCommand {
sandbox_policy_cwd,
command_cwd,
@@ -290,21 +294,25 @@ pub fn run_main() -> ! {
// Outer stage: bubblewrap first, then re-enter this binary in the
// sandboxed environment to apply seccomp. This path never falls back
// to legacy Landlock on failure.
let (proxy_route_spec, proxy_controls) = if let Some(context) = managed_network.as_ref() {
let (proxy_route_spec, proxy_bridges) = if let Some(context) = managed_network.as_ref() {
// Older explicit-proxy callers may omit ports. The new transparent
// switch always requires the executor's trusted ingress port list.
let ports = (transparent_tcp || !context.loopback_ports.is_empty())
.then_some(context.loopback_ports.as_slice());
let (proxy_route_spec, controls) = prepare_host_proxy_route_spec(ports, routing)
let (proxy_route_spec, bridges) = prepare_host_proxy_route_spec(ports, routing)
.unwrap_or_else(|err| panic!("failed to prepare host proxy routing bridge: {err}"));
(Some(proxy_route_spec), controls)
(Some(proxy_route_spec), bridges)
} else {
(None, Vec::new())
};
let options = BwrapOptions {
mount_proc: !no_proc && !inherit_pid_namespace,
inherit_pid_namespace,
network_mode: bwrap_network_mode(network_sandbox_policy, allow_network_for_proxy),
network_mode: if transparent_tcp {
BwrapNetworkMode::PreparedProxy
} else {
bwrap_network_mode(network_sandbox_policy, allow_network_for_proxy)
},
mask_wsl_interop: !file_system_sandbox_policy.has_full_disk_write_access()
&& Path::new(WSL_INTEROP_DIR).is_dir(),
mask_wslg_distro: (!file_system_sandbox_policy.has_full_disk_write_access()
@@ -336,13 +344,13 @@ pub fn run_main() -> ! {
command,
});
run_bwrap_with_proc_fallback(
caller_pid,
&sandbox_policy_cwd,
command_cwd.as_deref(),
&file_system_sandbox_policy,
options,
inner,
proxy_controls,
transparent_tcp,
proxy_bridges,
);
}
@@ -419,16 +427,26 @@ fn ensure_legacy_landlock_mode_supports_policy(
}
fn run_bwrap_with_proc_fallback(
caller_pid: libc::pid_t,
sandbox_policy_cwd: &Path,
command_cwd: Option<&Path>,
file_system_sandbox_policy: &FileSystemSandboxPolicy,
mut options: BwrapOptions,
inner: Vec<String>,
proxy_controls: Vec<File>,
transparent_tcp: bool,
proxy_bridges: Vec<HostProxyBridge>,
) -> ! {
let transparent_tcp = options.network_mode == BwrapNetworkMode::PreparedProxy;
let command_cwd = command_cwd.unwrap_or(sandbox_policy_cwd);
// An inherited SIG_IGN would auto-reap bubblewrap before we can collect
// its exit status. The preflight also waits for its own bubblewrap child.
if unsafe { libc::signal(libc::SIGCHLD, libc::SIG_DFL) } == libc::SIG_ERR {
panic!(
"failed to reset SIGCHLD: {}",
std::io::Error::last_os_error()
);
}
// Select before the preflight, whose synthetic policy omits writable roots.
initialize_bwrap_launcher(
file_system_sandbox_policy,
@@ -448,13 +466,7 @@ fn run_bwrap_with_proc_fallback(
options.mount_proc = false;
}
let tcp_files = transparent_tcp.then(|| {
let files = crate::tcp_namespace::create()
.unwrap_or_else(|err| panic!("failed to create private TCP namespace: {err}"));
options.network_mode = BwrapNetworkMode::PreparedProxy;
files
});
let mut bwrap_args = build_bwrap_argv(
let bwrap_args = build_bwrap_argv(
inner,
file_system_sandbox_policy,
sandbox_policy_cwd,
@@ -462,18 +474,24 @@ fn run_bwrap_with_proc_fallback(
options,
)
.unwrap_or_else(|err| exit_with_bwrap_build_error(err));
if let Some(files) = tcp_files {
crate::tcp_net_setup::prepare(
files,
if !proxy_bridges.is_empty() && bwrap_args.synthetic_mount_targets.is_empty() {
// This launcher would otherwise exec bubblewrap directly. Preserve
// --die-with-parent now that it stays alive to serve the proxy.
kill_on_parent_death(caller_pid);
}
// Host bridge threads must stay in the host network namespace. Only the
// bubblewrap child enters the private namespace and receives its TUN and resolver.
let mut bwrap_args = start_bwrap_supervisors(bwrap_args, proxy_bridges);
if transparent_tcp {
crate::tcp_net_setup::mount_resolver(
&mut bwrap_args,
file_system_sandbox_policy,
sandbox_policy_cwd,
)
.unwrap_or_else(|err| panic!("failed to prepare transparent TCP namespace: {err}"));
}
bwrap_args.preserved_files.extend(proxy_controls);
apply_inner_command_argv0(&mut bwrap_args.args);
run_or_exec_bwrap(bwrap_args);
exec_bwrap(bwrap_args.args, bwrap_args.preserved_files, transparent_tcp);
}
fn bwrap_network_mode(
@@ -596,47 +614,81 @@ fn resolve_true_command() -> String {
"true".to_string()
}
fn run_or_exec_bwrap(bwrap_args: crate::bwrap::BwrapArgs) -> ! {
if bwrap_args.synthetic_mount_targets.is_empty() {
exec_bwrap(bwrap_args.args, bwrap_args.preserved_files);
/// With both supervisors, the launcher runs `proxy supervisor -> mount cleanup supervisor -> bubblewrap`.
/// Each optional supervisor waits for its child. Separate descriptor tables
/// let mount cleanup proceed even when proxy connections exhaust theirs.
/// Returns only in the process that will exec bubblewrap.
fn start_bwrap_supervisors(
mut bwrap_args: crate::bwrap::BwrapArgs,
proxy_bridges: Vec<HostProxyBridge>,
) -> crate::bwrap::BwrapArgs {
if bwrap_args.synthetic_mount_targets.is_empty() && proxy_bridges.is_empty() {
return bwrap_args;
}
run_bwrap_in_child_with_synthetic_mount_cleanup(bwrap_args);
let setup_signal_mask = ForwardedSignalMask::block();
// Register before either fork so markers name the PID the caller will reap.
let registrations = register_synthetic_mount_targets(&bwrap_args.synthetic_mount_targets);
if !proxy_bridges.is_empty() {
let child_pid = fork_supervised_child();
if child_pid > 0 {
drop(bwrap_args.preserved_files);
// Bridge threads inherit the blocked signals. The supervising thread
// receives cancellation and forwards it to the child.
start_host_proxy_bridges(proxy_bridges);
supervise_child(child_pid, &setup_signal_mask, || {});
}
bwrap_args.preserved_files.extend(
proxy_bridges
.into_iter()
.map(HostProxyBridge::into_sandbox_control),
);
}
if !registrations.is_empty() {
let child_pid = fork_supervised_child();
if child_pid > 0 {
drop(bwrap_args.preserved_files);
supervise_child(child_pid, &setup_signal_mask, || {
cleanup_synthetic_mount_targets(&registrations);
});
}
}
setup_signal_mask.restore();
bwrap_args
}
fn run_bwrap_in_child_with_synthetic_mount_cleanup(bwrap_args: crate::bwrap::BwrapArgs) -> ! {
let crate::bwrap::BwrapArgs {
args,
preserved_files,
synthetic_mount_targets,
} = bwrap_args;
let setup_signal_mask = ForwardedSignalMask::block();
let synthetic_mount_registrations = register_synthetic_mount_targets(&synthetic_mount_targets);
fn fork_supervised_child() -> libc::pid_t {
let parent_pid = unsafe { libc::getpid() };
let pid = unsafe { libc::fork() };
if pid < 0 {
let err = std::io::Error::last_os_error();
panic!("failed to fork for bubblewrap: {err}");
}
if pid == 0 {
reset_forwarded_signal_handlers_to_default();
setup_signal_mask.restore();
let setpgid_res = unsafe { libc::setpgid(0, 0) };
if setpgid_res < 0 {
let err = std::io::Error::last_os_error();
panic!("failed to place bubblewrap child in its own process group: {err}");
}
terminate_with_parent(parent_pid);
exec_bwrap(args, preserved_files);
kill_on_parent_death(parent_pid);
}
pid
}
drop(preserved_files);
fn supervise_child(
pid: libc::pid_t,
setup_signal_mask: &ForwardedSignalMask,
cleanup: impl FnOnce(),
) -> ! {
let signal_forwarders = install_bwrap_signal_forwarders(pid);
setup_signal_mask.restore();
let status = wait_for_bwrap_child(pid);
let cleanup_signal_mask = ForwardedSignalMask::block();
BWRAP_CHILD_PID.store(0, Ordering::SeqCst);
cleanup_synthetic_mount_targets(&synthetic_mount_registrations);
cleanup();
signal_forwarders.restore();
cleanup_signal_mask.restore();
exit_with_wait_status(status);
@@ -681,15 +733,17 @@ impl ForwardedSignalMask {
}
}
fn terminate_with_parent(parent_pid: libc::pid_t) {
let res = unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) };
fn kill_on_parent_death(parent_pid: libc::pid_t) {
// Match bubblewrap's --die-with-parent even while the child has TERM blocked
// for mount registration or cleanup.
let res = unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) };
if res < 0 {
let err = std::io::Error::last_os_error();
panic!("failed to set bubblewrap child parent-death signal: {err}");
panic!("failed to set sandbox parent-death signal: {err}");
}
if unsafe { libc::getppid() } != parent_pid {
unsafe {
libc::raise(libc::SIGTERM);
libc::raise(libc::SIGKILL);
}
}
}
@@ -780,6 +834,9 @@ fn wait_for_bwrap_child(pid: libc::pid_t) -> libc::c_int {
fn register_synthetic_mount_targets(
targets: &[crate::bwrap::SyntheticMountTarget],
) -> Vec<SyntheticMountTargetRegistration> {
if targets.is_empty() {
return Vec::new();
}
with_synthetic_mount_registry_lock(|| {
targets
.iter()
@@ -1117,7 +1174,7 @@ fn run_bwrap_in_child_capture_stderr(bwrap_args: crate::bwrap::BwrapArgs) -> Str
close_fd_or_panic(write_fd, "close write end in bubblewrap child");
}
exec_bwrap(args, preserved_files);
exec_bwrap(args, preserved_files, /*transparent_tcp*/ false);
}
let signal_forwarders = install_bwrap_signal_forwarders(pid);
+46 -19
View File
@@ -1,4 +1,4 @@
//! Manage privileged bridge processes and transfer their loopback TCP listeners.
//! Isolate host proxy descriptors and transfer loopback listeners from the sandbox.
//!
//! Received descriptors are CLOEXEC and owned even when validation rejects them.
//! Shape validation does not authenticate the sender: the caller must retain exclusive
@@ -26,10 +26,54 @@ use std::os::fd::AsFd;
use std::os::fd::AsRawFd;
use std::os::fd::FromRawFd;
use std::os::fd::OwnedFd;
use std::os::fd::RawFd;
use std::os::unix::net::UnixStream;
const LISTENER_MESSAGE: u8 = 1;
/// Give one route its own descriptor budget. Its relay threads inherit this table.
///
/// # Safety
/// Call only on a fresh bridge thread with no owned descriptors. The supervisor must
/// keep the bootstrap socket open at `control` until the caller signals readiness.
pub(crate) unsafe fn private_bridge_control(control: RawFd) -> io::Result<UnixStream> {
// SAFETY: UNSHARE copies the calling thread's table before closing the upper range.
if unsafe {
libc::syscall(
libc::SYS_close_range,
control as u32 + 1,
u32::MAX,
libc::CLOSE_RANGE_UNSHARE,
)
} == 0
{
for fd in 3..control {
let _ = close_fd(fd);
}
} else {
// Older kernels support CLONE_FILES without close_range.
// SAFETY: unshare takes only flags and changes this thread's table.
if unsafe { libc::unshare(libc::CLONE_FILES) } == -1 {
return Err(io::Error::last_os_error());
}
let mut inherited = Vec::new();
for entry in std::fs::read_dir("/proc/thread-self/fd")? {
if let Ok(fd) = entry?.file_name().to_string_lossy().parse::<RawFd>()
&& fd > libc::STDERR_FILENO
&& fd != control
{
inherited.push(fd);
}
}
// The directory handle has dropped. No other thread can reuse these numbers.
for fd in inherited {
let _ = close_fd(fd);
}
}
// SAFETY: this thread owns the private copy of the supervisor's live socket.
Ok(unsafe { UnixStream::from_raw_fd(control) })
}
pub(crate) fn send_listener(channel: &UnixStream, listener: &TcpListener) -> io::Result<()> {
let descriptors = [listener.as_fd()];
let mut space = [MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(1))];
@@ -113,24 +157,7 @@ pub(crate) fn receive_listener(channel: &UnixStream) -> io::Result<TcpListener>
Ok(listener)
}
pub(crate) fn harden_bridge_process(expected_parent_pid: libc::pid_t) -> io::Result<()> {
detach_bridge_stdio()?;
set_parent_death_signal(expected_parent_pid)?;
codex_process_hardening::disable_process_dumping()
}
fn set_parent_death_signal(expected_parent_pid: libc::pid_t) -> io::Result<()> {
let res = unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) };
if res != 0 {
Err(io::Error::last_os_error())
} else if unsafe { libc::getppid() } != expected_parent_pid {
Err(io::Error::other("parent process already exited"))
} else {
Ok(())
}
}
fn detach_bridge_stdio() -> io::Result<()> {
pub(crate) fn detach_bridge_stdio() -> io::Result<()> {
let null_read_fd = unsafe { libc::open(c"/dev/null".as_ptr(), libc::O_RDONLY) };
if null_read_fd < 0 {
let err = io::Error::last_os_error();
+106 -81
View File
@@ -1,6 +1,7 @@
use crate::proxy_lifecycle::close_fd;
use crate::proxy_lifecycle::harden_bridge_process;
use crate::proxy_lifecycle::detach_bridge_stdio;
use crate::proxy_lifecycle::move_fd_above_stdio;
use crate::proxy_lifecycle::private_bridge_control;
use crate::proxy_lifecycle::receive_listener;
use crate::proxy_lifecycle::send_listener;
use crate::seccomp::ProxyRouting;
@@ -11,6 +12,7 @@ use serde::Deserialize;
use serde::Serialize;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::btree_map::Entry;
use std::fs::File;
use std::io;
use std::io::Read;
@@ -21,9 +23,9 @@ use std::net::Shutdown;
use std::net::SocketAddr;
use std::net::TcpListener;
use std::net::TcpStream;
use std::os::fd::AsFd;
use std::os::fd::AsRawFd;
use std::os::fd::FromRawFd;
use std::os::fd::OwnedFd;
use std::os::unix::net::UnixStream;
use std::sync::Arc;
use std::sync::mpsc;
@@ -57,13 +59,84 @@ struct ProxyRoutePlan {
has_proxy_config: bool,
}
/// One bootstrap socketpair per upstream endpoint. Only the sandbox end crosses exec.
pub(crate) struct HostProxyBridge {
endpoint: SocketAddr,
routing: ProxyRouting,
host_control: UnixStream,
sandbox_control: File,
attribution_token: Option<String>,
}
impl HostProxyBridge {
pub(crate) fn into_sandbox_control(self) -> File {
self.sandbox_control
}
}
/// Start bridge threads only in the outer supervisor, after the child inherits
/// the command's stdio. If setup fails, closing a host socket fails the handoff.
pub(crate) fn start_host_proxy_bridges(bridges: Vec<HostProxyBridge>) {
let mut startup_stderr = None;
let result: io::Result<()> = bridges
.into_iter()
.enumerate()
.try_for_each(|(index, bridge)| {
drop(bridge.sandbox_control);
if index == 0 {
// Reuse the slot freed above for diagnostics, even at the FD limit.
// The child already inherited its stdio; detach ours before starting any worker.
startup_stderr = io::stderr()
.as_fd()
.try_clone_to_owned()
.ok()
.map(File::from);
detach_bridge_stdio()?;
}
let control_fd = bridge.host_control.as_raw_fd();
let token = bridge.attribution_token.clone();
let (ready, initialized) = mpsc::channel();
let worker = std::thread::Builder::new()
.name("codex-proxy".to_string())
.spawn(move || {
// SAFETY: this new thread owns no FDs. The supervisor holds the socket
// until ready and its own descriptor table is unaffected.
let control = unsafe { private_bridge_control(control_fd)? };
ready.send(()).map_err(io::Error::other)?;
run_host_bridge(bridge.endpoint, bridge.routing, control, token)
})?;
if initialized.recv().is_ok() {
return Ok(());
}
// Reclaim the failed worker's task slot before falling back on hosts that
// prohibit private FD tables.
let _ = worker.join();
std::thread::Builder::new()
.name("codex-proxy".to_string())
.spawn(move || {
run_host_bridge(
bridge.endpoint,
bridge.routing,
bridge.host_control,
bridge.attribution_token,
)
})?;
Ok(())
});
if let (Err(err), Some(mut stderr)) = (result, startup_stderr) {
let _ = writeln!(stderr, "failed to start host proxy bridges: {err}");
}
}
pub(crate) fn prepare_host_proxy_route_spec(
port_allowlist: Option<&[u16]>,
routing: ProxyRouting,
) -> io::Result<(String, Vec<File>)> {
) -> io::Result<(String, Vec<HostProxyBridge>)> {
// The outer supervisor holds the upstream credential and host sockets.
codex_process_hardening::disable_process_dumping()?;
let (attribution_token, plan) = extract_attribution_token_and_plan(std::env::vars().collect());
// SAFETY: the sandbox helper is single-threaded here, before it forks bridge workers or
// executes the user command.
// SAFETY: bubblewrap and the host bridge threads have not been started yet.
unsafe {
std::env::remove_var(PROXY_ATTRIBUTION_TOKEN_ENV_KEY);
}
@@ -77,7 +150,7 @@ pub(crate) fn prepare_host_proxy_route_spec(
return Err(io::Error::new(io::ErrorKind::InvalidInput, message));
}
// Validate the entire plan before forking any host-side workers or sending
// Validate the entire plan before starting any host-side workers or sending
// attribution. An empty enforced list grants no routes.
if let Some(ports) = port_allowlist
&& plan
@@ -91,36 +164,30 @@ pub(crate) fn prepare_host_proxy_route_spec(
));
}
let mut control_by_endpoint = BTreeMap::new();
for route in &plan.routes {
if control_by_endpoint.contains_key(&route.endpoint) {
continue;
}
let control = spawn_host_bridge(
route.endpoint,
routing,
attribution_token.as_deref(),
&mut control_by_endpoint,
)?;
control_by_endpoint.insert(route.endpoint, control);
}
let mut bridges = BTreeMap::new();
let mut routes = Vec::with_capacity(plan.routes.len());
for route in plan.routes {
let Some(control) = control_by_endpoint.get(&route.endpoint) else {
return Err(io::Error::other(format!(
"missing bootstrap channel for endpoint {}",
route.endpoint
)));
let bridge = match bridges.entry(route.endpoint) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
let (host, sandbox) = UnixStream::pair()?;
entry.insert(HostProxyBridge {
endpoint: route.endpoint,
routing,
host_control: UnixStream::from(move_fd_above_stdio(host.into())?),
sandbox_control: File::from(move_fd_above_stdio(sandbox.into())?),
attribution_token: attribution_token.clone(),
})
}
};
routes.push(ProxyRouteEntry {
env_key: route.env_key,
control_fd: control.as_raw_fd(),
control_fd: bridge.sandbox_control.as_raw_fd(),
});
}
let spec = serde_json::to_string(&ProxyRouteSpec { routes }).map_err(io::Error::other)?;
Ok((spec, control_by_endpoint.into_values().collect()))
Ok((spec, bridges.into_values().collect()))
}
fn extract_attribution_token_and_plan(
@@ -329,51 +396,6 @@ fn rewrite_proxy_env_value(proxy_url: &str, local_port: u16) -> Option<String> {
Some(rewritten)
}
fn spawn_host_bridge(
endpoint: SocketAddr,
routing: ProxyRouting,
attribution_token: Option<&str>,
inherited_controls: &mut BTreeMap<SocketAddr, File>,
) -> io::Result<File> {
let (host_control, control) = UnixStream::pair()?;
let host_control = UnixStream::from(move_fd_above_stdio(host_control.into())?);
let mut control = UnixStream::from(move_fd_above_stdio(control.into())?);
control.set_read_timeout(Some(HANDOFF_TIMEOUT))?;
let parent_pid = unsafe { libc::getpid() };
let pid = unsafe { libc::fork() };
if pid < 0 {
return Err(io::Error::last_os_error());
}
if pid == 0 {
drop(control);
// A worker must not retain another route's inner bootstrap endpoint.
// Drop this child's inherited owners; the parent's copies are unaffected by fork.
inherited_controls.clear();
let result = run_host_bridge(
endpoint,
routing,
host_control,
attribution_token,
parent_pid,
);
if result.is_err() {
unsafe { libc::_exit(1) };
}
unsafe { libc::_exit(0) };
}
drop(host_control);
let mut ready = [0_u8; 1];
control.read_exact(&mut ready)?;
if ready != [HOST_BRIDGE_READY] {
return Err(io::Error::other(
"host bridge did not acknowledge readiness",
));
}
Ok(File::from(OwnedFd::from(control)))
}
/// Each connection worker holds a permit until it exits.
pub(super) struct ConnectionSlots(Arc<Semaphore>);
@@ -403,22 +425,25 @@ fn run_host_bridge(
endpoint: SocketAddr,
routing: ProxyRouting,
mut control: UnixStream,
attribution_token: Option<&str>,
parent_pid: libc::pid_t,
attribution_token: Option<String>,
) -> io::Result<()> {
harden_bridge_process(parent_pid)?;
control.write_all(&[HOST_BRIDGE_READY])?;
// The outer helper must launch bubblewrap before this listener can arrive.
// The socket keeps its isolated network namespace even though we accept in
// the host process; newly opened upstream sockets use our host namespace.
// The listener stays in the sandbox network namespace. Connections to the
// upstream proxy are opened here in the host network namespace.
let listener = receive_listener(&control)?;
control.write_all(&[HOST_BRIDGE_READY])?;
drop(control);
let attribution_token = attribution_token.map(str::to_owned);
let slots = ConnectionSlots::new();
loop {
let (client_stream, _) = listener.accept()?;
let client_stream = match listener.accept() {
Ok((stream, _)) => stream,
// Keep the listener available when descriptor pressure subsides.
Err(err) if matches!(err.raw_os_error(), Some(libc::EMFILE | libc::ENFILE)) => {
std::thread::sleep(Duration::from_millis(/*millis*/ 100));
continue;
}
Err(err) => return Err(err),
};
let attribution_token = attribution_token.clone();
slots.spawn(routing, move || {
let mut tcp_stream = match TcpStream::connect(endpoint) {
+55 -15
View File
@@ -2,8 +2,12 @@
//! takes ownership of all three received descriptors before any workload exec.
use crate::bwrap::BwrapArgs;
use crate::exec_util::make_files_inheritable;
use crate::proxy_lifecycle::move_fd_above_stdio;
use codex_protocol::protocol::FileSystemSandboxPolicy;
use codex_sandboxing::landlock::CODEX_LINUX_SANDBOX_ARG0;
use std::ffi::OsStr;
use std::ffi::OsString;
use std::fs::File;
use std::io;
use std::io::Seek;
@@ -11,10 +15,59 @@ use std::io::Write;
use std::os::fd::AsRawFd;
use std::os::fd::FromRawFd;
use std::os::fd::OwnedFd;
use std::os::unix::process::CommandExt;
use std::path::Path;
use std::process::Command;
pub(super) fn prepare(
files: [File; 3],
const SETUP_ARG: &str = "--setup-transparent-tcp";
/// Replace the forked supervisor image before creating user-namespace ID maps.
/// Exec drops the inherited attribution token from memory and restores normal
/// dumpability so an unprivileged caller can write its own UID/GID mappings.
pub(super) fn exec_setup(bwrap: &Path, args: &[String]) -> ! {
let err = Command::new("/proc/self/exe")
.arg0(CODEX_LINUX_SANDBOX_ARG0)
.arg(SETUP_ARG)
.arg(bwrap)
.args(args)
.exec();
panic!("failed to exec transparent TCP setup: {err}")
}
pub(super) fn dispatch() {
let mut args = std::env::args_os().skip(1);
if args.next().as_deref() != Some(OsStr::new(SETUP_ARG)) {
return;
}
let program = args
.next()
.unwrap_or_else(|| panic!("missing system bubblewrap path"));
let mut args: Vec<OsString> = args.collect();
let separator = args
.iter()
.position(|arg| arg == "--")
.filter(|separator| separator + 1 < args.len())
.unwrap_or_else(|| panic!("missing inner sandbox command"));
let files = crate::tcp_namespace::create()
.unwrap_or_else(|err| panic!("failed to create private TCP namespace: {err}"));
let numbers = files
.each_ref()
.map(|fd| fd.as_raw_fd().to_string())
.join(",");
args.splice(
separator + 2..separator + 2,
[
"--transparent-tcp".into(),
"--transparent-tcp-fd".into(),
numbers.into(),
],
);
make_files_inheritable(&files);
let err = Command::new(program).arg0("bwrap").args(args).exec();
panic!("failed to exec bubblewrap in transparent TCP namespace: {err}")
}
pub(super) fn mount_resolver(
bwrap: &mut BwrapArgs,
policy: &FileSystemSandboxPolicy,
cwd: &Path,
@@ -33,23 +86,11 @@ pub(super) fn prepare(
let mut resolv = File::from(move_fd_above_stdio(fd)?);
resolv.write_all(b"nameserver 127.0.0.53\n")?;
resolv.rewind()?;
let numbers = files
.each_ref()
.map(|f| f.as_raw_fd().to_string())
.join(",");
let separator = bwrap
.args
.iter()
.position(|arg| arg == "--")
.ok_or_else(|| io::Error::other("missing inner command"))?;
bwrap.args.splice(
separator + 2..separator + 2,
[
"--transparent-tcp".into(),
"--transparent-tcp-fd".into(),
numbers,
],
);
bwrap.args.splice(
separator..separator,
[
@@ -64,7 +105,6 @@ pub(super) fn prepare(
],
);
bwrap.preserved_files.push(resolv);
bwrap.preserved_files.extend(files);
Ok(())
}
@@ -37,6 +37,8 @@ use tempfile::NamedTempFile;
use tokio::process::Command;
use url::Url;
#[path = "managed_proxy_bridge_lifecycle_tests.rs"]
mod bridge_lifecycle;
#[path = "managed_tcp.rs"]
mod managed_tcp;
#[path = "managed_proxy_unix_sockets_tests.rs"]
@@ -189,32 +191,34 @@ fn linux_sandbox_command_with_options(
options: &[&str],
) -> Command {
let cwd = std::env::current_dir().expect("current directory should exist");
let permission_profile_json =
serde_json::to_string(permission_profile).expect("permission profile should serialize");
let mut args = vec![
"--sandbox-policy-cwd".to_string(),
cwd.to_string_lossy().to_string(),
"--permission-profile".to_string(),
permission_profile_json,
];
if let Some(context) = managed_network {
args.push("--managed-network".to_string());
args.push(context.to_string());
}
args.extend(options.iter().map(|arg| (*arg).to_string()));
args.push("--".to_string());
args.extend(command.iter().map(|entry| (*entry).to_string()));
let mut cmd = linux_sandbox_launcher(&cwd, permission_profile, managed_network, env);
cmd.args(options).arg("--").args(command);
cmd
}
fn linux_sandbox_launcher(
cwd: &Path,
permission_profile: &PermissionProfile,
managed_network: Option<&str>,
env: HashMap<String, String>,
) -> Command {
let mut cmd =
Command::new(cargo_bin("codex-linux-sandbox").expect("sandbox helper should exist"));
cmd.args(args)
cmd.arg("--sandbox-policy-cwd")
.arg(cwd)
.arg("--permission-profile")
.arg(
serde_json::to_string(permission_profile).expect("permission profile should serialize"),
)
.current_dir(cwd)
.env_clear()
.envs(env)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
if let Some(context) = managed_network {
cmd.args(["--managed-network", context]);
}
cmd
}
@@ -1139,79 +1143,3 @@ async fn handoff_isolates_concurrent_endpoints_and_closes_privileged_descriptors
server.join().expect("proxy responder should succeed");
}
}
#[tokio::test]
async fn cancelling_sandbox_closes_active_proxy_connection() {
if let Some(reason) = managed_proxy_skip_reason().await {
eprintln!("skipping managed proxy test: {reason}");
return;
}
for profile in [
PermissionProfile::workspace_write(),
PermissionProfile::Disabled,
] {
let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).expect("local proxy listener");
listener
.set_nonblocking(/*nonblocking*/ true)
.expect("bounded accept");
let mut env = create_env_from_core_vars();
strip_proxy_env(&mut env);
env.remove(PROXY_ATTRIBUTION_TOKEN_ENV_KEY);
env.insert(
"HTTP_PROXY".to_string(),
format!("http://{}", listener.local_addr().expect("proxy endpoint")),
);
let mut command = linux_sandbox_command(
&[
"bash",
"-c",
concat!(
"set -eu; proxy=${HTTP_PROXY#*://}; ",
"exec 3<>/dev/tcp/${proxy%:*}/${proxy##*:}; ",
"printf ready >&3; IFS= read -r response <&3",
),
],
&profile,
/*allow_network_for_proxy*/ true,
env,
);
command.kill_on_drop(true);
let mut child = command.spawn().expect("launch sandbox");
let (mut upstream, _) = tokio::time::timeout(OPERATION_TIMEOUT, async {
loop {
match listener.accept() {
Ok(connection) => break connection,
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
tokio::time::sleep(Duration::from_millis(/*millis*/ 10)).await;
}
Err(error) => panic!("accept proxy connection: {error}"),
}
}
})
.await
.expect("sandbox should establish its proxy connection");
upstream
.set_read_timeout(Some(OPERATION_TIMEOUT))
.expect("bounded read");
let mut ready = [0; 5];
upstream
.read_exact(&mut ready)
.expect("read client readiness");
assert_eq!(ready, *b"ready");
child.start_kill().expect("cancel sandbox");
let output = tokio::time::timeout(OPERATION_TIMEOUT, child.wait_with_output())
.await
.expect("cancelled sandbox should release output")
.expect("wait for cancelled sandbox");
assert_eq!(output.status.success(), false);
assert_eq!(
upstream
.read(&mut [0])
.expect("host bridge should close its active connection"),
0,
"cancelling the sandbox must stop its host bridge"
);
}
}
@@ -0,0 +1,274 @@
//! Verify bridge task lifetime and mount cleanup with live proxy connections.
use super::*;
use pretty_assertions::assert_eq;
use std::fs;
use std::io;
use std::os::unix::process::ExitStatusExt;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpListener;
use tokio::net::TcpStream;
use tokio::process::Child;
use tokio::time::timeout;
#[derive(Clone, Copy, Debug)]
enum Finish {
Exit(u8),
KillGroup,
KillLauncher,
}
struct LiveProxy {
child: Child,
upstreams: Vec<TcpStream>,
socks_listener: TcpListener,
}
fn managed_command(cwd: &Path, profile: &PermissionProfile) -> Command {
let mut env = create_env_from_core_vars();
strip_proxy_env(&mut env);
env.remove(PROXY_ATTRIBUTION_TOKEN_ENV_KEY);
linux_sandbox_launcher(cwd, profile, Some("{}"), env)
}
impl LiveProxy {
async fn start(mut command: Command, connections: usize) -> io::Result<Self> {
let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?;
let socks_listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?;
let script = r#"
set -eu
proxy=${HTTP_PROXY#*://}
while :; do
exec {fd}<>/dev/tcp/${proxy%:*}/${proxy##*:}
printf R >&"$fd"
IFS= read -r -u "$fd" reply
case $reply in
next) ;;
socks) proxy=${ALL_PROXY#*://} ;;
*) exit "$reply" ;;
esac
done
"#;
let child = command
.env("HTTP_PROXY", format!("http://{}", listener.local_addr()?))
.env(
"ALL_PROXY",
format!("socks5://{}", socks_listener.local_addr()?),
)
.process_group(/*pgroup*/ 0)
.kill_on_drop(true)
.args(["--", "bash", "-c", script])
.spawn()?;
let mut upstreams = Vec::new();
for n in 1..=connections {
let (mut stream, _) = listener.accept().await?;
assert_eq!(stream.read_u8().await?, b'R');
if n < connections {
stream.write_all(b"next\n").await?;
}
upstreams.push(stream);
}
Ok(Self {
child,
upstreams,
socks_listener,
})
}
fn pid(&self) -> i32 {
self.child.id().unwrap().try_into().unwrap()
}
async fn finish(mut self, action: Finish) -> io::Result<()> {
let pid = self.pid();
let stream = self.upstreams.last_mut().unwrap();
let expected = match action {
Finish::Exit(code) => {
stream.write_all(format!("{code}\n").as_bytes()).await?;
(Some(i32::from(code)), None)
}
Finish::KillGroup => {
assert_eq!(unsafe { libc::killpg(pid, libc::SIGKILL) }, 0);
(None, Some(libc::SIGKILL))
}
Finish::KillLauncher => {
self.child.start_kill()?;
(None, Some(libc::SIGKILL))
}
};
let output = self.child.wait_with_output().await?;
assert_eq!(
(output.status.code(), output.status.signal()),
expected,
"{action:?}: {}",
String::from_utf8_lossy(&output.stderr)
);
for stream in &mut self.upstreams {
assert_eq!(stream.read(&mut [0]).await?, 0, "proxy stayed open");
}
Ok(())
}
}
fn host_bridge_tasks(launcher: i32) -> io::Result<Vec<i32>> {
let mut tasks = Vec::new();
for thread in fs::read_dir(format!("/proc/{launcher}/task"))? {
let thread = thread?;
let tid = thread.file_name().to_str().unwrap().parse::<i32>().unwrap();
if tid != launcher {
tasks.push(tid);
}
// Include forked bridges. Bubblewrap runs in a separate process group.
for child in fs::read_to_string(thread.path().join("children"))?.split_whitespace() {
let pid = child.parse().unwrap();
if unsafe { libc::getpgid(pid) } == launcher {
tasks.push(pid);
}
}
}
Ok(tasks)
}
#[tokio::test]
async fn reaps_bridges_after_exit_and_cancellation() -> io::Result<()> {
const OBSERVER: &str = "CODEX_TEST_BRIDGE_OBSERVER";
if std::env::var_os(OBSERVER).is_some() {
timeout(SANDBOX_TIMEOUT, observe_bridge_teardown()).await??;
println!("verified bridge teardown");
return Ok(());
}
if let Some(reason) = managed_proxy_skip_reason().await {
eprintln!("skipping managed proxy test: {reason}");
return Ok(());
}
// Subreaper state is process-wide. Re-run this test in its own process so PID 1
// cannot collect leaked children before the test inspects them.
let name = std::thread::current().name().unwrap().to_owned();
let output = Command::new(std::env::current_exe()?)
.args(["--exact", &name, "--nocapture"])
.env(OBSERVER, "1")
.kill_on_drop(true)
.output()
.await?;
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(output.status.success(), "{stderr}");
assert!(String::from_utf8_lossy(&output.stdout).contains("verified bridge teardown"));
Ok(())
}
async fn observe_bridge_teardown() -> io::Result<()> {
assert_eq!(
unsafe { libc::prctl(libc::PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) },
0
);
let tmp = tempfile::tempdir()?;
let mut leaks = Vec::new();
for profile in [
PermissionProfile::workspace_write(),
PermissionProfile::Disabled,
] {
for ending in [Finish::Exit(0), Finish::KillGroup, Finish::KillLauncher] {
let command = managed_command(tmp.path(), &profile);
let proxy = LiveProxy::start(command, /*connections*/ 1).await?;
let tasks = host_bridge_tasks(proxy.pid())?;
assert!(tasks.len() >= 2, "expected active and idle bridge tasks");
proxy.finish(ending).await?;
for pid in tasks {
if Path::new(&format!("/proc/{pid}")).try_exists()? {
leaks.push((ending, pid));
}
}
while unsafe { libc::waitpid(-1, std::ptr::null_mut(), libc::WNOHANG) } > 0 {}
}
}
assert!(leaks.is_empty(), "bridge tasks survived: {leaks:?}");
// This caller previously exec'd bwrap directly. Its inherited signal
// disposition must still allow the supervisor to recover the command's exit status.
let mut command = managed_command(tmp.path(), &PermissionProfile::Disabled);
command.arg("--inherit-pid-namespace");
unsafe {
command.pre_exec(|| {
if libc::signal(libc::SIGCHLD, libc::SIG_IGN) == libc::SIG_ERR {
return Err(io::Error::last_os_error());
}
Ok(())
});
}
let proxy = LiveProxy::start(command, /*connections*/ 1).await?;
proxy.finish(Finish::Exit(37)).await?;
// Linux sends the parent-death signal when the thread that forked the launcher exits.
let mut command = managed_command(tmp.path(), &PermissionProfile::Disabled);
command.arg("--inherit-pid-namespace");
let runtime = tokio::runtime::Handle::current();
let (started, proxy) = tokio::sync::oneshot::channel();
let caller = std::thread::spawn(move || {
let proxy = runtime.block_on(async {
timeout(
OPERATION_TIMEOUT,
LiveProxy::start(command, /*connections*/ 1),
)
.await?
});
let _ = started.send(proxy);
});
let mut proxy = proxy.await.expect("caller returned the active proxy")?;
caller.join().expect("caller thread exited");
let output = timeout(OPERATION_TIMEOUT, proxy.child.wait_with_output())
.await
.expect("launcher survived its caller")?;
assert_eq!(output.status.signal(), Some(libc::SIGKILL));
assert_eq!(
proxy.upstreams[0].read(&mut [0]).await?,
0,
"proxy survived its caller"
);
Ok(())
}
#[tokio::test]
async fn proxy_fd_pressure_preserves_socks_and_mount_cleanup() -> io::Result<()> {
if let Some(reason) = managed_proxy_skip_reason().await {
eprintln!("skipping managed proxy test: {reason}");
return Ok(());
}
let temp = tempfile::tempdir()?;
let protected = temp.path().join(".codex");
timeout(SANDBOX_TIMEOUT, async {
for (http_connections, fd_limit, open_socks) in [(3, 20, true), (8, 16, false)] {
let command = managed_command(temp.path(), &PermissionProfile::workspace_write());
let mut proxy = LiveProxy::start(command, http_connections).await?;
assert!(
protected.try_exists()?,
"expected a live synthetic mount target"
);
let limit = libc::rlimit {
rlim_cur: fd_limit,
rlim_max: fd_limit,
};
let pid = proxy.pid();
let result =
unsafe { libc::prlimit(pid, libc::RLIMIT_NOFILE, &limit, std::ptr::null_mut()) };
assert_eq!(result, 0, "prlimit: {}", io::Error::last_os_error());
if open_socks {
proxy
.upstreams
.last_mut()
.unwrap()
.write_all(b"socks\n")
.await?;
let (mut socks, _) = timeout(OPERATION_TIMEOUT, proxy.socks_listener.accept())
.await
.expect("SOCKS upstream should accept while HTTP connections are open")?;
assert_eq!(socks.read_u8().await?, b'R');
proxy.upstreams.push(socks);
}
proxy.finish(Finish::Exit(37)).await?;
assert!(!protected.try_exists()?, "synthetic mount target survived");
}
Ok(())
})
.await?
}