mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Cache empty remote skill discovery with filesystem watches (#57598)
## Why Repeated repository skill discovery performs remote metadata requests even when no `.agents/skills` directories exist. Reuse verified empty results to avoid those round trips while watching for changes that require a new scan. ## What changed - Add the Linux executor capability `fs_watch_missing`, the `fs/watchMissing` request, and `fs/missingChanged` notifications. - Register watches in the background and reuse empty results only after the executor acknowledges coverage. Key reuse by filesystem, working directory, and project root markers. - Invalidate observations on relevant filesystem or mount changes, watch supersession, or connection loss. Clear cached observations on forced reloads and cache resets. - Retain live discovery while registration is pending or watching is unsupported, unavailable, or cannot verify the observation. ## Testing Add tests for empty-result reuse and invalidation, skill directory creation before watch acknowledgement, unsupported executors, idle WebSocket notifications, reconnection, case aliases, and bind mounts and unmounts. Include an ignored manual latency comparison for repeated empty remote scans. GitOrigin-RevId: 3ad499decefb184a99c068afa01d0dcdd8605c7f
This commit is contained in:
@@ -50,6 +50,7 @@ codex-websocket-client = { workspace = true }
|
||||
dirs = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
http = { workspace = true }
|
||||
notify = { workspace = true }
|
||||
opentelemetry = { workspace = true }
|
||||
prost = "0.14.3"
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -1068,6 +1068,21 @@ impl ExecServerClient {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn fs_watch_missing(
|
||||
&self,
|
||||
missing: Vec<codex_utils_path_uri::PathUri>,
|
||||
present: Option<codex_utils_path_uri::PathUri>,
|
||||
) -> Result<Option<tokio::sync::oneshot::Receiver<()>>, ExecServerError> {
|
||||
if !self.environment_info().await?.capabilities.fs_watch_missing {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(self
|
||||
.rpc_client()
|
||||
.await?
|
||||
.watch_missing(missing, present)
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub async fn fs_get_metadata(
|
||||
&self,
|
||||
params: FsGetMetadataParams,
|
||||
@@ -2744,6 +2759,10 @@ mod tests {
|
||||
provider_id: Some("sha256:fb4f62da3e84f6864dcec8ede7bc66f1c96ecaeaf55f8a786b85df994057c8ac".to_string()),
|
||||
..EnvironmentInfo::local()
|
||||
}); "with_executor_metadata")]
|
||||
#[test_case::test_case(Some(EnvironmentInfo {
|
||||
capabilities: Default::default(),
|
||||
..EnvironmentInfo::local()
|
||||
}); "without_optional_methods")]
|
||||
#[test_case::test_case(None; "legacy_server")]
|
||||
#[tokio::test]
|
||||
async fn environment_info_is_cached(
|
||||
@@ -2798,6 +2817,18 @@ mod tests {
|
||||
let cached_info = client.clone().environment_info().await?;
|
||||
assert_eq!(cached_info.as_ref(), &expected_info);
|
||||
assert!(Arc::ptr_eq(&info, &cached_info));
|
||||
if !info.capabilities.fs_watch_missing {
|
||||
// The old executor has disconnected: probing an unsupported RPC would fail.
|
||||
assert!(
|
||||
client
|
||||
.fs_watch_missing(
|
||||
vec![PathUri::parse("file:///workspace/.agents/skills")?],
|
||||
/*present*/ None
|
||||
)
|
||||
.await?
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -330,6 +330,22 @@ impl RemoteFileSystem {
|
||||
}
|
||||
|
||||
impl ExecutorFileSystem for RemoteFileSystem {
|
||||
fn watch_missing(
|
||||
&self,
|
||||
missing: Vec<PathUri>,
|
||||
present: Option<PathUri>,
|
||||
) -> ExecutorFileSystemFuture<'_, Option<tokio::sync::oneshot::Receiver<()>>> {
|
||||
Box::pin(async move {
|
||||
self.client
|
||||
.get()
|
||||
.await
|
||||
.map_err(map_remote_error)?
|
||||
.fs_watch_missing(missing, present)
|
||||
.await
|
||||
.map_err(map_remote_error)
|
||||
})
|
||||
}
|
||||
|
||||
fn write_file_stream<'a>(
|
||||
&'a self,
|
||||
path: &'a PathUri,
|
||||
|
||||
@@ -322,7 +322,14 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct MissingWatch {
|
||||
id: u64,
|
||||
invalidated: Option<oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
pub(crate) struct RpcClient {
|
||||
missing_watch: Arc<StdMutex<MissingWatch>>,
|
||||
metrics: Option<MetricsClient>,
|
||||
write_tx: mpsc::Sender<JSONRPCMessage>,
|
||||
pending: Arc<StdMutex<HashMap<RequestId, PendingRequest>>>,
|
||||
@@ -353,6 +360,8 @@ impl RpcClient {
|
||||
let closed = Arc::new(AtomicBool::new(false));
|
||||
let (event_tx, event_rx) = mpsc::channel(128);
|
||||
|
||||
let missing_watch = Arc::new(StdMutex::new(MissingWatch::default()));
|
||||
let reader_watch = Arc::clone(&missing_watch);
|
||||
let pending_for_reader = Arc::clone(&pending);
|
||||
let closed_for_reader = Arc::clone(&closed);
|
||||
let transport_for_reader = transport.clone();
|
||||
@@ -362,6 +371,25 @@ impl RpcClient {
|
||||
break None;
|
||||
};
|
||||
match event {
|
||||
JsonRpcConnectionEvent::Message(JSONRPCMessage::Notification(notification))
|
||||
if notification.method == crate::protocol::FS_MISSING_CHANGED_METHOD =>
|
||||
{
|
||||
let changed = serde_json::from_value::<
|
||||
crate::protocol::FsMissingChangedNotification,
|
||||
>(
|
||||
notification.params.unwrap_or(Value::Null)
|
||||
);
|
||||
let mut watch = reader_watch
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if changed
|
||||
.as_ref()
|
||||
.map_or(true, |changed| changed.watch_id == watch.id)
|
||||
&& let Some(invalidated) = watch.invalidated.take()
|
||||
{
|
||||
let _ = invalidated.send(());
|
||||
}
|
||||
}
|
||||
JsonRpcConnectionEvent::Message(message) => {
|
||||
if let Err(err) =
|
||||
handle_server_message(&pending_for_reader, &event_tx, message).await
|
||||
@@ -396,6 +424,11 @@ impl RpcClient {
|
||||
}
|
||||
};
|
||||
|
||||
reader_watch
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.invalidated
|
||||
.take();
|
||||
closed_for_reader.store(true, Ordering::Release);
|
||||
drain_pending(&pending_for_reader);
|
||||
let _ = event_tx
|
||||
@@ -409,6 +442,7 @@ impl RpcClient {
|
||||
(
|
||||
Self {
|
||||
metrics: codex_otel::global(),
|
||||
missing_watch,
|
||||
write_tx,
|
||||
pending,
|
||||
inbound_request_ids: Arc::new(StdMutex::new(HashSet::new())),
|
||||
@@ -516,12 +550,45 @@ impl RpcClient {
|
||||
.map_err(|_| RpcCallError::Closed)
|
||||
}
|
||||
|
||||
pub(crate) async fn watch_missing(
|
||||
&self,
|
||||
missing: Vec<codex_utils_path_uri::PathUri>,
|
||||
present: Option<codex_utils_path_uri::PathUri>,
|
||||
) -> Result<Option<oneshot::Receiver<()>>, RpcCallError> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let watch_id = {
|
||||
let mut watch = self
|
||||
.missing_watch
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
watch.id += 1;
|
||||
watch.invalidated = Some(tx);
|
||||
watch.id
|
||||
};
|
||||
let response: crate::protocol::FsWatchMissingResponse = self
|
||||
.call(
|
||||
crate::protocol::FS_WATCH_MISSING_METHOD,
|
||||
&crate::protocol::FsWatchMissingParams {
|
||||
watch_id,
|
||||
missing,
|
||||
present,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
Ok(response.watching.then_some(rx))
|
||||
}
|
||||
|
||||
pub(crate) fn is_disconnected(&self) -> bool {
|
||||
self.closed.load(Ordering::Acquire) || *self.disconnected_rx.borrow()
|
||||
}
|
||||
|
||||
pub(crate) async fn close_transport(&self) {
|
||||
self.closed.store(true, Ordering::Release);
|
||||
self.missing_watch
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.invalidated
|
||||
.take();
|
||||
self.transport.terminate();
|
||||
for task in &self.transport_tasks {
|
||||
task.abort();
|
||||
@@ -727,6 +794,11 @@ impl RpcClient {
|
||||
|
||||
impl Drop for RpcClient {
|
||||
fn drop(&mut self) {
|
||||
self.missing_watch
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.invalidated
|
||||
.take();
|
||||
self.transport.terminate();
|
||||
for task in &self.transport_tasks {
|
||||
task.abort();
|
||||
|
||||
@@ -79,6 +79,9 @@ use crate::server::session_registry::SessionHandle;
|
||||
use crate::server::session_registry::SessionRegistry;
|
||||
use crate::telemetry::ExecutorRegistration;
|
||||
|
||||
#[path = "missing_watch.rs"]
|
||||
mod missing_watch;
|
||||
|
||||
pub(crate) struct ExecServerHandler {
|
||||
pub(super) executor_registration: Option<Arc<ExecutorRegistration>>,
|
||||
session_registry: Arc<SessionRegistry>,
|
||||
@@ -86,6 +89,8 @@ pub(crate) struct ExecServerHandler {
|
||||
notifications: RpcNotificationSender,
|
||||
session: StdMutex<Option<SessionHandle>>,
|
||||
active_body_stream_ids: Mutex<HashSet<String>>,
|
||||
// Only the newest observation on this physical connection can be reused.
|
||||
missing_watch: Arc<StdMutex<missing_watch::MissingWatch>>,
|
||||
background_task_shutdown: CancellationToken,
|
||||
background_tasks: TaskTracker,
|
||||
file_system: FileSystemHandler,
|
||||
@@ -110,6 +115,7 @@ impl ExecServerHandler {
|
||||
notifications,
|
||||
session: StdMutex::new(None),
|
||||
active_body_stream_ids: Mutex::new(HashSet::new()),
|
||||
missing_watch: Arc::default(),
|
||||
background_task_shutdown: CancellationToken::new(),
|
||||
background_tasks: TaskTracker::new(),
|
||||
file_system: FileSystemHandler::new(runtime_paths.clone()),
|
||||
@@ -395,6 +401,24 @@ impl ExecServerHandler {
|
||||
self.file_system.create_directory(params).await
|
||||
}
|
||||
|
||||
pub(crate) async fn fs_watch_missing(
|
||||
&self,
|
||||
params: crate::protocol::FsWatchMissingParams,
|
||||
) -> Result<crate::protocol::FsWatchMissingResponse, JSONRPCErrorError> {
|
||||
self.require_initialized_for("filesystem")?;
|
||||
let watch = Arc::clone(&self.missing_watch);
|
||||
let notifications = self.notifications.clone();
|
||||
let shutdown = self.background_task_shutdown.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let Ok(mut active) = watch.try_lock() else {
|
||||
return Ok(crate::protocol::FsWatchMissingResponse { watching: false });
|
||||
};
|
||||
active.register(params, notifications, shutdown)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| crate::rpc::internal_error(error.to_string()))?
|
||||
}
|
||||
|
||||
pub(crate) async fn fs_get_metadata(
|
||||
&self,
|
||||
params: FsGetMetadataParams,
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
//! Watches one missing-path observation per physical executor connection. Changes,
|
||||
//! supersession and transport loss invalidate the observation without a clock deadline.
|
||||
|
||||
use codex_exec_server_protocol::JSONRPCErrorError;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use std::collections::BTreeSet;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::os::unix::fs::OpenOptionsExt as _;
|
||||
use std::path::Path;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
|
||||
use crate::protocol::FS_MISSING_CHANGED_METHOD;
|
||||
use crate::protocol::FsMissingChangedNotification;
|
||||
use crate::protocol::FsWatchMissingParams;
|
||||
use crate::protocol::FsWatchMissingResponse;
|
||||
use crate::rpc::RpcNotificationSender;
|
||||
|
||||
use notify::Event;
|
||||
use notify::EventKind;
|
||||
use notify::RecursiveMode;
|
||||
use notify::Watcher;
|
||||
#[cfg(target_os = "linux")]
|
||||
use tokio::io::Interest;
|
||||
#[cfg(target_os = "linux")]
|
||||
use tokio::io::unix::AsyncFd;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
const MAX_WATCH_PATHS: usize = 64;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct MissingWatch {
|
||||
id: u64,
|
||||
listener: Option<AbortOnDropHandle<()>>,
|
||||
}
|
||||
|
||||
impl MissingWatch {
|
||||
pub(super) fn register(
|
||||
&mut self,
|
||||
params: FsWatchMissingParams,
|
||||
notifications: RpcNotificationSender,
|
||||
shutdown: CancellationToken,
|
||||
) -> Result<FsWatchMissingResponse, JSONRPCErrorError> {
|
||||
let not_watching = FsWatchMissingResponse { watching: false };
|
||||
if params.watch_id <= self.id {
|
||||
return Ok(not_watching);
|
||||
}
|
||||
self.id = params.watch_id;
|
||||
self.listener.take();
|
||||
if !cfg!(target_os = "linux")
|
||||
|| params.missing.is_empty()
|
||||
|| params.missing.len() > MAX_WATCH_PATHS
|
||||
{
|
||||
return Ok(not_watching);
|
||||
}
|
||||
let watch_id = params.watch_id;
|
||||
let watch = match watch_missing(params) {
|
||||
Ok(Some(watch)) => watch,
|
||||
Ok(None) => return Ok(not_watching),
|
||||
Err(error) => {
|
||||
tracing::debug!(%error, "metadata watch unavailable; retaining live discovery");
|
||||
return Ok(not_watching);
|
||||
}
|
||||
};
|
||||
let changed = FsMissingChangedNotification { watch_id };
|
||||
self.listener = Some(AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
_ = shutdown.cancelled() => return,
|
||||
() = watch => {}
|
||||
}
|
||||
tracing::debug!(
|
||||
watch_id = changed.watch_id,
|
||||
"invalidating watched missing paths"
|
||||
);
|
||||
let _ = notifications
|
||||
.notify(FS_MISSING_CHANGED_METHOD, &changed)
|
||||
.await;
|
||||
})));
|
||||
Ok(FsWatchMissingResponse { watching: true })
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe directory and mount changes before verifying the paths. Ancestor changes,
|
||||
/// mount changes, watcher errors and queue overflows invalidate the observation.
|
||||
fn watch_missing(
|
||||
params: FsWatchMissingParams,
|
||||
) -> io::Result<Option<impl Future<Output = ()> + Send>> {
|
||||
let missing = params
|
||||
.missing
|
||||
.iter()
|
||||
.map(|path| path.to_abs_path().map(AbsolutePathBuf::into_path_buf))
|
||||
.collect::<io::Result<Vec<_>>>()?;
|
||||
let present = params
|
||||
.present
|
||||
.map(|path| path.to_abs_path().map(AbsolutePathBuf::into_path_buf))
|
||||
.transpose()?;
|
||||
let mut parents = BTreeSet::new();
|
||||
let targets: Vec<_> = missing.iter().chain(&present).cloned().collect();
|
||||
for target in &targets {
|
||||
for parent in target.ancestors().skip(1) {
|
||||
parents.insert(parent.to_owned());
|
||||
if parents.len() > MAX_WATCH_PATHS {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
// inotify stays attached to a covered directory. Arm mount-table notifications
|
||||
// before installing directory watches and verifying the observed paths.
|
||||
#[cfg(target_os = "linux")]
|
||||
let mounts = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags(libc::O_NONBLOCK)
|
||||
.open("/proc/self/mounts")
|
||||
.and_then(|file| AsyncFd::with_interest(file, Interest::PRIORITY))
|
||||
.inspect_err(
|
||||
|error| tracing::debug!(%error, "mount changes unavailable for missing-path watch"),
|
||||
)
|
||||
.ok();
|
||||
let (changed, invalidated) = oneshot::channel();
|
||||
let mut changed = Some(changed);
|
||||
let mut watcher = notify::recommended_watcher(move |event: notify::Result<Event>| {
|
||||
if event.as_ref().map_or(true, |event| {
|
||||
event.need_rescan()
|
||||
|| (!matches!(event.kind, EventKind::Access(_))
|
||||
&& (event.paths.is_empty()
|
||||
|| event.paths.iter().any(|path| {
|
||||
targets
|
||||
.iter()
|
||||
.any(|target| event_may_affect_target(path, target))
|
||||
})))
|
||||
}) && let Some(changed) = changed.take()
|
||||
{
|
||||
let _ = changed.send(());
|
||||
}
|
||||
})
|
||||
.map_err(io::Error::other)?;
|
||||
|
||||
// Ancestors sort before their descendants. If a missing parent is created during
|
||||
// setup its already-watched parent invalidates the observation. Reject symlinks:
|
||||
// their targets can change outside any of these directories.
|
||||
for parent in parents {
|
||||
match std::fs::symlink_metadata(&parent) {
|
||||
Ok(metadata) if metadata.is_dir() => {
|
||||
// inotify can accept a watch on a network mount yet never see changes
|
||||
// made by another host. Only reuse observations on local Linux filesystems.
|
||||
#[cfg(target_os = "linux")]
|
||||
if !matches!(
|
||||
rustix::fs::statfs(&parent)?.f_type as _,
|
||||
libc::EXT4_SUPER_MAGIC
|
||||
| 0x5846_5342 /* XFS_SUPER_MAGIC from linux/magic.h; absent in libc on musl */
|
||||
| libc::BTRFS_SUPER_MAGIC
|
||||
| libc::TMPFS_MAGIC
|
||||
| libc::OVERLAYFS_SUPER_MAGIC
|
||||
) {
|
||||
return Ok(None);
|
||||
}
|
||||
watcher
|
||||
.watch(&parent, RecursiveMode::NonRecursive)
|
||||
.map_err(io::Error::other)?;
|
||||
}
|
||||
Ok(_) => return Ok(None),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
for path in missing {
|
||||
match std::fs::symlink_metadata(path) {
|
||||
Ok(_) => return Ok(None),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
if let Some(path) = present {
|
||||
match std::fs::symlink_metadata(path) {
|
||||
Ok(metadata) if !metadata.is_symlink() => {}
|
||||
Ok(_) => return Ok(None),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
Ok(Some(async move {
|
||||
let _directory_watcher = watcher;
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Some(mounts) = mounts {
|
||||
tokio::select! {
|
||||
_ = invalidated => {}
|
||||
_ = mounts.ready(Interest::PRIORITY) => {}
|
||||
}
|
||||
return;
|
||||
}
|
||||
let _ = invalidated.await;
|
||||
}))
|
||||
}
|
||||
|
||||
/// Events preserve the spelling that was written, even in casefolded directories.
|
||||
/// When either component is non-ASCII, let the next scan resolve Unicode aliases
|
||||
/// using the filesystem's own rules.
|
||||
fn event_may_affect_target(path: &Path, target: &Path) -> bool {
|
||||
let mut components = target.iter();
|
||||
path.iter().all(|name| {
|
||||
components.next().is_some_and(|expected| {
|
||||
name.eq_ignore_ascii_case(expected) || !name.is_ascii() || !expected.is_ascii()
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -187,6 +187,12 @@ pub(crate) fn build_router() -> RpcRouter<ExecServerHandler> {
|
||||
handler.fs_create_directory(params).await
|
||||
},
|
||||
);
|
||||
router.request(
|
||||
crate::protocol::FS_WATCH_MISSING_METHOD,
|
||||
|handler: Arc<ExecServerHandler>, params: crate::protocol::FsWatchMissingParams| async move {
|
||||
handler.fs_watch_missing(params).await
|
||||
},
|
||||
);
|
||||
router.request(
|
||||
FS_GET_METADATA_METHOD,
|
||||
|handler: Arc<ExecServerHandler>, params: WireFsGetMetadataParams| async move {
|
||||
|
||||
@@ -2,6 +2,9 @@ mod common;
|
||||
#[cfg(target_os = "linux")]
|
||||
#[path = "common/fake_bwrap.rs"]
|
||||
mod fake_bwrap;
|
||||
#[cfg(target_os = "linux")]
|
||||
#[path = "capability_discovery/mount_watch.rs"]
|
||||
mod mount_watch;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use anyhow::Context as _;
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
//! Exercises missing-path watches across mount changes on an idle executor WebSocket.
|
||||
|
||||
use super::common::exec_server::ExecServerHarness;
|
||||
use super::common::exec_server::exec_server;
|
||||
use super::discover_root;
|
||||
use super::initialize;
|
||||
use super::write_file;
|
||||
use anyhow::Context as _;
|
||||
use codex_exec_server_protocol::FS_MISSING_CHANGED_METHOD;
|
||||
use codex_exec_server_protocol::FS_WATCH_MISSING_METHOD;
|
||||
use codex_exec_server_protocol::JSONRPCMessage;
|
||||
use codex_exec_server_protocol::JSONRPCNotification;
|
||||
use codex_exec_server_protocol::JSONRPCResponse;
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
const PARENT_NAMESPACE: &str = "CODEX_TEST_MISSING_WATCH_PARENT_MOUNT_NS";
|
||||
|
||||
#[test]
|
||||
fn bind_mount_and_unmount_reveal_skills() -> anyhow::Result<()> {
|
||||
// The parent removes the fixtures after the child's private mounts disappear.
|
||||
let tmp = tempfile::tempdir()?;
|
||||
let output = Command::new("unshare")
|
||||
.args([
|
||||
"--user",
|
||||
"--map-root-user",
|
||||
"--mount",
|
||||
"--propagation",
|
||||
"private",
|
||||
"--",
|
||||
])
|
||||
.arg(std::env::current_exe()?)
|
||||
.args([
|
||||
"--exact",
|
||||
"mount_watch::in_private_namespace",
|
||||
"--ignored",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(PARENT_NAMESPACE, std::fs::read_link("/proc/self/ns/mnt")?)
|
||||
.env("LC_ALL", "C")
|
||||
.env("TMPDIR", tmp.path())
|
||||
.output();
|
||||
if let Err(error) = &output
|
||||
&& error.kind() == std::io::ErrorKind::NotFound
|
||||
{
|
||||
eprintln!("skipping mount watch test: unshare unavailable");
|
||||
return Ok(());
|
||||
}
|
||||
let output = output?;
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
if output.status.code() == Some(1)
|
||||
&& stderr.starts_with("unshare:")
|
||||
&& [
|
||||
"Operation not permitted",
|
||||
"Permission denied",
|
||||
"Read-only file system",
|
||||
"Function not implemented",
|
||||
]
|
||||
.iter()
|
||||
.any(|reason| stderr.contains(reason))
|
||||
{
|
||||
eprintln!("skipping mount watch test: {stderr}");
|
||||
return Ok(());
|
||||
}
|
||||
assert!(
|
||||
output.status.success() && String::from_utf8_lossy(&output.stdout).contains("1 passed;"),
|
||||
"{output:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "run by bind_mount_and_unmount_reveal_skills in a private Linux mount namespace"]
|
||||
async fn in_private_namespace() -> anyhow::Result<()> {
|
||||
let parent = std::env::var_os(PARENT_NAMESPACE).context("use the namespace wrapper")?;
|
||||
assert_ne!(std::fs::read_link("/proc/self/ns/mnt")?.as_os_str(), parent);
|
||||
let fixture = tempfile::tempdir()?;
|
||||
let empty = fixture.path().join("empty/.agents");
|
||||
let populated = fixture.path().join("populated/.agents");
|
||||
std::fs::create_dir_all(&empty)?;
|
||||
write_file(
|
||||
&populated.join("skills/mounted/SKILL.md"),
|
||||
"---\nname: mounted\ndescription: Mount test\n---\n",
|
||||
)?;
|
||||
let mut server = exec_server().await?;
|
||||
initialize(&mut server).await?;
|
||||
|
||||
assert_reveals_skills(
|
||||
&mut server,
|
||||
&empty,
|
||||
/*watch_id*/ 1,
|
||||
Command::new("mount")
|
||||
.arg("--bind")
|
||||
.arg(&populated)
|
||||
.arg(&empty),
|
||||
)
|
||||
.await?;
|
||||
run(Command::new("umount").arg(&empty))?;
|
||||
|
||||
run(Command::new("mount")
|
||||
.arg("--bind")
|
||||
.arg(&empty)
|
||||
.arg(&populated))?;
|
||||
assert_reveals_skills(
|
||||
&mut server,
|
||||
&populated,
|
||||
/*watch_id*/ 2,
|
||||
Command::new("umount").arg(&populated),
|
||||
)
|
||||
.await?;
|
||||
server.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_reveals_skills(
|
||||
server: &mut ExecServerHarness,
|
||||
agents: &Path,
|
||||
watch_id: u64,
|
||||
change: &mut Command,
|
||||
) -> anyhow::Result<()> {
|
||||
let root = PathUri::from_host_native_path(agents)?;
|
||||
let before = discover_root(server, "repository", root.clone()).await?;
|
||||
assert_eq!((before.error, before.skills), (None, vec![]));
|
||||
|
||||
let id = server
|
||||
.send_request(
|
||||
FS_WATCH_MISSING_METHOD,
|
||||
serde_json::json!({ "watchId": watch_id, "missing": [root.join("skills")?] }),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
server.next_event().await?,
|
||||
JSONRPCMessage::Response(JSONRPCResponse {
|
||||
id,
|
||||
result: serde_json::json!({ "watching": true })
|
||||
})
|
||||
);
|
||||
run(change)?;
|
||||
assert_eq!(
|
||||
server
|
||||
.next_event()
|
||||
.await
|
||||
.context("expected a mount change on the idle socket")?,
|
||||
JSONRPCMessage::Notification(JSONRPCNotification {
|
||||
method: FS_MISSING_CHANGED_METHOD.to_string(),
|
||||
params: Some(serde_json::json!({ "watchId": watch_id })),
|
||||
})
|
||||
);
|
||||
|
||||
let after = discover_root(server, "repository", root.clone()).await?;
|
||||
let skills: Vec<_> = after
|
||||
.skills
|
||||
.into_iter()
|
||||
.map(|skill| skill.instructions.path)
|
||||
.collect();
|
||||
assert_eq!(
|
||||
(after.error, skills),
|
||||
(None, vec![root.join("skills/mounted/SKILL.md")?])
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run(command: &mut Command) -> anyhow::Result<()> {
|
||||
let output = command.output()?;
|
||||
anyhow::ensure!(
|
||||
output.status.success(),
|
||||
"{command:?}: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -2,13 +2,23 @@
|
||||
|
||||
mod common;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use anyhow::Context;
|
||||
#[cfg(target_os = "linux")]
|
||||
use codex_exec_server::Environment;
|
||||
use codex_exec_server::InitializeParams;
|
||||
use codex_exec_server::InitializeResponse;
|
||||
use codex_exec_server_protocol::JSONRPCError;
|
||||
use codex_exec_server_protocol::JSONRPCMessage;
|
||||
use codex_exec_server_protocol::JSONRPCResponse;
|
||||
#[cfg(target_os = "linux")]
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
#[cfg(target_os = "linux")]
|
||||
use codex_utils_path_uri::PathUri;
|
||||
use common::exec_server::exec_server;
|
||||
use pretty_assertions::assert_eq;
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::time::Duration;
|
||||
use tokio_tungstenite::connect_async;
|
||||
use tokio_tungstenite::tungstenite::Error as WebSocketError;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
@@ -143,3 +153,99 @@ async fn exec_server_rejects_browser_origin_websocket_handshake() -> anyhow::Res
|
||||
server.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn missing_metadata_watch_invalidates_on_idle_websocket_and_recovery() -> anyhow::Result<()> {
|
||||
let mut server = exec_server().await?;
|
||||
let mut proxy = server.disconnectable_websocket_proxy().await?;
|
||||
let environment = Environment::create_for_tests(Some(proxy.websocket_url().to_owned()))?;
|
||||
let fs = environment.get_filesystem();
|
||||
let repo = tempfile::tempdir()?;
|
||||
let root = AbsolutePathBuf::try_from(repo.path().canonicalize()?)?;
|
||||
let skills = root.join(".agents/skills");
|
||||
let marker = root.join(".git");
|
||||
std::fs::write(&marker, "")?;
|
||||
let watch = || {
|
||||
fs.watch_missing(
|
||||
vec![PathUri::from_abs_path(&skills)],
|
||||
Some(PathUri::from_abs_path(&marker)),
|
||||
)
|
||||
};
|
||||
let superseded = watch().await?.context("first watch")?;
|
||||
let mut invalidated = watch()
|
||||
.await?
|
||||
.context("executor accepted empty observation")?;
|
||||
assert!(superseded.await.is_err());
|
||||
std::fs::write(root.join(".agents-backup"), "")?;
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut invalidated)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
std::fs::create_dir_all(&skills)?;
|
||||
// The executor pushes this across the already open socket, without another RPC.
|
||||
tokio::time::timeout(Duration::from_secs(5), invalidated).await??;
|
||||
assert!(watch().await?.is_none());
|
||||
|
||||
std::fs::remove_dir(&skills)?;
|
||||
let invalidated = watch().await?.context("watch after deletion")?;
|
||||
std::fs::remove_file(&marker)?;
|
||||
tokio::time::timeout(Duration::from_secs(5), invalidated).await??;
|
||||
assert!(watch().await?.is_none());
|
||||
|
||||
std::fs::write(&marker, "")?;
|
||||
let old = watch().await?.context("watch before disconnect")?;
|
||||
proxy.pause_and_disconnect().await?;
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(5), old)
|
||||
.await?
|
||||
.is_err()
|
||||
);
|
||||
std::fs::create_dir(&skills)?;
|
||||
proxy.resume()?;
|
||||
assert!(watch().await?.is_none());
|
||||
std::fs::remove_dir(&skills)?;
|
||||
let new = watch()
|
||||
.await?
|
||||
.context("new watch on recovered connection")?;
|
||||
std::fs::create_dir(&skills)?;
|
||||
tokio::time::timeout(Duration::from_secs(5), new).await??;
|
||||
|
||||
server.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn missing_metadata_watch_invalidates_on_possible_casefold_aliases() -> anyhow::Result<()> {
|
||||
let mut server = exec_server().await?;
|
||||
let environment = Environment::create_for_tests(Some(server.websocket_url().to_owned()))?;
|
||||
let fs = environment.get_filesystem();
|
||||
|
||||
// U+017F (long s) folds to ASCII s in a casefolded ext4 directory.
|
||||
for (parent, missing, created) in [
|
||||
("", ".agents/skills", ".Agents"),
|
||||
(".agents", "skills", "Skills"),
|
||||
(".agents", "skills", "\u{017f}kills"),
|
||||
(".agents", "\u{017f}kills", "skills"),
|
||||
] {
|
||||
let repo = tempfile::tempdir()?;
|
||||
let root = AbsolutePathBuf::try_from(repo.path().canonicalize()?)?;
|
||||
let parent = root.join(parent);
|
||||
std::fs::create_dir_all(&parent)?;
|
||||
let missing = PathUri::from_abs_path(&parent.join(missing));
|
||||
let invalidated = fs
|
||||
.watch_missing(vec![missing], None)
|
||||
.await?
|
||||
.context("executor accepted empty observation")?;
|
||||
|
||||
std::fs::create_dir(parent.join(created))?;
|
||||
tokio::time::timeout(Duration::from_secs(5), invalidated)
|
||||
.await
|
||||
.with_context(|| format!("watch missed {created}"))??;
|
||||
}
|
||||
|
||||
server.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user