Backport alpha.7 source to the alpha.6.1 hotfix line

This commit is contained in:
jif-oai
2026-08-04 13:15:28 +01:00
parent 51d4aa946c
commit d38c809843
135 changed files with 23925 additions and 1482 deletions
+181 -58
View File
@@ -21,6 +21,7 @@ import re
import subprocess
import sys
import tempfile
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any, NamedTuple, NoReturn
from urllib.parse import quote
@@ -30,6 +31,7 @@ PREFIX = "codex"
REPOSITORY = "openai/codex"
RELEASE_METADATA_NAME = "release.json"
INSTALLER_NAMES = ("install.sh", "install.ps1")
MAX_UPLOAD_WORKERS = 8
# Keep this pattern in sync with release-tag validation in
# .github/workflows/rust-release.yml.
VERSION_RE = re.compile(
@@ -38,6 +40,7 @@ VERSION_RE = re.compile(
)
CRC64_RE = re.compile(r"^[A-Za-z0-9+/]{11}=$")
SHA256_RE = re.compile(r"^sha256:([0-9a-f]{64})$")
MISSING_OBJECT_RE = re.compile(r"\((?:404|NoSuchKey|NotFound)\)")
class PublishError(RuntimeError):
@@ -65,7 +68,7 @@ def run_command(args: list[str]) -> str:
return result.stdout or ""
def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:
def get_release_metadata(tag: str, directory: Path, stage: str) -> list[ReleaseAsset]:
try:
metadata = json.loads(
run_command(
@@ -83,21 +86,9 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:
]
)
)
run_command(
[
"gh",
"release",
"download",
tag,
"--repo",
REPOSITORY,
"--dir",
str(directory),
]
)
except (OSError, subprocess.CalledProcessError) as error:
raise PublishError(
f"GitHub release download failed for {tag}: {error}"
f"GitHub release metadata request failed for {tag}: {error}"
) from error
except json.JSONDecodeError as error:
raise PublishError(
@@ -112,6 +103,9 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:
raise PublishError(
f"GitHub returned invalid release metadata for {tag}: {asset!r}"
)
# DotSlash runs concurrently, so defer its in-progress assets to finalization.
if stage == "assets" and asset.get("state") != "uploaded":
continue
name = asset.get("name")
size = asset.get("size")
digest = asset.get("digest")
@@ -131,14 +125,37 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:
)
expected[name] = ReleaseAsset(directory / name, size, match.group(1))
assets = sorted(directory.iterdir(), key=lambda path: path.name)
if not assets:
if not expected:
raise PublishError(f"GitHub Release {tag} has no assets")
if any(not path.is_file() for path in assets) or {
path.name for path in assets
} != set(expected):
return sorted(expected.values(), key=lambda asset: asset.path.name)
def download_assets(tag: str, directory: Path, assets: list[ReleaseAsset]) -> None:
if not assets:
return
command = [
"gh",
"release",
"download",
tag,
"--repo",
REPOSITORY,
"--dir",
str(directory),
]
for asset in assets:
command.extend(["--pattern", asset.path.name])
try:
run_command(command)
except (OSError, subprocess.CalledProcessError) as error:
raise PublishError(
f"GitHub release download failed for {tag}: {error}"
) from error
if any(not asset.path.is_file() for asset in assets):
raise PublishError("GitHub returned invalid release assets")
return [expected[path.name] for path in assets]
def stream_digest(source: Any) -> tuple[int, str]:
@@ -150,6 +167,17 @@ def stream_digest(source: Any) -> tuple[int, str]:
return size, digest.hexdigest()
def validate_asset(asset: ReleaseAsset) -> None:
with asset.path.open("rb") as source:
size, sha256 = stream_digest(source)
if size != asset.size or sha256 != asset.sha256:
raise PublishError(
f"GitHub asset mismatch for {asset.path.name}: expected "
f"size={asset.size} sha256={asset.sha256}, got "
f"size={size} sha256={sha256}"
)
def raise_s3(
action: str, key: str, error: Exception, detail: str | None = None
) -> NoReturn:
@@ -246,6 +274,7 @@ def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> N
)
for name in INSTALLER_NAMES:
asset = installers[name]
validate_asset(asset)
installer_key = f"{PREFIX}/{name}"
put_object(endpoint, installer_key, asset.path, asset.sha256, extra_args=[])
verify_remote(endpoint, installer_key, asset.size, asset.sha256)
@@ -256,11 +285,92 @@ def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> N
)
def publish_asset(
endpoint: str, version: str, asset: ReleaseAsset, *, verify: bool = True
) -> dict[str, Any]:
validate_asset(asset)
key = f"{PREFIX}/releases/{version}/{asset.path.name}"
put_object(endpoint, key, asset.path, asset.sha256, extra_args=["--no-overwrite"])
if verify:
verify_remote(endpoint, key, asset.size, asset.sha256)
status = "published and verified" if verify else "published"
print(
f"{status} s3://{BUCKET}/{key} size={asset.size} sha256={asset.sha256}",
file=sys.stderr,
)
return {
"key": key,
"name": asset.path.name,
"sha256": asset.sha256,
"size": asset.size,
}
def publish_assets(
endpoint: str, version: str, assets: list[ReleaseAsset], *, verify: bool = True
) -> list[dict[str, Any]]:
if not assets:
return []
published = {}
with ThreadPoolExecutor(
max_workers=min(MAX_UPLOAD_WORKERS, len(assets))
) as executor:
futures = {
executor.submit(
publish_asset, endpoint, version, asset, verify=verify
): asset
for asset in assets
}
for future in as_completed(futures):
asset = futures[future]
published[asset.path.name] = future.result()
return [published[asset.path.name] for asset in assets]
def find_published_assets(
endpoint: str, version: str, assets: list[ReleaseAsset]
) -> dict[str, dict[str, Any]]:
published = {}
with ThreadPoolExecutor(
max_workers=min(MAX_UPLOAD_WORKERS, len(assets))
) as executor:
futures = {
executor.submit(
verify_remote,
endpoint,
f"{PREFIX}/releases/{version}/{asset.path.name}",
asset.size,
asset.sha256,
): asset
for asset in assets
}
for future in as_completed(futures):
asset = futures[future]
try:
future.result()
except PublishError as error:
cause = error.__cause__
if isinstance(cause, subprocess.CalledProcessError) and (
MISSING_OBJECT_RE.search(cause.stderr or "")
):
continue
raise
published[asset.path.name] = {
"key": f"{PREFIX}/releases/{version}/{asset.path.name}",
"name": asset.path.name,
"sha256": asset.sha256,
"size": asset.size,
}
return published
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--tag", required=True)
parser.add_argument("--make-latest", choices=("true", "false"), required=True)
parser.add_argument("--prerelease", choices=("true", "false"), required=True)
parser.add_argument("--stage", choices=("assets", "finalize"), required=True)
return parser.parse_args()
@@ -276,50 +386,62 @@ def main() -> int:
version = args.tag.removeprefix("rust-v")
if args.tag == version or not VERSION_RE.fullmatch(version):
raise PublishError(f"invalid rust release tag: {args.tag}")
published = []
metadata_assets = []
with tempfile.TemporaryDirectory() as temp_dir:
assets_directory = Path(temp_dir) / "assets"
assets_directory.mkdir()
assets = download_assets(args.tag, assets_directory)
for asset in assets:
with asset.path.open("rb") as source:
size, sha256 = stream_digest(source)
if size != asset.size or sha256 != asset.sha256:
raise PublishError(
f"GitHub asset mismatch for {asset.path.name}: expected "
f"size={asset.size} sha256={asset.sha256}, got "
f"size={size} sha256={sha256}"
)
for asset in assets:
path = asset.path
size = asset.size
sha256 = asset.sha256
key = f"{PREFIX}/releases/{version}/{path.name}"
put_object(endpoint, key, path, sha256, extra_args=["--no-overwrite"])
verify_remote(endpoint, key, size, sha256)
assets = get_release_metadata(args.tag, assets_directory, args.stage)
if args.stage == "assets":
download_assets(args.tag, assets_directory, assets)
published = publish_assets(endpoint, version, assets, verify=False)
print(
f"published and verified s3://{BUCKET}/{key} "
f"size={size} sha256={sha256}",
file=sys.stderr,
json.dumps(
{
"assetCount": len(published),
"assets": published,
"releasePrefix": f"{PREFIX}/releases/{version}/",
"stage": args.stage,
"tag": args.tag,
"version": version,
},
sort_keys=True,
)
)
published.append(
{
"key": key,
"sha256": sha256,
"size": size,
}
)
metadata_assets.append(
{
"name": path.name,
"digest": f"sha256:{sha256}",
"browser_download_url": (
f"https://releases.openai.com/{PREFIX}/releases/"
f"{version}/{quote(path.name, safe='')}"
),
}
return 0
previously_published = find_published_assets(endpoint, version, assets)
remaining = [
asset for asset in assets if asset.path.name not in previously_published
]
required_downloads = list(remaining)
if args.prerelease == "false":
required_downloads.extend(
asset
for asset in assets
if asset.path.name in INSTALLER_NAMES
and asset.path.name in previously_published
)
download_assets(args.tag, assets_directory, required_downloads)
additionally_published = {
asset["name"]: asset
for asset in publish_assets(endpoint, version, remaining)
}
published = [
previously_published.get(asset.path.name)
or additionally_published[asset.path.name]
for asset in assets
]
metadata_assets = [
{
"name": asset.path.name,
"digest": f"sha256:{asset.sha256}",
"browser_download_url": (
f"https://releases.openai.com/{PREFIX}/releases/"
f"{version}/{quote(asset.path.name, safe='')}"
),
}
for asset in assets
]
metadata_path = Path(temp_dir) / RELEASE_METADATA_NAME
metadata_path.write_text(
@@ -393,6 +515,7 @@ def main() -> int:
"size": metadata_size,
},
"releasePrefix": f"{PREFIX}/releases/{version}/",
"stage": args.stage,
"tag": args.tag,
"version": version,
},
+6 -1
View File
@@ -12,6 +12,9 @@ on:
prerelease:
required: true
type: boolean
stage:
required: true
type: string
permissions: {}
@@ -40,9 +43,11 @@ jobs:
RELEASE_TAG: ${{ inputs.tag }}
RELEASE_MAKE_LATEST: ${{ inputs.make_latest }}
RELEASE_PRERELEASE: ${{ inputs.prerelease }}
RELEASE_STAGE: ${{ inputs.stage }}
run: |
set -euo pipefail
python3 .github/scripts/publish_r2_release.py \
--tag "${RELEASE_TAG}" \
--make-latest "${RELEASE_MAKE_LATEST}" \
--prerelease "${RELEASE_PRERELEASE}"
--prerelease "${RELEASE_PRERELEASE}" \
--stage "${RELEASE_STAGE}"
+17 -3
View File
@@ -1356,9 +1356,9 @@ jobs:
# (e.g. -alpha, -beta). Otherwise publish a normal release.
prerelease: ${{ steps.release_name.outputs.prerelease }}
publish-r2:
name: publish-r2
needs: [release, publish-dotslash]
publish-r2-assets:
name: publish-r2-assets
needs: release
permissions:
contents: read
uses: ./.github/workflows/r2-release.yml
@@ -1367,6 +1367,20 @@ jobs:
tag: ${{ needs.release.outputs.tag }}
make_latest: ${{ fromJSON(needs.release.outputs.make_latest) }}
prerelease: ${{ fromJSON(needs.release.outputs.prerelease) }}
stage: assets
publish-r2:
name: publish-r2
needs: [release, publish-dotslash, publish-r2-assets]
permissions:
contents: read
uses: ./.github/workflows/r2-release.yml
secrets: inherit
with:
tag: ${{ needs.release.outputs.tag }}
make_latest: ${{ fromJSON(needs.release.outputs.make_latest) }}
prerelease: ${{ fromJSON(needs.release.outputs.prerelease) }}
stage: finalize
publish-dotslash:
name: publish-dotslash
+24 -1
View File
@@ -2571,6 +2571,7 @@ dependencies = [
"tokio-util",
"tracing",
"tracing-subscriber",
"uuid",
]
[[package]]
@@ -2761,6 +2762,7 @@ dependencies = [
"codex-thread-store",
"codex-tools",
"codex-utils-absolute-path",
"codex-utils-audio",
"codex-utils-cache",
"codex-utils-cargo-bin",
"codex-utils-home-dir",
@@ -2802,7 +2804,6 @@ dependencies = [
"sha1 0.10.6",
"shlex",
"similar",
"symphonia",
"tempfile",
"test-case",
"test-log",
@@ -4086,6 +4087,9 @@ dependencies = [
"codex-utils-absolute-path",
"codex-utils-path-uri",
"include_dir",
"pretty_assertions",
"serde",
"serde_yaml",
"thiserror 2.0.18",
]
@@ -4103,14 +4107,19 @@ dependencies = [
"codex-skills",
"codex-tools",
"codex-utils-absolute-path",
"codex-utils-cargo-bin",
"codex-utils-path-uri",
"codex-utils-plugins",
"codex-utils-string",
"futures",
"insta",
"opentelemetry_sdk",
"pretty_assertions",
"schemars 0.8.22",
"serde",
"serde_json",
"serde_yaml",
"tempfile",
"tokio",
"tracing",
"url",
@@ -4377,6 +4386,20 @@ dependencies = [
"codex-protocol",
]
[[package]]
name = "codex-utils-audio"
version = "0.0.0"
dependencies = [
"base64 0.22.1",
"codex-protocol",
"codex-utils-cache",
"codex-utils-string",
"pretty_assertions",
"symphonia",
"thiserror 2.0.18",
"tracing",
]
[[package]]
name = "codex-utils-cache"
version = "0.0.0"
+2
View File
@@ -94,6 +94,7 @@ members = [
"v8-poc",
"websocket-client",
"utils/absolute-path",
"utils/audio",
"utils/path-uri",
"utils/cargo-bin",
"git-utils",
@@ -246,6 +247,7 @@ codex-tui = { path = "tui" }
codex-uds = { path = "uds" }
codex-utils-absolute-path = { path = "utils/absolute-path" }
codex-utils-approval-presets = { path = "utils/approval-presets" }
codex-utils-audio = { path = "utils/audio" }
codex-utils-cache = { path = "utils/cache" }
codex-utils-cargo-bin = { path = "utils/cargo-bin" }
codex-utils-cli = { path = "utils/cli" }
+1
View File
@@ -27,6 +27,7 @@ tokio = { workspace = true, features = ["io-std", "io-util", "macros", "net", "p
tokio-util = { workspace = true, features = ["rt"] }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
[dev-dependencies]
codex-code-mode = { workspace = true }
+108 -5
View File
@@ -12,6 +12,7 @@ use codex_code_mode_protocol::host::Capability;
use codex_code_mode_protocol::host::CapabilitySet;
use codex_code_mode_protocol::host::ClientHello;
use codex_code_mode_protocol::host::ClientToHost;
use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY;
use codex_code_mode_protocol::host::EncodedFrame;
use codex_code_mode_protocol::host::FramedReader;
use codex_code_mode_protocol::host::FramedWriter;
@@ -33,6 +34,7 @@ use tokio::sync::mpsc;
use tokio::sync::oneshot;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
use uuid::Uuid;
use super::HostState;
use super::MAX_ACTIVE_CELLS;
@@ -41,8 +43,12 @@ use super::MAX_RECENT_REQUEST_IDS;
use super::RequestKind;
use super::RequestRegistry;
use super::SeenSessionIds;
use super::negotiate;
use super::peer::HostPeer;
use super::run;
use super::transport::BulkConnectionRegistry;
use super::transport::ConnectionReader;
use super::transport::ConnectionWriter;
fn client_hello(
versions: impl IntoIterator<Item = ProtocolVersion>,
@@ -250,6 +256,107 @@ impl AsyncWrite for BlockingWriter {
}
}
#[tokio::test]
async fn failed_host_hello_removes_the_bulk_pairing_reservation() {
let (host_reader, client_writer) = tokio::io::duplex(/*max_buf_size*/ 4096);
let capability = Capability::new(DUAL_WEBSOCKET_CAPABILITY).expect("dual websocket capability");
let hello = ClientToHost::ClientHello(
ClientHello::new(
SupportedProtocolVersions::try_new([ProtocolVersion::V1]).expect("supported versions"),
CapabilitySet::empty(),
CapabilitySet::try_new([capability]).expect("optional capabilities"),
)
.expect("client hello"),
);
FramedWriter::new(client_writer)
.write(&hello)
.await
.expect("write client hello");
let bytes = Arc::new(Mutex::new(Vec::new()));
let registry = BulkConnectionRegistry::default();
let mut reader = ConnectionReader::from_reader(host_reader);
let mut writer = ConnectionWriter::from_writer(FailingHandshakeWriter {
bytes: Arc::clone(&bytes),
});
let result = negotiate(&mut reader, &mut writer, Some(&registry)).await;
assert!(result.is_err());
let message = EncodedFrame::decode_framed::<HostToClient>(
&bytes.lock().unwrap_or_else(PoisonError::into_inner),
)
.expect("decode failed host hello");
let HostToClient::HostHello(hello) = message else {
panic!("expected a host hello");
};
let token = hello
.bulk_connection_token()
.expect("dual websocket pairing token");
let token = Uuid::parse_str(token).expect("UUID pairing token");
assert!(registry.remove(token).is_none());
}
struct FailingHandshakeWriter {
bytes: Arc<Mutex<Vec<u8>>>,
}
impl AsyncWrite for FailingHandshakeWriter {
fn poll_write(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
bytes: &[u8],
) -> Poll<std::io::Result<usize>> {
self.bytes
.lock()
.unwrap_or_else(PoisonError::into_inner)
.extend_from_slice(bytes);
Poll::Ready(Ok(bytes.len()))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Err(std::io::Error::other("host hello write failed")))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
}
#[tokio::test]
async fn optional_dual_websocket_capability_falls_back_to_a_single_connection() {
let (host_stream, client_stream) = tokio::io::duplex(/*max_buf_size*/ 1024);
let (host_reader, host_writer) = tokio::io::split(host_stream);
let (client_reader, client_writer) = tokio::io::split(client_stream);
let host = tokio::spawn(run(host_reader, host_writer));
let mut reader = FramedReader::new(client_reader);
let mut writer = FramedWriter::new(client_writer);
let capability = Capability::new(DUAL_WEBSOCKET_CAPABILITY).expect("dual websocket capability");
writer
.write(&ClientToHost::ClientHello(
ClientHello::new(
SupportedProtocolVersions::try_new([ProtocolVersion::V1])
.expect("supported versions"),
CapabilitySet::empty(),
CapabilitySet::try_new([capability]).expect("optional capabilities"),
)
.expect("client hello"),
))
.await
.expect("write hello");
assert_eq!(
reader.read::<HostToClient>().await.expect("host hello"),
Some(HostToClient::HostHello(HostHello::new(
ProtocolVersion::V1,
CapabilitySet::empty(),
)))
);
drop(writer);
drop(reader);
host.await.expect("host task").expect("host connection");
}
#[tokio::test]
async fn incompatible_or_invalid_handshake_is_rejected() {
let (host_stream, client_stream) = tokio::io::duplex(/*max_buf_size*/ 1024);
@@ -395,7 +502,7 @@ async fn session_id_cannot_be_reused_after_shutdown() {
}
#[test]
fn request_cancellation_tombstones_are_bounded() {
fn request_history_is_bounded() {
let mut requests = RequestRegistry::default();
let duplicate = request_id(/*value*/ -1);
requests
@@ -411,10 +518,6 @@ fn request_cancellation_tombstones_are_bounded() {
requests.cancel(id);
requests.finish(id);
}
for value in 10_000..20_000 {
requests.cancel(request_id(value));
}
assert!(requests.active.is_empty());
assert_eq!(requests.recent.len(), MAX_RECENT_REQUEST_IDS);
assert_eq!(requests.recent_order.len(), MAX_RECENT_REQUEST_IDS);
+138 -40
View File
@@ -10,18 +10,22 @@ use std::time::Duration;
use anyhow::Context;
use anyhow::Result;
use codex_code_mode_protocol::host::Capability;
use codex_code_mode_protocol::host::CapabilitySet;
use codex_code_mode_protocol::host::ClientToHost;
use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY;
use codex_code_mode_protocol::host::EncodedFrame;
use codex_code_mode_protocol::host::HandshakeRejectReason;
use codex_code_mode_protocol::host::HostHello;
use codex_code_mode_protocol::host::HostRequest;
use codex_code_mode_protocol::host::HostResponse;
use codex_code_mode_protocol::host::HostToClient;
use codex_code_mode_protocol::host::MAX_PENDING_DELEGATE_CALLS;
use codex_code_mode_protocol::host::ProtocolVersion;
use codex_code_mode_protocol::host::RequestId;
use codex_code_mode_protocol::host::SessionId;
use codex_code_mode_protocol::host::SupportedProtocolVersions;
use codex_code_mode_protocol::host::TransportLane;
use codex_code_mode_runtime::InProcessCodeModeSession;
use tokio::io::AsyncRead;
use tokio::io::AsyncWrite;
@@ -32,6 +36,8 @@ use tokio_util::task::TaskTracker;
use self::delegate::RemoteDelegate;
use self::peer::HostPeer;
use self::transport::BulkConnectionRegistration;
use self::transport::BulkConnectionRegistry;
use self::transport::ConnectionReader;
use self::transport::ConnectionWriter;
@@ -46,6 +52,14 @@ const MAX_ACTIVE_CELLS: usize = 128;
const MAX_RECENT_REQUEST_IDS: usize = 4096;
const MAX_RECENT_SESSION_IDS: usize = 4096;
const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
const OUTGOING_CHANNEL_CAPACITY: usize = 128;
const BULK_PAIRING_TIMEOUT: Duration = Duration::from_secs(10);
enum NegotiatedConnection {
Rejected,
Single,
Dual(BulkConnectionRegistration),
}
struct HostLimits {
request_permits: Arc<Semaphore>,
@@ -81,6 +95,7 @@ where
ConnectionReader::from_reader(reader),
ConnectionWriter::from_writer(writer),
Arc::new(HostLimits::new()),
/*bulk_connections*/ None,
)
.await
}
@@ -89,13 +104,39 @@ async fn run_connection(
mut reader: ConnectionReader,
mut writer: ConnectionWriter,
limits: Arc<HostLimits>,
bulk_connections: Option<BulkConnectionRegistry>,
) -> Result<()> {
if !negotiate(&mut reader, &mut writer).await? {
return Ok(());
}
let (outgoing_tx, mut outgoing_rx) = mpsc::channel::<EncodedFrame>(/*max_capacity*/ 128);
let peer = Arc::new(HostPeer::new(outgoing_tx));
let negotiated = negotiate(&mut reader, &mut writer, bulk_connections.as_ref()).await?;
let bulk_connection = match negotiated {
NegotiatedConnection::Rejected => return Ok(()),
NegotiatedConnection::Single => None,
NegotiatedConnection::Dual(mut registration) => {
match tokio::time::timeout(BULK_PAIRING_TIMEOUT, registration.receive()).await {
Ok(Ok(connection)) => Some(connection),
Ok(Err(_)) => {
anyhow::bail!("code-mode host bulk websocket pairing was abandoned");
}
Err(_) => {
anyhow::bail!("timed out pairing code-mode host bulk websocket");
}
}
}
};
let (mut bulk_reader, bulk_writer) = match bulk_connection {
Some(connection) => (Some(connection.reader), Some(connection.writer)),
None => (None, None),
};
let (outgoing_tx, outgoing_rx) = mpsc::channel::<EncodedFrame>(OUTGOING_CHANNEL_CAPACITY);
let (bulk_tx, bulk_rx) = if bulk_writer.is_some() {
let (sender, receiver) = mpsc::channel::<EncodedFrame>(MAX_PENDING_DELEGATE_CALLS);
(Some(sender), Some(receiver))
} else {
(None, None)
};
let peer = match bulk_tx {
Some(sender) => Arc::new(HostPeer::new(outgoing_tx).with_bulk_sender(sender)),
None => Arc::new(HostPeer::new(outgoing_tx)),
};
let state = Arc::new(HostState {
sessions: Mutex::new(HashMap::new()),
seen_session_ids: Mutex::new(SeenSessionIds::default()),
@@ -108,25 +149,14 @@ async fn run_connection(
});
let writer_disconnected = peer.disconnection_token();
let writer_task = tokio::spawn(async move {
loop {
tokio::select! {
_ = writer_disconnected.cancelled() => return Ok::<(), anyhow::Error>(()),
frame = outgoing_rx.recv() => {
let Some(frame) = frame else {
return Ok(());
};
let result = tokio::select! {
_ = writer_disconnected.cancelled() => return Ok(()),
result = writer.write_frame(frame) => result,
};
if let Err(err) = result {
return Err(
anyhow::Error::new(err)
.context("failed to write code-mode host message")
);
}
}
}
if let (Some(bulk_writer), Some(bulk_rx)) = (bulk_writer, bulk_rx) {
tokio::try_join!(
drive_writer(writer, outgoing_rx, writer_disconnected.clone()),
drive_writer(bulk_writer, bulk_rx, writer_disconnected)
)?;
Ok(())
} else {
drive_writer(writer, outgoing_rx, writer_disconnected).await
}
});
let writer_peer = Arc::clone(&peer);
@@ -147,14 +177,30 @@ async fn run_connection(
let input_result = async {
loop {
let message = tokio::select! {
let (message, lane) = tokio::select! {
// Session operations and shutdown must make progress even when bulk callbacks are ready.
biased;
_ = peer.disconnected() => break,
message = reader.read() => message
.context("failed to read code-mode client message")?,
message = reader.read() => (
message.context("failed to read code-mode client control message")?,
TransportLane::Control,
),
message = async {
match &mut bulk_reader {
Some(reader) => reader.read().await,
None => Ok(None),
}
}, if bulk_reader.is_some() => (
message.context("failed to read code-mode client bulk message")?,
TransportLane::Bulk,
),
};
let Some(message) = message else {
break;
};
if bulk_reader.is_some() && !message.allows_transport_lane(lane) {
anyhow::bail!("code-mode client sent a message on the wrong websocket lane");
}
match message {
ClientToHost::ClientHello(_) => {
anyhow::bail!("received a second code-mode client hello");
@@ -195,13 +241,40 @@ async fn run_connection(
Ok(())
}
async fn negotiate(reader: &mut ConnectionReader, writer: &mut ConnectionWriter) -> Result<bool> {
async fn drive_writer(
mut writer: ConnectionWriter,
mut outgoing: mpsc::Receiver<EncodedFrame>,
disconnected: CancellationToken,
) -> Result<()> {
loop {
tokio::select! {
_ = disconnected.cancelled() => return Ok(()),
frame = outgoing.recv() => {
let Some(frame) = frame else {
return Ok(());
};
tokio::select! {
_ = disconnected.cancelled() => return Ok(()),
result = writer.write_frame(frame) => {
result.context("failed to write code-mode host message")?;
}
}
}
}
}
}
async fn negotiate(
reader: &mut ConnectionReader,
writer: &mut ConnectionWriter,
bulk_connections: Option<&BulkConnectionRegistry>,
) -> Result<NegotiatedConnection> {
let Some(first_message) = reader
.read()
.await
.context("failed to read code-mode client hello")?
else {
return Ok(false);
return Ok(NegotiatedConnection::Rejected);
};
let ClientToHost::ClientHello(client_hello) = first_message else {
writer
@@ -212,7 +285,7 @@ async fn negotiate(reader: &mut ConnectionReader, writer: &mut ConnectionWriter)
})
.await
.context("failed to reject invalid code-mode client hello")?;
return Ok(false);
return Ok(NegotiatedConnection::Rejected);
};
let supported_versions = SupportedProtocolVersions::try_new([ProtocolVersion::V1])?;
@@ -226,10 +299,26 @@ async fn negotiate(reader: &mut ConnectionReader, writer: &mut ConnectionWriter)
})
.await
.context("failed to reject incompatible code-mode client")?;
return Ok(false);
return Ok(NegotiatedConnection::Rejected);
}
let host_capabilities = CapabilitySet::empty();
let dual_capability = Capability::new(DUAL_WEBSOCKET_CAPABILITY)?;
let dual_requested = client_hello
.required_capabilities()
.contains(&dual_capability)
|| client_hello
.optional_capabilities()
.contains(&dual_capability);
let registration = if dual_requested {
bulk_connections.and_then(BulkConnectionRegistry::reserve)
} else {
None
};
let host_capabilities = if registration.is_some() {
CapabilitySet::try_new([dual_capability])?
} else {
CapabilitySet::empty()
};
if let Some(capability) = client_hello
.required_capabilities()
.iter()
@@ -243,17 +332,26 @@ async fn negotiate(reader: &mut ConnectionReader, writer: &mut ConnectionWriter)
})
.await
.context("failed to reject unsupported code-mode capability")?;
return Ok(false);
return Ok(NegotiatedConnection::Rejected);
}
let (hello, negotiated) = if let Some(registration) = registration {
(
HostHello::new(ProtocolVersion::V1, host_capabilities)
.with_bulk_connection_token(registration.token().to_string()),
NegotiatedConnection::Dual(registration),
)
} else {
(
HostHello::new(ProtocolVersion::V1, host_capabilities),
NegotiatedConnection::Single,
)
};
writer
.write(&HostToClient::HostHello(HostHello::new(
ProtocolVersion::V1,
host_capabilities,
)))
.write(&HostToClient::HostHello(hello))
.await
.context("failed to write code-mode host hello")?;
Ok(true)
Ok(negotiated)
}
struct HostState {
@@ -584,7 +682,7 @@ impl RequestRegistry {
Ok(cancellation)
}
fn cancel(&self, request_id: RequestId) {
fn cancel(&mut self, request_id: RequestId) {
if let Some(request) = self.active.get(&request_id)
&& request.kind.is_cancellable()
{
+16 -3
View File
@@ -16,6 +16,7 @@ use codex_code_mode_protocol::host::HostToClient;
use codex_code_mode_protocol::host::MAX_PENDING_DELEGATE_CALLS;
use codex_code_mode_protocol::host::RequestId;
use codex_code_mode_protocol::host::SessionId;
use codex_code_mode_protocol::host::TransportLane;
use codex_code_mode_protocol::host::WireResult;
use tokio::sync::Mutex;
use tokio::sync::Notify;
@@ -29,6 +30,7 @@ const CELL_MESSAGE_CAPACITY: usize = 128;
pub(super) struct HostPeer {
outgoing_tx: mpsc::Sender<EncodedFrame>,
bulk_tx: Option<mpsc::Sender<EncodedFrame>>,
pending: Mutex<HashMap<DelegateRequestId, PendingDelegate>>,
delegate_permits: Arc<Semaphore>,
cell_routes: StdMutex<HashMap<(SessionId, CellId), CellRoute>>,
@@ -62,6 +64,7 @@ impl HostPeer {
pub(super) fn new(outgoing_tx: mpsc::Sender<EncodedFrame>) -> Self {
Self {
outgoing_tx,
bulk_tx: None,
pending: Mutex::new(HashMap::new()),
delegate_permits: Arc::new(Semaphore::new(MAX_PENDING_DELEGATE_CALLS)),
cell_routes: StdMutex::new(HashMap::new()),
@@ -72,10 +75,16 @@ impl HostPeer {
}
}
pub(super) fn with_bulk_sender(mut self, bulk_tx: mpsc::Sender<EncodedFrame>) -> Self {
self.bulk_tx = Some(bulk_tx);
self
}
pub(super) fn send(&self, message: HostToClient) -> Result<(), PeerSendError> {
let frame = EncodedFrame::encode(&message)
.map_err(|err| PeerSendError::Payload(err.to_string()))?;
self.send_frame(frame)
let lane = message.transport_lane();
self.send_frame(frame, lane)
}
pub(super) fn respond(
@@ -389,8 +398,12 @@ impl HostPeer {
});
}
fn send_frame(&self, frame: EncodedFrame) -> Result<(), PeerSendError> {
match self.outgoing_tx.try_send(frame) {
fn send_frame(&self, frame: EncodedFrame, lane: TransportLane) -> Result<(), PeerSendError> {
let sender = match lane {
TransportLane::Control => &self.outgoing_tx,
TransportLane::Bulk => self.bulk_tx.as_ref().unwrap_or(&self.outgoing_tx),
};
match sender.try_send(frame) {
Ok(()) => Ok(()),
Err(mpsc::error::TrySendError::Full(_)) => {
self.disconnect();
+101
View File
@@ -1,13 +1,17 @@
use std::collections::HashMap;
use std::io;
use std::io::Write as _;
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::PoisonError;
use anyhow::Context;
use anyhow::Result;
use axum::Router;
use axum::body::Body;
use axum::extract::ConnectInfo;
use axum::extract::Path;
use axum::extract::State;
use axum::extract::ws::Message;
use axum::extract::ws::WebSocket;
@@ -34,15 +38,19 @@ use futures::stream::SplitStream;
use tokio::io::AsyncRead;
use tokio::io::AsyncWrite;
use tokio::net::TcpListener;
use tokio::sync::oneshot;
use tracing::info;
use tracing::warn;
use uuid::Uuid;
use crate::HostLimits;
use crate::MAX_IN_FLIGHT_REQUESTS;
/// The default transport retains the standalone host's original stdio behavior.
pub const DEFAULT_LISTEN_URL: &str = "stdio";
const MAX_WEBSOCKET_FRAME_BYTES: usize = MAX_FRAME_BYTES + std::mem::size_of::<u32>();
const MAX_PENDING_BULK_CONNECTIONS: usize = MAX_IN_FLIGHT_REQUESTS;
type BoxedReader = Box<dyn AsyncRead + Send + Unpin>;
type BoxedWriter = Box<dyn AsyncWrite + Send + Unpin>;
@@ -66,6 +74,64 @@ pub(crate) enum ConnectionWriter {
#[derive(Clone)]
struct WebSocketListenerState {
limits: Arc<HostLimits>,
bulk_connections: BulkConnectionRegistry,
}
pub(crate) struct BulkConnection {
pub(crate) reader: ConnectionReader,
pub(crate) writer: ConnectionWriter,
}
#[derive(Clone, Default)]
pub(crate) struct BulkConnectionRegistry {
pending: Arc<Mutex<HashMap<Uuid, oneshot::Sender<BulkConnection>>>>,
}
pub(crate) struct BulkConnectionRegistration {
registry: BulkConnectionRegistry,
token: Uuid,
receiver: oneshot::Receiver<BulkConnection>,
}
impl BulkConnectionRegistry {
pub(crate) fn reserve(&self) -> Option<BulkConnectionRegistration> {
let mut pending = self.pending.lock().unwrap_or_else(PoisonError::into_inner);
if pending.len() >= MAX_PENDING_BULK_CONNECTIONS {
return None;
}
let token = Uuid::new_v4();
let (sender, receiver) = oneshot::channel();
pending.insert(token, sender);
Some(BulkConnectionRegistration {
registry: self.clone(),
token,
receiver,
})
}
pub(crate) fn remove(&self, token: Uuid) -> Option<oneshot::Sender<BulkConnection>> {
self.pending
.lock()
.unwrap_or_else(PoisonError::into_inner)
.remove(&token)
}
}
impl BulkConnectionRegistration {
pub(crate) fn token(&self) -> Uuid {
self.token
}
pub(crate) async fn receive(&mut self) -> Result<BulkConnection, oneshot::error::RecvError> {
(&mut self.receiver).await
}
}
impl Drop for BulkConnectionRegistration {
fn drop(&mut self) {
self.registry.remove(self.token);
}
}
impl ConnectionReader {
@@ -165,6 +231,7 @@ async fn run_websocket_listener(bind_address: SocketAddr) -> Result<()> {
.context("failed to read code-mode host websocket listen address")?;
let state = WebSocketListenerState {
limits: Arc::new(HostLimits::new()),
bulk_connections: BulkConnectionRegistry::default(),
};
info!("codex-code-mode-host listening on ws://{local_addr}");
println!("ws://{local_addr}");
@@ -174,6 +241,7 @@ async fn run_websocket_listener(bind_address: SocketAddr) -> Result<()> {
let router = Router::new()
.route("/", any(websocket_upgrade_handler))
.route("/bulk/{token}", any(bulk_websocket_upgrade_handler))
.route("/readyz", get(readiness_handler))
.layer(middleware::from_fn(reject_requests_with_origin_header))
.with_state(state);
@@ -220,6 +288,7 @@ async fn websocket_upgrade_handler(
ConnectionReader::WebSocket(reader),
ConnectionWriter::WebSocket(writer),
state.limits,
Some(state.bulk_connections),
)
.await
{
@@ -228,6 +297,38 @@ async fn websocket_upgrade_handler(
})
}
async fn bulk_websocket_upgrade_handler(
websocket: WebSocketUpgrade,
Path(token): Path<String>,
ConnectInfo(peer_addr): ConnectInfo<SocketAddr>,
State(state): State<WebSocketListenerState>,
) -> Response {
let Ok(token) = Uuid::parse_str(&token) else {
return StatusCode::NOT_FOUND.into_response();
};
let Some(pending) = state.bulk_connections.remove(token) else {
return StatusCode::NOT_FOUND.into_response();
};
websocket
.max_frame_size(MAX_WEBSOCKET_FRAME_BYTES)
.max_message_size(MAX_WEBSOCKET_FRAME_BYTES)
.on_upgrade(move |stream| async move {
info!(%peer_addr, "code-mode host bulk websocket client connected");
let (writer, reader) = stream.split();
if pending
.send(BulkConnection {
reader: ConnectionReader::WebSocket(reader),
writer: ConnectionWriter::WebSocket(writer),
})
.is_err()
{
warn!(%peer_addr, "code-mode host bulk websocket pairing expired");
}
})
.into_response()
}
#[cfg(test)]
#[path = "transport_tests.rs"]
mod tests;
@@ -2,9 +2,34 @@ use std::net::SocketAddr;
use pretty_assertions::assert_eq;
use super::BulkConnectionRegistry;
use super::ListenTransport;
use super::MAX_PENDING_BULK_CONNECTIONS;
use super::parse_listen_url;
#[test]
fn bulk_connection_registration_cleans_up_when_dropped() {
let registry = BulkConnectionRegistry::default();
let registration = registry.reserve().expect("bulk connection registration");
let token = registration.token();
drop(registration);
assert!(registry.remove(token).is_none());
}
#[test]
fn bulk_connection_registrations_are_bounded_and_released() {
let registry = BulkConnectionRegistry::default();
let mut registrations = (0..MAX_PENDING_BULK_CONNECTIONS)
.map(|_| registry.reserve().expect("bulk connection registration"))
.collect::<Vec<_>>();
assert!(registry.reserve().is_none());
drop(registrations.pop());
assert!(registry.reserve().is_some());
}
#[test]
fn parse_listen_url_accepts_stdio_transports() {
assert_eq!(
+497
View File
@@ -1,12 +1,27 @@
use std::process::Stdio;
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context;
use anyhow::Result;
use codex_code_mode::CellId;
use codex_code_mode::CodeModeNestedToolCall;
use codex_code_mode::CodeModeSessionDelegate;
use codex_code_mode::CodeModeSessionProvider;
use codex_code_mode::CodeModeToolKind;
use codex_code_mode::ExecuteRequest;
use codex_code_mode::FunctionCallOutputContentItem;
use codex_code_mode::NoopCodeModeSessionDelegate;
use codex_code_mode::NotificationFuture;
use codex_code_mode::RuntimeResponse;
use codex_code_mode::ToolDefinition;
use codex_code_mode::ToolInvocationFuture;
use codex_code_mode::WebSocketCodeModeSessionProvider;
use codex_code_mode_protocol::host::Capability;
use codex_code_mode_protocol::host::CapabilitySet;
use codex_code_mode_protocol::host::ClientHello;
use codex_code_mode_protocol::host::ClientToHost;
use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY;
use codex_code_mode_protocol::host::DelegateRequest;
use codex_code_mode_protocol::host::DelegateResponse;
use codex_code_mode_protocol::host::EncodedFrame;
@@ -26,6 +41,8 @@ use codex_code_mode_protocol::host::WireRuntimeResponse;
use codex_code_mode_protocol::host::WireToolDefinition;
use codex_code_mode_protocol::host::WireToolKind;
use codex_code_mode_protocol::host::WireToolName;
use codex_code_mode_protocol::host::WireWaitRequest;
use codex_protocol::ToolName;
use futures::SinkExt;
use futures::StreamExt;
use pretty_assertions::assert_eq;
@@ -37,6 +54,7 @@ use tokio::io::BufReader;
use tokio::net::TcpStream;
use tokio::process::Child;
use tokio::process::Command;
use tokio::sync::Semaphore;
use tokio::time::timeout;
use tokio_tungstenite::MaybeTlsStream;
use tokio_tungstenite::WebSocketStream;
@@ -49,6 +67,8 @@ use tokio_tungstenite::tungstenite::http::HeaderValue;
use tokio_tungstenite::tungstenite::http::StatusCode;
use tokio_tungstenite::tungstenite::http::header::ORIGIN;
use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const TEST_TIMEOUT: Duration = Duration::from_secs(10);
const MAX_WEBSOCKET_FRAME_BYTES: usize = MAX_FRAME_BYTES + std::mem::size_of::<u32>();
@@ -62,6 +82,43 @@ struct HostClient {
websocket: WebSocketStream<MaybeTlsStream<TcpStream>>,
}
struct LargeToolResultDelegate {
started: Semaphore,
release: Semaphore,
}
impl CodeModeSessionDelegate for LargeToolResultDelegate {
fn invoke_tool<'a>(
&'a self,
invocation: CodeModeNestedToolCall,
_cancellation_token: CancellationToken,
) -> ToolInvocationFuture<'a> {
Box::pin(async move {
assert_eq!(invocation.tool_name, ToolName::plain("large"));
self.started.add_permits(1);
let permit = self
.release
.acquire()
.await
.map_err(|_| "large tool release closed".to_string())?;
permit.forget();
Ok(json!({ "value": "x".repeat(8 * 1024 * 1024) }))
})
}
fn notify<'a>(
&'a self,
_call_id: String,
_cell_id: CellId,
_text: String,
_cancellation_token: CancellationToken,
) -> NotificationFuture<'a> {
Box::pin(async { Ok(()) })
}
fn cell_closed(&self, _cell_id: &CellId) {}
}
impl HostHarness {
async fn start() -> Result<Self> {
let host_program = codex_utils_cargo_bin::cargo_bin("codex-code-mode-host")?;
@@ -166,6 +223,34 @@ impl HostClient {
Ok(())
}
async fn negotiate_dual(&mut self, websocket_url: &str) -> Result<HostClient> {
let capability = Capability::new(DUAL_WEBSOCKET_CAPABILITY)?;
let hello = ClientHello::new(
SupportedProtocolVersions::try_new([ProtocolVersion::V1])?,
CapabilitySet::empty(),
CapabilitySet::try_new([capability.clone()])?,
)?;
self.send(&ClientToHost::ClientHello(hello)).await?;
let HostToClient::HostHello(hello) = self.read().await? else {
anyhow::bail!("expected code-mode host hello");
};
assert!(hello.capabilities().contains(&capability));
let token = hello
.bulk_connection_token()
.context("dual websocket handshake omitted its pairing token")?;
let bulk_url = format!("{}/bulk/{token}", websocket_url.trim_end_matches('/'));
let config = WebSocketConfig::default()
.max_frame_size(Some(MAX_WEBSOCKET_FRAME_BYTES))
.max_message_size(Some(MAX_WEBSOCKET_FRAME_BYTES));
let (websocket, _) = timeout(
TEST_TIMEOUT,
connect_async_with_config(bulk_url, Some(config), /*disable_nagle*/ false),
)
.await
.context("timed out connecting to code-mode host bulk websocket")??;
Ok(HostClient { websocket })
}
async fn open_session(&mut self, session_id: SessionId) -> Result<()> {
let id = RequestId::new(/*value*/ 1);
self.send(&ClientToHost::Request {
@@ -317,6 +402,418 @@ async fn websocket_listener_executes_cells_and_forwards_tool_callbacks() -> Resu
Ok(())
}
#[tokio::test]
async fn production_websocket_client_runs_nested_tools_while_other_sessions_progress() -> Result<()>
{
let host = HostHarness::start().await?;
let provider = WebSocketCodeModeSessionProvider::new(host.websocket_url.clone());
let delegate = Arc::new(LargeToolResultDelegate {
started: Semaphore::new(/*permits*/ 0),
release: Semaphore::new(/*permits*/ 0),
});
let slow_session = provider
.create_session(delegate.clone())
.await
.map_err(anyhow::Error::msg)?;
let fast_session = provider
.create_session(Arc::new(NoopCodeModeSessionDelegate))
.await
.map_err(anyhow::Error::msg)?;
let slow_cell = slow_session
.execute(ExecuteRequest {
tool_call_id: "large-tool".to_string(),
enabled_tools: vec![ToolDefinition {
name: "large".to_string(),
tool_name: ToolName::plain("large"),
description: String::new(),
kind: CodeModeToolKind::Function,
input_schema: None,
output_schema: None,
}],
source: r#"const result = await tools.large({ value: "request" }); text(String(result.value.length));"#
.to_string(),
yield_time_ms: Some(20_000),
max_output_tokens: Some(1_000),
})
.await
.map_err(anyhow::Error::msg)?;
let started = timeout(TEST_TIMEOUT, delegate.started.acquire())
.await
.context("large tool callback did not start")??;
started.forget();
let fast_response = timeout(TEST_TIMEOUT, async {
fast_session
.execute(ExecuteRequest {
tool_call_id: "fast-before-transfer".to_string(),
enabled_tools: Vec::new(),
source: r#"text("fast-before");"#.to_string(),
yield_time_ms: Some(5_000),
max_output_tokens: Some(1_000),
})
.await
.map_err(anyhow::Error::msg)?
.initial_response()
.await
.map_err(anyhow::Error::msg)
})
.await
.context("unrelated execution was blocked by the pending tool callback")??;
assert_eq!(
fast_response,
RuntimeResponse::Result {
cell_id: CellId::new("1".to_string()),
content_items: vec![FunctionCallOutputContentItem::InputText {
text: "fast-before".to_string(),
}],
error_text: None,
}
);
delegate.release.add_permits(1);
let slow_response = async {
timeout(TEST_TIMEOUT, slow_cell.initial_response())
.await
.context("large tool result did not finish")?
.map_err(anyhow::Error::msg)
};
let concurrent_response = async {
timeout(TEST_TIMEOUT, async {
fast_session
.execute(ExecuteRequest {
tool_call_id: "fast-during-transfer".to_string(),
enabled_tools: Vec::new(),
source: r#"text("fast-during");"#.to_string(),
yield_time_ms: Some(5_000),
max_output_tokens: Some(1_000),
})
.await
.map_err(anyhow::Error::msg)?
.initial_response()
.await
.map_err(anyhow::Error::msg)
})
.await
.context("unrelated execution was blocked by the large tool transfer")?
};
tokio::pin!(slow_response);
tokio::pin!(concurrent_response);
let concurrent_response = tokio::select! {
response = &mut concurrent_response => response?,
response = &mut slow_response => {
response?;
anyhow::bail!("large tool result completed before the unrelated control response");
}
};
let slow_response = slow_response.await?;
assert_eq!(
slow_response,
RuntimeResponse::Result {
cell_id: CellId::new("1".to_string()),
content_items: vec![FunctionCallOutputContentItem::InputText {
text: "8388608".to_string(),
}],
error_text: None,
}
);
assert_eq!(
concurrent_response,
RuntimeResponse::Result {
cell_id: CellId::new("2".to_string()),
content_items: vec![FunctionCallOutputContentItem::InputText {
text: "fast-during".to_string(),
}],
error_text: None,
}
);
slow_session.shutdown().await.map_err(anyhow::Error::msg)?;
fast_session.shutdown().await.map_err(anyhow::Error::msg)?;
Ok(())
}
#[tokio::test]
async fn websocket_dual_connections_route_notifications_and_tool_callbacks_to_separate_lanes()
-> Result<()> {
let host = HostHarness::start().await?;
let mut control = host.connect().await?;
let mut bulk = control.negotiate_dual(&host.websocket_url).await?;
let session_id = SessionId::new("dual-websocket-session")?;
control.open_session(session_id.clone()).await?;
let execute_id = RequestId::new(/*value*/ 2);
control
.send(&ClientToHost::Request {
id: execute_id,
request: HostRequest::Execute {
session_id: session_id.clone(),
request: WireExecuteRequest {
tool_call_id: "dual-websocket-call".to_string(),
enabled_tools: vec![WireToolDefinition {
name: "echo".to_string(),
tool_name: WireToolName {
name: "echo".to_string(),
namespace: None,
},
description: String::new(),
kind: WireToolKind::Function,
input_schema: None,
output_schema: None,
}],
source: r#"notify("important"); const result = await tools.echo({ value: "ping" }); text(result.value);"#
.to_string(),
yield_time_ms: Some(5_000),
max_output_tokens: Some(1_000),
},
},
})
.await?;
let started = control.read().await?;
let HostToClient::Response {
id,
result:
WireResult::Ok {
value: HostResponse::ExecutionStarted { cell_id },
},
} = started
else {
anyhow::bail!("expected execution-started response on control lane, got {started:?}");
};
assert_eq!(id, execute_id);
let notification = control.read().await?;
let HostToClient::DelegateRequest {
id: notification_id,
..
} = &notification
else {
anyhow::bail!("expected notification on control lane, got {notification:?}");
};
let notification_id = *notification_id;
assert_eq!(
notification,
HostToClient::DelegateRequest {
id: notification_id,
session_id: session_id.clone(),
request: DelegateRequest::Notify {
call_id: "dual-websocket-call".to_string(),
cell_id: cell_id.clone(),
text: "important".to_string(),
},
}
);
control
.send(&ClientToHost::DelegateResponse {
id: notification_id,
result: WireResult::Ok {
value: DelegateResponse::NotificationDelivered,
},
})
.await?;
let callback = bulk.read().await?;
let HostToClient::DelegateRequest {
id: delegate_id,
session_id: callback_session_id,
request: DelegateRequest::InvokeTool { invocation },
} = callback
else {
anyhow::bail!("expected tool callback on bulk lane, got {callback:?}");
};
assert_eq!(callback_session_id, session_id);
assert_eq!(invocation.input, Some(json!({ "value": "ping" })));
bulk.send(&ClientToHost::DelegateResponse {
id: delegate_id,
result: WireResult::Ok {
value: DelegateResponse::ToolResult {
result: json!({ "value": "pong" }),
},
},
})
.await?;
assert_eq!(
control.read().await?,
HostToClient::InitialResponse {
id: execute_id,
result: WireResult::Ok {
value: WireRuntimeResponse::Result {
cell_id: cell_id.clone(),
content_items: vec![WireContentItem::InputText {
text: "pong".to_string(),
}],
error_text: None,
},
},
}
);
assert_eq!(
control.read().await?,
HostToClient::CellClosed {
session_id: session_id.clone(),
cell_id,
}
);
let shutdown_id = RequestId::new(/*value*/ 3);
control
.send(&ClientToHost::Request {
id: shutdown_id,
request: HostRequest::ShutdownSession {
session_id: session_id.clone(),
},
})
.await?;
assert_eq!(
control.read().await?,
HostToClient::Response {
id: shutdown_id,
result: WireResult::Ok {
value: HostResponse::SessionClosed { session_id },
},
}
);
Ok(())
}
#[tokio::test]
async fn websocket_control_operations_bypass_an_incomplete_bulk_frame() -> Result<()> {
let host = HostHarness::start().await?;
let mut control = host.connect().await?;
let mut bulk = control.negotiate_dual(&host.websocket_url).await?;
let session_id = SessionId::new("bulk-priority-session")?;
control.open_session(session_id.clone()).await?;
let bulk_message = ClientToHost::DelegateResponse {
id: codex_code_mode_protocol::host::DelegateRequestId::new(/*value*/ 999),
result: WireResult::Ok {
value: DelegateResponse::ToolResult {
result: json!({ "image": "x".repeat(1024 * 1024) }),
},
},
};
let payload = EncodedFrame::encode(&bulk_message)?.into_framed_bytes();
let mask = [0x13_u8, 0x37, 0xc0, 0xde];
let mut partial_frame = vec![0x82_u8, 0xff];
partial_frame.extend_from_slice(&(payload.len() as u64).to_be_bytes());
partial_frame.extend_from_slice(&mask);
partial_frame.extend(
payload[..64 * 1024]
.iter()
.enumerate()
.map(|(index, byte)| byte ^ mask[index % mask.len()]),
);
bulk.websocket
.get_mut()
.write_all(&partial_frame)
.await
.context("failed to start the intentionally incomplete bulk frame")?;
let request_id = RequestId::new(/*value*/ 42);
control
.send(&ClientToHost::Request {
id: request_id,
request: HostRequest::Wait {
session_id: session_id.clone(),
request: WireWaitRequest {
cell_id: codex_code_mode_protocol::host::WireCellId::new("missing-cell"),
yield_time_ms: 10,
},
},
})
.await?;
let response = timeout(TEST_TIMEOUT, control.read())
.await
.context("control wait was blocked behind the incomplete bulk frame")??;
assert!(matches!(
response,
HostToClient::Response {
id,
result: WireResult::Ok {
value: HostResponse::WaitCompleted { .. },
},
} if id == request_id
));
let execute_id = RequestId::new(/*value*/ 43);
control
.send(&ClientToHost::Request {
id: execute_id,
request: HostRequest::Execute {
session_id,
request: WireExecuteRequest {
tool_call_id: "control-execute".to_string(),
enabled_tools: Vec::new(),
source: r#"text("fast");"#.to_string(),
yield_time_ms: Some(5_000),
max_output_tokens: Some(1_000),
},
},
})
.await?;
for _ in 0..2 {
let response = timeout(TEST_TIMEOUT, control.read())
.await
.context("control execute was blocked behind the incomplete bulk frame")??;
assert!(matches!(
response,
HostToClient::Response { id, .. } | HostToClient::InitialResponse { id, .. }
if id == execute_id
));
}
Ok(())
}
#[tokio::test]
async fn websocket_bulk_lane_rejects_control_messages() -> Result<()> {
let host = HostHarness::start().await?;
let mut control = host.connect().await?;
let mut bulk = control.negotiate_dual(&host.websocket_url).await?;
let session_id = SessionId::new("wrong-lane-session")?;
control.open_session(session_id.clone()).await?;
bulk.send(&ClientToHost::Request {
id: RequestId::new(/*value*/ 42),
request: HostRequest::Wait {
session_id,
request: WireWaitRequest {
cell_id: codex_code_mode_protocol::host::WireCellId::new("missing-cell"),
yield_time_ms: 10,
},
},
})
.await?;
let result = timeout(TEST_TIMEOUT, control.websocket.next())
.await
.context("wrong-lane control message did not disconnect the paired sockets")?;
assert!(
matches!(result, None | Some(Ok(Message::Close(_))) | Some(Err(_))),
"wrong-lane message unexpectedly returned {result:?}"
);
Ok(())
}
#[tokio::test]
async fn websocket_bulk_pairing_rejects_unknown_tokens() -> Result<()> {
let host = HostHarness::start().await?;
let unknown_token = Uuid::new_v4();
let url = format!("{}/bulk/{unknown_token}", host.websocket_url);
let error = match connect_async(url).await {
Ok(_) => anyhow::bail!("unknown bulk pairing token should be rejected"),
Err(error) => error,
};
let WebSocketError::Http(response) = error else {
anyhow::bail!("bulk pairing failed unexpectedly: {error}");
};
assert_eq!(response.status(), StatusCode::NOT_FOUND);
Ok(())
}
#[tokio::test]
async fn websocket_listener_accepts_frames_larger_than_default_websocket_limit() -> Result<()> {
let host = HostHarness::start().await?;
@@ -22,6 +22,7 @@ use super::ProtocolVersion;
use super::RequestId;
use super::SessionId;
use super::SupportedProtocolVersions;
use super::TransportLane;
use super::WireCellId;
use super::WireContentItem;
use super::WireExecuteRequest;
@@ -72,6 +73,130 @@ where
);
}
#[test]
fn dual_websocket_hello_preserves_the_pairing_token() {
assert_wire_round_trip(
HostToClient::HostHello(
HostHello::new(
ProtocolVersion::V1,
CapabilitySet::try_new([capability("dual-websocket-v1")])
.expect("valid capabilities"),
)
.with_bulk_connection_token("pairing-token".to_string()),
),
json!({
"type": "connection/ready",
"selectedVersion": 1,
"capabilities": ["dual-websocket-v1"],
"bulkConnectionToken": "pairing-token",
}),
);
}
#[test]
fn message_families_use_dedicated_transport_lanes() {
for (message, lane) in [
(
ClientToHost::CancelRequest {
id: request_id(/*value*/ 1),
},
TransportLane::Control,
),
(
ClientToHost::DelegateResponse {
id: delegate_request_id(/*value*/ 1),
result: WireResult::Ok {
value: DelegateResponse::NotificationDelivered,
},
},
TransportLane::Control,
),
(
ClientToHost::DelegateResponse {
id: delegate_request_id(/*value*/ 2),
result: WireResult::Ok {
value: DelegateResponse::ToolResult {
result: json!({ "value": "tool result" }),
},
},
},
TransportLane::Bulk,
),
(
ClientToHost::DelegateResponse {
id: delegate_request_id(/*value*/ 3),
result: WireResult::Err {
message: "delegate failed".to_string(),
},
},
TransportLane::Bulk,
),
] {
assert_eq!(message.transport_lane(), lane);
assert!(message.allows_transport_lane(lane));
assert!(!message.allows_transport_lane(match lane {
TransportLane::Control => TransportLane::Bulk,
TransportLane::Bulk => TransportLane::Control,
}));
}
for (message, lane) in [
(
HostToClient::Response {
id: request_id(/*value*/ 1),
result: WireResult::Err {
message: "x".repeat(128 * 1024),
},
},
TransportLane::Control,
),
(
HostToClient::DelegateRequest {
id: delegate_request_id(/*value*/ 1),
session_id: session_id(),
request: DelegateRequest::Notify {
call_id: "call-1".to_string(),
cell_id: cell_id("cell-1"),
text: "important".to_string(),
},
},
TransportLane::Control,
),
(
HostToClient::DelegateRequest {
id: delegate_request_id(/*value*/ 2),
session_id: session_id(),
request: DelegateRequest::InvokeTool {
invocation: WireNestedToolCall {
cell_id: cell_id("cell-1"),
runtime_tool_call_id: "runtime-call-1".to_string(),
tool_name: WireToolName {
name: "tool".to_string(),
namespace: None,
},
tool_kind: WireToolKind::Function,
input: None,
},
},
},
TransportLane::Bulk,
),
(
HostToClient::CancelDelegateRequest {
id: delegate_request_id(/*value*/ 1),
},
TransportLane::Bulk,
),
] {
assert_eq!(message.transport_lane(), lane);
assert!(message.allows_transport_lane(lane));
assert!(!message.allows_transport_lane(match lane {
TransportLane::Control => TransportLane::Bulk,
TransportLane::Bulk => TransportLane::Control,
}));
}
}
fn execute_request() -> WireExecuteRequest {
WireExecuteRequest {
tool_call_id: "call-1".to_string(),
@@ -12,6 +12,7 @@ use super::ProtocolVersion;
use super::RequestId;
use super::SessionId;
use super::SupportedProtocolVersions;
use super::TransportLane;
use super::WireCellId;
use super::WireExecuteRequest;
use super::WireNestedToolCall;
@@ -105,6 +106,8 @@ impl std::error::Error for ClientHelloError {}
pub struct HostHello {
selected_version: ProtocolVersion,
capabilities: CapabilitySet,
#[serde(default, skip_serializing_if = "Option::is_none")]
bulk_connection_token: Option<String>,
}
impl HostHello {
@@ -112,9 +115,15 @@ impl HostHello {
Self {
selected_version,
capabilities,
bulk_connection_token: None,
}
}
pub fn with_bulk_connection_token(mut self, token: String) -> Self {
self.bulk_connection_token = Some(token);
self
}
pub fn selected_version(&self) -> ProtocolVersion {
self.selected_version
}
@@ -122,6 +131,10 @@ impl HostHello {
pub fn capabilities(&self) -> &CapabilitySet {
&self.capabilities
}
pub fn bulk_connection_token(&self) -> Option<&str> {
self.bulk_connection_token.as_deref()
}
}
/// Messages sent from a client to the code-mode host.
@@ -141,6 +154,30 @@ pub enum ClientToHost {
},
}
impl ClientToHost {
/// Keeps notification acknowledgments with control traffic and tool results on the bulk lane.
pub fn transport_lane(&self) -> TransportLane {
match self {
Self::DelegateResponse {
result:
WireResult::Ok {
value: DelegateResponse::NotificationDelivered,
},
..
}
| Self::ClientHello(_)
| Self::Request { .. }
| Self::CancelRequest { .. } => TransportLane::Control,
Self::DelegateResponse { .. } => TransportLane::Bulk,
}
}
/// Validates the message families accepted by each paired socket.
pub fn allows_transport_lane(&self, lane: TransportLane) -> bool {
self.transport_lane() == lane
}
}
/// Messages sent from the code-mode host to a client.
#[derive(Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields, tag = "type", rename_all_fields = "camelCase")]
@@ -174,6 +211,33 @@ pub enum HostToClient {
},
}
impl HostToClient {
/// Keeps notifications with control traffic and nested-tool callbacks on the bulk lane.
pub fn transport_lane(&self) -> TransportLane {
match self {
Self::DelegateRequest {
request: DelegateRequest::InvokeTool { .. },
..
}
| Self::CancelDelegateRequest { .. } => TransportLane::Bulk,
Self::DelegateRequest {
request: DelegateRequest::Notify { .. },
..
}
| Self::HostHello(_)
| Self::HandshakeRejected { .. }
| Self::Response { .. }
| Self::InitialResponse { .. }
| Self::CellClosed { .. } => TransportLane::Control,
}
}
/// Rejects messages received on the wrong paired socket.
pub fn allows_transport_lane(&self, lane: TransportLane) -> bool {
self.transport_lane() == lane
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields, tag = "method", rename_all_fields = "camelCase")]
pub enum HostRequest {
+12 -3
View File
@@ -1,9 +1,8 @@
//! Messages and framing for the code-mode host boundary.
//!
//! Protocol version 1 multiplexes session operations and delegate callbacks by
//! request ID over one ordered connection. It defines no optional capabilities
//! yet; capability names provide an extension point for later versions without
//! weakening the v1 decoder.
//! request ID over one ordered connection. WebSocket peers can negotiate a
//! separate bulk connection without changing the existing inner messages.
mod codec;
mod error;
@@ -14,6 +13,16 @@ mod types;
/// Maximum number of unresolved delegate callbacks allowed per host connection.
pub const MAX_PENDING_DELEGATE_CALLS: usize = 1_024;
/// Optional second WebSocket carrying delegate callbacks and their responses.
pub const DUAL_WEBSOCKET_CAPABILITY: &str = "dual-websocket-v1";
/// Selects one socket of a negotiated dual-WebSocket connection.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum TransportLane {
Control,
Bulk,
}
pub use codec::EncodedFrame;
pub use codec::FramedReader;
pub use codec::FramedWriter;
@@ -1,4 +1,5 @@
use std::fmt;
use std::future::Future;
use std::io;
use std::path::Path;
use std::path::PathBuf;
@@ -14,17 +15,21 @@ use codex_code_mode_protocol::ExecuteRequest;
use codex_code_mode_protocol::StartedCell;
use codex_code_mode_protocol::WaitOutcome;
use codex_code_mode_protocol::WaitRequest;
use codex_code_mode_protocol::host::Capability;
use codex_code_mode_protocol::host::CapabilitySet;
use codex_code_mode_protocol::host::ClientHello;
use codex_code_mode_protocol::host::ClientToHost;
use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY;
use codex_code_mode_protocol::host::EncodedFrame;
use codex_code_mode_protocol::host::FramedReader;
use codex_code_mode_protocol::host::FramedWriter;
use codex_code_mode_protocol::host::HostToClient;
use codex_code_mode_protocol::host::MAX_FRAME_BYTES;
use codex_code_mode_protocol::host::MAX_PENDING_DELEGATE_CALLS;
use codex_code_mode_protocol::host::ProtocolVersion;
use codex_code_mode_protocol::host::RequestId;
use codex_code_mode_protocol::host::SupportedProtocolVersions;
use codex_code_mode_protocol::host::TransportLane;
use codex_http_client::HttpClientFactory;
use codex_websocket_client::WebSocketConnector;
use futures::StreamExt;
@@ -36,6 +41,7 @@ use tokio::sync::mpsc;
use tokio::sync::oneshot;
use tokio::task::JoinHandle;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::http::Uri;
use tokio_tungstenite::tungstenite::protocol::WebSocketConfig;
use tokio_util::sync::CancellationToken;
use tracing::debug;
@@ -57,6 +63,8 @@ mod transport;
const IPC_CHANNEL_CAPACITY: usize = 128;
const HOST_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
// TODO(anp) make this timeout configurable if 60 seconds is insufficient.
const DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT: Duration = Duration::from_secs(60);
const MAX_WEBSOCKET_FRAME_BYTES: usize = MAX_FRAME_BYTES + std::mem::size_of::<u32>();
// Host spawn errors become model-visible tool output. Bound configured paths
// while preserving the executable-bearing suffix needed to diagnose failures.
@@ -68,6 +76,7 @@ pub(super) enum ConnectionError {
host_program: PathBuf,
error: io::Error,
},
BulkConnectionUnavailable(String),
Other(String),
}
@@ -98,7 +107,9 @@ impl fmt::Display for ConnectionError {
&host_program[suffix_start..]
)
}
Self::Other(message) => formatter.write_str(message),
Self::BulkConnectionUnavailable(message) | Self::Other(message) => {
formatter.write_str(message)
}
}
}
}
@@ -132,6 +143,45 @@ enum ConnectionOwner {
WebSocket,
}
struct BulkConnectionOptions {
websocket_url: String,
http_client_factory: HttpClientFactory,
}
impl BulkConnectionOptions {
async fn connect(
&self,
token: &str,
) -> Result<(ConnectionReader, ConnectionWriter), ConnectionError> {
let control_uri = self.websocket_url.parse::<Uri>().map_err(|error| {
ConnectionError::Other(format!(
"failed to build code-mode host bulk websocket URL: {error}"
))
})?;
let bulk_path = format!("{}/bulk/{token}", control_uri.path().trim_end_matches('/'));
let bulk_path_and_query = match control_uri.query() {
Some(query) => format!("{bulk_path}?{query}"),
None => bulk_path,
};
let mut bulk_uri_parts = control_uri.into_parts();
bulk_uri_parts.path_and_query = Some(bulk_path_and_query.parse().map_err(|error| {
ConnectionError::Other(format!(
"failed to build code-mode host bulk websocket path: {error}"
))
})?);
let bulk_url = Uri::from_parts(bulk_uri_parts)
.map_err(|error| {
ConnectionError::Other(format!(
"failed to build code-mode host bulk websocket URL: {error}"
))
})?
.to_string();
connect_websocket_transport(&bulk_url, &self.http_client_factory)
.await
.map_err(|error| ConnectionError::BulkConnectionUnavailable(error.to_string()))
}
}
impl CallerCancellation {
fn new() -> Self {
Self {
@@ -202,6 +252,7 @@ impl Connection {
ConnectionReader::Stdio(FramedReader::new(stdout)),
ConnectionWriter::Stdio(FramedWriter::new(stdin)),
ConnectionOwner::Process(Box::new(child)),
/*bulk_connection_options*/ None,
)
.await
}
@@ -210,38 +261,17 @@ impl Connection {
websocket_url: &str,
http_client_factory: &HttpClientFactory,
) -> Result<Self, ConnectionError> {
let request = websocket_url.into_client_request().map_err(|error| {
ConnectionError::Other(format!(
"failed to build code-mode host websocket request: {error}"
))
})?;
let connector = WebSocketConnector::new(http_client_factory).map_err(|error| {
ConnectionError::Other(format!(
"failed to configure code-mode host websocket TLS: {error}"
))
})?;
let websocket_config = WebSocketConfig::default()
.max_frame_size(Some(MAX_WEBSOCKET_FRAME_BYTES))
.max_message_size(Some(MAX_WEBSOCKET_FRAME_BYTES));
let (websocket, _) = tokio::time::timeout(
HOST_HANDSHAKE_TIMEOUT,
connector.connect(request, websocket_config),
)
.await
.map_err(|_| {
ConnectionError::Other("timed out connecting to the code-mode host websocket".into())
})?
.map_err(|error| {
ConnectionError::Other(format!(
"failed to connect to the code-mode host websocket: {error}"
))
})?;
let (writer, reader) = websocket.split();
let (reader, writer) =
connect_websocket_transport(websocket_url, http_client_factory).await?;
Self::establish(
ConnectionReader::WebSocket(reader),
ConnectionWriter::WebSocket(writer),
reader,
writer,
ConnectionOwner::WebSocket,
Some(BulkConnectionOptions {
websocket_url: websocket_url.to_string(),
http_client_factory: http_client_factory.clone(),
}),
)
.await
}
@@ -250,13 +280,22 @@ impl Connection {
mut reader: ConnectionReader,
mut writer: ConnectionWriter,
mut owner: ConnectionOwner,
bulk_connection_options: Option<BulkConnectionOptions>,
) -> Result<Self, ConnectionError> {
let handshake = async {
let dual_capability =
Capability::new(DUAL_WEBSOCKET_CAPABILITY).map_err(|error| error.to_string())?;
let optional_capabilities = if bulk_connection_options.is_some() {
CapabilitySet::try_new([dual_capability.clone()])
.map_err(|error| error.to_string())?
} else {
CapabilitySet::empty()
};
let hello = ClientHello::new(
SupportedProtocolVersions::try_new([ProtocolVersion::V1])
.map_err(|err| err.to_string())?,
CapabilitySet::empty(),
CapabilitySet::empty(),
optional_capabilities,
)
.map_err(|err| err.to_string())?;
writer
@@ -271,7 +310,20 @@ impl Connection {
Some(HostToClient::HostHello(hello))
if hello.selected_version() == ProtocolVersion::V1 =>
{
Ok(())
if hello.capabilities().contains(&dual_capability) {
hello
.bulk_connection_token()
.map(str::to_string)
.ok_or_else(|| {
"code-mode host advertised dual websockets without a pairing token"
.to_string()
})
.map(Some)
} else if hello.bulk_connection_token().is_some() {
Err("code-mode host returned an unexpected bulk pairing token".to_string())
} else {
Ok(None)
}
}
Some(HostToClient::HandshakeRejected { reason }) => {
Err(format!("code-mode host rejected the handshake: {reason:?}"))
@@ -292,51 +344,85 @@ impl Connection {
));
}
};
if let Err(err) = handshake_result {
let _ = writer.close().await;
owner.close().await;
return Err(ConnectionError::Other(err));
}
let bulk_token = match handshake_result {
Ok(token) => token,
Err(err) => {
let _ = writer.close().await;
owner.close().await;
return Err(ConnectionError::Other(err));
}
};
let (bulk_reader, bulk_writer) = if let Some(token) = bulk_token {
let Some(options) = bulk_connection_options else {
let _ = writer.close().await;
owner.close().await;
return Err(ConnectionError::Other(
"code-mode host negotiated an unsupported bulk websocket".to_string(),
));
};
match options.connect(&token).await {
Ok((reader, writer)) => (Some(reader), Some(writer)),
Err(error) => {
let _ = writer.close().await;
owner.close().await;
return Err(error);
}
}
} else {
(None, None)
};
let (command_tx, command_rx) = mpsc::channel(IPC_CHANNEL_CAPACITY);
let (event_tx, event_rx) = mpsc::channel(IPC_CHANNEL_CAPACITY);
let (outgoing_tx, mut outgoing_rx) = mpsc::channel::<EncodedFrame>(IPC_CHANNEL_CAPACITY);
let (outgoing_tx, outgoing_rx) = mpsc::channel::<EncodedFrame>(IPC_CHANNEL_CAPACITY);
let (bulk_tx, bulk_rx) = if bulk_writer.is_some() {
let (sender, receiver) = mpsc::channel::<EncodedFrame>(MAX_PENDING_DELEGATE_CALLS);
(Some(sender), Some(receiver))
} else {
(None, None)
};
let dual_websocket = bulk_writer.is_some();
let cancellation = CancellationToken::new();
let alive = Arc::new(AtomicBool::new(true));
let failure = Arc::new(std::sync::Mutex::new(None));
let writer_cancellation = cancellation.clone();
let writer_task = tokio::spawn(async move {
loop {
tokio::select! {
_ = writer_cancellation.cancelled() => {
return writer
.close()
.await
.map_err(|error| format!("failed to close code-mode host connection: {error}"));
}
frame = outgoing_rx.recv() => {
let Some(frame) = frame else {
return Err("code-mode host outgoing stream closed".to_string());
};
let result = tokio::select! {
_ = writer_cancellation.cancelled() => return Ok(()),
result = writer.write_frame(frame) => result,
};
if let Err(err) = result {
return Err(format!("failed to write code-mode host message: {err}"));
}
}
}
if let (Some(bulk_writer), Some(bulk_rx)) = (bulk_writer, bulk_rx) {
tokio::try_join!(
drive_writer(writer, outgoing_rx, writer_cancellation.clone()),
drive_writer(bulk_writer, bulk_rx, writer_cancellation)
)?;
Ok(())
} else {
drive_writer(writer, outgoing_rx, writer_cancellation).await
}
});
let reader_events = event_tx.clone();
let reader_cancellation = cancellation.clone();
let reader_task =
tokio::spawn(
async move { drive_reader(reader, reader_events, reader_cancellation).await },
);
let reader_task = tokio::spawn(async move {
let lane = dual_websocket.then_some(TransportLane::Control);
if let Some(bulk_reader) = bulk_reader {
tokio::try_join!(
drive_reader(
reader,
reader_events.clone(),
reader_cancellation.clone(),
lane,
),
drive_reader(
bulk_reader,
reader_events,
reader_cancellation,
Some(TransportLane::Bulk),
)
)?;
Ok(())
} else {
drive_reader(reader, reader_events, reader_cancellation, lane).await
}
});
let (driver, execute_claim_tx) = ConnectionDriver::new(
command_rx,
@@ -349,6 +435,10 @@ impl Connection {
cancellation: cancellation.clone(),
},
);
let driver = match bulk_tx {
Some(sender) => driver.with_bulk_sender(sender),
None => driver,
};
let driver_task = tokio::spawn(driver.run());
tokio::spawn(
ConnectionSupervisor {
@@ -439,16 +529,23 @@ impl Connection {
session: RemoteSession,
request: WaitRequest,
) -> Result<WaitOutcome, String> {
// Account for the runtime's one-second yield grace separately from transport.
let runtime_timeout =
Duration::from_millis(request.yield_time_ms).saturating_add(Duration::from_secs(1));
let cancellation = CallerCancellation::new();
let (response_tx, response_rx) = oneshot::channel();
self.send(DriverCommand::Wait {
session,
request,
caller_cancellation: cancellation.token(),
response_tx,
})
.await?;
let result = self.receive(response_rx).await;
let result = self
.with_transport_deadline(runtime_timeout, "wait", async {
self.send(DriverCommand::Wait {
session,
request,
caller_cancellation: cancellation.token(),
response_tx,
})
.await?;
self.receive(response_rx).await
})
.await;
cancellation.disarm();
result
}
@@ -459,13 +556,16 @@ impl Connection {
cell_id: CellId,
) -> Result<WaitOutcome, String> {
let (response_tx, response_rx) = oneshot::channel();
self.send(DriverCommand::Terminate {
session,
cell_id,
response_tx,
self.with_transport_deadline(Duration::ZERO, "terminate", async {
self.send(DriverCommand::Terminate {
session,
cell_id,
response_tx,
})
.await?;
self.receive(response_rx).await
})
.await?;
self.receive(response_rx).await
.await
}
pub(super) async fn shutdown_session(&self, session: RemoteSession) -> Result<(), String> {
@@ -478,6 +578,26 @@ impl Connection {
self.receive(response_rx).await
}
async fn with_transport_deadline<T>(
&self,
runtime_timeout: Duration,
request_type: &str,
request: impl Future<Output = Result<T, String>>,
) -> Result<T, String> {
let deadline = runtime_timeout.saturating_add(DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT);
match tokio::time::timeout(deadline, request).await {
Ok(result) => result,
Err(_) => {
warn!(request_type, "code-mode host request exceeded its deadline");
let reason =
format!("code-mode host timed out waiting for {request_type} response");
mark_connection_dead(&self.alive, &self.failure, reason.clone());
self.cancellation.cancel();
Err(reason)
}
}
}
async fn send(&self, command: DriverCommand) -> Result<(), String> {
if !self.is_alive() {
return Err(self.failure_message());
@@ -504,6 +624,73 @@ impl Connection {
}
}
async fn connect_websocket_transport(
websocket_url: &str,
http_client_factory: &HttpClientFactory,
) -> Result<(ConnectionReader, ConnectionWriter), ConnectionError> {
let request = websocket_url.into_client_request().map_err(|error| {
ConnectionError::Other(format!(
"failed to build code-mode host websocket request: {error}"
))
})?;
let connector = WebSocketConnector::new(http_client_factory).map_err(|error| {
ConnectionError::Other(format!(
"failed to configure code-mode host websocket TLS: {error}"
))
})?;
let websocket_config = WebSocketConfig::default()
.max_frame_size(Some(MAX_WEBSOCKET_FRAME_BYTES))
.max_message_size(Some(MAX_WEBSOCKET_FRAME_BYTES));
let (websocket, _) = tokio::time::timeout(
HOST_HANDSHAKE_TIMEOUT,
connector.connect(request, websocket_config),
)
.await
.map_err(|_| {
ConnectionError::Other("timed out connecting to the code-mode host websocket".into())
})?
.map_err(|error| {
ConnectionError::Other(format!(
"failed to connect to the code-mode host websocket: {error}"
))
})?;
let (writer, reader) = websocket.split();
Ok((
ConnectionReader::WebSocket(reader),
ConnectionWriter::WebSocket(writer),
))
}
async fn drive_writer(
mut writer: ConnectionWriter,
mut outgoing: mpsc::Receiver<EncodedFrame>,
cancellation: CancellationToken,
) -> Result<(), String> {
loop {
tokio::select! {
_ = cancellation.cancelled() => {
return writer
.close()
.await
.map_err(|error| format!("failed to close code-mode host connection: {error}"));
}
frame = outgoing.recv() => {
let Some(frame) = frame else {
return Err("code-mode host outgoing stream closed".to_string());
};
tokio::select! {
_ = cancellation.cancelled() => return Ok(()),
result = writer.write_frame(frame) => {
result.map_err(|error| {
format!("failed to write code-mode host message: {error}")
})?;
}
}
}
}
}
}
impl Drop for Connection {
fn drop(&mut self) {
mark_connection_dead(
@@ -1,3 +1,4 @@
use std::collections::VecDeque;
use std::panic::AssertUnwindSafe;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
@@ -6,7 +7,9 @@ use std::sync::atomic::Ordering;
use codex_code_mode_protocol::CellId;
use codex_code_mode_protocol::CodeModeSessionDelegate;
use codex_code_mode_protocol::host::EncodedFrame;
use codex_code_mode_protocol::host::HostToClient;
use codex_code_mode_protocol::host::RequestId;
use codex_code_mode_protocol::host::TransportLane;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
@@ -39,7 +42,9 @@ pub(super) struct ConnectionDriver {
event_tx: mpsc::Sender<DriverEvent>,
execute_claim_rx: mpsc::UnboundedReceiver<RequestId>,
outgoing_tx: mpsc::Sender<EncodedFrame>,
bulk_tx: Option<mpsc::Sender<EncodedFrame>>,
requests: RequestTracker,
deferred_host_messages: VecDeque<HostToClient>,
sessions: SessionRegistry,
delegates: DelegateRuntime,
alive: Arc<AtomicBool>,
@@ -64,7 +69,9 @@ impl ConnectionDriver {
event_tx: event_tx.clone(),
execute_claim_rx,
outgoing_tx,
bulk_tx: None,
requests: RequestTracker::new(),
deferred_host_messages: VecDeque::new(),
sessions: SessionRegistry::new(),
delegates: DelegateRuntime::new(event_tx),
alive: lifecycle.alive,
@@ -130,8 +137,17 @@ impl ConnectionDriver {
}
}
fn queue_frame(&mut self, frame: EncodedFrame) -> bool {
match self.outgoing_tx.try_send(frame) {
pub(super) fn with_bulk_sender(mut self, sender: mpsc::Sender<EncodedFrame>) -> Self {
self.bulk_tx = Some(sender);
self
}
fn queue_frame(&mut self, frame: EncodedFrame, lane: TransportLane) -> bool {
let sender = match lane {
TransportLane::Control => &self.outgoing_tx,
TransportLane::Bulk => self.bulk_tx.as_ref().unwrap_or(&self.outgoing_tx),
};
match sender.try_send(frame) {
Ok(()) => true,
Err(mpsc::error::TrySendError::Full(_)) => {
self.fail("code-mode host outgoing queue is full".to_string());
@@ -105,7 +105,8 @@ impl ConnectionDriver {
},
&self.event_tx,
);
self.queue_frame(frame)
let lane = message.transport_lane();
self.queue_frame(frame, lane)
}
fn execute(
@@ -164,7 +165,8 @@ impl ConnectionDriver {
},
&self.event_tx,
);
self.queue_frame(frame)
let lane = message.transport_lane();
self.queue_frame(frame, lane)
}
fn wait(
@@ -293,6 +295,7 @@ impl ConnectionDriver {
};
self.requests
.insert_pending(request_id, pending, &self.event_tx);
self.queue_frame(frame)
let lane = message.transport_lane();
self.queue_frame(frame, lane)
}
}
@@ -261,10 +261,7 @@ impl ConnectionDriver {
};
let target = match self.sessions.delegate_target(&session_id, wire_cell_id) {
Ok(target) => target,
Err(err) => {
self.fail(err);
return false;
}
Err(err) => return self.send_delegate_response(id, Err(err)),
};
match self.delegates.start(id, target, request) {
Ok(()) => true,
@@ -321,7 +318,8 @@ impl ConnectionDriver {
}
}
};
self.queue_frame(frame)
let lane = message.transport_lane();
self.queue_frame(frame, lane)
}
pub(super) fn close_cell(&mut self, session_id: SessionId, cell_id: WireCellId) -> bool {
@@ -52,6 +52,15 @@ impl RequestTracker {
})
}
pub(super) fn has_pending_execute_for_session(&self, session_id: &SessionId) -> bool {
self.pending.values().any(|request| {
matches!(
request,
PendingRequest::Execute { session, .. } if session.id == *session_id
)
})
}
pub(super) fn allocate_id(&mut self) -> Result<RequestId, String> {
let id = self.next_request_id;
self.next_request_id = self
@@ -56,6 +56,62 @@ impl ConnectionDriver {
}
pub(super) fn handle_host_message(&mut self, message: HostToClient) -> bool {
if self.should_defer_host_message(&message) {
if self.deferred_host_messages.len()
>= codex_code_mode_protocol::host::MAX_PENDING_DELEGATE_CALLS
{
self.fail(
"code-mode host exceeded deferred cross-socket message limit".to_string(),
);
return false;
}
self.deferred_host_messages.push_back(message);
return true;
}
if !self.dispatch_host_message(message) {
return false;
}
for _ in 0..self.deferred_host_messages.len() {
let Some(message) = self.deferred_host_messages.pop_front() else {
break;
};
if self.should_defer_host_message(&message) {
self.deferred_host_messages.push_back(message);
} else if !self.dispatch_host_message(message) {
return false;
}
}
true
}
fn should_defer_host_message(&self, message: &HostToClient) -> bool {
match message {
HostToClient::DelegateRequest {
session_id,
request,
..
} => {
let cell_id = match request {
codex_code_mode_protocol::host::DelegateRequest::InvokeTool { invocation } => {
&invocation.cell_id
}
codex_code_mode_protocol::host::DelegateRequest::Notify { cell_id, .. } => {
cell_id
}
};
!self.sessions.contains_cell(session_id, cell_id)
&& self.requests.has_pending_execute_for_session(session_id)
}
HostToClient::Response { .. }
| HostToClient::InitialResponse { .. }
| HostToClient::CellClosed { .. }
| HostToClient::CancelDelegateRequest { .. }
| HostToClient::HostHello(_)
| HostToClient::HandshakeRejected { .. } => false,
}
}
fn dispatch_host_message(&mut self, message: HostToClient) -> bool {
match message {
HostToClient::Response { id, result } => {
self.complete_request(id, result.into_result())
@@ -69,6 +125,12 @@ impl ConnectionDriver {
request,
} => self.start_delegate(id, session_id, request),
HostToClient::CancelDelegateRequest { id } => {
self.deferred_host_messages.retain(|message| {
!matches!(
message,
HostToClient::DelegateRequest { id: deferred_id, .. } if *deferred_id == id
)
});
self.delegates.cancel(id);
true
}
@@ -302,7 +364,8 @@ impl ConnectionDriver {
}
fn send_cancel_request(&mut self, id: RequestId) -> bool {
let frame = match EncodedFrame::encode(&ClientToHost::CancelRequest { id }) {
let message = ClientToHost::CancelRequest { id };
let frame = match EncodedFrame::encode(&message) {
Ok(frame) => frame,
Err(err) => {
self.fail(format!(
@@ -311,7 +374,8 @@ impl ConnectionDriver {
return false;
}
};
self.queue_frame(frame)
let lane = message.transport_lane();
self.queue_frame(frame, lane)
}
fn shutdown_abandoned_session(&mut self, session: RemoteSession) -> bool {
@@ -61,6 +61,12 @@ impl SessionRegistry {
self.records.contains_key(session_id)
}
pub(super) fn contains_cell(&self, session_id: &SessionId, cell_id: &WireCellId) -> bool {
self.records
.get(session_id)
.is_some_and(|session| session.cells.contains_key(cell_id))
}
pub(super) fn insert_ready(
&mut self,
session: RemoteSession,
@@ -15,6 +15,7 @@ use codex_code_mode_protocol::WaitRequest;
use codex_code_mode_protocol::host::ClientToHost;
use codex_code_mode_protocol::host::DelegateRequest;
use codex_code_mode_protocol::host::DelegateRequestId;
use codex_code_mode_protocol::host::DelegateResponse;
use codex_code_mode_protocol::host::EncodedFrame;
use codex_code_mode_protocol::host::HostResponse;
use codex_code_mode_protocol::host::HostToClient;
@@ -31,6 +32,8 @@ use tokio::sync::mpsc;
use tokio::sync::oneshot;
use tokio_util::sync::CancellationToken;
use super::super::Connection;
use super::super::DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT;
use super::ConnectionDriver;
use super::DriverCommand;
use super::DriverEvent;
@@ -45,6 +48,7 @@ struct DriverHarness {
outgoing_rx: mpsc::Receiver<codex_code_mode_protocol::host::EncodedFrame>,
cancellation: CancellationToken,
alive: Arc<AtomicBool>,
failure: Arc<StdMutex<Option<String>>>,
driver_task: tokio::task::JoinHandle<()>,
}
@@ -55,6 +59,7 @@ impl DriverHarness {
let (outgoing_tx, outgoing_rx) = mpsc::channel(/*max_capacity*/ 16);
let cancellation = CancellationToken::new();
let alive = Arc::new(AtomicBool::new(true));
let failure = Arc::new(StdMutex::new(None));
let (driver, execute_claim_tx) = ConnectionDriver::new(
command_rx,
event_rx,
@@ -62,7 +67,7 @@ impl DriverHarness {
outgoing_tx,
DriverLifecycle {
alive: Arc::clone(&alive),
failure: Arc::new(StdMutex::new(None)),
failure: Arc::clone(&failure),
cancellation: cancellation.clone(),
},
);
@@ -74,6 +79,7 @@ impl DriverHarness {
outgoing_rx,
cancellation,
alive,
failure,
driver_task,
}
}
@@ -193,6 +199,11 @@ struct RecordingDelegate {
struct PanickingDelegate;
struct LargeResultBurstDelegate {
started: AtomicUsize,
release: CancellationToken,
}
#[derive(Debug, Eq, PartialEq)]
enum HeldDelegateEvent {
Started,
@@ -282,6 +293,35 @@ impl CodeModeSessionDelegate for PanickingDelegate {
fn cell_closed(&self, _cell_id: &CellId) {}
}
impl CodeModeSessionDelegate for LargeResultBurstDelegate {
fn invoke_tool<'a>(
&'a self,
_invocation: CodeModeNestedToolCall,
cancellation_token: CancellationToken,
) -> ToolInvocationFuture<'a> {
self.started.fetch_add(1, Ordering::Release);
let release = self.release.clone();
Box::pin(async move {
tokio::select! {
_ = cancellation_token.cancelled() => Err("cancelled".to_string()),
_ = release.cancelled() => Ok("x".repeat(256 * 1024).into()),
}
})
}
fn notify<'a>(
&'a self,
_call_id: String,
_cell_id: CellId,
_text: String,
_cancellation_token: CancellationToken,
) -> NotificationFuture<'a> {
Box::pin(async { Ok(()) })
}
fn cell_closed(&self, _cell_id: &CellId) {}
}
impl CodeModeSessionDelegate for RecordingDelegate {
fn invoke_tool<'a>(
&'a self,
@@ -330,6 +370,143 @@ async fn next_held_delegate_event(
.expect("delegate event stream")
}
#[tokio::test]
async fn deferred_delegates_follow_cell_readiness_and_cancellation() {
let mut harness = DriverHarness::start();
let session = remote_session();
let delegate = Arc::new(RecordingDelegate::default());
harness.open(session.clone(), delegate.clone()).await;
let (first_response_tx, first_response_rx) = oneshot::channel();
let (second_response_tx, second_response_rx) = oneshot::channel();
for (tool_call_id, response_tx) in [
("first-cell", first_response_tx),
("second-cell", second_response_tx),
] {
harness
.command_tx
.send(DriverCommand::Execute {
session: session.clone(),
request: ExecuteRequest {
tool_call_id: tool_call_id.to_string(),
enabled_tools: Vec::new(),
source: "text('done')".to_string(),
yield_time_ms: Some(/*yield_time_ms*/ 1),
max_output_tokens: None,
},
caller_cancellation: CancellationToken::new(),
response_tx,
})
.await
.expect("execute command");
harness.outgoing_rx.recv().await.expect("execute frame");
}
let first_cell_id = CellId::new("first-cell".to_string());
let second_cell_id = CellId::new("second-cell".to_string());
let first_delegate_id = DelegateRequestId::new(/*value*/ 7);
let second_delegate_id = DelegateRequestId::new(/*value*/ 8);
let cancelled_delegate_id = DelegateRequestId::new(/*value*/ 9);
for (delegate_id, cell_id) in [
(first_delegate_id, first_cell_id.clone()),
(second_delegate_id, second_cell_id.clone()),
(cancelled_delegate_id, first_cell_id.clone()),
] {
harness
.event_tx
.send(DriverEvent::HostMessage(HostToClient::DelegateRequest {
id: delegate_id,
session_id: session.id.clone(),
request: DelegateRequest::Notify {
call_id: format!("notify-{}", cell_id.as_str()),
cell_id: (&cell_id).into(),
text: "hello".to_string(),
},
}))
.await
.expect("early delegate request");
}
harness
.event_tx
.send(DriverEvent::HostMessage(
HostToClient::CancelDelegateRequest {
id: cancelled_delegate_id,
},
))
.await
.expect("deferred delegate cancellation");
harness
.event_tx
.send(DriverEvent::HostMessage(HostToClient::Response {
id: RequestId::new(/*value*/ 3),
result: WireResult::Ok {
value: HostResponse::ExecutionStarted {
cell_id: (&second_cell_id).into(),
},
},
}))
.await
.expect("second execution-started response");
let _second_started = second_response_rx
.await
.expect("second execute response")
.expect("second started cell");
let second_response = tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv())
.await
.expect("second delegate response timeout")
.expect("second delegate response frame");
assert_eq!(
EncodedFrame::decode_framed::<ClientToHost>(&second_response.into_framed_bytes())
.expect("decode second delegate response"),
ClientToHost::DelegateResponse {
id: second_delegate_id,
result: WireResult::Ok {
value: codex_code_mode_protocol::host::DelegateResponse::NotificationDelivered,
},
}
);
assert_eq!(delegate.notifications.load(Ordering::Relaxed), 1);
harness
.event_tx
.send(DriverEvent::HostMessage(HostToClient::Response {
id: RequestId::new(/*value*/ 2),
result: WireResult::Ok {
value: HostResponse::ExecutionStarted {
cell_id: (&first_cell_id).into(),
},
},
}))
.await
.expect("first execution-started response");
let _first_started = first_response_rx
.await
.expect("first execute response")
.expect("first started cell");
let first_response = tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv())
.await
.expect("first delegate response timeout")
.expect("first delegate response frame");
assert_eq!(
EncodedFrame::decode_framed::<ClientToHost>(&first_response.into_framed_bytes())
.expect("decode first delegate response"),
ClientToHost::DelegateResponse {
id: first_delegate_id,
result: WireResult::Ok {
value: codex_code_mode_protocol::host::DelegateResponse::NotificationDelivered,
},
}
);
assert_eq!(delegate.notifications.load(Ordering::Relaxed), 2);
assert!(matches!(
harness.outgoing_rx.try_recv(),
Err(mpsc::error::TryRecvError::Empty)
));
assert!(harness.alive.load(Ordering::Relaxed));
}
#[tokio::test]
async fn dropped_open_waiter_shuts_down_committed_session() {
let mut harness = DriverHarness::start();
@@ -465,6 +642,97 @@ async fn delegate_cancel_is_best_effort_and_sends_no_late_response() {
assert_eq!(delegate.notifications.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn concurrent_large_delegate_results_do_not_disconnect_a_backpressured_bulk_lane() {
const CONCURRENT_RESULTS: usize = 129;
let (command_tx, command_rx) = mpsc::channel(/*max_capacity*/ 16);
let (event_tx, event_rx) = mpsc::channel(/*max_capacity*/ 16);
let (outgoing_tx, outgoing_rx) = mpsc::channel(/*max_capacity*/ 16);
let (bulk_tx, mut bulk_rx) = mpsc::channel(MAX_PENDING_DELEGATE_CALLS);
let cancellation = CancellationToken::new();
let alive = Arc::new(AtomicBool::new(true));
let failure = Arc::new(StdMutex::new(None));
let (driver, execute_claim_tx) = ConnectionDriver::new(
command_rx,
event_rx,
event_tx.clone(),
outgoing_tx,
DriverLifecycle {
alive: Arc::clone(&alive),
failure: Arc::clone(&failure),
cancellation: cancellation.clone(),
},
);
let driver_task = tokio::spawn(driver.with_bulk_sender(bulk_tx).run());
let mut harness = DriverHarness {
command_tx,
event_tx,
execute_claim_tx,
outgoing_rx,
cancellation,
alive,
failure,
driver_task,
};
let session = remote_session();
let delegate = Arc::new(LargeResultBurstDelegate {
started: AtomicUsize::new(0),
release: CancellationToken::new(),
});
harness.open(session.clone(), delegate.clone()).await;
let _started = harness
.start_cell(session.clone(), /*request_id*/ 2, "1")
.await;
for value in 1..=CONCURRENT_RESULTS {
harness
.start_tool_delegate(&session, DelegateRequestId::new(value as i64))
.await;
}
tokio::time::timeout(Duration::from_secs(10), async {
while delegate.started.load(Ordering::Acquire) < CONCURRENT_RESULTS {
tokio::task::yield_now().await;
}
})
.await
.expect("concurrent delegate calls should all start");
delegate.release.cancel();
tokio::time::timeout(Duration::from_secs(10), async {
while bulk_rx.len() < CONCURRENT_RESULTS {
assert!(
harness.alive.load(Ordering::Acquire),
"bulk queue disconnected before accepting all concurrent tool results"
);
tokio::task::yield_now().await;
}
})
.await
.expect("concurrent large results should queue behind the blocked bulk writer");
let _unrelated = harness.start_cell(session, /*request_id*/ 3, "2").await;
assert!(harness.alive.load(Ordering::Acquire));
for _ in 0..CONCURRENT_RESULTS {
let frame = bulk_rx.recv().await.expect("queued bulk delegate result");
let message = EncodedFrame::decode_framed::<ClientToHost>(&frame.into_framed_bytes())
.expect("decode queued delegate result");
let ClientToHost::DelegateResponse {
result:
WireResult::Ok {
value: DelegateResponse::ToolResult { result },
},
..
} = message
else {
panic!("expected a successful large delegate result");
};
assert_eq!(result.as_str().map(str::len), Some(256 * 1024));
}
assert!(harness.alive.load(Ordering::Acquire));
}
#[tokio::test]
async fn delegate_limit_returns_an_error_without_disconnecting() {
let mut harness = DriverHarness::start();
@@ -796,16 +1064,17 @@ async fn delegate_task_panic_becomes_tool_error_without_killing_connection() {
}
#[tokio::test]
async fn delegate_for_unknown_cell_fails_connection_without_invocation() {
async fn delegate_for_unknown_cell_returns_error_without_invocation() {
let mut harness = DriverHarness::start();
let session = remote_session();
let delegate = Arc::new(RecordingDelegate::default());
harness.open(session.clone(), delegate.clone()).await;
let id = DelegateRequestId::new(/*value*/ 7);
harness
.event_tx
.send(DriverEvent::HostMessage(HostToClient::DelegateRequest {
id: DelegateRequestId::new(/*value*/ 7),
id,
session_id: session.id,
request: DelegateRequest::InvokeTool {
invocation: WireNestedToolCall {
@@ -819,14 +1088,28 @@ async fn delegate_for_unknown_cell_fails_connection_without_invocation() {
}))
.await
.expect("delegate request");
tokio::task::yield_now().await;
let response = tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv())
.await
.expect("delegate response timeout")
.expect("delegate response frame");
assert!(!harness.alive.load(Ordering::Acquire));
assert_eq!(
EncodedFrame::decode_framed::<ClientToHost>(&response.into_framed_bytes())
.expect("decode delegate response"),
ClientToHost::DelegateResponse {
id,
result: WireResult::Err {
message: "code-mode host delegated for unknown cell missing in session session-1"
.to_string(),
},
}
);
assert!(harness.alive.load(Ordering::Acquire));
assert_eq!(delegate.invocations.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn delegate_after_cell_close_fails_connection_without_invocation() {
async fn delegate_after_cell_close_returns_error_without_invocation() {
let mut harness = DriverHarness::start();
let session = remote_session();
let delegate = Arc::new(RecordingDelegate::default());
@@ -842,10 +1125,11 @@ async fn delegate_after_cell_close_fails_connection_without_invocation() {
}))
.await
.expect("cell close");
let id = DelegateRequestId::new(/*value*/ 7);
harness
.event_tx
.send(DriverEvent::HostMessage(HostToClient::DelegateRequest {
id: DelegateRequestId::new(/*value*/ 7),
id,
session_id: session.id,
request: DelegateRequest::Notify {
call_id: "notify-1".to_string(),
@@ -855,10 +1139,24 @@ async fn delegate_after_cell_close_fails_connection_without_invocation() {
}))
.await
.expect("delegate request");
tokio::task::yield_now().await;
let response = tokio::time::timeout(Duration::from_secs(1), harness.outgoing_rx.recv())
.await
.expect("delegate response timeout")
.expect("delegate response frame");
assert!(!harness.alive.load(Ordering::Acquire));
assert_eq!(delegate.invocations.load(Ordering::Relaxed), 0);
assert_eq!(
EncodedFrame::decode_framed::<ClientToHost>(&response.into_framed_bytes())
.expect("decode delegate response"),
ClientToHost::DelegateResponse {
id,
result: WireResult::Err {
message: "code-mode host delegated for unknown cell 1 in session session-1"
.to_string(),
},
}
);
assert!(harness.alive.load(Ordering::Acquire));
assert_eq!(delegate.notifications.load(Ordering::Relaxed), 0);
}
#[tokio::test]
@@ -1037,6 +1335,90 @@ async fn remote_wait_accepts_durations_longer_than_five_minutes() {
);
}
/// A stalled remote wait must expire even when its request never reaches the host.
#[tokio::test(start_paused = true)]
async fn queued_remote_wait_times_out_and_invalidates_the_connection() {
let mut harness = DriverHarness::start();
let session = remote_session();
harness
.open(session.clone(), Arc::new(RecordingDelegate::default()))
.await;
let _started = harness
.start_cell(session.clone(), /*request_id*/ 2, "1")
.await;
let connection = Connection {
command_tx: harness.command_tx.clone(),
execute_claim_tx: harness.execute_claim_tx.clone(),
alive: Arc::clone(&harness.alive),
failure: Arc::clone(&harness.failure),
cancellation: harness.cancellation.clone(),
};
let response = tokio::spawn(async move {
let result = connection
.wait(
session,
WaitRequest {
cell_id: CellId::new("1".to_string()),
yield_time_ms: 1,
},
)
.await;
(connection, result)
});
while harness.outgoing_rx.is_empty() {
tokio::task::yield_now().await;
}
tokio::time::advance(DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT + Duration::from_secs(2)).await;
let (connection, result) = response.await.expect("wait task");
assert_eq!(
result,
Err("code-mode host timed out waiting for wait response".to_string())
);
assert!(!harness.alive.load(Ordering::Acquire));
assert!(harness.cancellation.is_cancelled());
drop(connection);
}
/// A stalled termination must expire and invalidate the same connection as a stalled wait.
#[tokio::test(start_paused = true)]
async fn queued_remote_termination_times_out_and_invalidates_the_connection() {
let mut harness = DriverHarness::start();
let session = remote_session();
harness
.open(session.clone(), Arc::new(RecordingDelegate::default()))
.await;
let _started = harness
.start_cell(session.clone(), /*request_id*/ 2, "1")
.await;
let connection = Connection {
command_tx: harness.command_tx.clone(),
execute_claim_tx: harness.execute_claim_tx.clone(),
alive: Arc::clone(&harness.alive),
failure: Arc::clone(&harness.failure),
cancellation: harness.cancellation.clone(),
};
let response = tokio::spawn(async move {
let result = connection
.terminate(session, CellId::new("1".to_string()))
.await;
(connection, result)
});
while harness.outgoing_rx.is_empty() {
tokio::task::yield_now().await;
}
tokio::time::advance(DEFAULT_HOST_WAIT_TRANSPORT_TIMEOUT + Duration::from_secs(1)).await;
let (connection, result) = response.await.expect("termination task");
assert_eq!(
result,
Err("code-mode host timed out waiting for terminate response".to_string())
);
assert!(!harness.alive.load(Ordering::Acquire));
assert!(harness.cancellation.is_cancelled());
drop(connection);
}
#[tokio::test]
async fn cancelled_wait_is_retired_before_next_wait_is_sent() {
let mut harness = DriverHarness::start();
@@ -1,3 +1,4 @@
use codex_code_mode_protocol::host::TransportLane;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
@@ -8,6 +9,7 @@ pub(super) async fn drive_reader(
mut reader: ConnectionReader,
events: mpsc::Sender<DriverEvent>,
cancellation: CancellationToken,
lane: Option<TransportLane>,
) -> Result<(), String> {
loop {
let message = tokio::select! {
@@ -19,6 +21,11 @@ pub(super) async fn drive_reader(
Ok(None) => return Err("code-mode host closed its stdout".to_string()),
Err(err) => return Err(format!("failed to read code-mode host message: {err}")),
};
if let Some(lane) = lane
&& !message.allows_transport_lane(lane)
{
return Err("code-mode host sent a message on the wrong websocket lane".to_string());
}
events
.send(DriverEvent::HostMessage(message))
.await
@@ -7,8 +7,10 @@ use codex_code_mode_protocol::CodeModeSessionProvider;
use codex_code_mode_protocol::ExecuteRequest;
use codex_code_mode_protocol::FunctionCallOutputContentItem;
use codex_code_mode_protocol::RuntimeResponse;
use codex_code_mode_protocol::host::Capability;
use codex_code_mode_protocol::host::CapabilitySet;
use codex_code_mode_protocol::host::ClientToHost;
use codex_code_mode_protocol::host::DUAL_WEBSOCKET_CAPABILITY;
use codex_code_mode_protocol::host::EncodedFrame;
use codex_code_mode_protocol::host::HostHello;
use codex_code_mode_protocol::host::HostRequest;
@@ -24,6 +26,8 @@ use codex_http_client::OutboundProxyPolicy;
use futures::SinkExt;
use futures::StreamExt;
use pretty_assertions::assert_eq;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpListener;
use tokio::time::timeout;
use tokio_tungstenite::accept_async;
@@ -224,6 +228,91 @@ async fn websocket_provider_executes_over_shared_connector() {
.expect("websocket test host task should succeed");
}
#[tokio::test]
async fn websocket_provider_fails_when_a_negotiated_bulk_connection_is_unavailable() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("websocket test listener should bind");
let websocket_url = format!(
"ws://{}/?access_token=shared-token",
listener
.local_addr()
.expect("websocket test listener should have an address")
);
let server = tokio::spawn(async move {
let (stream, _) = listener
.accept()
.await
.expect("websocket test host should accept the first connection");
let mut control = accept_async(stream)
.await
.expect("first control websocket should connect");
let frame = control
.next()
.await
.expect("first client hello")
.expect("first websocket frame")
.into_data();
let ClientToHost::ClientHello(hello) =
EncodedFrame::decode_framed(&frame).expect("decode first client hello")
else {
panic!("expected first client hello");
};
let capability =
Capability::new(DUAL_WEBSOCKET_CAPABILITY).expect("dual websocket capability");
assert!(hello.optional_capabilities().contains(&capability));
let hello = HostToClient::HostHello(
HostHello::new(
ProtocolVersion::V1,
CapabilitySet::try_new([capability.clone()]).expect("host capabilities"),
)
.with_bulk_connection_token("fallback-token".to_string()),
);
let frame = EncodedFrame::encode(&hello).expect("encode dual host hello");
control
.send(Message::Binary(frame.into_framed_bytes().into()))
.await
.expect("send dual host hello");
let (mut bulk, _) = listener
.accept()
.await
.expect("websocket test host should accept the bulk connection");
let mut request = [0_u8; 1024];
let request_len = bulk
.read(&mut request)
.await
.expect("read bulk websocket handshake");
let request = std::str::from_utf8(&request[..request_len]).expect("bulk HTTP request");
assert!(
request.starts_with("GET /bulk/fallback-token?access_token=shared-token HTTP/1.1\r\n")
);
bulk.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.await
.expect("reject unavailable bulk websocket");
drop(bulk);
drop(control);
});
let provider = WebSocketCodeModeSessionProvider::new(websocket_url);
let error = match provider
.create_session(Arc::new(NoopCodeModeSessionDelegate))
.await
{
Ok(_) => panic!("provider should reject an unavailable negotiated bulk websocket"),
Err(error) => error,
};
assert!(
error.contains("404"),
"unexpected negotiated bulk websocket error: {error}"
);
drop(provider);
timeout(Duration::from_secs(5), server)
.await
.expect("negotiated bulk websocket test host should disconnect promptly")
.expect("negotiated bulk websocket test host task should succeed");
}
#[tokio::test]
async fn provider_returns_missing_host_error() {
let provider = ProcessOwnedCodeModeSessionProvider::with_host_program(
+18 -202
View File
@@ -4,9 +4,6 @@ mod namespace;
pub use environment::EnvironmentSkillLoadOutcome;
pub use environment::EnvironmentSkillMetadata;
pub use environment::EnvironmentSkillSnapshot;
pub use environment::EnvironmentSkillSnapshotOutcome;
pub use environment::load_environment_skills_from_discovery;
pub use environment::load_environment_skills_from_root;
use crate::model::SkillDependencies;
@@ -26,6 +23,9 @@ use codex_exec_server::ExecutorFileSystem;
use codex_exec_server::LOCAL_FS;
use codex_protocol::protocol::Product;
use codex_protocol::protocol::SkillScope;
use codex_skills::ParsedSkillFrontmatter;
use codex_skills::SkillParseError;
use codex_skills::parse_skill_frontmatter_metadata;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_absolute_path::AbsolutePathBufGuard;
use codex_utils_path_uri::PathUri;
@@ -60,22 +60,6 @@ use tracing::error;
// TODO(anp): Tune this eight-scan limit after revisiting byte-based backpressure.
pub const MAX_CONCURRENT_ROOT_SCANS: usize = 8;
#[derive(Debug, Deserialize)]
struct SkillFrontmatter {
#[serde(default)]
name: Option<String>,
#[serde(default)]
description: Option<String>,
#[serde(default)]
metadata: SkillFrontmatterMetadata,
}
#[derive(Debug, Default, Deserialize)]
struct SkillFrontmatterMetadata {
#[serde(default, rename = "short-description")]
short_description: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
struct SkillMetadataFile {
#[serde(default)]
@@ -128,13 +112,6 @@ struct DependencyTool {
url: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct ParsedSkillFrontmatter {
name: String,
description: String,
short_description: Option<String>,
}
const SKILLS_FILENAME: &str = "SKILL.md";
const AGENTS_DIR_NAME: &str = ".agents";
const SKILLS_METADATA_DIR: &str = "agents";
@@ -165,31 +142,27 @@ struct ResolvedDiscoveredSkill {
}
#[derive(Debug)]
enum SkillParseError {
enum SkillLoadError {
Read(std::io::Error),
MissingFrontmatter,
InvalidYaml(serde_yaml::Error),
MissingField(&'static str),
InvalidField { field: &'static str, reason: String },
Parse(SkillParseError),
}
impl fmt::Display for SkillParseError {
impl fmt::Display for SkillLoadError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
SkillParseError::Read(e) => write!(f, "failed to read file: {e}"),
SkillParseError::MissingFrontmatter => {
write!(f, "missing YAML frontmatter delimited by ---")
}
SkillParseError::InvalidYaml(e) => write!(f, "invalid YAML: {e}"),
SkillParseError::MissingField(field) => write!(f, "missing field `{field}`"),
SkillParseError::InvalidField { field, reason } => {
write!(f, "invalid {field}: {reason}")
}
Self::Read(error) => write!(f, "failed to read file: {error}"),
Self::Parse(error) => write!(f, "{error}"),
}
}
}
impl Error for SkillParseError {}
impl Error for SkillLoadError {}
impl From<SkillParseError> for SkillLoadError {
fn from(error: SkillParseError) -> Self {
Self::Parse(error)
}
}
pub struct SkillRoot {
pub path: AbsolutePathBuf,
@@ -693,7 +666,7 @@ async fn parse_skill_file(
scope: SkillScope,
plugin_identity: Option<&PluginIdentity>,
plugin_root: Option<&AbsolutePathBuf>,
) -> Result<SkillMetadata, SkillParseError> {
) -> Result<SkillMetadata, SkillLoadError> {
let metadata_path = path_uri
.parent()
.and_then(|parent| parent.join(SKILLS_METADATA_DIR).ok())
@@ -708,12 +681,12 @@ async fn parse_skill_file(
fs.read_file_text(path_uri, /*sandbox*/ None),
load_skill_metadata(fs, path, &metadata, plugin_root),
);
let contents = contents.map_err(SkillParseError::Read)?;
let contents = contents.map_err(SkillLoadError::Read)?;
let ParsedSkillFrontmatter {
name: base_name,
description,
short_description,
} = parse_skill_frontmatter_metadata_inner(&contents, || default_skill_name(path))?;
} = parse_skill_frontmatter_metadata(&contents, || default_skill_name(path))?;
let LoadedSkillMetadata {
interface,
dependencies,
@@ -734,56 +707,6 @@ async fn parse_skill_file(
})
}
fn parse_skill_frontmatter_metadata_inner(
contents: &str,
default_name: impl FnOnce() -> String,
) -> Result<ParsedSkillFrontmatter, SkillParseError> {
let frontmatter = extract_frontmatter(contents).ok_or(SkillParseError::MissingFrontmatter)?;
let parsed: SkillFrontmatter = match serde_yaml::from_str(&frontmatter) {
Ok(parsed) => Ok(parsed),
Err(original_error) => match repair_frontmatter_scalar_fields(&frontmatter) {
// Some third-party skills use prose like `description: Build for AWS: ECS`
// or `argument-hint: <duration: e.g. 7d>`. Keep the repair line-oriented
// so unrelated invalid YAML still surfaces.
Some(repaired_frontmatter) => {
serde_yaml::from_str(&repaired_frontmatter).map_err(|_| original_error)
}
None => Err(original_error),
},
}
.map_err(SkillParseError::InvalidYaml)?;
let name = parsed
.name
.as_deref()
.map(sanitize_single_line)
.filter(|value| !value.is_empty())
.unwrap_or_else(default_name);
let description = parsed
.description
.as_deref()
.map(sanitize_single_line)
.unwrap_or_default();
let short_description = parsed
.metadata
.short_description
.as_deref()
.map(sanitize_single_line)
.filter(|value| !value.is_empty());
validate_len(&name, MAX_NAME_LEN, "name")?;
if description.is_empty() {
return Err(SkillParseError::MissingField("description"));
}
Ok(ParsedSkillFrontmatter {
name,
description,
short_description,
})
}
fn default_skill_name(path: &AbsolutePathBuf) -> String {
path.parent()
.and_then(|parent| {
@@ -1067,91 +990,6 @@ fn sanitize_single_line(raw: &str) -> String {
raw.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn repair_frontmatter_scalar_fields(frontmatter: &str) -> Option<String> {
let mut changed = false;
let mut block_scalar_indent: Option<usize> = None;
let mut repaired_lines: Vec<String> = Vec::new();
for line in frontmatter.lines() {
let indent = line
.chars()
.take_while(|character| *character == ' ')
.count();
if let Some(block_indent) = block_scalar_indent {
if line.trim().is_empty() || indent > block_indent {
repaired_lines.push(line.to_string());
continue;
}
block_scalar_indent = None;
}
let Some((key, value)) = line.split_once(':') else {
repaired_lines.push(line.to_string());
continue;
};
if key.trim().is_empty() || !value.chars().next().is_none_or(char::is_whitespace) {
repaired_lines.push(line.to_string());
continue;
}
let trimmed_start = value.trim_start();
let leading_whitespace = &value[..value.len() - trimmed_start.len()];
let mut scalar = trimmed_start;
let mut comment = "";
for (index, character) in trimmed_start.char_indices() {
if character == '#'
&& (index == 0
|| trimmed_start[..index]
.chars()
.next_back()
.is_some_and(char::is_whitespace))
{
let comment_start = trimmed_start[..index].trim_end().len();
scalar = &trimmed_start[..comment_start];
comment = &trimmed_start[comment_start..];
break;
}
}
let scalar = scalar.trim_end();
let Some(first_char) = scalar.chars().next() else {
repaired_lines.push(line.to_string());
continue;
};
if matches!(first_char, '|' | '>') {
block_scalar_indent = Some(indent);
repaired_lines.push(line.to_string());
continue;
}
if matches!(first_char, '\'' | '"') {
repaired_lines.push(line.to_string());
continue;
}
let mut has_colon_separator = false;
let mut chars = scalar.chars().peekable();
while let Some(character) = chars.next() {
if character == ':'
&& matches!(chars.peek(), Some(next_character) if next_character.is_whitespace())
{
has_colon_separator = true;
break;
}
}
let invalid_flow_like_scalar = matches!(first_char, '[' | '{' | '@' | '`')
&& serde_yaml::from_str::<serde_yaml::Value>(scalar).is_err();
if !has_colon_separator && !invalid_flow_like_scalar {
repaired_lines.push(line.to_string());
continue;
}
let quoted_scalar = format!("'{}'", scalar.replace('\'', "''"));
repaired_lines.push(format!(
"{key}:{leading_whitespace}{quoted_scalar}{comment}"
));
changed = true;
}
changed.then(|| repaired_lines.join("\n"))
}
fn validate_len(
value: &str,
max_len: usize,
@@ -1211,28 +1049,6 @@ fn resolve_color_str(value: Option<String>, field: &'static str) -> Option<Strin
}
}
fn extract_frontmatter(contents: &str) -> Option<String> {
let mut lines = contents.lines();
if !matches!(lines.next(), Some(line) if line.trim() == "---") {
return None;
}
let mut frontmatter_lines: Vec<&str> = Vec::new();
let mut found_closing = false;
for line in lines.by_ref() {
if line.trim() == "---" {
found_closing = true;
break;
}
frontmatter_lines.push(line);
}
if frontmatter_lines.is_empty() || !found_closing {
return None;
}
Some(frontmatter_lines.join("\n"))
}
#[cfg(test)]
pub(crate) async fn skill_roots_from_layer_stack(
fs: Arc<dyn ExecutorFileSystem>,
+3 -142
View File
@@ -1,10 +1,10 @@
use std::collections::HashMap;
use std::io;
use codex_exec_server::CapabilityRootDiscovery;
use codex_exec_server::ExecutorFileSystem;
use codex_protocol::protocol::Product;
pub use codex_skills::EnvironmentSkillMetadata;
use codex_skills::ParsedSkillFrontmatter;
use codex_skills::parse_skill_frontmatter_metadata;
use codex_utils_path_uri::PathUri;
use futures::StreamExt;
@@ -12,7 +12,6 @@ use crate::model::SkillDependencies;
use crate::model::SkillPolicy;
use super::MAX_QUALIFIED_NAME_LEN;
use super::ParsedSkillFrontmatter;
use super::SkillMetadataFile;
use super::discovery::DirectorySymlinkPolicy;
use super::discovery::DiscoveredSkill;
@@ -22,7 +21,6 @@ use super::discovery::SkillDiscoveryOptions;
use super::discovery::SkillMetadataDiscovery;
use super::discovery::discover_skills;
use super::namespace::SkillNamespaceResolver;
use super::parse_skill_frontmatter_metadata_inner;
use super::resolve_dependencies;
use super::resolve_policy;
use super::sanitize_single_line;
@@ -37,19 +35,6 @@ struct ParsedEnvironmentSkill {
policy: Option<SkillPolicy>,
}
/// Parsed executor skill plus the instructions already materialized by capability discovery.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EnvironmentSkillSnapshot {
pub metadata: EnvironmentSkillMetadata,
pub instructions: String,
}
#[derive(Debug, Default)]
pub struct EnvironmentSkillSnapshotOutcome {
pub skills: Vec<EnvironmentSkillSnapshot>,
pub warnings: Vec<String>,
}
impl ParsedEnvironmentSkill {
async fn load(
file_system: &dyn ExecutorFileSystem,
@@ -72,7 +57,7 @@ impl ParsedEnvironmentSkill {
name: base_name,
description,
short_description,
} = parse_skill_frontmatter_metadata_inner(&contents, || default_skill_name(&skill.path))
} = parse_skill_frontmatter_metadata(&contents, || default_skill_name(&skill.path))
.map_err(|err| err.to_string())?;
let (dependencies, policy) = match &skill.metadata {
SkillMetadataDiscovery::Present(_) | SkillMetadataDiscovery::Absent => {
@@ -197,130 +182,6 @@ pub async fn load_environment_skills_from_root(
outcome
}
/// Parses an executor-produced manifest bundle without issuing additional filesystem requests.
pub fn load_environment_skills_from_discovery(
discovery: &CapabilityRootDiscovery,
restriction_product: Option<Product>,
) -> EnvironmentSkillSnapshotOutcome {
let mut outcome = EnvironmentSkillSnapshotOutcome {
warnings: discovery.warnings.clone(),
..Default::default()
};
if let Some(error) = &discovery.error {
outcome.warnings.push(error.clone());
return outcome;
}
let mut plugin_namespaces = HashMap::new();
for (plugin_root, name) in discovery.namespace_manifests.iter().filter_map(|manifest| {
#[derive(serde::Deserialize)]
struct ManifestName {
#[serde(default)]
name: String,
}
let plugin_root = manifest.path.parent()?.parent()?;
let parsed = serde_json::from_str::<ManifestName>(&manifest.contents).ok()?;
let name = if parsed.name.trim().is_empty() {
plugin_root.basename()?
} else {
parsed.name
};
Some((plugin_root, name))
}) {
// Exec-server orders manifests by the same precedence as local discovery. Preserve the
// first manifest if an older or alternate server returns duplicates for one plugin root.
plugin_namespaces.entry(plugin_root).or_insert(name);
}
for skill in &discovery.skills {
let plugin_namespace =
nearest_plugin_namespace(&skill.instructions.path, &plugin_namespaces);
let ParsedSkillFrontmatter {
name: base_name,
description,
short_description,
} = match parse_skill_frontmatter_metadata_inner(&skill.instructions.contents, || {
default_skill_name(&skill.instructions.path)
}) {
Ok(frontmatter) => frontmatter,
Err(error) => {
outcome.warnings.push(format!(
"Failed to load environment skill at {}: {error}",
skill.instructions.path
));
continue;
}
};
let name = plugin_namespace
.map(|namespace| format!("{namespace}:{base_name}"))
.unwrap_or(base_name);
if let Err(error) = validate_len(&name, MAX_QUALIFIED_NAME_LEN, "qualified name") {
outcome.warnings.push(format!(
"Failed to load environment skill at {}: {error}",
skill.instructions.path
));
continue;
}
let (dependencies, policy) = skill
.metadata
.as_ref()
.and_then(|metadata| {
serde_yaml::from_str::<SkillMetadataFile>(&metadata.contents)
.map_err(|error| {
tracing::warn!(
path = %metadata.path,
"ignoring invalid discovered skill metadata: {error}"
);
})
.ok()
})
.map(|metadata| {
(
resolve_dependencies(metadata.dependencies),
resolve_policy(metadata.policy),
)
})
.unwrap_or((None, None));
let metadata = EnvironmentSkillMetadata {
path_to_skills_md: skill.instructions.path.clone(),
name,
description,
short_description,
dependencies,
policy,
};
if metadata.matches_product_restriction(restriction_product) {
outcome.skills.push(EnvironmentSkillSnapshot {
metadata,
instructions: skill.instructions.contents.clone(),
});
}
}
outcome.skills.sort_by(|left, right| {
left.metadata.name.cmp(&right.metadata.name).then_with(|| {
left.metadata
.path_to_skills_md
.to_string()
.cmp(&right.metadata.path_to_skills_md.to_string())
})
});
outcome
}
fn nearest_plugin_namespace<'a>(
skill_path: &PathUri,
plugin_namespaces: &'a HashMap<PathUri, String>,
) -> Option<&'a str> {
let mut ancestor = skill_path.parent();
while let Some(path) = ancestor {
if let Some(namespace) = plugin_namespaces.get(&path) {
return Some(namespace);
}
ancestor = path.parent();
}
None
}
async fn read_skill_contents(
file_system: &dyn ExecutorFileSystem,
skill_path: &PathUri,
@@ -1,15 +1,10 @@
#![cfg(unix)]
use std::fs;
use std::sync::Mutex;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::time::Duration;
use codex_core_skills::loader::EnvironmentSkillMetadata;
use codex_core_skills::loader::load_environment_skills_from_discovery;
use codex_core_skills::loader::load_environment_skills_from_root;
use codex_exec_server::CapabilityRootDiscoverRequest;
use codex_exec_server::CapabilityRootsDiscoverParams;
use codex_exec_server::CopyOptions;
use codex_exec_server::CreateDirectoryOptions;
use codex_exec_server::ExecutorFileSystem;
@@ -22,26 +17,15 @@ use codex_exec_server::ReadDirectoryEntry;
use codex_exec_server::RemoveOptions;
use codex_exec_server::WalkOptions;
use codex_exec_server::WalkOutcome;
use codex_exec_server::discover_capability_roots;
use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
use tempfile::tempdir;
use tokio::sync::Notify;
#[derive(Clone, Copy)]
enum ManifestMetadataBehavior {
Immediate,
WaitForSkillRead,
}
struct RecordingFileSystem<'a> {
inner: &'a dyn ExecutorFileSystem,
read_files: Mutex<Vec<PathUri>>,
metadata_files: Mutex<Vec<PathUri>>,
walks: AtomicUsize,
manifest_metadata_behavior: ManifestMetadataBehavior,
skill_read_started: AtomicBool,
skill_read_started_notify: Notify,
}
#[derive(Debug, PartialEq, Eq)]
@@ -52,18 +36,12 @@ struct FileSystemCalls {
}
impl<'a> RecordingFileSystem<'a> {
fn new(
inner: &'a dyn ExecutorFileSystem,
manifest_metadata_behavior: ManifestMetadataBehavior,
) -> Self {
fn new(inner: &'a dyn ExecutorFileSystem) -> Self {
Self {
inner,
read_files: Mutex::new(Vec::new()),
metadata_files: Mutex::new(Vec::new()),
walks: AtomicUsize::new(0),
manifest_metadata_behavior,
skill_read_started: AtomicBool::new(false),
skill_read_started_notify: Notify::new(),
}
}
@@ -106,10 +84,6 @@ impl ExecutorFileSystem for RecordingFileSystem<'_> {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(path.clone());
if path.basename().as_deref() == Some("SKILL.md") {
self.skill_read_started.store(true, Ordering::Release);
self.skill_read_started_notify.notify_waiters();
}
self.inner.read_file(path, sandbox)
}
@@ -148,22 +122,6 @@ impl ExecutorFileSystem for RecordingFileSystem<'_> {
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(path.clone());
if matches!(
self.manifest_metadata_behavior,
ManifestMetadataBehavior::WaitForSkillRead
) && path.basename().as_deref() == Some("plugin.json")
{
return Box::pin(async move {
loop {
let notified = self.skill_read_started_notify.notified();
if self.skill_read_started.load(Ordering::Acquire) {
break;
}
notified.await;
}
self.inner.get_metadata(path, sandbox).await
});
}
self.inner.get_metadata(path, sandbox)
}
@@ -206,216 +164,6 @@ impl ExecutorFileSystem for RecordingFileSystem<'_> {
}
}
#[tokio::test]
async fn loads_nearest_plugin_namespaces_without_reading_unused_sibling_manifests() {
let root = tempdir().expect("tempdir");
let standalone_skill = root.path().join("standalone/SKILL.md");
let outer_root = root.path().join("plugins/outer");
let outer_skill = outer_root.join("skills/deploy/SKILL.md");
let inner_root = outer_root.join("nested/inner");
let inner_skill = inner_root.join("skills/audit/SKILL.md");
let unused_root = root.path().join("plugins/unused");
for path in [&standalone_skill, &outer_skill, &inner_skill] {
fs::create_dir_all(path.parent().expect("skill parent")).expect("skill dir");
}
for (plugin_root, name) in [
(&outer_root, "outer"),
(&inner_root, "inner"),
(&unused_root, "unused"),
] {
fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("manifest dir");
fs::write(
plugin_root.join(".codex-plugin/plugin.json"),
format!(r#"{{"name":"{name}"}}"#),
)
.expect("manifest");
}
for (path, name) in [
(&standalone_skill, "standalone"),
(&outer_skill, "deploy"),
(&inner_skill, "audit"),
] {
fs::write(
path,
format!("---\nname: {name}\ndescription: {name} skill.\n---\n"),
)
.expect("skill");
}
let file_system =
RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate);
let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI");
let outcome = load_environment_skills_from_root(
&file_system,
&root_uri,
/*restriction_product*/ None,
)
.await;
assert_eq!(outcome.warnings, Vec::<String>::new());
assert_eq!(
outcome.skills,
vec![
EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(&inner_skill).unwrap(),
name: "inner:audit".to_string(),
description: "audit skill.".to_string(),
short_description: None,
dependencies: None,
policy: None,
},
EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(&outer_skill).unwrap(),
name: "outer:deploy".to_string(),
description: "deploy skill.".to_string(),
short_description: None,
dependencies: None,
policy: None,
},
EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(&standalone_skill).unwrap(),
name: "standalone".to_string(),
description: "standalone skill.".to_string(),
short_description: None,
dependencies: None,
policy: None,
},
]
);
let mut manifest_reads = file_system
.calls()
.read_files
.into_iter()
.filter(|path| path.basename().as_deref() == Some("plugin.json"))
.collect::<Vec<_>>();
manifest_reads.sort_by_key(ToString::to_string);
let mut expected_manifest_reads = [&outer_root, &inner_root]
.into_iter()
.map(|plugin_root| {
PathUri::from_host_native_path(plugin_root.join(".codex-plugin/plugin.json")).unwrap()
})
.collect::<Vec<_>>();
expected_manifest_reads.sort_by_key(ToString::to_string);
assert_eq!(manifest_reads, expected_manifest_reads);
}
#[tokio::test]
async fn reuses_walk_inventory_for_missing_skill_metadata() {
const SKILL_COUNT: usize = 66;
let root = tempdir().expect("tempdir");
let manifest_path = root.path().join(".codex-plugin/plugin.json");
fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir");
fs::write(&manifest_path, r#"{"name":"inventory"}"#).expect("manifest");
let mut skill_paths = Vec::new();
for index in 0..SKILL_COUNT {
let name = format!("skill-{index}");
let skill_path = root.path().join(&name).join("SKILL.md");
fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir");
fs::write(
&skill_path,
format!("---\nname: {name}\ndescription: {name} skill.\n---\n"),
)
.expect("skill");
skill_paths.push(skill_path);
}
let file_system =
RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate);
let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI");
let outcome = load_environment_skills_from_root(
&file_system,
&root_uri,
/*restriction_product*/ None,
)
.await;
let mut expected_skills = skill_paths
.iter()
.enumerate()
.map(|(index, skill_path)| EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(),
name: format!("inventory:skill-{index}"),
description: format!("skill-{index} skill."),
short_description: None,
dependencies: None,
policy: None,
})
.collect::<Vec<_>>();
expected_skills.sort_by(|left, right| {
left.name.cmp(&right.name).then_with(|| {
left.path_to_skills_md
.to_string()
.cmp(&right.path_to_skills_md.to_string())
})
});
assert_eq!(outcome.skills, expected_skills);
assert_eq!(outcome.warnings, Vec::<String>::new());
let mut expected_read_files = skill_paths
.iter()
.map(|path| PathUri::from_host_native_path(path).unwrap())
.collect::<Vec<_>>();
let manifest_uri = PathUri::from_host_native_path(manifest_path).unwrap();
expected_read_files.push(manifest_uri.clone());
expected_read_files.sort_by_key(ToString::to_string);
assert_eq!(
file_system.calls(),
FileSystemCalls {
walks: 1,
read_files: expected_read_files,
metadata_files: vec![manifest_uri],
}
);
}
#[tokio::test]
async fn reads_skill_files_while_resolving_plugin_namespaces() {
let root = tempdir().expect("tempdir");
let manifest_path = root.path().join(".codex-plugin/plugin.json");
fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir");
fs::write(&manifest_path, r#"{"name":"parallel"}"#).expect("manifest");
let skill_path = root.path().join("demo/SKILL.md");
fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir");
fs::write(
&skill_path,
"---\nname: demo\ndescription: demo skill.\n---\n",
)
.expect("skill");
let file_system = RecordingFileSystem::new(
LOCAL_FS.as_ref(),
ManifestMetadataBehavior::WaitForSkillRead,
);
let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI");
let outcome = tokio::time::timeout(
Duration::from_secs(5),
load_environment_skills_from_root(
&file_system,
&root_uri,
/*restriction_product*/ None,
),
)
.await
.expect("skill reads should start before namespace resolution finishes");
assert_eq!(outcome.warnings, Vec::<String>::new());
assert_eq!(
outcome.skills,
vec![EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(),
name: "parallel:demo".to_string(),
description: "demo skill.".to_string(),
short_description: None,
dependencies: None,
policy: None,
}]
);
}
#[cfg(unix)]
#[tokio::test]
async fn host_loading_reuses_walk_inventory_for_symlinked_skill_pack() {
@@ -459,10 +207,7 @@ async fn host_loading_reuses_walk_inventory_for_symlinked_skill_pack() {
let linked_root = host_root.join("linked-plugin");
symlink(&skills_root, &linked_root).expect("skill pack symlink");
let recording = Arc::new(RecordingFileSystem::new(
LOCAL_FS.as_ref(),
ManifestMetadataBehavior::Immediate,
));
let recording = Arc::new(RecordingFileSystem::new(LOCAL_FS.as_ref()));
let file_system: Arc<dyn ExecutorFileSystem> = recording.clone();
let future = load_skills_from_roots(
[SkillRoot {
@@ -547,142 +292,3 @@ async fn host_loading_reuses_walk_inventory_for_symlinked_skill_pack() {
1
);
}
#[tokio::test]
async fn executor_bundle_parser_matches_the_existing_environment_loader() {
let root = tempdir().expect("tempdir");
let plugin_manifest = root.path().join(".codex-plugin/plugin.json");
let nested_manifest = root.path().join("nested/.claude-plugin/plugin.json");
let deploy_skill = root.path().join("skills/deploy/SKILL.md");
let deploy_metadata = root.path().join("skills/deploy/agents/openai.yaml");
let audit_skill = root.path().join("nested/skills/audit/SKILL.md");
for (path, contents) in [
(&plugin_manifest, r#"{"name":"demo"}"#),
(&nested_manifest, r#"{"name":"nested"}"#),
(
&deploy_skill,
"---\nname: deploy\ndescription: Deploy the service.\n---\n\nDeploy.\n",
),
(
&deploy_metadata,
"policy:\n allow_implicit_invocation: false\n",
),
(
&audit_skill,
"---\nname: audit\ndescription: Audit the service.\n---\n\nAudit.\n",
),
] {
fs::create_dir_all(path.parent().expect("test file parent")).expect("test directory");
fs::write(path, contents).expect("test file");
}
let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI");
let existing = load_environment_skills_from_root(
LOCAL_FS.as_ref(),
&root_uri,
/*restriction_product*/ None,
)
.await;
let response = discover_capability_roots(
LOCAL_FS.as_ref(),
CapabilityRootsDiscoverParams {
roots: vec![CapabilityRootDiscoverRequest {
id: "demo@1".to_string(),
path: root_uri,
sandbox: None,
}],
},
)
.await
.expect("capability discovery");
let bundled = load_environment_skills_from_discovery(
response.roots.first().expect("discovered root"),
/*restriction_product*/ None,
);
assert_eq!(bundled.warnings, existing.warnings);
assert_eq!(
bundled
.skills
.iter()
.map(|skill| skill.metadata.clone())
.collect::<Vec<_>>(),
existing.skills
);
assert_eq!(
bundled
.skills
.iter()
.map(|skill| skill.instructions.as_str())
.collect::<Vec<_>>(),
vec![
"---\nname: deploy\ndescription: Deploy the service.\n---\n\nDeploy.\n",
"---\nname: audit\ndescription: Audit the service.\n---\n\nAudit.\n",
]
);
}
#[tokio::test]
async fn executor_bundle_preserves_parent_namespace_and_manifest_precedence() {
let plugin = tempdir().expect("tempdir");
for (relative_path, name) in [
(".codex-plugin/plugin.json", "codex-name"),
(".claude-plugin/plugin.json", "claude-name"),
(".cursor-plugin/plugin.json", "cursor-name"),
] {
let manifest = plugin.path().join(relative_path);
fs::create_dir_all(manifest.parent().expect("manifest parent"))
.expect("manifest directory");
fs::write(&manifest, format!(r#"{{"name":"{name}"}}"#)).expect("manifest");
}
let skills_root = plugin.path().join("skills");
let skill_path = skills_root.join("search/SKILL.md");
fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill directory");
fs::write(
&skill_path,
"---\nname: search\ndescription: Search the project.\n---\n\nSearch.\n",
)
.expect("skill");
let root_uri = PathUri::from_host_native_path(&skills_root).expect("skills root URI");
let existing = load_environment_skills_from_root(
LOCAL_FS.as_ref(),
&root_uri,
/*restriction_product*/ None,
)
.await;
let response = discover_capability_roots(
LOCAL_FS.as_ref(),
CapabilityRootsDiscoverParams {
roots: vec![CapabilityRootDiscoverRequest {
id: "skills-only".to_string(),
path: root_uri,
sandbox: None,
}],
},
)
.await
.expect("capability discovery");
let discovery = response.roots.first().expect("discovered root");
let bundled =
load_environment_skills_from_discovery(discovery, /*restriction_product*/ None);
assert_eq!(discovery.plugin, None);
assert_eq!(discovery.namespace_manifests.len(), 1);
assert!(
discovery.namespace_manifests[0]
.path
.to_string()
.ends_with("/.codex-plugin/plugin.json")
);
assert_eq!(bundled.warnings, existing.warnings);
assert_eq!(
bundled
.skills
.iter()
.map(|skill| skill.metadata.clone())
.collect::<Vec<_>>(),
existing.skills
);
assert_eq!(bundled.skills[0].metadata.name, "codex-name:search");
}
+1 -1
View File
@@ -69,6 +69,7 @@ codex-terminal-detection = { workspace = true }
codex-thread-store = { workspace = true }
codex-tools = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-audio = { workspace = true }
codex-utils-cache = { workspace = true }
codex-utils-image = { workspace = true }
codex-utils-home-dir = { workspace = true }
@@ -103,7 +104,6 @@ serde_json = { workspace = true }
sha1 = { workspace = true }
shlex = { workspace = true }
similar = { workspace = true }
symphonia = { workspace = true }
tempfile = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = [
+4
View File
@@ -3,6 +3,7 @@ use crate::config::ConfigBuilder;
use crate::context::ContextualUserFragment;
use crate::environment_selection::TurnEnvironmentSnapshot;
use crate::environment_selection::TurnEnvironmentState;
use crate::session::turn_context::EnvironmentConfig;
use crate::session::turn_context::TurnEnvironment;
use codex_config::ConfigLayerEntry;
use codex_config::ConfigLayerStack;
@@ -337,6 +338,9 @@ fn resolved_local_environments<const N: usize>(
PathUri::from_abs_path(&cwd),
Vec::new(),
/*shell*/ None,
EnvironmentConfig {
allow_login_shell: true,
},
))
})
.collect(),
@@ -1,5 +1,8 @@
use super::ContextualUserFragment;
pub(crate) const APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX: &str =
"Approved command prefix saved:";
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ApprovedCommandPrefixSaved {
prefixes: String,
@@ -27,6 +30,9 @@ impl ContextualUserFragment for ApprovedCommandPrefixSaved {
}
fn body(&self) -> String {
format!("Approved command prefix saved:\n{}", self.prefixes)
format!(
"{APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX}\n{}",
self.prefixes
)
}
}
+1
View File
@@ -34,6 +34,7 @@ mod user_instructions;
mod user_shell_command;
pub(crate) mod world_state;
pub(crate) use approved_command_prefix_saved::APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX;
pub(crate) use approved_command_prefix_saved::ApprovedCommandPrefixSaved;
pub(crate) use apps_instructions::AppsInstructions;
pub(crate) use available_plugins_instructions::AvailablePluginsInstructions;
@@ -1,4 +1,5 @@
use super::ContextualUserFragment;
use codex_protocol::AgentPath;
use codex_protocol::ThreadId;
use codex_protocol::protocol::CONTEXT_WINDOW_CLOSE_TAG;
use codex_protocol::protocol::CONTEXT_WINDOW_GUIDANCE_CLOSE_TAG;
@@ -9,6 +10,7 @@ use uuid::Uuid;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct TokenBudgetContext {
thread_id: ThreadId,
agent_path: AgentPath,
first_window_id: Uuid,
previous_window_id: Option<Uuid>,
window_id: Uuid,
@@ -18,6 +20,7 @@ pub(crate) struct TokenBudgetContext {
impl TokenBudgetContext {
pub(crate) fn new(
thread_id: ThreadId,
agent_path: AgentPath,
first_window_id: Uuid,
previous_window_id: Option<Uuid>,
window_id: Uuid,
@@ -25,6 +28,7 @@ impl TokenBudgetContext {
) -> Self {
Self {
thread_id,
agent_path,
first_window_id,
previous_window_id,
window_id,
@@ -47,11 +51,11 @@ impl ContextualUserFragment for TokenBudgetContext {
}
fn body(&self) -> String {
let thread_id = self.thread_id;
let agent_path = &self.agent_path;
let first_window_id = self.first_window_id;
let window_id = self.window_id;
let mut lines = vec![
format!("Thread id: {thread_id}"),
format!("Agent name: {agent_path}"),
format!("First context window id: {first_window_id}"),
format!("Current context window id: {window_id}"),
];
@@ -0,0 +1,59 @@
use super::PreviousSectionState;
use super::WorldStateSection;
use crate::context::APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX;
use crate::context::ApprovedCommandPrefixSaved;
use crate::context::ContextualUserFragment;
use codex_execpolicy::Policy;
use codex_protocol::models::format_allow_prefixes;
use std::collections::BTreeSet;
/// Newly approved command prefixes visible without the full permissions instructions.
#[derive(Clone, Debug)]
pub(crate) struct CompactPermissionsState {
prefixes: BTreeSet<Vec<String>>,
}
impl CompactPermissionsState {
pub(crate) fn new(exec_policy: &Policy) -> Self {
Self {
prefixes: exec_policy.get_allowed_prefixes().into_iter().collect(),
}
}
}
impl WorldStateSection for CompactPermissionsState {
const ID: &'static str = "approved_command_prefixes";
type Snapshot = BTreeSet<Vec<String>>;
fn snapshot(&self) -> Self::Snapshot {
self.prefixes.clone()
}
fn matches_legacy_fragment(role: &str, text: &str) -> bool {
role == "developer"
&& text
.trim_start()
.starts_with(APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX)
}
fn render_diff(
&self,
previous: PreviousSectionState<'_, Self::Snapshot>,
) -> Option<Box<dyn ContextualUserFragment>> {
let added_prefixes = match previous {
PreviousSectionState::Known(previous) => self
.prefixes
.difference(previous)
.cloned()
.collect::<Vec<_>>(),
PreviousSectionState::Absent | PreviousSectionState::Unknown => return None,
};
format_allow_prefixes(added_prefixes)
.filter(|prefixes| !prefixes.is_empty())
.map(|prefixes| Box::new(ApprovedCommandPrefixSaved::new(prefixes)) as _)
}
}
#[cfg(test)]
#[path = "compact_permissions_tests.rs"]
mod tests;
@@ -0,0 +1,64 @@
use super::*;
use codex_execpolicy::Decision;
use pretty_assertions::assert_eq;
#[test]
fn renders_only_newly_approved_prefixes() {
use PreviousSectionState::Known;
let mut exec_policy = Policy::empty();
exec_policy
.add_prefix_rule(&["git".to_string(), "pull".to_string()], Decision::Allow)
.expect("test prefix should be valid");
let with_existing_prefix = CompactPermissionsState::new(&exec_policy);
exec_policy
.add_prefix_rule(&["cargo".to_string(), "test".to_string()], Decision::Allow)
.expect("test prefix should be valid");
let with_new_prefix = CompactPermissionsState::new(&exec_policy);
let existing_snapshot = with_existing_prefix.snapshot();
let current_snapshot = with_new_prefix.snapshot();
assert_eq!(
with_new_prefix
.render_diff(Known(&existing_snapshot))
.map(|fragment| fragment.render()),
Some("Approved command prefix saved:\n- [\"cargo\", \"test\"]".to_string())
);
assert!(
with_new_prefix
.render_diff(Known(&current_snapshot))
.is_none()
);
}
#[test]
fn does_not_duplicate_a_retained_legacy_update() {
use PreviousSectionState::Unknown;
let mut exec_policy = Policy::empty();
exec_policy
.add_prefix_rule(&["touch".to_string()], Decision::Allow)
.expect("test prefix should be valid");
let state = CompactPermissionsState::new(&exec_policy);
assert_eq!(
state.render_diff(Unknown).map(|fragment| fragment.render()),
None
);
}
#[test]
fn does_not_render_existing_prefixes_without_a_previous_snapshot() {
use PreviousSectionState::Absent;
let mut exec_policy = Policy::empty();
exec_policy
.add_prefix_rule(&["touch".to_string()], Decision::Allow)
.expect("test prefix should be valid");
assert!(
CompactPermissionsState::new(&exec_policy)
.render_diff(Absent)
.is_none()
);
}
@@ -1,6 +1,7 @@
mod agents_md;
mod apps_instructions;
mod collaboration_mode;
mod compact_permissions;
mod context_window_guidance;
mod environment;
mod environments_instructions;
@@ -33,6 +34,7 @@ use std::fmt;
pub(crate) use agents_md::AgentsMdState;
pub(crate) use apps_instructions::AppsInstructionsState;
pub(crate) use collaboration_mode::CollaborationModeState;
pub(crate) use compact_permissions::CompactPermissionsState;
pub(crate) use context_window_guidance::ContextWindowGuidanceState;
pub(crate) use environment::EnvironmentsState;
pub(crate) use environments_instructions::EnvironmentsInstructionsState;
@@ -2,20 +2,35 @@ use super::PreviousSectionState;
use super::WorldStateHash;
use super::WorldStateSection;
use crate::context::ApprovalPromptContext;
use crate::context::ApprovedCommandPrefixSaved;
use crate::context::ContextualUserFragment;
use crate::context::PermissionsInstructions;
use codex_execpolicy::Policy;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::format_allow_prefixes;
use codex_protocol::protocol::AskForApproval;
use serde::Deserialize;
use serde::Serialize;
use std::collections::BTreeSet;
use std::path::Path;
/// Permission instructions currently visible to the model.
#[derive(Clone, Debug)]
pub(crate) struct PermissionsState {
snapshot: WorldStateHash,
snapshot: PermissionsSnapshot,
instructions: PermissionsInstructions,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
pub(crate) enum PermissionsSnapshot {
Current {
instructions: WorldStateHash,
approved_command_prefixes: BTreeSet<Vec<String>>,
},
Legacy(WorldStateHash),
}
impl PermissionsState {
pub(crate) fn new(
permission_profile: &PermissionProfile,
@@ -26,16 +41,23 @@ impl PermissionsState {
exec_permission_approvals_enabled: bool,
request_permissions_tool_enabled: bool,
) -> Self {
let instructions = PermissionsInstructions::from_permission_profile(
permission_profile,
approval_policy,
approval_context,
exec_policy,
cwd,
exec_permission_approvals_enabled,
request_permissions_tool_enabled,
);
let snapshot = WorldStateHash::from_fragment(&instructions);
let build_instructions = |exec_policy| {
PermissionsInstructions::from_permission_profile(
permission_profile,
approval_policy,
approval_context,
exec_policy,
cwd,
exec_permission_approvals_enabled,
request_permissions_tool_enabled,
)
};
let instructions = build_instructions(exec_policy);
let instructions_without_approved_prefixes = build_instructions(&Policy::empty());
let snapshot = PermissionsSnapshot::Current {
instructions: WorldStateHash::from_fragment(&instructions_without_approved_prefixes),
approved_command_prefixes: exec_policy.get_allowed_prefixes().into_iter().collect(),
};
Self {
snapshot,
instructions,
@@ -45,7 +67,7 @@ impl PermissionsState {
impl WorldStateSection for PermissionsState {
const ID: &'static str = "permissions";
type Snapshot = WorldStateHash;
type Snapshot = PermissionsSnapshot;
fn snapshot(&self) -> Self::Snapshot {
self.snapshot.clone()
@@ -67,8 +89,35 @@ impl WorldStateSection for PermissionsState {
&self,
previous: PreviousSectionState<'_, Self::Snapshot>,
) -> Option<Box<dyn ContextualUserFragment>> {
if matches!(previous, PreviousSectionState::Known(previous) if previous == &self.snapshot) {
return None;
match (previous, &self.snapshot) {
(
PreviousSectionState::Known(PermissionsSnapshot::Current {
instructions: previous_instructions,
approved_command_prefixes: previous_prefixes,
}),
PermissionsSnapshot::Current {
instructions,
approved_command_prefixes,
},
) if previous_instructions == instructions => {
if previous_prefixes == approved_command_prefixes {
return None;
}
if previous_prefixes.is_subset(approved_command_prefixes) {
let added_prefixes = approved_command_prefixes
.difference(previous_prefixes)
.cloned()
.collect();
if let Some(prefixes) = format_allow_prefixes(added_prefixes) {
return Some(Box::new(ApprovedCommandPrefixSaved::new(prefixes)));
}
}
}
(
PreviousSectionState::Known(PermissionsSnapshot::Legacy(previous)),
PermissionsSnapshot::Current { .. },
) if previous == &WorldStateHash::from_fragment(&self.instructions) => return None,
_ => {}
}
Some(Box::new(self.instructions.clone()))
@@ -1,5 +1,6 @@
use super::*;
use crate::context::world_state::test_support::render_section_cases;
use codex_execpolicy::Decision;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::models::ContentItem;
use codex_protocol::models::PermissionProfile;
@@ -8,6 +9,7 @@ use codex_protocol::openai_models::ApprovalMessages;
use codex_protocol::openai_models::PermissionMessages;
use codex_protocol::protocol::AskForApproval;
use pretty_assertions::assert_eq;
use serde_json::json;
use std::path::Path;
#[test]
@@ -30,6 +32,145 @@ fn snapshots() {
]));
}
#[test]
fn approved_prefix_is_rendered_without_reinjecting_permissions() {
use PreviousSectionState::Known;
let without_approved_prefix = permissions_state_with_default_messages(&Policy::empty());
let mut exec_policy = Policy::empty();
exec_policy
.add_prefix_rule(
&["touch".to_string(), "allow-prefix.txt".to_string()],
Decision::Allow,
)
.expect("test prefix should be valid");
let with_approved_prefix = permissions_state_with_default_messages(&exec_policy);
let approved_prefix = r#"["touch", "allow-prefix.txt"]"#;
let without_snapshot = without_approved_prefix.snapshot();
let with_snapshot = with_approved_prefix.snapshot();
let rendered_update = with_approved_prefix
.render_diff(Known(&without_snapshot))
.expect("approving a prefix should render a world-state update")
.render();
assert_ne!(without_snapshot, with_snapshot);
assert_ne!(
without_approved_prefix.instructions,
with_approved_prefix.instructions
);
assert!(
!without_approved_prefix
.instructions
.body()
.contains(approved_prefix)
);
assert!(
with_approved_prefix
.instructions
.body()
.contains(approved_prefix)
);
assert_eq!(
rendered_update,
"Approved command prefix saved:\n- [\"touch\", \"allow-prefix.txt\"]"
);
assert!(!rendered_update.contains("<permissions instructions>"));
let before_approval_permissions =
permissions_state(PermissionProfile::read_only(), AskForApproval::OnRequest)
.instructions
.render();
let model_visible_before_and_after = format!(
"BEFORE APPROVAL — PERMISSIONS ALREADY IN CONTEXT (CONDENSED)\n{before_approval_permissions}\n\nAFTER APPROVAL — WORLD-STATE DIFF APPENDS\n{rendered_update}\n\nFULL PERMISSIONS BLOCK APPENDED AFTER APPROVAL\nNone",
);
insta::assert_snapshot!(
"approved_prefix_is_rendered_without_reinjecting_permissions",
model_visible_before_and_after
);
}
#[test]
fn renders_only_newly_approved_prefixes() {
use PreviousSectionState::Known;
let mut exec_policy = Policy::empty();
exec_policy
.add_prefix_rule(&["git".to_string(), "pull".to_string()], Decision::Allow)
.expect("test prefix should be valid");
let with_existing_prefix = permissions_state_with_default_messages(&exec_policy);
exec_policy
.add_prefix_rule(&["cargo".to_string(), "test".to_string()], Decision::Allow)
.expect("test prefix should be valid");
let with_new_prefix = permissions_state_with_default_messages(&exec_policy);
let existing_snapshot = with_existing_prefix.snapshot();
let current_snapshot = with_new_prefix.snapshot();
assert_eq!(
with_new_prefix
.render_diff(Known(&existing_snapshot))
.map(|fragment| fragment.render()),
Some("Approved command prefix saved:\n- [\"cargo\", \"test\"]".to_string())
);
assert!(
with_new_prefix
.render_diff(Known(&current_snapshot))
.is_none()
);
}
#[test]
fn legacy_snapshot_deserializes_and_only_suppresses_matching_full_permissions() {
use super::super::WorldState;
use super::super::WorldStateSnapshot;
let without_approved_prefix = permissions_state_with_default_messages(&Policy::empty());
let mut exec_policy = Policy::empty();
exec_policy
.add_prefix_rule(&["touch".to_string()], Decision::Allow)
.expect("test prefix should be valid");
let with_approved_prefix = permissions_state_with_default_messages(&exec_policy);
let matching_legacy: WorldStateSnapshot = serde_json::from_value(json!({
"permissions": WorldStateHash::from_fragment(&with_approved_prefix.instructions),
}))
.expect("legacy world-state snapshot should deserialize");
let stale_legacy: WorldStateSnapshot = serde_json::from_value(json!({
"permissions": WorldStateHash::from_fragment(&without_approved_prefix.instructions),
}))
.expect("legacy world-state snapshot should deserialize");
let expected_permissions = with_approved_prefix.instructions.render();
let mut world_state = WorldState::default();
world_state.add_section(with_approved_prefix);
assert!(world_state.render_diff(&matching_legacy).is_empty());
assert_eq!(
world_state
.render_diff(&stale_legacy)
.into_iter()
.map(|fragment| fragment.render())
.collect::<Vec<_>>(),
vec![expected_permissions]
);
}
#[test]
fn removing_an_approved_prefix_renders_full_permissions() {
use PreviousSectionState::Known;
let mut exec_policy = Policy::empty();
exec_policy
.add_prefix_rule(&["touch".to_string()], Decision::Allow)
.expect("test prefix should be valid");
let with_approved_prefix = permissions_state_with_default_messages(&exec_policy);
let without_approved_prefix = permissions_state_with_default_messages(&Policy::empty());
let rendered = without_approved_prefix
.render_diff(Known(&with_approved_prefix.snapshot()))
.expect("removing a prefix should refresh permissions")
.render();
assert_eq!(rendered, without_approved_prefix.instructions.render());
}
#[test]
fn persisted_permissions_are_detected_inside_bundled_developer_messages() {
let state = permissions_state(PermissionProfile::read_only(), AskForApproval::OnRequest);
@@ -94,3 +235,19 @@ fn permissions_state(
/*request_permissions_tool_enabled*/ false,
)
}
fn permissions_state_with_default_messages(exec_policy: &Policy) -> PermissionsState {
PermissionsState::new(
&PermissionProfile::read_only(),
AskForApproval::OnRequest,
ApprovalPromptContext::new(
ApprovalsReviewer::User,
/*messages*/ None,
/*permission_messages*/ None,
),
exec_policy,
Path::new("/workspace"),
/*exec_permission_approvals_enabled*/ false,
/*request_permissions_tool_enabled*/ false,
)
}
@@ -0,0 +1,16 @@
---
source: core/src/context/world_state/permissions_tests.rs
expression: model_visible_before_and_after
---
BEFORE APPROVAL — PERMISSIONS ALREADY IN CONTEXT (CONDENSED)
<permissions instructions>
Read only.
Ask for approval.
</permissions instructions>
AFTER APPROVAL — WORLD-STATE DIFF APPENDS
Approved command prefix saved:
- ["touch", "allow-prefix.txt"]
FULL PERMISSIONS BLOCK APPENDED AFTER APPROVAL
None
@@ -5,28 +5,28 @@ expression: "render_section_cases(&[(Absent, Absent), (Absent, Known(&read_only)
Absent -> Absent
None
Absent -> "0ccde536dd8b4cfebb1df573f679dfe86b5718e3" (role - developer)
Absent -> {"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} (role - developer)
<permissions instructions>
Read only.
Ask for approval.
</permissions instructions>
"0ccde536dd8b4cfebb1df573f679dfe86b5718e3" -> "0ccde536dd8b4cfebb1df573f679dfe86b5718e3"
{"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} -> {"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"}
None
"0ccde536dd8b4cfebb1df573f679dfe86b5718e3" -> "7e17d9a0df61e5f0c032ed7f0bf1dda58c9d5e85" (role - developer)
{"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} -> {"approved_command_prefixes":[],"instructions":"7e17d9a0df61e5f0c032ed7f0bf1dda58c9d5e85"} (role - developer)
<permissions instructions>
Full access.
Ask for approval.
</permissions instructions>
"0ccde536dd8b4cfebb1df573f679dfe86b5718e3" -> "d96f228104cb899a8d3788b08e1d64b3bbb6ac84" (role - developer)
{"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} -> {"approved_command_prefixes":[],"instructions":"d96f228104cb899a8d3788b08e1d64b3bbb6ac84"} (role - developer)
<permissions instructions>
Read only.
Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected.
</permissions instructions>
Unknown -> "0ccde536dd8b4cfebb1df573f679dfe86b5718e3" (role - developer)
Unknown -> {"approved_command_prefixes":[],"instructions":"0ccde536dd8b4cfebb1df573f679dfe86b5718e3"} (role - developer)
<permissions instructions>
Read only.
Ask for approval.
+1 -1
View File
@@ -1,4 +1,3 @@
use crate::audio_preparation::estimate_audio_token_count;
use crate::context::ContextualUserFragment;
use crate::context::world_state::WorldState;
use crate::context::world_state::WorldStateSnapshot;
@@ -23,6 +22,7 @@ use codex_protocol::protocol::TokenUsage;
use codex_protocol::protocol::TokenUsageInfo;
use codex_protocol::protocol::TurnContextItem;
use codex_protocol::protocol::WorldStateItem;
use codex_utils_audio::estimate_audio_token_count;
use codex_utils_cache::BlockingLruCache;
use codex_utils_cache::sha1_digest;
use codex_utils_output_truncation::TruncationPolicy;
@@ -1,4 +1,5 @@
use super::*;
use crate::context::APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX;
use crate::context::UserInstructions;
use crate::context::world_state::WorldState;
use crate::context::world_state::WorldStateSection;
@@ -1102,6 +1103,33 @@ fn drop_last_n_user_turns_trims_context_updates_above_rolled_back_turn() {
);
}
#[test]
fn drop_last_n_user_turns_trims_saved_prefix_update_above_rolled_back_turn() {
let items = vec![
assistant_msg("session prefix item"),
user_input_text_msg("turn 1 user"),
assistant_msg("turn 1 assistant"),
developer_msg(&format!(
"{APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX}\n- [\"touch\"]"
)),
user_input_text_msg("turn 2 user"),
assistant_msg("turn 2 assistant"),
];
let modalities = default_input_modalities();
let mut history = create_history_with_items(items);
history.drop_last_n_user_turns(/*num_turns*/ 1);
assert_eq!(
history.for_prompt(&modalities),
vec![
assistant_msg("session prefix item"),
user_input_text_msg("turn 1 user"),
assistant_msg("turn 1 assistant"),
]
);
}
#[test]
fn drop_last_n_user_turns_clears_reference_context_for_mixed_developer_context_bundles() {
let items = vec![
+49 -16
View File
@@ -22,6 +22,7 @@ use futures::future::BoxFuture;
use futures::future::Shared;
use tokio_util::task::AbortOnDropHandle;
use crate::session::turn_context::EnvironmentConfig;
use crate::session::turn_context::TurnEnvironment;
use crate::shell::Shell;
use crate::shell_snapshot::ShellSnapshot;
@@ -79,6 +80,8 @@ impl StartingTurnEnvironment {
pub(crate) struct ThreadEnvironments {
environment_manager: Arc<EnvironmentManager>,
local_shell: Shell,
// Temporary thread-wide config source until environments supply their own config.
fallback_environment_config: EnvironmentConfig,
shell_snapshot: ShellSnapshot,
non_blocking_snapshots: bool,
environments: ArcSwap<Vec<SelectedTurnEnvironment>>,
@@ -89,6 +92,7 @@ impl ThreadEnvironments {
pub(crate) fn new(
environment_manager: Arc<EnvironmentManager>,
local_shell: Shell,
fallback_environment_config: EnvironmentConfig,
shell_snapshot: ShellSnapshot,
current: TurnEnvironmentSnapshot,
non_blocking_snapshots: bool,
@@ -98,9 +102,11 @@ impl ThreadEnvironments {
.environments
.into_iter()
.filter_map(|environment| {
let TurnEnvironmentState::Ready(environment) = environment else {
let TurnEnvironmentState::Ready(mut environment) = environment else {
return None;
};
// Child threads can have a different shell policy from their parent.
environment.config = fallback_environment_config.clone();
let selection = environment.selection();
let selected_environment = Arc::clone(&environment.environment);
let resolution: TurnEnvironmentResolution =
@@ -116,6 +122,7 @@ impl ThreadEnvironments {
Self {
environment_manager,
local_shell,
fallback_environment_config,
shell_snapshot,
non_blocking_snapshots,
environments: ArcSwap::from_pointee(environments),
@@ -166,6 +173,7 @@ impl ThreadEnvironments {
selected_environment.clone(),
Arc::clone(&environment),
self.local_shell.clone(),
self.fallback_environment_config.clone(),
self.shell_snapshot.clone(),
)
.remote_handle();
@@ -265,6 +273,7 @@ impl ThreadEnvironments {
selection: TurnEnvironmentSelection,
environment: Arc<Environment>,
local_shell: Shell,
config: EnvironmentConfig,
shell_snapshot: ShellSnapshot,
) -> BoxFuture<'static, TurnEnvironmentResult> {
async move {
@@ -300,6 +309,7 @@ impl ThreadEnvironments {
selection.cwd,
selection.workspace_roots,
shell,
config,
);
let task = shell_snapshot
.build(turn_environment.clone())
@@ -506,6 +516,9 @@ mod tests {
let turn_environments = Arc::new(ThreadEnvironments::new(
environment_manager,
crate::shell::default_user_shell(),
EnvironmentConfig {
allow_login_shell: true,
},
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
@@ -649,7 +662,7 @@ url = "ws://127.0.0.1:8765"
}
#[tokio::test]
async fn local_environment_uses_configured_shell() {
async fn local_environment_uses_configured_shell_and_login_policy() {
let cwd = AbsolutePathBuf::current_dir().expect("cwd");
let local_shell = Shell {
shell_type: crate::shell::ShellType::Zsh,
@@ -658,6 +671,9 @@ url = "ws://127.0.0.1:8765"
let turn_environments = ThreadEnvironments::new(
Arc::new(EnvironmentManager::default_for_tests()),
local_shell.clone(),
EnvironmentConfig {
allow_login_shell: false,
},
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
@@ -669,13 +685,10 @@ url = "ws://127.0.0.1:8765"
}]);
let snapshot = turn_environments.snapshot().await;
let environment = snapshot.primary().expect("local environment");
assert_eq!(
snapshot
.primary()
.and_then(|environment| environment.shell.as_ref()),
Some(&local_shell)
);
assert_eq!(environment.shell.as_ref(), Some(&local_shell));
assert!(!environment.config.allow_login_shell);
}
#[tokio::test]
@@ -797,6 +810,9 @@ url = "ws://127.0.0.1:8765"
let environments = Arc::new(ThreadEnvironments::new(
manager,
crate::shell::default_user_shell(),
EnvironmentConfig {
allow_login_shell: true,
},
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
@@ -850,6 +866,9 @@ url = "ws://127.0.0.1:8765"
let turn_environments = ThreadEnvironments::new(
manager,
crate::shell::default_user_shell(),
EnvironmentConfig {
allow_login_shell: true,
},
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ true,
@@ -924,6 +943,9 @@ url = "ws://127.0.0.1:8765"
let environments = ThreadEnvironments::new(
Arc::clone(&manager),
crate::shell::default_user_shell(),
EnvironmentConfig {
allow_login_shell: true,
},
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ true,
@@ -978,6 +1000,9 @@ url = "ws://127.0.0.1:8765"
let environments = Arc::new(ThreadEnvironments::new(
Arc::clone(&manager),
crate::shell::default_user_shell(),
EnvironmentConfig {
allow_login_shell: true,
},
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ true,
@@ -1050,7 +1075,7 @@ url = "ws://127.0.0.1:8765"
}
#[tokio::test]
async fn inherited_environment_reuses_parent_handle() {
async fn inherited_environment_reuses_parent_handle_and_uses_child_login_policy() {
let cwd = AbsolutePathBuf::current_dir().expect("cwd");
let selection = TurnEnvironmentSelection {
environment_id: REMOTE_ENVIRONMENT_ID.to_string(),
@@ -1067,6 +1092,9 @@ url = "ws://127.0.0.1:8765"
selection.cwd.clone(),
Vec::new(),
/*shell*/ None,
EnvironmentConfig {
allow_login_shell: true,
},
);
let manager = Arc::new(environment_manager_without_environments());
manager
@@ -1079,6 +1107,9 @@ url = "ws://127.0.0.1:8765"
let environments = ThreadEnvironments::new(
manager,
crate::shell::default_user_shell(),
EnvironmentConfig {
allow_login_shell: false,
},
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot {
environments: vec![TurnEnvironmentState::Ready(inherited)],
@@ -1089,13 +1120,9 @@ url = "ws://127.0.0.1:8765"
environments.update_selections(std::slice::from_ref(&selection));
let snapshot = environments.snapshot().await;
assert!(Arc::ptr_eq(
&snapshot
.primary()
.expect("inherited environment")
.environment,
&inherited_environment,
));
let inherited = snapshot.primary().expect("inherited environment");
assert!(Arc::ptr_eq(&inherited.environment, &inherited_environment));
assert!(!inherited.config.allow_login_shell);
}
#[tokio::test]
@@ -1124,6 +1151,9 @@ url = "ws://127.0.0.1:8765"
cwd_uri.clone(),
Vec::new(),
/*shell*/ None,
EnvironmentConfig {
allow_login_shell: true,
},
))],
};
let multiple = TurnEnvironmentSnapshot {
@@ -1135,6 +1165,9 @@ url = "ws://127.0.0.1:8765"
cwd_uri,
Vec::new(),
/*shell*/ None,
EnvironmentConfig {
allow_login_shell: true,
},
)),
],
};
+2
View File
@@ -32,12 +32,14 @@ use codex_protocol::user_input::UserInput;
use tracing::warn;
use uuid::Uuid;
use crate::context::APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX;
use crate::context::is_contextual_user_fragment;
use crate::context::parse_visible_hook_prompt_message;
use crate::web_search::web_search_action_detail;
const CONTEXTUAL_DEVELOPER_PREFIXES: &[&str] = &[
"<permissions instructions>",
APPROVED_COMMAND_PREFIX_SAVED_MESSAGE_PREFIX,
"<model_switch>",
APPS_INSTRUCTIONS_OPEN_TAG,
COLLABORATION_MODE_OPEN_TAG,
+1 -1
View File
@@ -49,7 +49,7 @@ fn recognizes_context_window_as_contextual_developer_content() {
let content = vec![ContentItem::InputText {
text: format!(
r#"{CONTEXT_WINDOW_OPEN_TAG}
Thread id: 00000000-0000-0000-0000-000000000000
Agent name: /root
{CONTEXT_WINDOW_CLOSE_TAG}"#
),
}];
-1
View File
@@ -7,7 +7,6 @@
mod apply_patch;
mod apps;
mod audio_preparation;
mod client;
mod client_common;
mod realtime_context;
+5
View File
@@ -217,6 +217,7 @@ mod tests {
use super::*;
use crate::environment_selection::TurnEnvironmentState;
use crate::session::tests::make_session_and_context;
use crate::session::turn_context::EnvironmentConfig;
use crate::session::turn_context::TurnContext;
use crate::session::turn_context::TurnEnvironment;
use codex_utils_absolute_path::AbsolutePathBuf;
@@ -239,6 +240,7 @@ mod tests {
PathUri::from_abs_path(&cwd),
Vec::new(),
primary.shell.clone(),
primary.config.clone(),
);
}
@@ -323,6 +325,9 @@ mod tests {
let environments = crate::environment_selection::ThreadEnvironments::new(
session.services.turn_environments.environment_manager(),
crate::shell::default_user_shell(),
EnvironmentConfig {
allow_login_shell: turn_context.config.permissions.allow_login_shell,
},
crate::shell_snapshot::ShellSnapshot::disabled(),
Default::default(),
/*non_blocking_snapshots*/ true,
+4
View File
@@ -6,6 +6,7 @@ use crate::session::step_context::StepContext;
use crate::session::tests::make_session_and_context;
use crate::session::tests::make_session_and_context_with_rx;
use crate::session::tests::mcp_config_for_test;
use crate::session::turn_context::EnvironmentConfig;
use crate::session::turn_context::TurnEnvironment;
use crate::state::ActiveTurn;
use crate::test_support::models_manager_with_provider;
@@ -1102,6 +1103,9 @@ async fn mcp_sandbox_cwd_uses_matching_server_environment_uri() -> anyhow::Resul
secondary_cwd.clone(),
Vec::new(),
/*shell*/ None,
EnvironmentConfig {
allow_login_shell: true,
},
)));
let step_context = StepContext::for_test(Arc::new(turn_context));
+10 -21
View File
@@ -381,29 +381,18 @@ pub async fn exec_approval(
if let ReviewDecision::ApprovedExecpolicyAmendment {
proposed_execpolicy_amendment,
} = &decision
{
match sess
&& let Err(err) = sess
.persist_execpolicy_amendment(proposed_execpolicy_amendment)
.await
{
Ok(()) => {
sess.record_execpolicy_amendment_message(
&event_turn_id,
proposed_execpolicy_amendment,
)
.await;
}
Err(err) => {
let message = format!("Failed to apply execpolicy amendment: {err}");
tracing::warn!("{message}");
let warning = EventMsg::Warning(WarningEvent { message });
sess.send_event_raw(Event {
id: event_turn_id.clone(),
msg: warning,
})
.await;
}
}
{
let message = format!("Failed to apply execpolicy amendment: {err}");
tracing::warn!("{message}");
let warning = EventMsg::Warning(WarningEvent { message });
sess.send_event_raw(Event {
id: event_turn_id.clone(),
msg: warning,
})
.await;
}
match decision {
ReviewDecision::Abort => {
+4 -19
View File
@@ -17,12 +17,10 @@ use crate::agent::status::is_final;
use crate::agent_communication::AgentCommunicationContext;
use crate::agent_communication::AgentCommunicationKind;
use crate::attestation::AttestationProvider;
use crate::audio_preparation::prepare_response_items as prepare_audio_response_items;
use crate::compact;
use crate::compact::CompactedHistoryMetadata;
use crate::config::ManagedFeatures;
use crate::config::resolve_tool_suggest_config_from_layer_stack;
use crate::context::ApprovedCommandPrefixSaved;
use crate::context::ContextualUserFragment;
use crate::context::ModelSwitchInstructions;
use crate::context::NetworkRuleSaved;
@@ -104,7 +102,6 @@ use codex_protocol::models::AdditionalPermissionProfile;
use codex_protocol::models::BaseInstructions;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::SandboxEnforcement;
use codex_protocol::models::format_allow_prefixes;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::openai_models::ModelPreset;
use codex_protocol::permissions::FileSystemSandboxPolicy;
@@ -154,6 +151,7 @@ use codex_thread_store::ReadThreadParams;
use codex_thread_store::ResumeThreadParams;
use codex_thread_store::ThreadPersistenceMetadata;
use codex_thread_store::ThreadStore;
use codex_utils_audio::prepare_response_items as prepare_audio_response_items;
use codex_utils_path_uri::PathUri;
use futures::future::BoxFuture;
use futures::future::Shared;
@@ -2164,22 +2162,6 @@ impl Session {
))
}
pub(crate) async fn record_execpolicy_amendment_message(
&self,
sub_id: &str,
amendment: &ExecPolicyAmendment,
) {
let Some(prefixes) = format_allow_prefixes(vec![amendment.command.clone()]) else {
warn!("execpolicy amendment for {sub_id} had no command prefix");
return;
};
let message: ResponseItem =
ContextualUserFragment::into(ApprovedCommandPrefixSaved::new(prefixes));
let turn_context = self.turn_context_for_sub_id(sub_id).await;
self.inject_no_new_turn(vec![message], turn_context.as_deref())
.await;
}
pub(crate) async fn persist_network_policy_amendment(
&self,
amendment: &NetworkPolicyAmendment,
@@ -3511,6 +3493,9 @@ impl Session {
developer_sections.push(
crate::context::TokenBudgetContext::new(
self.thread_id(),
session_source
.get_agent_path()
.unwrap_or_else(codex_protocol::AgentPath::root),
auto_compact_window_ids.first_window_id,
auto_compact_window_ids.previous_window_id,
auto_compact_window_ids.window_id,
+5
View File
@@ -5,6 +5,7 @@ use crate::agents_md_manager::AgentsMdManager;
use crate::config::ConstraintError;
use crate::environment_selection::ThreadEnvironments;
use crate::environment_selection::TurnEnvironmentSnapshot;
use crate::session::turn_context::EnvironmentConfig;
use crate::shell_snapshot::ShellSnapshot;
use crate::skills::SkillError;
use crate::state::ActiveTurn;
@@ -939,6 +940,10 @@ impl Session {
let turn_environments = Arc::new(ThreadEnvironments::new(
environment_manager,
default_shell.clone(),
// Temporary: preserve thread-level behavior until environments supply config.
EnvironmentConfig {
allow_login_shell: config.permissions.allow_login_shell,
},
shell_snapshot,
inherited_environments.unwrap_or_default(),
config.features.enabled(Feature::DeferredExecutor),
+33 -3
View File
@@ -1,4 +1,5 @@
use super::mcp_refresh::McpRefresh;
use super::turn_context::EnvironmentConfig;
use super::turn_context::TurnEnvironment;
use super::*;
use crate::agents_md_manager::AgentsMdManager;
@@ -4869,6 +4870,12 @@ async fn resolved_environments_for_configuration(
let turn_environments = ThreadEnvironments::new(
Arc::clone(&environment_manager),
default_user_shell(),
EnvironmentConfig {
allow_login_shell: session_configuration
.original_config_do_not_use
.permissions
.allow_login_shell,
},
ShellSnapshot::disabled(),
TurnEnvironmentSnapshot::default(),
/*non_blocking_snapshots*/ false,
@@ -5667,6 +5674,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) {
let turn_environments = Arc::new(ThreadEnvironments::new(
environment_manager,
default_user_shell(),
EnvironmentConfig {
allow_login_shell: config.permissions.allow_login_shell,
},
ShellSnapshot::disabled(),
resolved_environments,
/*non_blocking_snapshots*/ false,
@@ -6470,6 +6480,7 @@ async fn request_permissions_tool_resolves_relative_paths_against_selected_envir
PathUri::from_abs_path(&environment_cwd),
Vec::new(),
current_environment.shell,
current_environment.config,
));
let call_id = "call-1".to_string();
@@ -7131,6 +7142,7 @@ async fn primary_environment_uses_first_turn_environment() {
second_cwd_uri.clone(),
Vec::new(),
/*shell*/ None,
first_environment.config.clone(),
)));
assert_eq!(
@@ -7856,6 +7868,9 @@ where
let turn_environments = Arc::new(ThreadEnvironments::new(
environment_manager,
default_user_shell(),
EnvironmentConfig {
allow_login_shell: config.permissions.allow_login_shell,
},
ShellSnapshot::disabled(),
resolved_turn_environments.clone(),
/*non_blocking_snapshots*/ false,
@@ -8446,6 +8461,7 @@ async fn conflicting_ready_environment_root_ids_keep_first_location() {
local_environment.cwd().clone(),
local_environment.workspace_roots().to_vec(),
local_environment.shell.clone(),
local_environment.config.clone(),
));
}
let environments = TurnEnvironmentSnapshot {
@@ -8656,6 +8672,7 @@ async fn record_context_updates_emits_environment_item_for_cwd_changes() {
PathUri::from_abs_path(&cwd),
Vec::new(),
environment.shell,
environment.config,
));
let update_items =
@@ -8721,6 +8738,7 @@ async fn record_context_updates_omits_environment_item_when_disabled() {
PathUri::from_abs_path(&test_path_buf("/new-repo").abs()),
Vec::new(),
environment.shell,
environment.config,
));
let update_items =
@@ -9048,13 +9066,24 @@ async fn build_initial_context_adds_multi_agent_v2_subagent_usage_hint_as_develo
.await
.session_configuration
.session_source = session_source.clone();
Arc::get_mut(&mut turn_context)
.expect("thread settings should not be shared")
.session_source = session_source;
let turn_context_mut =
Arc::get_mut(&mut turn_context).expect("thread settings should not be shared");
turn_context_mut.session_source = session_source;
Arc::make_mut(&mut turn_context_mut.config)
.features
.enable(Feature::TokenBudget)
.expect("test config should allow token budget");
let initial_context = build_initial_context(&session, &turn_context).await;
let developer_messages = developer_message_texts(&initial_context);
assert!(
developer_messages
.iter()
.flatten()
.any(|text| text.contains("<context_window>\nAgent name: /root/worker\n")),
"expected subagent context window to include its canonical name, got {developer_messages:?}"
);
assert!(
developer_messages
.iter()
@@ -9172,6 +9201,7 @@ async fn turn_context_item_stores_local_cwd() {
cwd,
Vec::new(),
environment.shell,
environment.config,
));
#[allow(deprecated)]
+10
View File
@@ -27,6 +27,12 @@ use tracing::instrument;
pub(crate) type ShellSnapshotTask = Shared<BoxFuture<'static, Option<Arc<ShellSnapshotFile>>>>;
/// Effective per-environment config; fields move here as executor config is migrated.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct EnvironmentConfig {
pub(crate) allow_login_shell: bool,
}
#[derive(Clone)]
pub(crate) struct TurnEnvironment {
pub(crate) environment_id: String,
@@ -34,6 +40,7 @@ pub(crate) struct TurnEnvironment {
cwd: PathUri,
workspace_roots: Vec<PathUri>,
pub(crate) shell: Option<shell::Shell>,
pub(crate) config: EnvironmentConfig,
pub(crate) shell_snapshot: ShellSnapshotTask,
}
@@ -44,6 +51,7 @@ impl TurnEnvironment {
cwd: PathUri,
workspace_roots: Vec<PathUri>,
shell: Option<shell::Shell>,
config: EnvironmentConfig,
) -> Self {
Self {
environment_id,
@@ -51,6 +59,7 @@ impl TurnEnvironment {
cwd,
workspace_roots,
shell,
config,
shell_snapshot: futures::future::ready(None).boxed().shared(),
}
}
@@ -90,6 +99,7 @@ impl std::fmt::Debug for TurnEnvironment {
.field("cwd", &self.cwd)
.field("workspace_roots", &self.workspace_roots)
.field("shell", &self.shell)
.field("config", &self.config)
.finish_non_exhaustive()
}
}
+4
View File
@@ -7,6 +7,7 @@ use crate::context::ApprovalPromptContext;
use crate::context::world_state::AgentsMdState;
use crate::context::world_state::AppsInstructionsState;
use crate::context::world_state::CollaborationModeState;
use crate::context::world_state::CompactPermissionsState;
use crate::context::world_state::ContextWindowGuidanceState;
use crate::context::world_state::EnvironmentsInstructionsState;
use crate::context::world_state::EnvironmentsState;
@@ -135,6 +136,9 @@ impl Session {
.features
.enabled(Feature::RequestPermissionsTool),
));
} else {
let exec_policy = self.services.exec_policy.current();
world_state.add_section(CompactPermissionsState::new(exec_policy.as_ref()));
}
if turn_context.config.include_collaboration_mode_instructions {
world_state.add_section(CollaborationModeState::from_collaboration_mode(
+144 -279
View File
@@ -232,6 +232,38 @@ impl StartThreadOptions {
}
}
struct ThreadSpawnRequest {
options: StartThreadOptions,
auth_manager: Arc<AuthManager>,
agent_control: AgentControl,
parent_thread_id: Option<ThreadId>,
forked_from_thread_id: Option<ThreadId>,
fork_persistence: ForkPersistence,
inherited_environments: Option<TurnEnvironmentSnapshot>,
inherited_exec_policy: Option<Arc<crate::exec_policy::ExecPolicyManager>>,
user_shell_override: Option<crate::shell::Shell>,
}
impl ThreadSpawnRequest {
fn new(
options: StartThreadOptions,
auth_manager: Arc<AuthManager>,
agent_control: AgentControl,
) -> Self {
Self {
options,
auth_manager,
agent_control,
parent_thread_id: None,
forked_from_thread_id: None,
fork_persistence: ForkPersistence::Copied,
inherited_environments: None,
inherited_exec_policy: None,
user_shell_override: None,
}
}
}
fn originator_from_service_name(service_name: Option<&str>) -> Option<String> {
let service_name = service_name?.trim();
for originator in [
@@ -777,46 +809,25 @@ impl ThreadManager {
async fn start_thread_inner(
&self,
options: StartThreadOptions,
mut options: StartThreadOptions,
forked_from_thread_id: Option<ThreadId>,
) -> CodexResult<NewThread> {
let environments = options.environments.unwrap_or_else(|| {
default_thread_environment_selections(
self.state.environment_manager.as_ref(),
&options.config.cwd,
&options.config.workspace_roots,
)
});
let agent_control = self.agent_control_for_config(&options.config);
let (resumed_session_source, resumed_thread_source) = options
.initial_history
.get_resumed_session_sources()
.unwrap_or_else(|| (self.state.session_source.clone(), None));
let session_source = options.session_source.unwrap_or(resumed_session_source);
let thread_source = options.thread_source.or(resumed_thread_source);
Box::pin(self.state.spawn_thread_with_source(
options.config,
options.initial_history,
options.history_mode,
options.allow_provider_model_fallback,
Arc::clone(&self.state.auth_manager),
agent_control,
session_source,
/*parent_thread_id*/ None,
forked_from_thread_id,
ForkPersistence::Copied,
thread_source,
options.dynamic_tools,
options.metrics_service_name,
/*inherited_environments*/ None,
/*inherited_exec_policy*/ None,
options.parent_trace,
environments,
options.thread_extension_init,
options.supports_openai_form_elicitation,
/*user_shell_override*/ None,
))
.await
options.session_source = Some(
options
.session_source
.take()
.unwrap_or(resumed_session_source),
);
options.thread_source = options.thread_source.take().or(resumed_thread_source);
let mut request =
ThreadSpawnRequest::new(options, Arc::clone(&self.state.auth_manager), agent_control);
request.forked_from_thread_id = forked_from_thread_id;
Box::pin(self.state.spawn_thread(request)).await
}
// TODO(jif) merge with fork_agent
@@ -885,11 +896,6 @@ impl ThreadManager {
supports_openai_form_elicitation: bool,
) -> CodexResult<NewThread> {
let agent_control = self.agent_control_for_config(&config);
let environments = default_thread_environment_selections(
self.state.environment_manager.as_ref(),
&config.cwd,
&config.workspace_roots,
);
let (session_source, thread_source) = initial_history
.get_resumed_session_sources()
.unwrap_or_else(|| (self.state.session_source.clone(), None));
@@ -901,28 +907,19 @@ impl ThreadManager {
.restore_v2_agent_metadata(&config, resumed.conversation_id)
.await;
}
Box::pin(self.state.spawn_thread_with_source(
config,
let options = StartThreadOptions {
initial_history,
/*history_mode*/ None,
/*allow_provider_model_fallback*/ false,
session_source: Some(session_source),
thread_source,
parent_trace,
supports_openai_form_elicitation,
..StartThreadOptions::new(config)
};
Box::pin(self.state.spawn_thread(ThreadSpawnRequest::new(
options,
auth_manager,
agent_control,
session_source,
/*parent_thread_id*/ None,
/*forked_from_thread_id*/ None,
ForkPersistence::Copied,
thread_source,
Vec::new(),
/*metrics_service_name*/ None,
/*inherited_environments*/ None,
/*inherited_exec_policy*/ None,
parent_trace,
environments,
/*thread_extension_init*/ ExtensionDataInit::default(),
supports_openai_form_elicitation,
/*user_shell_override*/ None,
))
)))
.await
}
@@ -933,29 +930,14 @@ impl ThreadManager {
supports_openai_form_elicitation: bool,
) -> CodexResult<NewThread> {
let agent_control = self.agent_control_for_config(&config);
let environments = default_thread_environment_selections(
self.state.environment_manager.as_ref(),
&config.cwd,
&config.workspace_roots,
);
Box::pin(self.state.spawn_thread(
config,
InitialHistory::New,
Arc::clone(&self.state.auth_manager),
agent_control,
/*parent_thread_id*/ None,
/*forked_from_thread_id*/ None,
ForkPersistence::Copied,
/*thread_source*/ None,
Vec::new(),
/*metrics_service_name*/ None,
/*parent_trace*/ None,
environments,
/*thread_extension_init*/ ExtensionDataInit::default(),
let options = StartThreadOptions {
supports_openai_form_elicitation,
/*user_shell_override*/ Some(user_shell_override),
))
.await
..StartThreadOptions::new(config)
};
let mut request =
ThreadSpawnRequest::new(options, Arc::clone(&self.state.auth_manager), agent_control);
request.user_shell_override = Some(user_shell_override);
Box::pin(self.state.spawn_thread(request)).await
}
pub(crate) async fn resume_thread_from_rollout_with_user_shell_override_for_tests(
@@ -968,37 +950,19 @@ impl ThreadManager {
) -> CodexResult<NewThread> {
let agent_control = self.agent_control_for_config(&config);
let initial_history = self.initial_history_from_rollout_path(rollout_path).await?;
let environments = default_thread_environment_selections(
self.state.environment_manager.as_ref(),
&config.cwd,
&config.workspace_roots,
);
let (session_source, thread_source) = initial_history
.get_resumed_session_sources()
.unwrap_or_else(|| (self.state.session_source.clone(), None));
Box::pin(self.state.spawn_thread_with_source(
config,
let options = StartThreadOptions {
initial_history,
/*history_mode*/ None,
/*allow_provider_model_fallback*/ false,
auth_manager,
agent_control,
session_source,
/*parent_thread_id*/ None,
/*forked_from_thread_id*/ None,
ForkPersistence::Copied,
session_source: Some(session_source),
thread_source,
Vec::new(),
/*metrics_service_name*/ None,
/*inherited_environments*/ None,
/*inherited_exec_policy*/ None,
/*parent_trace*/ None,
environments,
/*thread_extension_init*/ ExtensionDataInit::default(),
supports_openai_form_elicitation,
/*user_shell_override*/ Some(user_shell_override),
))
.await
..StartThreadOptions::new(config)
};
let mut request = ThreadSpawnRequest::new(options, auth_manager, agent_control);
request.user_shell_override = Some(user_shell_override);
Box::pin(self.state.spawn_thread(request)).await
}
/// Removes the thread from the manager's internal map, though the thread is stored
@@ -1200,30 +1164,19 @@ impl ThreadManager {
let interrupted_marker =
InterruptedTurnHistoryMarker::from_config_and_version(&config, multi_agent_version);
let history = fork_history_from_snapshot(snapshot, history, interrupted_marker);
let environments = default_thread_environment_selections(
self.state.environment_manager.as_ref(),
&config.cwd,
&config.workspace_roots,
);
let agent_control = self.agent_control_for_config(&config);
Box::pin(self.state.spawn_thread(
config,
history,
Arc::clone(&self.state.auth_manager),
agent_control,
/*parent_thread_id*/ None,
source_thread_id,
fork_persistence,
let options = StartThreadOptions {
initial_history: history,
thread_source,
Vec::new(),
/*metrics_service_name*/ None,
parent_trace,
environments,
/*thread_extension_init*/ ExtensionDataInit::default(),
supports_openai_form_elicitation,
/*user_shell_override*/ None,
))
.await
..StartThreadOptions::new(config)
};
let mut request =
ThreadSpawnRequest::new(options, Arc::clone(&self.state.auth_manager), agent_control);
request.forked_from_thread_id = source_thread_id;
request.fork_persistence = fork_persistence;
Box::pin(self.state.spawn_thread(request)).await
}
pub(crate) fn agent_control(&self) -> AgentControl {
@@ -1551,36 +1504,21 @@ impl ThreadManagerState {
inherited_exec_policy: Option<Arc<crate::exec_policy::ExecPolicyManager>>,
environments: Option<Vec<TurnEnvironmentSelection>>,
) -> CodexResult<NewThread> {
let environments = environments.unwrap_or_else(|| {
default_thread_environment_selections(
self.environment_manager.as_ref(),
&config.cwd,
&config.workspace_roots,
)
});
Box::pin(self.spawn_thread_with_source(
config,
InitialHistory::New,
let options = StartThreadOptions {
history_mode,
/*allow_provider_model_fallback*/ false,
Arc::clone(&self.auth_manager),
agent_control,
session_source,
parent_thread_id,
forked_from_thread_id,
ForkPersistence::Copied,
session_source: Some(session_source),
thread_source,
Vec::new(),
metrics_service_name,
inherited_environments,
inherited_exec_policy,
/*parent_trace*/ None,
environments,
/*thread_extension_init*/ ExtensionDataInit::default(),
/*supports_openai_form_elicitation*/ false,
/*user_shell_override*/ None,
))
.await
..StartThreadOptions::new(config)
};
let mut request =
ThreadSpawnRequest::new(options, Arc::clone(&self.auth_manager), agent_control);
request.parent_thread_id = parent_thread_id;
request.forked_from_thread_id = forked_from_thread_id;
request.inherited_environments = inherited_environments;
request.inherited_exec_policy = inherited_exec_policy;
Box::pin(self.spawn_thread(request)).await
}
pub(crate) async fn resume_thread_with_history_with_source(
@@ -1596,35 +1534,19 @@ impl ThreadManagerState {
inherited_environments,
inherited_exec_policy,
} = options;
let environments = default_thread_environment_selections(
self.environment_manager.as_ref(),
&config.cwd,
&config.workspace_roots,
);
let thread_source = initial_history.get_resumed_thread_source();
Box::pin(self.spawn_thread_with_source(
config,
let options = StartThreadOptions {
initial_history,
/*history_mode*/ None,
/*allow_provider_model_fallback*/ false,
Arc::clone(&self.auth_manager),
agent_control,
session_source,
parent_thread_id,
/*forked_from_thread_id*/ None,
ForkPersistence::Copied,
session_source: Some(session_source),
thread_source,
Vec::new(),
/*metrics_service_name*/ None,
inherited_environments,
inherited_exec_policy,
/*parent_trace*/ None,
environments,
/*thread_extension_init*/ ExtensionDataInit::default(),
/*supports_openai_form_elicitation*/ false,
/*user_shell_override*/ None,
))
.await
..StartThreadOptions::new(config)
};
let mut request =
ThreadSpawnRequest::new(options, Arc::clone(&self.auth_manager), agent_control);
request.parent_thread_id = parent_thread_id;
request.inherited_environments = inherited_environments;
request.inherited_exec_policy = inherited_exec_policy;
Box::pin(self.spawn_thread(request)).await
}
#[allow(clippy::too_many_arguments)]
@@ -1643,6 +1565,52 @@ impl ThreadManagerState {
environments: Option<Vec<TurnEnvironmentSelection>>,
thread_extension_init: ExtensionDataInit,
) -> CodexResult<NewThread> {
let options = StartThreadOptions {
initial_history,
history_mode,
session_source: Some(session_source),
thread_source,
environments,
thread_extension_init,
..StartThreadOptions::new(config)
};
let mut request =
ThreadSpawnRequest::new(options, Arc::clone(&self.auth_manager), agent_control);
request.parent_thread_id = parent_thread_id;
request.forked_from_thread_id = forked_from_thread_id;
request.inherited_environments = inherited_environments;
request.inherited_exec_policy = inherited_exec_policy;
Box::pin(self.spawn_thread(request)).await
}
/// Spawn a new thread with optional history and register it with the manager.
async fn spawn_thread(&self, request: ThreadSpawnRequest) -> CodexResult<NewThread> {
let ThreadSpawnRequest {
options,
auth_manager,
agent_control,
parent_thread_id,
forked_from_thread_id,
fork_persistence,
inherited_environments,
inherited_exec_policy,
user_shell_override,
} = request;
let StartThreadOptions {
config,
allow_provider_model_fallback,
initial_history,
history_mode,
session_source,
thread_source,
dynamic_tools,
metrics_service_name,
parent_trace,
environments,
thread_extension_init,
supports_openai_form_elicitation,
} = options;
let session_source = session_source.unwrap_or_else(|| self.session_source.clone());
let environments = environments.unwrap_or_else(|| {
default_thread_environment_selections(
self.environment_manager.as_ref(),
@@ -1650,109 +1618,6 @@ impl ThreadManagerState {
&config.workspace_roots,
)
});
Box::pin(self.spawn_thread_with_source(
config,
initial_history,
history_mode,
/*allow_provider_model_fallback*/ false,
Arc::clone(&self.auth_manager),
agent_control,
session_source,
parent_thread_id,
forked_from_thread_id,
ForkPersistence::Copied,
thread_source,
Vec::new(),
/*metrics_service_name*/ None,
inherited_environments,
inherited_exec_policy,
/*parent_trace*/ None,
environments,
thread_extension_init,
/*supports_openai_form_elicitation*/ false,
/*user_shell_override*/ None,
))
.await
}
/// Spawn a new thread with optional history and register it with the manager.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn spawn_thread(
&self,
config: Config,
initial_history: InitialHistory,
auth_manager: Arc<AuthManager>,
agent_control: AgentControl,
parent_thread_id: Option<ThreadId>,
forked_from_thread_id: Option<ThreadId>,
fork_persistence: ForkPersistence,
thread_source: Option<ThreadSource>,
dynamic_tools: Vec<codex_protocol::dynamic_tools::DynamicToolSpec>,
metrics_service_name: Option<String>,
parent_trace: Option<W3cTraceContext>,
environments: Vec<TurnEnvironmentSelection>,
thread_extension_init: ExtensionDataInit,
supports_openai_form_elicitation: bool,
user_shell_override: Option<crate::shell::Shell>,
) -> CodexResult<NewThread> {
Box::pin(self.spawn_thread_with_source(
config,
initial_history,
/*history_mode*/ None,
/*allow_provider_model_fallback*/ false,
auth_manager,
agent_control,
self.session_source.clone(),
parent_thread_id,
forked_from_thread_id,
fork_persistence,
thread_source,
dynamic_tools,
metrics_service_name,
/*inherited_environments*/ None,
/*inherited_exec_policy*/ None,
parent_trace,
environments,
thread_extension_init,
supports_openai_form_elicitation,
user_shell_override,
))
.await
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn spawn_thread_with_source(
&self,
config: Config,
initial_history: InitialHistory,
history_mode: Option<ThreadHistoryMode>,
allow_provider_model_fallback: bool,
auth_manager: Arc<AuthManager>,
agent_control: AgentControl,
session_source: SessionSource,
parent_thread_id: Option<ThreadId>,
forked_from_thread_id: Option<ThreadId>,
fork_persistence: ForkPersistence,
thread_source: Option<ThreadSource>,
dynamic_tools: Vec<codex_protocol::dynamic_tools::DynamicToolSpec>,
metrics_service_name: Option<String>,
inherited_environments: Option<TurnEnvironmentSnapshot>,
inherited_exec_policy: Option<Arc<crate::exec_policy::ExecPolicyManager>>,
parent_trace: Option<W3cTraceContext>,
environments: Vec<TurnEnvironmentSelection>,
thread_extension_init: ExtensionDataInit,
supports_openai_form_elicitation: bool,
user_shell_override: Option<crate::shell::Shell>,
) -> CodexResult<NewThread> {
let source_changed_during_startup = Arc::new(AtomicBool::new(false));
{
let mut starting = self
.starting_mcp_runtimes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
starting.retain(|runtime| runtime.strong_count() != 0);
starting.push(Arc::downgrade(&source_changed_during_startup));
}
let is_resumed_thread = matches!(&initial_history, InitialHistory::Resumed(_));
if let InitialHistory::Resumed(resumed) = &initial_history {
let mut threads = self.threads.write().await;
+9 -16
View File
@@ -9,7 +9,6 @@ use crate::sandboxing::SandboxPermissions;
use crate::session::turn_context::TurnContext;
use crate::tools::flat_tool_name;
use crate::tools::sandboxing::ApprovalCtx;
use crate::tools::sandboxing::ToolCtx;
use crate::tools::sandboxing::ToolError;
use crate::tools::sandboxing::ToolRuntime;
use codex_config::types::ApprovalsReviewer;
@@ -188,14 +187,12 @@ impl ApprovalResolution {
}
}
pub(super) async fn resolve_tool_apporval<Rq, Out, T>(
pub(super) async fn resolve_tool_approval<Rq, Out, T>(
tool: &mut T,
req: &Rq,
permission_request_run_id: &str,
ctx: ApprovalCtx<'_>,
tool_ctx: &ToolCtx,
reviewer: ApprovalReviewer,
otel: &codex_otel::SessionTelemetry,
) -> Result<ReviewDecision, ToolError>
where
T: ToolRuntime<Rq, Out>,
@@ -214,7 +211,7 @@ where
decision: ReviewDecision::Approved,
source: ApprovalResolutionSource::Hook,
};
record_resolution(otel, tool_ctx, &resolution);
record_resolution(&ctx, &resolution);
return resolution.into_tool_result();
}
Some(PermissionRequestDecision::Deny { message }) => {
@@ -222,7 +219,7 @@ where
decision: ReviewDecision::denied(message),
source: ApprovalResolutionSource::Hook,
};
record_resolution(otel, tool_ctx, &resolution);
record_resolution(&ctx, &resolution);
return resolution.into_tool_result();
}
None => {}
@@ -245,7 +242,7 @@ where
),
source: ApprovalResolutionSource::Guardian,
};
record_resolution(otel, tool_ctx, &resolution);
record_resolution(&ctx, &resolution);
return resolution.into_tool_result();
}
};
@@ -268,24 +265,20 @@ where
decision: resolution,
source,
};
record_resolution(otel, tool_ctx, &resolution);
record_resolution(&ctx, &resolution);
resolution.into_tool_result()
}
fn record_resolution(
otel: &codex_otel::SessionTelemetry,
tool_ctx: &ToolCtx,
resolution: &ApprovalResolution,
) {
fn record_resolution(ctx: &ApprovalCtx<'_>, resolution: &ApprovalResolution) {
let source = match resolution.source {
ApprovalResolutionSource::Hook => ToolDecisionSource::Config,
ApprovalResolutionSource::Guardian => ToolDecisionSource::AutomatedReviewer,
ApprovalResolutionSource::User => ToolDecisionSource::User,
};
let tool_name = flat_tool_name(&tool_ctx.tool_name);
otel.tool_decision(
let tool_name = flat_tool_name(ctx.tool_name);
ctx.turn.session_telemetry.tool_decision(
tool_name.as_ref(),
&tool_ctx.call_id,
ctx.call_id,
&resolution.decision,
source,
);
@@ -30,6 +30,7 @@ SOURCE: /[\s\S]+/
default_exec_yield_time_ms,
code_mode_only,
),
defer_loading: None,
format: FreeformToolFormat {
r#type: "grammar".to_string(),
syntax: "lark".to_string(),
@@ -72,6 +73,7 @@ mod tests {
codex_code_mode::DEFAULT_EXEC_YIELD_TIME_MS,
/*code_mode_only*/ true,
),
defer_loading: None,
format: FreeformToolFormat {
r#type: "grammar".to_string(),
syntax: "lark".to_string(),
+1 -1
View File
@@ -24,7 +24,6 @@ use serde_json::Value as JsonValue;
use tokio::sync::OnceCell;
use tokio_util::sync::CancellationToken;
use crate::audio_preparation::estimate_audio_token_count;
use crate::function_tool::FunctionCallError;
use crate::original_image_detail::can_request_original_image_detail;
use crate::original_image_detail::sanitize_original_image_detail as sanitize_image_detail_items;
@@ -41,6 +40,7 @@ use crate::tools::router::ToolCallSource;
use crate::unified_exec::resolve_max_tokens;
use codex_protocol::openai_models::ToolMode;
use codex_tools::ToolName;
use codex_utils_audio::estimate_audio_token_count;
use codex_utils_output_truncation::TruncationPolicy;
use codex_utils_output_truncation::formatted_truncate_text_content_items_with_policy;
use codex_utils_output_truncation::truncate_function_output_items_with_policy;
@@ -18,6 +18,7 @@ pub fn create_apply_patch_freeform_tool(include_environment_id: bool) -> ToolSpe
ToolSpec::Freeform(FreeformTool {
name: "apply_patch".to_string(),
description: "The `apply_patch` tool can be used to edit files. This is a FREEFORM tool, so do not wrap the patch in JSON.".to_string(),
defer_loading: None,
format: FreeformToolFormat {
r#type: "grammar".to_string(),
syntax: "lark".to_string(),
@@ -10,6 +10,7 @@ fn create_apply_patch_freeform_tool_matches_expected_spec() {
description:
"The `apply_patch` tool can be used to edit files. This is a FREEFORM tool, so do not wrap the patch in JSON."
.to_string(),
defer_loading: None,
format: FreeformToolFormat {
r#type: "grammar".to_string(),
syntax: "lark".to_string(),
@@ -6,6 +6,7 @@ use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_tools::ConversationHistory;
use codex_tools::ExtensionTurnItem;
use codex_tools::ResponsesApiNamespaceTool;
use codex_tools::ToolCall as ExtensionToolCall;
use codex_tools::ToolEnvironment;
use codex_tools::ToolName;
@@ -61,7 +62,23 @@ impl ToolExecutor<ToolInvocation> for ExtensionToolAdapter {
impl CoreToolRuntime for ExtensionToolAdapter {
fn matches_kind(&self, payload: &ToolPayload) -> bool {
matches!(payload, ToolPayload::Function { .. })
match payload {
ToolPayload::Function { .. } => true,
ToolPayload::Custom { .. } => match self.0.spec() {
ToolSpec::Freeform(_) => true,
ToolSpec::Namespace(namespace) => namespace.tools.iter().any(|tool| {
matches!(
tool,
ResponsesApiNamespaceTool::Custom(tool)
if tool.name == self.0.tool_name().name
)
}),
ToolSpec::Function(_)
| ToolSpec::ToolSearch { .. }
| ToolSpec::WebSearch { .. } => false,
},
ToolPayload::ToolSearch { .. } => false,
}
}
}
@@ -286,6 +303,18 @@ mod tests {
}
}
#[test]
fn function_extensions_reject_custom_payloads() {
let handler = ExtensionToolAdapter::new(Arc::new(StubExtensionExecutor));
assert!(handler.matches_kind(&ToolPayload::Function {
arguments: "{}".to_string(),
}));
assert!(!handler.matches_kind(&ToolPayload::Custom {
input: "raw input".to_string(),
}));
}
#[tokio::test]
async fn exposes_generic_hook_payloads() {
let handler = ExtensionToolAdapter::new(Arc::new(StubExtensionExecutor));
@@ -91,14 +91,14 @@ impl ShellCommandHandler {
turn_context: &TurnContext,
turn_environment: &TurnEnvironment,
cwd: AbsolutePathBuf,
allow_login_shell: bool,
) -> Result<ExecParams, FunctionCallError> {
let session_shell = session.user_shell();
let shell = turn_environment
.shell
.as_ref()
.unwrap_or(session_shell.as_ref());
let use_login_shell = Self::resolve_use_login_shell(params.login, allow_login_shell)?;
let use_login_shell =
Self::resolve_use_login_shell(params.login, turn_environment.config.allow_login_shell)?;
let command = Self::base_command(shell, &params.command, use_login_shell);
let mut env = create_env(
@@ -214,7 +214,6 @@ impl ShellCommandHandler {
turn.as_ref(),
&turn_environment,
cwd,
turn.config.permissions.allow_login_shell,
)?;
let shell_type = Some(
turn_environment
@@ -12,6 +12,7 @@ use crate::exec_env::inject_permission_profile_env;
use crate::sandboxing::SandboxPermissions;
use crate::session::step_context::StepContext;
use crate::session::tests::make_session_and_context;
use crate::session::turn_context::EnvironmentConfig;
use crate::session::turn_context::TurnEnvironment;
use crate::shell::Shell;
use crate::shell::ShellType;
@@ -111,6 +112,9 @@ async fn shell_command_handler_to_exec_params_uses_selected_environment() {
PathUri::from_abs_path(&selected_cwd),
Vec::new(),
Some(selected_shell),
EnvironmentConfig {
allow_login_shell: true,
},
);
let mut expected_env = create_env(
&turn_context.config.permissions.shell_environment_policy,
@@ -136,7 +140,6 @@ async fn shell_command_handler_to_exec_params_uses_selected_environment() {
&turn_context,
&selected_environment,
expected_cwd.clone(),
/*allow_login_shell*/ true,
)
.expect("login shells should be allowed");
@@ -192,10 +195,12 @@ fn shell_command_handler_respects_explicit_login_flag() {
#[tokio::test]
async fn shell_command_handler_defaults_to_non_login_when_disallowed() {
let (session, turn_context) = make_session_and_context().await;
let turn_environment = turn_context
let mut turn_environment = turn_context
.environments
.primary()
.expect("primary environment");
.expect("primary environment")
.clone();
turn_environment.config.allow_login_shell = false;
let cwd = turn_environment
.cwd()
.to_abs_path()
@@ -215,9 +220,8 @@ async fn shell_command_handler_defaults_to_non_login_when_disallowed() {
&params,
&session,
&turn_context,
turn_environment,
&turn_environment,
cwd,
/*allow_login_shell*/ false,
)
.expect("non-login shells should still be allowed");
@@ -257,7 +261,6 @@ async fn shell_command_handler_rejects_justification_without_sandbox_permissions
&turn_context,
turn_environment,
cwd,
/*allow_login_shell*/ false,
)
.expect_err("justification without sandbox permissions should be rejected");
@@ -235,7 +235,7 @@ impl ExecCommandHandler {
&args,
shell,
&shell_mode,
turn.config.permissions.allow_login_shell,
turn_environment.config.allow_login_shell,
)
.map_err(FunctionCallError::RespondToModel)?;
let command = resolved_command.command;
@@ -9,6 +9,8 @@ use codex_utils_output_truncation::TruncationPolicy;
use pretty_assertions::assert_eq;
use std::sync::Arc;
use crate::environment_selection::TurnEnvironmentState;
use crate::function_tool::FunctionCallError;
use crate::session::step_context::StepContext;
use crate::session::tests::make_session_and_context;
use crate::tools::context::ExecCommandToolOutput;
@@ -17,6 +19,7 @@ use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolPayload;
use crate::tools::hook_names::HookToolName;
use crate::tools::registry::CoreToolRuntime;
use crate::tools::registry::ToolExecutor;
use crate::turn_diff_tracker::TurnDiffTracker;
use tokio::sync::Mutex;
@@ -168,6 +171,46 @@ fn test_get_command_rejects_explicit_login_when_disallowed() -> anyhow::Result<(
Ok(())
}
#[tokio::test]
async fn exec_command_rejects_login_when_selected_environment_disallows_it() {
let (session, mut turn) = make_session_and_context().await;
assert!(turn.config.permissions.allow_login_shell);
let TurnEnvironmentState::Ready(environment) = turn
.environments
.environments
.first_mut()
.expect("primary environment")
else {
panic!("primary environment should be ready");
};
environment.config.allow_login_shell = false;
let turn = Arc::new(turn);
let invocation = ToolInvocation {
session: session.into(),
step_context: StepContext::for_test(Arc::clone(&turn)),
turn,
cancellation_token: tokio_util::sync::CancellationToken::new(),
tracker: Arc::new(Mutex::new(TurnDiffTracker::new())),
call_id: "login-disallowed".to_string(),
tool_name: codex_tools::ToolName::plain("exec_command"),
source: ToolCallSource::Direct,
payload: ToolPayload::Function {
arguments: serde_json::json!({ "cmd": "echo hello", "login": true }).to_string(),
},
};
let Err(FunctionCallError::RespondToModel(message)) =
ExecCommandHandler::default().handle(invocation).await
else {
panic!("expected login-shell rejection");
};
assert_eq!(
message,
"login shell is disabled by config; omit `login` or set it to false."
);
}
#[test]
fn test_get_command_rejects_explicit_shell_in_zsh_fork_mode() -> anyhow::Result<()> {
let json = r#"{"cmd": "echo hello", "shell": "/bin/bash"}"#;
@@ -273,6 +273,7 @@ mod tests {
PathUri::from_abs_path(&cwd),
Vec::new(),
current.shell,
current.config,
));
}
+7 -10
View File
@@ -7,7 +7,7 @@ retry with an escalated sandbox strategy on denial (no re‑approval thanks to
caching).
*/
use super::approvals::ApprovalReviewer;
use super::approvals::resolve_tool_apporval;
use super::approvals::resolve_tool_approval;
use crate::network_policy_decision::network_approval_context_from_payload;
use crate::tools::flat_tool_name;
use crate::tools::network_approval::ActiveNetworkApproval;
@@ -171,17 +171,16 @@ impl ToolOrchestrator {
session: &tool_ctx.session,
turn: &tool_ctx.turn,
call_id: &tool_ctx.call_id,
tool_name: &tool_ctx.tool_name,
retry_reason: None,
network_approval_context: None,
};
resolve_tool_apporval(
resolve_tool_approval(
tool,
req,
tool_ctx.call_id.as_str(),
approval_ctx,
tool_ctx,
ApprovalReviewer::Guardian,
&otel,
)
.await?;
already_approved = true;
@@ -202,21 +201,20 @@ impl ToolOrchestrator {
session: &tool_ctx.session,
turn: &tool_ctx.turn,
call_id: &tool_ctx.call_id,
tool_name: &tool_ctx.tool_name,
retry_reason: reason.clone(),
network_approval_context: None,
};
resolve_tool_apporval(
resolve_tool_approval(
tool,
req,
tool_ctx.call_id.as_str(),
approval_ctx,
tool_ctx,
if strict_auto_review {
ApprovalReviewer::Guardian
} else {
ApprovalReviewer::for_turn(turn_ctx)
},
&otel,
)
.await?;
already_approved = true;
@@ -397,23 +395,22 @@ impl ToolOrchestrator {
session: &tool_ctx.session,
turn: &tool_ctx.turn,
call_id: &tool_ctx.call_id,
tool_name: &tool_ctx.tool_name,
retry_reason: Some(retry_reason),
network_approval_context: network_approval_context.clone(),
};
let permission_request_run_id = format!("{}:retry", tool_ctx.call_id);
resolve_tool_apporval(
resolve_tool_approval(
tool,
req,
&permission_request_run_id,
approval_ctx,
tool_ctx,
if strict_auto_review {
ApprovalReviewer::Guardian
} else {
ApprovalReviewer::for_turn(turn_ctx)
},
&otel,
)
.await?;
}
+1
View File
@@ -582,6 +582,7 @@ fn namespace_function_names(specs: &[ToolSpec], namespace_name: &str) -> Vec<Str
.iter()
.map(|tool| match tool {
ResponsesApiNamespaceTool::Function(tool) => tool.name.clone(),
ResponsesApiNamespaceTool::Custom(tool) => tool.name.clone(),
})
.collect(),
),
@@ -1,4 +1,5 @@
use super::*;
use crate::session::turn_context::EnvironmentConfig;
use crate::tools::sandboxing::SandboxAttempt;
use codex_protocol::config_types::WindowsSandboxLevel;
use codex_protocol::models::AdditionalPermissionProfile;
@@ -21,6 +22,9 @@ fn test_turn_environment(environment_id: &str) -> crate::session::turn_context::
PathUri::from_abs_path(&std::env::temp_dir().abs()),
Vec::new(),
/*shell*/ None,
EnvironmentConfig {
allow_login_shell: true,
},
)
}
@@ -1,4 +1,5 @@
use super::*;
use crate::session::turn_context::EnvironmentConfig;
use codex_exec_server::Environment;
use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
@@ -16,6 +17,9 @@ async fn approval_key_uses_path_uri_and_includes_environment_id() {
PathUri::from_abs_path(&cwd),
Vec::new(),
/*shell*/ None,
EnvironmentConfig {
allow_login_shell: true,
},
),
shell_type: None,
hook_command: "echo hello".to_string(),
@@ -549,6 +549,7 @@ impl<'a> ToolRuntime<UnifiedExecRequest, UnifiedExecProcess> for UnifiedExecRunt
mod tests {
use super::*;
use crate::exec::DEFAULT_EXEC_COMMAND_TIMEOUT_MS;
use crate::session::turn_context::EnvironmentConfig;
use crate::tools::sandboxing::ToolRuntime;
use codex_exec_server::Environment;
use codex_exec_server::LOCAL_ENVIRONMENT_ID;
@@ -566,6 +567,9 @@ mod tests {
cwd,
Vec::new(),
/*shell*/ None,
EnvironmentConfig {
allow_login_shell: true,
},
)
}
+1
View File
@@ -121,6 +121,7 @@ pub(crate) struct ApprovalCtx<'a> {
pub session: &'a Arc<Session>,
pub turn: &'a Arc<TurnContext>,
pub call_id: &'a str,
pub tool_name: &'a ToolName,
pub retry_reason: Option<String>,
pub network_approval_context: Option<NetworkApprovalContext>,
}
+18 -7
View File
@@ -713,11 +713,16 @@ fn merge_into_namespaces(specs: Vec<ToolSpec>) -> Vec<ToolSpec> {
continue;
};
namespace.tools.sort_by(|left, right| match (left, right) {
(
ResponsesApiNamespaceTool::Function(left),
ResponsesApiNamespaceTool::Function(right),
) => left.name.cmp(&right.name),
namespace.tools.sort_by(|left, right| {
let left_name = match left {
ResponsesApiNamespaceTool::Function(tool) => &tool.name,
ResponsesApiNamespaceTool::Custom(tool) => &tool.name,
};
let right_name = match right {
ResponsesApiNamespaceTool::Function(tool) => &tool.name,
ResponsesApiNamespaceTool::Custom(tool) => &tool.name,
};
left_name.cmp(right_name)
});
if namespace.description.trim().is_empty() {
@@ -760,7 +765,7 @@ fn add_core_tool_sources(context: &CoreToolPlanContext<'_>, registry: &mut ToolR
if environment_mode.has_environment() {
let include_environment_id = matches!(environment_mode, ToolEnvironmentMode::Multiple);
registry.add(ExecCommandHandler::new(ExecCommandHandlerOptions {
allow_login_shell: turn_context.config.permissions.allow_login_shell,
allow_login_shell: any_environment_allows_login_shell(context.environments),
exec_permission_approvals_enabled: false,
include_environment_id,
include_shell_parameter: unified_exec_should_include_shell_parameter(
@@ -800,6 +805,12 @@ fn tool_environment_mode(environments: &TurnEnvironmentSnapshot) -> ToolEnvironm
ToolEnvironmentMode::from_count(environments.turn_environments().count())
}
fn any_environment_allows_login_shell(environments: &TurnEnvironmentSnapshot) -> bool {
environments
.turn_environments()
.any(|environment| environment.config.allow_login_shell)
}
#[instrument(level = "trace", skip_all)]
fn add_shell_tools(context: &CoreToolPlanContext<'_>, registry: &mut ToolRegistry) {
let turn_context = context.turn_context;
@@ -809,7 +820,7 @@ fn add_shell_tools(context: &CoreToolPlanContext<'_>, registry: &mut ToolRegistr
return;
}
let allow_login_shell = turn_context.config.permissions.allow_login_shell;
let allow_login_shell = any_environment_allows_login_shell(context.environments);
let exec_permission_approvals_enabled = features.enabled(Feature::ExecPermissionApprovals);
let include_environment_id = matches!(environment_mode, ToolEnvironmentMode::Multiple);
let supports_shell_command = context.environments.single_local_environment().is_some();
+71 -1
View File
@@ -84,6 +84,7 @@ impl ToolPlanProbe {
.iter()
.map(|tool| match tool {
ResponsesApiNamespaceTool::Function(tool) => tool.name.clone(),
ResponsesApiNamespaceTool::Custom(tool) => tool.name.clone(),
})
.collect::<Vec<_>>(),
)),
@@ -661,6 +662,70 @@ async fn shell_family_registers_visible_unified_exec_and_hidden_legacy_shell() {
assert!(has_parameter(plan.visible_spec("exec_command"), "shell"));
}
#[tokio::test]
async fn login_shell_parameter_follows_selected_environment() {
for (tool_name, guardian) in [
("shell_command", false),
("exec_command", false),
("exec_command", true),
] {
for allow_login_shell in [false, true] {
let plan = probe(|turn| {
set_feature(turn, Feature::ShellTool, /*enabled*/ true);
set_feature(turn, Feature::UnifiedExec, tool_name == "exec_command");
set_feature(turn, Feature::ShellZshFork, /*enabled*/ false);
turn.model_info.shell_type = ConfigShellToolType::ShellCommand;
update_config(turn, |config| {
config.permissions.allow_login_shell = !allow_login_shell;
});
let TurnEnvironmentState::Ready(environment) = turn
.environments
.environments
.first_mut()
.expect("primary environment")
else {
panic!("primary environment should be ready");
};
environment.config.allow_login_shell = allow_login_shell;
if guardian {
turn.session_source = codex_protocol::protocol::SessionSource::SubAgent(
codex_protocol::protocol::SubAgentSource::Other(
crate::guardian::GUARDIAN_REVIEWER_NAME.to_string(),
),
);
}
})
.await;
assert_eq!(
has_parameter(plan.visible_spec(tool_name), "login"),
allow_login_shell
);
}
}
}
#[tokio::test]
async fn login_shell_parameter_is_available_when_any_environment_allows_it() {
let plan = probe(|turn| {
set_features(turn, &[Feature::ShellTool, Feature::UnifiedExec]);
set_feature(turn, Feature::ShellZshFork, /*enabled*/ false);
update_config(turn, |config| {
config.permissions.allow_login_shell = false;
});
duplicate_primary_environment(turn);
for (index, environment) in turn.environments.environments.iter_mut().enumerate() {
let TurnEnvironmentState::Ready(environment) = environment else {
panic!("environment should be ready");
};
environment.config.allow_login_shell = index == 1;
}
})
.await;
assert!(has_parameter(plan.visible_spec("exec_command"), "login"));
}
#[tokio::test]
async fn shell_command_is_not_registered_without_a_single_local_environment() {
let remote_environment = probe(|turn| {
@@ -836,6 +901,9 @@ async fn zsh_fork_unified_exec_keeps_shell_parameter_when_remote_environment_ava
remote_cwd,
Vec::new(),
/*shell*/ None,
crate::session::turn_context::EnvironmentConfig {
allow_login_shell: true,
},
),
));
})
@@ -1599,7 +1667,9 @@ async fn code_mode_only_exposes_configured_dynamic_namespace_directly() {
let ToolSpec::Namespace(namespace) = plan.visible_spec("direct_only") else {
panic!("expected direct-only namespace spec");
};
let ResponsesApiNamespaceTool::Function(tool) = &namespace.tools[0];
let ResponsesApiNamespaceTool::Function(tool) = &namespace.tools[0] else {
panic!("expected direct-only namespace function tool");
};
assert_eq!(tool.defer_loading, None);
let ToolSpec::Freeform(exec) = plan.visible_spec(codex_code_mode::PUBLIC_TOOL_NAME) else {
panic!("expected code mode exec tool");
+121
View File
@@ -2139,6 +2139,127 @@ async fn approving_apply_patch_for_session_skips_future_prompts_for_same_file()
Ok(())
}
#[tokio::test(flavor = "current_thread")]
#[cfg(unix)]
async fn approving_execpolicy_amendment_does_not_reinject_permissions_instructions() -> Result<()> {
assert_execpolicy_amendment_context(PermissionsInstructionsExpectation::Included).await
}
#[tokio::test(flavor = "current_thread")]
#[cfg(unix)]
async fn approving_execpolicy_amendment_is_reported_when_permissions_are_disabled() -> Result<()> {
assert_execpolicy_amendment_context(PermissionsInstructionsExpectation::Omitted).await
}
#[derive(Clone, Copy)]
enum PermissionsInstructionsExpectation {
Included,
Omitted,
}
async fn assert_execpolicy_amendment_context(
expectation: PermissionsInstructionsExpectation,
) -> Result<()> {
let (include_permissions_instructions, expected_permissions_block_count) = match expectation {
PermissionsInstructionsExpectation::Included => (true, 1),
PermissionsInstructionsExpectation::Omitted => (false, 0),
};
let server = start_mock_server().await;
let approval_policy = AskForApproval::OnRequest;
let sandbox_policy = SandboxPolicy::new_read_only_policy();
let sandbox_policy_for_config = sandbox_policy.clone();
let mut builder = test_codex()
.with_model_info_override("gpt-5.4", |model_info| {
model_info.model_messages = None;
})
.with_config(move |config| {
config.include_permissions_instructions = include_permissions_instructions;
config.permissions.approval_policy = Constrained::allow_any(approval_policy);
config
.features
.disable(Feature::DeferredExecutor)
.expect("test config should allow feature update");
config
.set_legacy_sandbox_policy(sandbox_policy_for_config)
.expect("set sandbox policy");
});
let test = builder.build(&server).await?;
let call_id = "permissions-prefix";
let (event, expected_command) = ActionKind::RunCommand {
command: "touch permissions-prefix.txt",
}
.prepare(
&test,
&server,
call_id,
SandboxPermissions::RequireEscalated,
)
.await?;
let expected_command =
expected_command.expect("execpolicy amendment scenario should produce a shell command");
let expected_execpolicy_amendment = ExecPolicyAmendment::new(vec![
"touch".to_string(),
"permissions-prefix.txt".to_string(),
]);
let _ = mount_sse_once(
&server,
sse(vec![
ev_response_created("resp-permissions-prefix-1"),
event,
ev_completed("resp-permissions-prefix-1"),
]),
)
.await;
let follow_up = mount_sse_once(
&server,
sse(vec![
ev_assistant_message("msg-permissions-prefix", "done"),
ev_completed("resp-permissions-prefix-2"),
]),
)
.await;
submit_turn(&test, "permissions-prefix", approval_policy, sandbox_policy).await?;
let approval = expect_exec_approval(&test, expected_command.as_str()).await;
assert_eq!(
approval.proposed_execpolicy_amendment,
Some(expected_execpolicy_amendment.clone())
);
test.codex
.submit(Op::ExecApproval {
id: approval.effective_approval_id(),
turn_id: None,
decision: ReviewDecision::ApprovedExecpolicyAmendment {
proposed_execpolicy_amendment: expected_execpolicy_amendment,
},
})
.await?;
wait_for_completion(&test).await;
let developer_messages = follow_up.single_request().message_input_texts("developer");
let permissions_block_count = developer_messages
.iter()
.map(|message| message.matches("<permissions instructions>").count())
.sum::<usize>();
assert_eq!(
permissions_block_count, expected_permissions_block_count,
"saving an execpolicy amendment must respect the full permissions rendering mode: \
{developer_messages:#?}"
);
assert!(
developer_messages
.iter()
.any(|message| message.contains("Approved command prefix saved:")
&& message.contains(r#"["touch", "permissions-prefix.txt"]"#)),
"expected developer message documenting saved rule, got: {developer_messages:?}"
);
Ok(())
}
#[tokio::test(flavor = "current_thread")]
#[cfg(unix)]
async fn approving_execpolicy_amendment_persists_policy_and_skips_future_prompts() -> Result<()> {
+314
View File
@@ -8,7 +8,9 @@ use codex_config::types::McpServerTransportConfig;
use codex_core::StartThreadOptions;
use codex_core::config::Config;
use codex_core::config::CurrentTimeReminderConfig;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::ToolContributor;
use codex_features::CurrentTimeSource;
use codex_features::Feature;
use codex_login::CodexAuth;
@@ -21,11 +23,25 @@ use codex_protocol::dynamic_tools::DynamicToolNamespaceTool;
use codex_protocol::dynamic_tools::DynamicToolResponse;
use codex_protocol::dynamic_tools::DynamicToolSpec;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ToolMode;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::Op;
use codex_protocol::user_input::UserInput;
use codex_tools::FreeformTool;
use codex_tools::FreeformToolFormat;
use codex_tools::FunctionCallError;
use codex_tools::JsonToolOutput;
use codex_tools::ResponsesApiNamespace;
use codex_tools::ResponsesApiNamespaceTool;
use codex_tools::ToolCall;
use codex_tools::ToolExecutor;
use codex_tools::ToolExecutorFuture;
use codex_tools::ToolName;
use codex_tools::ToolOutput;
use codex_tools::ToolPayload;
use codex_tools::ToolSpec;
use codex_web_search_extension::install as install_web_search_extension;
use core_test_support::apps_test_server::AppsTestServer;
use core_test_support::apps_test_server::AppsTestToolLoading;
@@ -2053,6 +2069,140 @@ try {
Ok(())
}
/// A stalled host wait must return its timeout to the model and reconnect for the next exec.
#[tokio::test(flavor = "current_thread")]
async fn code_mode_wait_timeout_reconnects_on_next_exec() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let mut builder = test_codex()
.with_model("test-gpt-5.1-codex")
.with_config(|config| {
config
.features
.enable(Feature::CodeMode)
.expect("code mode should be enabled");
});
let test = builder.build_with_auto_env(&server).await?;
let first_turn = responses::mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
ev_custom_tool_call(
"call-1",
"exec",
"yield_control(); await new Promise(() => {});",
),
ev_completed("resp-1"),
]),
sse(vec![
ev_assistant_message("msg-1", "waiting"),
ev_completed("resp-2"),
]),
],
)
.await;
test.submit_turn("start a stalled code-mode cell").await?;
let first_request = first_turn
.last_request()
.expect("initial exec should be returned to the model");
let first_items = custom_tool_output_items(&first_request, "call-1");
let cell_id = extract_running_cell_id(text_item(&first_items, /*index*/ 0));
let timeout_completion = responses::mount_function_call_agent_response(
&server,
"call-2",
&serde_json::to_string(&serde_json::json!({
"cell_id": cell_id,
"yield_time_ms": 60_000,
}))?,
"wait",
)
.await
.completion;
test.codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "wait for the stalled cell".to_string(),
text_elements: Vec::new(),
}],
final_output_json_schema: None,
responsesapi_client_metadata: None,
additional_context: Default::default(),
thread_settings: Default::default(),
})
.await?;
wait_for_event_match(&test.codex, |event| match event {
EventMsg::RawResponseItem(raw) => match &raw.item {
ResponseItem::FunctionCall { call_id, .. } if call_id == "call-2" => Some(()),
_ => None,
},
_ => None,
})
.await;
tokio::time::pause();
for _ in 0..130 {
tokio::time::advance(Duration::from_secs(1)).await;
if timeout_completion
.function_call_output_text("call-2")
.is_some()
{
break;
}
}
tokio::time::resume();
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
let timeout_output = timeout_completion
.function_call_output_text("call-2")
.expect("timed-out wait should be returned to the model");
assert!(
timeout_output.contains("code-mode host timed out waiting for wait response"),
"unexpected wait output: {timeout_output}"
);
let reconnect_turn = responses::mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-5"),
ev_custom_tool_call(
"call-3",
"exec",
r#"text("reconnected"); yield_control(); await new Promise(() => {});"#,
),
ev_completed("resp-5"),
]),
sse(vec![
ev_assistant_message("msg-3", "reconnected"),
ev_completed("resp-6"),
]),
],
)
.await;
test.submit_turn("run a cell after the timeout").await?;
let reconnect_request = reconnect_turn
.last_request()
.expect("replacement exec should be returned to the model");
let reconnect_items = custom_tool_output_items(&reconnect_request, "call-3");
assert_eq!(
extract_running_cell_id(text_item(&reconnect_items, /*index*/ 0)),
"g2:1"
);
assert_eq!(text_item(&reconnect_items, /*index*/ 1), "reconnected");
Ok(())
}
#[cfg_attr(windows, ignore = "no exec_command on Windows")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_can_yield_and_resume_with_wait() -> Result<()> {
@@ -3900,6 +4050,170 @@ text(JSON.stringify({
Ok(())
}
struct NamespacedCustomTool;
impl ToolContributor for NamespacedCustomTool {
fn tools(
&self,
_session_store: &ExtensionData,
_thread_store: &ExtensionData,
) -> Vec<Arc<dyn ToolExecutor<ToolCall>>> {
vec![Arc::new(Self)]
}
}
impl ToolExecutor<ToolCall> for NamespacedCustomTool {
fn tool_name(&self) -> ToolName {
ToolName::namespaced("editor", "apply_patch")
}
fn spec(&self) -> ToolSpec {
ToolSpec::Namespace(ResponsesApiNamespace {
name: "editor".to_string(),
description: "Editing tools.".to_string(),
tools: vec![ResponsesApiNamespaceTool::Custom(FreeformTool {
name: "apply_patch".to_string(),
description: "Apply a raw editor patch.".to_string(),
defer_loading: None,
format: FreeformToolFormat {
r#type: "grammar".to_string(),
syntax: "lark".to_string(),
definition: "start: /.+/".to_string(),
},
})],
})
}
fn handle(&self, call: ToolCall) -> ToolExecutorFuture<'_> {
Box::pin(async move {
let ToolPayload::Custom { input } = call.payload else {
return Err(FunctionCallError::Fatal(
"expected custom tool payload".to_string(),
));
};
Ok(Box::new(JsonToolOutput::new(serde_json::json!({
"namespace": call.tool_name.namespace,
"name": call.tool_name.name,
"input": input,
}))) as Box<dyn ToolOutput>)
})
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_exposes_and_dispatches_namespaced_custom_tools() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = responses::start_mock_server().await;
let mut extensions = ExtensionRegistryBuilder::<Config>::new();
extensions.tool_contributor(Arc::new(NamespacedCustomTool));
let mut builder = test_codex()
.with_model("test-gpt-5.1-codex")
.with_extensions(Arc::new(extensions.build()))
.with_config(|config| {
let _ = config.features.enable(Feature::CodeMode);
});
let test = builder.build(&server).await?;
let code = r#"
const tool = ALL_TOOLS.find(({ name }) => name === "editor__apply_patch");
const result = await tools.editor__apply_patch("nested patch");
text(JSON.stringify({
name: tool?.name ?? null,
description: tool?.description ?? null,
result,
}));
"#;
let response_mock = responses::mount_sse_sequence(
&server,
vec![
sse(vec![
ev_response_created("resp-1"),
responses::ev_custom_tool_call_with_namespace(
"call-direct",
"editor",
"apply_patch",
"direct patch",
),
ev_completed("resp-1"),
]),
sse(vec![
ev_response_created("resp-2"),
ev_custom_tool_call("call-exec", "exec", code),
ev_completed("resp-2"),
]),
sse(vec![
ev_assistant_message("msg-1", "done"),
ev_completed("resp-3"),
]),
],
)
.await;
test.submit_turn("call the namespaced custom editor tool directly and through exec")
.await?;
let requests = response_mock.requests();
assert_eq!(requests.len(), 3);
let declaration =
"declare const tools: { editor__apply_patch(input: string): Promise<unknown>; };";
let description =
format!("Apply a raw editor patch.\n\nexec tool declaration:\n```ts\n{declaration}\n```");
let first_body = requests[0].body_json();
let namespaced_custom_tool = namespace_child_tool(&first_body, "editor", "apply_patch")
.expect("namespaced custom tool should be included in the model request");
assert_eq!(
namespaced_custom_tool,
&serde_json::json!({
"type": "custom",
"name": "apply_patch",
"description": description,
"format": {
"type": "grammar",
"syntax": "lark",
"definition": "start: /.+/",
},
})
);
let (direct_output, direct_success) =
custom_tool_output_body_and_success(&requests[1], "call-direct");
assert_ne!(direct_success, Some(false));
let direct_output = serde_json::from_str::<Value>(&direct_output).unwrap_or_else(|error| {
panic!("invalid direct custom tool output `{direct_output}`: {error}")
});
assert_eq!(
direct_output,
serde_json::json!({
"namespace": "editor",
"name": "apply_patch",
"input": "direct patch",
})
);
let (exec_output, exec_success) =
custom_tool_output_body_and_success(&requests[2], "call-exec");
assert_ne!(exec_success, Some(false));
let exec_output = serde_json::from_str::<Value>(&exec_output).unwrap_or_else(|error| {
panic!("invalid code mode custom tool output `{exec_output}`: {error}")
});
assert_eq!(
exec_output,
serde_json::json!({
"name": "editor__apply_patch",
"description": format!("Editing tools.\n\n{description}"),
"result": {
"namespace": "editor",
"name": "apply_patch",
"input": "nested patch",
},
})
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn code_mode_exposes_namespaced_mcp_tools_on_global_tools_object() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -6,7 +6,7 @@ Scenario: New context window tool installs fresh full context before the next fo
## Final Follow-Up Request
00:message/developer[3]:
[01] <context_window>\nThread id: <THREAD_ID>\nFirst context window id: <FIRST_WINDOW_ID>\nCurrent context window id: <WINDOW_ID>\nPrevious context window id: <FIRST_WINDOW_ID>\n</context_window>
[01] <context_window>\nAgent name: /root\nFirst context window id: <FIRST_WINDOW_ID>\nCurrent context window id: <WINDOW_ID>\nPrevious context window id: <FIRST_WINDOW_ID>\n</context_window>
[02] <SKILLS_INSTRUCTIONS>
[03] <PERMISSIONS_INSTRUCTIONS>
01:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
+14 -24
View File
@@ -74,7 +74,7 @@ fn model_token_budget_config() -> ModelTokenBudgetConfig {
}
fn token_budget_contexts(request: &ResponsesRequest) -> Vec<String> {
let context_window_prefix = format!("{CONTEXT_WINDOW_OPEN_TAG}\nThread id: ");
let context_window_prefix = format!("{CONTEXT_WINDOW_OPEN_TAG}\nAgent name: ");
request
.message_input_texts("developer")
.into_iter()
@@ -82,13 +82,10 @@ fn token_budget_contexts(request: &ResponsesRequest) -> Vec<String> {
.collect()
}
fn token_budget_window_ids(
text: &str,
thread_id: codex_protocol::ThreadId,
) -> (String, Option<String>, String) {
fn token_budget_window_ids(text: &str, agent_name: &str) -> (String, Option<String>, String) {
let captures = assert_regex_match(
&format!(
r"^{CONTEXT_WINDOW_OPEN_TAG}\nThread id: {thread_id}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})(?:\nPrevious context window id: ([0-9a-f-]{{36}}))?\n{CONTEXT_WINDOW_CLOSE_TAG}$"
r"^{CONTEXT_WINDOW_OPEN_TAG}\nAgent name: {agent_name}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})(?:\nPrevious context window id: ([0-9a-f-]{{36}}))?\n{CONTEXT_WINDOW_CLOSE_TAG}$"
),
text,
);
@@ -223,11 +220,10 @@ async fn token_budget_context_is_only_emitted_with_full_context() -> Result<()>
let requests = responses.requests();
assert_eq!(requests.len(), 2);
let thread_id = test.session_configured.thread_id;
let initial_token_budget = token_budget_contexts(&requests[0]);
assert_eq!(initial_token_budget.len(), 1);
let (first_window_id, previous_window_id, window_id) =
token_budget_window_ids(&initial_token_budget[0], thread_id);
token_budget_window_ids(&initial_token_budget[0], "/root");
assert_eq!(previous_window_id, None);
assert_eq!(first_window_id, window_id);
assert_eq!(
@@ -731,7 +727,7 @@ async fn token_budget_context_injects_plain_thread_hint_text() -> Result<()> {
assert_eq!(token_budgets.len(), 1);
let captures = assert_regex_match(
&format!(
r"^{CONTEXT_WINDOW_OPEN_TAG}\nThread id: {thread_id}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})\nmanual history hint for thread {thread_id}\nunstructured notes/thread_hint fixture result\n{CONTEXT_WINDOW_CLOSE_TAG}$"
r"^{CONTEXT_WINDOW_OPEN_TAG}\nAgent name: /root\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})\nmanual history hint for thread {thread_id}\nunstructured notes/thread_hint fixture result\n{CONTEXT_WINDOW_CLOSE_TAG}$"
),
&token_budgets[0],
);
@@ -915,11 +911,10 @@ async fn get_context_remaining_returns_token_budget_remaining_fragment() -> Resu
"get_context_remaining should be exposed when token budget is enabled"
);
let thread_id = test.session_configured.thread_id;
let remaining_context = "You have 6500 tokens left in this context window.".to_string();
let token_budgets = token_budget_contexts(&requests[1]);
assert_eq!(token_budgets.len(), 1);
token_budget_window_ids(&token_budgets[0], thread_id);
token_budget_window_ids(&token_budgets[0], "/root");
assert_eq!(
requests[2].function_call_output_content_and_success(call_id),
Some((Some(remaining_context), None))
@@ -1099,18 +1094,17 @@ async fn token_budget_context_uses_new_window_after_compaction() -> Result<()> {
"token budget compaction should not call server-side compaction"
);
let thread_id = test.session_configured.thread_id;
let initial_token_budget = token_budget_contexts(&requests[0]);
assert_eq!(initial_token_budget.len(), 1);
let (initial_first_window_id, initial_previous_window_id, initial_window_id) =
token_budget_window_ids(&initial_token_budget[0], thread_id);
token_budget_window_ids(&initial_token_budget[0], "/root");
let post_compaction_token_budget = token_budget_contexts(&requests[1]);
assert_eq!(post_compaction_token_budget.len(), 1);
let (
post_compaction_first_window_id,
post_compaction_previous_window_id,
post_compaction_window_id,
) = token_budget_window_ids(&post_compaction_token_budget[0], thread_id);
) = token_budget_window_ids(&post_compaction_token_budget[0], "/root");
assert_eq!(initial_previous_window_id, None);
assert_eq!(initial_first_window_id, initial_window_id);
assert_eq!(post_compaction_first_window_id, initial_first_window_id);
@@ -1264,15 +1258,14 @@ async fn token_budget_mid_turn_auto_compaction_resets_before_active_follow_up()
"fresh token-budget windows should drop active tool output with the prior history"
);
let thread_id = test.session_configured.thread_id;
let initial_token_budget = token_budget_contexts(&requests[0]);
assert_eq!(initial_token_budget.len(), 1);
let (initial_first_window_id, _, initial_window_id) =
token_budget_window_ids(&initial_token_budget[0], thread_id);
token_budget_window_ids(&initial_token_budget[0], "/root");
let follow_up_token_budget = token_budget_contexts(&requests[1]);
assert_eq!(follow_up_token_budget.len(), 1);
let (follow_up_first_window_id, follow_up_previous_window_id, follow_up_window_id) =
token_budget_window_ids(&follow_up_token_budget[0], thread_id);
token_budget_window_ids(&follow_up_token_budget[0], "/root");
assert_eq!(follow_up_first_window_id, initial_first_window_id);
assert_eq!(
follow_up_previous_window_id.as_deref(),
@@ -1364,16 +1357,15 @@ async fn token_budget_auto_compact_fallback_uses_buffer_until_new_context() -> R
fallback_request.function_call_output_text(trigger_call_id),
Some("You have 0 tokens left in this context window.".to_string())
);
let thread_id = test.session_configured.thread_id;
let initial_context = token_budget_contexts(&requests[0]);
assert_eq!(token_budget_contexts(fallback_request), initial_context);
assert_eq!(token_budget_contexts(&requests[2]), initial_context);
assert!(requests[2].body_contains_text(AUTO_COMPACT_FALLBACK_PROMPT));
assert!(requests[2].body_contains_text(fallback_call_id));
let (_, _, initial_window_id) = token_budget_window_ids(&initial_context[0], thread_id);
let (_, _, initial_window_id) = token_budget_window_ids(&initial_context[0], "/root");
let follow_up_context = token_budget_contexts(&requests[3]);
let (_, previous_window_id, follow_up_window_id) =
token_budget_window_ids(&follow_up_context[0], thread_id);
token_budget_window_ids(&follow_up_context[0], "/root");
assert_eq!(
previous_window_id.as_deref(),
Some(initial_window_id.as_str())
@@ -1523,15 +1515,14 @@ async fn new_context_tool_skips_auto_compact_fallback() -> Result<()> {
.any(|name| name == "new_context"),
"new_context should be exposed when token budget is enabled"
);
let thread_id = test.session_configured.thread_id;
let initial_token_budget = token_budget_contexts(&requests[0]);
assert_eq!(initial_token_budget.len(), 1);
let (initial_first_window_id, _, initial_window_id) =
token_budget_window_ids(&initial_token_budget[0], thread_id);
token_budget_window_ids(&initial_token_budget[0], "/root");
let new_window_token_budget = token_budget_contexts(&requests[2]);
assert_eq!(new_window_token_budget.len(), 1);
let (new_first_window_id, new_previous_window_id, new_window_id) =
token_budget_window_ids(&new_window_token_budget[0], thread_id);
token_budget_window_ids(&new_window_token_budget[0], "/root");
assert_eq!(new_first_window_id, initial_first_window_id);
assert_eq!(
new_previous_window_id.as_deref(),
@@ -1552,7 +1543,6 @@ async fn new_context_tool_skips_auto_compact_fallback() -> Result<()> {
&ContextSnapshotOptions::default(),
);
let snapshot = snapshot
.replace(&thread_id.to_string(), "<THREAD_ID>")
.replace(&new_first_window_id, "<FIRST_WINDOW_ID>")
.replace(&new_window_id, "<WINDOW_ID>");
insta::assert_snapshot!(
+52
View File
@@ -242,6 +242,58 @@ async fn create_workspace_directory(
Ok(abs_path.into_path_buf())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn exec_command_hides_and_rejects_login_when_disabled() -> Result<()> {
skip_if_no_network!(Ok(()));
let builder = test_codex().with_model("gpt-5.4").with_config(|config| {
config.permissions.allow_login_shell = false;
config.use_experimental_unified_exec_tool = true;
config
.features
.enable(Feature::UnifiedExec)
.expect("test config should allow feature update");
});
let harness = TestCodexHarness::with_builder(builder).await?;
let call_id = "exec-command-login-disabled";
let arguments = json!({
"cmd": "echo should not run",
"login": true,
});
let responses = mount_sse_sequence(
harness.server(),
vec![
sse(vec![
ev_response_created("resp-1"),
ev_function_call(call_id, "exec_command", &serde_json::to_string(&arguments)?),
ev_completed("resp-1"),
]),
sse(vec![
ev_assistant_message("msg-1", "done"),
ev_completed("resp-2"),
]),
],
)
.await;
harness.submit("run the command with login").await?;
assert_eq!(
harness.function_call_stdout(call_id).await,
"login shell is disabled by config; omit `login` or set it to false."
);
let request = responses.requests()[0].body_json();
let exec_tool = request["tools"]
.as_array()
.expect("tools should be an array")
.iter()
.find(|tool| tool["name"] == "exec_command")
.expect("exec_command should be available");
assert!(exec_tool["parameters"]["properties"].get("login").is_none());
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn unified_exec_intercepts_apply_patch_exec_command() -> Result<()> {
skip_if_no_network!(Ok(()));
+19 -1
View File
@@ -72,6 +72,7 @@ use codex_core::config::resolve_profile_v2_config_path;
use codex_core::find_thread_meta_by_name_str;
use codex_core::format_exec_policy_error_with_source;
use codex_core::path_utils;
use codex_core::read_session_meta_line;
use codex_feedback::CodexFeedback;
use codex_git_utils::get_git_repo_root;
use codex_login::AuthConfig;
@@ -1466,6 +1467,7 @@ async fn resolve_resume_thread_id(
let model_providers = resume_lookup_model_providers(config, args);
if args.last {
let mut use_state_db_only = state_db.is_some();
let mut cursor = None;
loop {
let response: ThreadListResponse = send_request_with_response(
@@ -1484,7 +1486,7 @@ async fn resolve_resume_thread_id(
parent_thread_id: None,
ancestor_thread_id: None,
cwd: None,
use_state_db_only: false,
use_state_db_only,
search_term: None,
},
},
@@ -1493,12 +1495,28 @@ async fn resolve_resume_thread_id(
.await
.map_err(anyhow::Error::msg)?;
for thread in response.data {
if use_state_db_only && let Some(path) = thread.path.as_deref() {
let Ok(session_meta) = read_session_meta_line(path).await else {
continue;
};
if session_meta.meta.id.to_string() != thread.id {
continue;
}
}
let latest_cwd = latest_thread_cwd(&thread).await;
if args.all || cwds_match(config.cwd.as_path(), latest_cwd.as_path()) {
// A usable SQLite candidate is authoritative. Scanning is reserved for a
// complete miss so successful `--last` lookups avoid auditing every rollout.
return Ok(Some(thread.id));
}
}
let Some(next_cursor) = response.next_cursor else {
if use_state_db_only {
// Repair from rollouts before giving up on a missing SQLite match.
use_state_db_only = false;
cursor = None;
continue;
}
return Ok(None);
};
cursor = Some(next_cursor);
+190
View File
@@ -1,5 +1,8 @@
#![allow(clippy::unwrap_used)]
use anyhow::Context;
use codex_core::config::ConfigBuilder;
use codex_core::init_state_db;
use codex_protocol::ThreadId;
use core_test_support::responses;
use core_test_support::skip_if_no_network;
use core_test_support::test_codex_exec::test_codex_exec;
@@ -202,6 +205,193 @@ async fn exec_resume_last_appends_to_existing_file() -> anyhow::Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn exec_resume_last_repairs_rollout_missing_from_state_db() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let test = test_codex_exec();
let server = MockServer::start().await;
let _response_mock = mount_exec_responses(&server, /*count*/ 2).await;
let repo_root = exec_repo_root()?;
let marker = format!("resume-last-repair-{}", Uuid::new_v4());
test.cmd_with_server(&server)
.arg("--skip-git-repo-check")
.arg("-C")
.arg(&repo_root)
.arg(format!("echo {marker}"))
.assert()
.success();
let sessions_dir = test.home_path().join("sessions");
let path = find_session_file_containing_marker(&sessions_dir, &marker)
.expect("no session file found after first run");
let thread_id = ThreadId::from_string(&extract_conversation_id(&path))?;
let config = ConfigBuilder::default()
.codex_home(test.home_path().to_path_buf())
.build()
.await?;
let state_db = init_state_db(&config)
.await
.expect("state DB should initialize");
assert_eq!(state_db.delete_thread(thread_id).await?, 1);
state_db
.mark_backfill_complete(/*last_watermark*/ None)
.await?;
let resumed_marker = format!("resume-last-repaired-{}", Uuid::new_v4());
test.cmd_with_server(&server)
.arg("--skip-git-repo-check")
.arg("-C")
.arg(&repo_root)
.arg("resume")
.arg("--last")
.arg(format!("echo {resumed_marker}"))
.assert()
.success();
let resumed_path = find_session_file_containing_marker(&sessions_dir, &resumed_marker)
.expect("no resumed session file after SQLite repair");
assert_eq!(resumed_path, path);
assert!(state_db.get_thread(thread_id).await?.is_some());
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn exec_resume_last_trusts_usable_state_db_candidate() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let test = test_codex_exec();
let server = MockServer::start().await;
let _response_mock = mount_exec_responses(&server, /*count*/ 3).await;
let repo_root = exec_repo_root()?;
let sessions_dir = test.home_path().join("sessions");
let older_marker = format!("resume-last-indexed-{}", Uuid::new_v4());
test.cmd_with_server(&server)
.arg("--skip-git-repo-check")
.arg("-C")
.arg(&repo_root)
.arg(format!("echo {older_marker}"))
.assert()
.success();
let older_path = find_session_file_containing_marker(&sessions_dir, &older_marker)
.expect("no indexed session file after first run");
let newer_marker = format!("resume-last-unindexed-{}", Uuid::new_v4());
test.cmd_with_server(&server)
.arg("--skip-git-repo-check")
.arg("-C")
.arg(&repo_root)
.arg(format!("echo {newer_marker}"))
.assert()
.success();
let newer_path = find_session_file_containing_marker(&sessions_dir, &newer_marker)
.expect("no unindexed session file after second run");
let newer_thread_id = ThreadId::from_string(&extract_conversation_id(&newer_path))?;
let config = ConfigBuilder::default()
.codex_home(test.home_path().to_path_buf())
.build()
.await?;
let state_db = init_state_db(&config)
.await
.expect("state DB should initialize");
assert_eq!(state_db.delete_thread(newer_thread_id).await?, 1);
state_db
.mark_backfill_complete(/*last_watermark*/ None)
.await?;
let resumed_marker = format!("resume-last-authoritative-{}", Uuid::new_v4());
test.cmd_with_server(&server)
.arg("--skip-git-repo-check")
.arg("-C")
.arg(&repo_root)
.arg("resume")
.arg("--last")
.arg(format!("echo {resumed_marker}"))
.assert()
.success();
let resumed_path = find_session_file_containing_marker(&sessions_dir, &resumed_marker)
.expect("no resumed session file after SQLite lookup");
assert_eq!(
(
resumed_path,
state_db
.get_thread(newer_thread_id)
.await?
.map(|metadata| metadata.id),
),
(older_path, None),
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn exec_resume_last_skips_mismatched_state_db_candidate() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let test = test_codex_exec();
let server = MockServer::start().await;
let _response_mock = mount_exec_responses(&server, /*count*/ 3).await;
let repo_root = exec_repo_root()?;
let sessions_dir = test.home_path().join("sessions");
let older_marker = format!("resume-last-valid-{}", Uuid::new_v4());
test.cmd_with_server(&server)
.arg("--skip-git-repo-check")
.arg("-C")
.arg(&repo_root)
.arg(format!("echo {older_marker}"))
.assert()
.success();
let older_path = find_session_file_containing_marker(&sessions_dir, &older_marker)
.expect("no valid session file after first run");
let newer_marker = format!("resume-last-mismatched-{}", Uuid::new_v4());
test.cmd_with_server(&server)
.arg("--skip-git-repo-check")
.arg("-C")
.arg(&repo_root)
.arg(format!("echo {newer_marker}"))
.assert()
.success();
let newer_path = find_session_file_containing_marker(&sessions_dir, &newer_marker)
.expect("no mismatched session file after second run");
let newer_thread_id = ThreadId::from_string(&extract_conversation_id(&newer_path))?;
let config = ConfigBuilder::default()
.codex_home(test.home_path().to_path_buf())
.build()
.await?;
let state_db = init_state_db(&config)
.await
.expect("state DB should initialize");
let mut mismatched = state_db
.get_thread(newer_thread_id)
.await?
.expect("newer thread should be indexed");
mismatched.rollout_path = older_path.clone();
state_db.upsert_thread(&mismatched).await?;
let resumed_marker = format!("resume-last-valid-resumed-{}", Uuid::new_v4());
test.cmd_with_server(&server)
.arg("--skip-git-repo-check")
.arg("-C")
.arg(&repo_root)
.arg("resume")
.arg("--last")
.arg(format!("echo {resumed_marker}"))
.assert()
.success();
let resumed_path = find_session_file_containing_marker(&sessions_dir, &resumed_marker)
.expect("no resumed session file after skipping mismatched SQLite row");
assert_eq!(resumed_path, older_path);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn exec_resume_last_accepts_prompt_after_flag_in_json_mode() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
+3 -1
View File
@@ -49,7 +49,9 @@ fn uses_reserved_image_gen_namespace() {
panic!("imagegen should advertise a namespace tool");
};
assert_eq!(spec.name, IMAGE_GEN_NAMESPACE);
let ResponsesApiNamespaceTool::Function(function) = &spec.tools[0];
let ResponsesApiNamespaceTool::Function(function) = &spec.tools[0] else {
panic!("imagegen should advertise a function tool");
};
assert_eq!(function.name, IMAGEGEN_TOOL_NAME);
}
+5
View File
@@ -22,11 +22,13 @@ codex-protocol = { workspace = true }
codex-skills = { workspace = true }
codex-tools = { workspace = true }
codex-utils-path-uri = { workspace = true }
codex-utils-plugins = { workspace = true }
codex-utils-string = { workspace = true }
futures = { workspace = true }
schemars = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
tokio = { workspace = true, features = ["sync", "time"] }
tracing = { workspace = true }
url = { workspace = true }
@@ -34,6 +36,9 @@ url = { workspace = true }
[dev-dependencies]
codex-models-manager = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-cargo-bin = { workspace = true }
insta = { workspace = true }
opentelemetry_sdk = { workspace = true }
pretty_assertions = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+1
View File
@@ -4,6 +4,7 @@ mod config;
mod dynamic_skill_selector;
mod extension;
mod fragments;
mod loader;
pub mod provider;
mod render;
mod render_observability;
+167
View File
@@ -0,0 +1,167 @@
use std::collections::HashSet;
use std::io;
use codex_exec_server::ExecutorFileSystem;
use codex_exec_server::WalkEntryKind;
use codex_exec_server::WalkOptions;
use codex_utils_path_uri::PathUri;
use codex_utils_plugins::DISCOVERABLE_PLUGIN_MANIFEST_PATHS;
use super::MAX_SCAN_DEPTH;
use super::MAX_SKILLS_DIRS_PER_ROOT;
use super::SKILLS_FILENAME;
use super::SKILLS_METADATA_DIR;
use super::SKILLS_METADATA_FILENAME;
const MAX_SKILLS_ENTRIES_PER_ROOT: usize = 20_000;
pub(super) const MAX_CONCURRENT_SKILL_LOADS: usize = 64;
pub(super) struct SkillDiscovery {
pub skills: Vec<DiscoveredSkill>,
pub plugin_roots: HashSet<PathUri>,
pub namespace_roots: HashSet<PathUri>,
pub warnings: Vec<String>,
}
pub(super) struct DiscoveredSkill {
pub path: PathUri,
pub metadata: SkillMetadataDiscovery,
}
pub(super) enum SkillMetadataDiscovery {
Present(PathUri),
Absent,
Probe(PathUri),
}
pub(super) async fn discover_skills(
file_system: &dyn ExecutorFileSystem,
root: &PathUri,
) -> SkillDiscovery {
let empty_discovery = || SkillDiscovery {
skills: Vec::new(),
plugin_roots: HashSet::new(),
namespace_roots: HashSet::new(),
warnings: Vec::new(),
};
let walk = match file_system
.walk(
root,
WalkOptions {
max_depth: MAX_SCAN_DEPTH,
max_directories: MAX_SKILLS_DIRS_PER_ROOT,
max_entries: MAX_SKILLS_ENTRIES_PER_ROOT,
follow_directory_symlinks: true,
prune_hidden_directories: false,
},
/*sandbox*/ None,
)
.await
{
Ok(walk) => walk,
Err(error) if error.kind() == io::ErrorKind::NotFound => return empty_discovery(),
Err(error) => {
let mut discovery = empty_discovery();
discovery
.warnings
.push(format!("failed to walk skills root {root}: {error:#}"));
return discovery;
}
};
let inventory_complete = !walk.truncated && walk.errors.is_empty();
let mut warnings = walk
.errors
.into_iter()
.map(|error| {
format!(
"failed to scan skill path {}: {}",
error.path, error.message
)
})
.collect::<Vec<_>>();
if walk.truncated {
warnings.push(format!(
"skills scan reached its traversal limit (root: {root})"
));
}
let mut skill_files = Vec::new();
let mut file_paths = HashSet::new();
let mut metadata_directory_parents = HashSet::new();
let mut plugin_roots = HashSet::new();
for entry in walk.entries {
match entry.kind {
WalkEntryKind::Directory => {
if entry
.path
.basename()
.is_some_and(|name| name.eq_ignore_ascii_case(SKILLS_METADATA_DIR))
&& let Some(skill_dir) = entry.path.parent()
{
metadata_directory_parents.insert(skill_dir);
}
if DISCOVERABLE_PLUGIN_MANIFEST_PATHS
.iter()
.any(|path| path.split('/').next() == entry.path.basename().as_deref())
&& let Some(plugin_root) = entry.path.parent()
{
plugin_roots.insert(plugin_root);
}
}
WalkEntryKind::File => {
file_paths.insert(entry.path.clone());
if entry.path.basename().as_deref() == Some(SKILLS_FILENAME) {
skill_files.push(entry.path);
}
}
}
}
let skills = skill_files
.into_iter()
.map(|path| DiscoveredSkill {
metadata: discover_skill_metadata(
&path,
&file_paths,
&metadata_directory_parents,
inventory_complete,
),
path,
})
.collect();
SkillDiscovery {
skills,
plugin_roots,
namespace_roots: HashSet::from([root.clone()]),
warnings,
}
}
fn discover_skill_metadata(
skill_path: &PathUri,
file_paths: &HashSet<PathUri>,
metadata_directory_parents: &HashSet<PathUri>,
inventory_complete: bool,
) -> SkillMetadataDiscovery {
let Some(skill_dir) = skill_path.parent() else {
return SkillMetadataDiscovery::Absent;
};
let Ok(metadata_dir) = skill_dir.join(SKILLS_METADATA_DIR) else {
return SkillMetadataDiscovery::Absent;
};
let Ok(metadata_path) = metadata_dir.join(SKILLS_METADATA_FILENAME) else {
return SkillMetadataDiscovery::Absent;
};
if file_paths.contains(&metadata_path) {
return SkillMetadataDiscovery::Present(metadata_path);
}
if inventory_complete && !metadata_directory_parents.contains(&skill_dir) {
SkillMetadataDiscovery::Absent
} else {
// A complete walk proves ordinary absence, but keep a filesystem probe for case aliases,
// file symlinks omitted by the walk, and incomplete inventories.
SkillMetadataDiscovery::Probe(metadata_path)
}
}
@@ -0,0 +1,382 @@
use std::collections::HashMap;
use std::io;
use codex_exec_server::CapabilityRootDiscovery;
use codex_exec_server::ExecutorFileSystem;
use codex_protocol::protocol::Product;
use codex_skills::EnvironmentSkillMetadata;
use codex_skills::ParsedSkillFrontmatter;
use codex_skills::SkillDependencies;
use codex_skills::SkillPolicy;
use codex_skills::parse_skill_frontmatter_metadata;
use codex_utils_path_uri::PathUri;
use futures::StreamExt;
use super::MAX_QUALIFIED_NAME_LEN;
use super::discovery::DiscoveredSkill;
use super::discovery::MAX_CONCURRENT_SKILL_LOADS;
use super::discovery::SkillMetadataDiscovery;
use super::discovery::discover_skills;
use super::metadata::SkillMetadataFile;
use super::metadata::resolve_dependencies;
use super::metadata::resolve_policy;
use super::metadata::sanitize_single_line;
use super::metadata::validate_len;
use super::namespace::SkillNamespaceResolver;
struct ParsedEnvironmentSkill {
path_to_skills_md: PathUri,
base_name: String,
description: String,
short_description: Option<String>,
dependencies: Option<SkillDependencies>,
policy: Option<SkillPolicy>,
}
/// Parsed executor skill plus the instructions already materialized by capability discovery.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EnvironmentSkillSnapshot {
pub metadata: EnvironmentSkillMetadata,
pub instructions: String,
}
#[derive(Debug, Default)]
pub struct EnvironmentSkillSnapshotOutcome {
pub skills: Vec<EnvironmentSkillSnapshot>,
pub warnings: Vec<String>,
}
impl ParsedEnvironmentSkill {
async fn load(
file_system: &dyn ExecutorFileSystem,
skill: &DiscoveredSkill,
) -> Result<Self, String> {
let (contents, discovered_metadata) = match &skill.metadata {
SkillMetadataDiscovery::Present(metadata_path) => {
let (contents, metadata) = tokio::join!(
read_skill_contents(file_system, &skill.path),
read_skill_metadata(file_system, metadata_path),
);
(contents?, metadata)
}
SkillMetadataDiscovery::Absent | SkillMetadataDiscovery::Probe(_) => (
read_skill_contents(file_system, &skill.path).await?,
(None, None),
),
};
let ParsedSkillFrontmatter {
name: base_name,
description,
short_description,
} = parse_skill_frontmatter_metadata(&contents, || default_skill_name(&skill.path))
.map_err(|err| err.to_string())?;
let (dependencies, policy) = match &skill.metadata {
SkillMetadataDiscovery::Present(_) | SkillMetadataDiscovery::Absent => {
discovered_metadata
}
SkillMetadataDiscovery::Probe(metadata_path) => {
probe_skill_metadata(file_system, metadata_path).await
}
};
Ok(Self {
path_to_skills_md: skill.path.clone(),
base_name,
description,
short_description,
dependencies,
policy,
})
}
}
#[derive(Debug, Default)]
pub struct EnvironmentSkillLoadOutcome {
pub skills: Vec<EnvironmentSkillMetadata>,
pub warnings: Vec<String>,
}
/// Discovers skills without converting environment-owned paths to host paths.
#[tracing::instrument(
name = "skills.environment.load",
level = "info",
skip_all,
fields(skill_count = tracing::field::Empty)
)]
pub async fn load_environment_skills_from_root(
file_system: &dyn ExecutorFileSystem,
root: &PathUri,
restriction_product: Option<Product>,
) -> EnvironmentSkillLoadOutcome {
let mut outcome = EnvironmentSkillLoadOutcome::default();
// Preserve environment discovery behavior by following directory aliases and including
// hidden directories exposed by the executor.
let discovery = discover_skills(file_system, root).await;
tracing::Span::current().record("skill_count", discovery.skills.len());
outcome.warnings.extend(discovery.warnings);
if discovery.skills.is_empty() {
return outcome;
}
let skill_paths = discovery
.skills
.iter()
.map(|skill| skill.path.clone())
.collect::<Vec<_>>();
let namespace_resolver = SkillNamespaceResolver::discover(
file_system,
root,
&skill_paths,
discovery.plugin_roots,
discovery.namespace_roots,
);
// Remote executors can multiplex these independent per-skill reads, so polling a bounded
// number together allows the I/O for each skill and its metadata to happen concurrently.
let skill_results = futures::stream::iter(discovery.skills)
.map(|skill| {
let path = skill.path.clone();
async move {
(
path,
ParsedEnvironmentSkill::load(file_system, &skill).await,
)
}
})
.buffered(MAX_CONCURRENT_SKILL_LOADS)
.collect::<Vec<_>>();
let (namespace_resolver, skill_results) = tokio::join!(namespace_resolver, skill_results);
for (path, result) in skill_results {
let result = result.and_then(|skill| {
let name = namespace_resolver
.for_skill(root, &skill.path_to_skills_md)
.qualify(&skill.base_name);
validate_len(&name, MAX_QUALIFIED_NAME_LEN, "qualified name")
.map_err(|err| err.to_string())?;
Ok(EnvironmentSkillMetadata {
path_to_skills_md: skill.path_to_skills_md,
name,
description: skill.description,
short_description: skill.short_description,
dependencies: skill.dependencies,
policy: skill.policy,
})
});
match result {
Ok(skill) if skill.matches_product_restriction(restriction_product) => {
outcome.skills.push(skill);
}
Ok(_) => {}
Err(message) => outcome.warnings.push(format!(
"Failed to load environment skill at {path}: {message}"
)),
}
}
outcome.skills.sort_by(|left, right| {
left.name.cmp(&right.name).then_with(|| {
left.path_to_skills_md
.to_string()
.cmp(&right.path_to_skills_md.to_string())
})
});
outcome
}
/// Parses an executor-produced manifest bundle without issuing additional filesystem requests.
pub fn load_environment_skills_from_discovery(
discovery: &CapabilityRootDiscovery,
restriction_product: Option<Product>,
) -> EnvironmentSkillSnapshotOutcome {
let mut outcome = EnvironmentSkillSnapshotOutcome {
warnings: discovery.warnings.clone(),
..Default::default()
};
if let Some(error) = &discovery.error {
outcome.warnings.push(error.clone());
return outcome;
}
let mut plugin_namespaces = HashMap::new();
for (plugin_root, name) in discovery.namespace_manifests.iter().filter_map(|manifest| {
#[derive(serde::Deserialize)]
struct ManifestName {
#[serde(default)]
name: String,
}
let plugin_root = manifest.path.parent()?.parent()?;
let parsed = serde_json::from_str::<ManifestName>(&manifest.contents).ok()?;
let name = if parsed.name.trim().is_empty() {
plugin_root.basename()?
} else {
parsed.name
};
Some((plugin_root, name))
}) {
// Exec-server orders manifests by the same precedence as local discovery. Preserve the
// first manifest if an older or alternate server returns duplicates for one plugin root.
plugin_namespaces.entry(plugin_root).or_insert(name);
}
for skill in &discovery.skills {
let plugin_namespace =
nearest_plugin_namespace(&skill.instructions.path, &plugin_namespaces);
let ParsedSkillFrontmatter {
name: base_name,
description,
short_description,
} = match parse_skill_frontmatter_metadata(&skill.instructions.contents, || {
default_skill_name(&skill.instructions.path)
}) {
Ok(frontmatter) => frontmatter,
Err(error) => {
outcome.warnings.push(format!(
"Failed to load environment skill at {}: {error}",
skill.instructions.path
));
continue;
}
};
let name = plugin_namespace
.map(|namespace| format!("{namespace}:{base_name}"))
.unwrap_or(base_name);
if let Err(error) = validate_len(&name, MAX_QUALIFIED_NAME_LEN, "qualified name") {
outcome.warnings.push(format!(
"Failed to load environment skill at {}: {error}",
skill.instructions.path
));
continue;
}
let (dependencies, policy) = skill
.metadata
.as_ref()
.and_then(|metadata| {
serde_yaml::from_str::<SkillMetadataFile>(&metadata.contents)
.map_err(|error| {
tracing::warn!(
path = %metadata.path,
"ignoring invalid discovered skill metadata: {error}"
);
})
.ok()
})
.map(|metadata| {
(
resolve_dependencies(metadata.dependencies),
resolve_policy(metadata.policy),
)
})
.unwrap_or((None, None));
let metadata = EnvironmentSkillMetadata {
path_to_skills_md: skill.instructions.path.clone(),
name,
description,
short_description,
dependencies,
policy,
};
if metadata.matches_product_restriction(restriction_product) {
outcome.skills.push(EnvironmentSkillSnapshot {
metadata,
instructions: skill.instructions.contents.clone(),
});
}
}
outcome.skills.sort_by(|left, right| {
left.metadata.name.cmp(&right.metadata.name).then_with(|| {
left.metadata
.path_to_skills_md
.to_string()
.cmp(&right.metadata.path_to_skills_md.to_string())
})
});
outcome
}
fn nearest_plugin_namespace<'a>(
skill_path: &PathUri,
plugin_namespaces: &'a HashMap<PathUri, String>,
) -> Option<&'a str> {
let mut ancestor = skill_path.parent();
while let Some(path) = ancestor {
if let Some(namespace) = plugin_namespaces.get(&path) {
return Some(namespace);
}
ancestor = path.parent();
}
None
}
async fn read_skill_contents(
file_system: &dyn ExecutorFileSystem,
skill_path: &PathUri,
) -> Result<String, String> {
file_system
.read_file_text(skill_path, /*sandbox*/ None)
.await
.map_err(|err| format!("failed to read file: {err}"))
}
async fn probe_skill_metadata(
file_system: &dyn ExecutorFileSystem,
metadata_path: &PathUri,
) -> (Option<SkillDependencies>, Option<SkillPolicy>) {
match file_system
.get_metadata(metadata_path, /*sandbox*/ None)
.await
{
Ok(metadata) if metadata.is_file => {}
Ok(_) => return (None, None),
Err(error) if error.kind() == io::ErrorKind::NotFound => return (None, None),
Err(error) => {
tracing::warn!("ignoring {metadata_path}: failed to stat metadata: {error}");
return (None, None);
}
}
read_skill_metadata(file_system, metadata_path).await
}
async fn read_skill_metadata(
file_system: &dyn ExecutorFileSystem,
metadata_path: &PathUri,
) -> (Option<SkillDependencies>, Option<SkillPolicy>) {
let contents = match file_system
.read_file_text(metadata_path, /*sandbox*/ None)
.await
{
Ok(contents) => contents,
Err(error) => {
tracing::warn!("ignoring {metadata_path}: failed to read metadata: {error}");
return (None, None);
}
};
let parsed: SkillMetadataFile = match serde_yaml::from_str(&contents) {
Ok(parsed) => parsed,
Err(error) => {
tracing::warn!("ignoring {metadata_path}: invalid metadata: {error}");
return (None, None);
}
};
(
resolve_dependencies(parsed.dependencies),
resolve_policy(parsed.policy),
)
}
fn default_skill_name(path: &PathUri) -> String {
path.parent()
.and_then(|parent| parent.basename())
.map(|name| sanitize_single_line(&name))
.filter(|name| !name.is_empty())
.unwrap_or_else(|| "skill".to_string())
}
#[cfg(test)]
#[path = "environment_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "environment_io_tests.rs"]
mod io_tests;
@@ -0,0 +1,414 @@
use std::fs;
use std::sync::Mutex;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::time::Duration;
use codex_exec_server::CopyOptions;
use codex_exec_server::CreateDirectoryOptions;
use codex_exec_server::ExecutorFileSystem;
use codex_exec_server::ExecutorFileSystemFuture;
use codex_exec_server::FileMetadata;
use codex_exec_server::FileSystemReadStream;
use codex_exec_server::FileSystemSandboxContext;
use codex_exec_server::LOCAL_FS;
use codex_exec_server::ReadDirectoryEntry;
use codex_exec_server::RemoveOptions;
use codex_exec_server::WalkOptions;
use codex_exec_server::WalkOutcome;
use codex_skills::EnvironmentSkillMetadata;
use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
use tempfile::tempdir;
use tokio::sync::Notify;
use super::load_environment_skills_from_root;
#[derive(Clone, Copy)]
enum ManifestMetadataBehavior {
Immediate,
WaitForSkillRead,
}
struct RecordingFileSystem<'a> {
inner: &'a dyn ExecutorFileSystem,
read_files: Mutex<Vec<PathUri>>,
metadata_files: Mutex<Vec<PathUri>>,
walks: AtomicUsize,
manifest_metadata_behavior: ManifestMetadataBehavior,
skill_read_started: AtomicBool,
skill_read_started_notify: Notify,
}
#[derive(Debug, PartialEq, Eq)]
struct FileSystemCalls {
walks: usize,
read_files: Vec<PathUri>,
metadata_files: Vec<PathUri>,
}
impl<'a> RecordingFileSystem<'a> {
fn new(
inner: &'a dyn ExecutorFileSystem,
manifest_metadata_behavior: ManifestMetadataBehavior,
) -> Self {
Self {
inner,
read_files: Mutex::new(Vec::new()),
metadata_files: Mutex::new(Vec::new()),
walks: AtomicUsize::new(0),
manifest_metadata_behavior,
skill_read_started: AtomicBool::new(false),
skill_read_started_notify: Notify::new(),
}
}
fn calls(&self) -> FileSystemCalls {
let mut read_files = self
.read_files
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
read_files.sort_by_key(ToString::to_string);
let mut metadata_files = self
.metadata_files
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
metadata_files.sort_by_key(ToString::to_string);
FileSystemCalls {
walks: self.walks.load(Ordering::Relaxed),
read_files,
metadata_files,
}
}
}
impl ExecutorFileSystem for RecordingFileSystem<'_> {
fn canonicalize<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, PathUri> {
self.inner.canonicalize(path, sandbox)
}
fn read_file<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, Vec<u8>> {
self.read_files
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(path.clone());
if path.basename().as_deref() == Some("SKILL.md") {
self.skill_read_started.store(true, Ordering::Release);
self.skill_read_started_notify.notify_waiters();
}
self.inner.read_file(path, sandbox)
}
fn read_file_stream<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, FileSystemReadStream> {
self.inner.read_file_stream(path, sandbox)
}
fn write_file<'a>(
&'a self,
path: &'a PathUri,
contents: Vec<u8>,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
self.inner.write_file(path, contents, sandbox)
}
fn create_directory<'a>(
&'a self,
path: &'a PathUri,
options: CreateDirectoryOptions,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
self.inner.create_directory(path, options, sandbox)
}
fn get_metadata<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, FileMetadata> {
self.metadata_files
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(path.clone());
if matches!(
self.manifest_metadata_behavior,
ManifestMetadataBehavior::WaitForSkillRead
) && path.basename().as_deref() == Some("plugin.json")
{
return Box::pin(async move {
loop {
let notified = self.skill_read_started_notify.notified();
if self.skill_read_started.load(Ordering::Acquire) {
break;
}
notified.await;
}
self.inner.get_metadata(path, sandbox).await
});
}
self.inner.get_metadata(path, sandbox)
}
fn read_directory<'a>(
&'a self,
path: &'a PathUri,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, Vec<ReadDirectoryEntry>> {
self.inner.read_directory(path, sandbox)
}
fn walk<'a>(
&'a self,
path: &'a PathUri,
options: WalkOptions,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, WalkOutcome> {
self.walks.fetch_add(1, Ordering::Relaxed);
self.inner.walk(path, options, sandbox)
}
fn remove<'a>(
&'a self,
path: &'a PathUri,
options: RemoveOptions,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
self.inner.remove(path, options, sandbox)
}
fn copy<'a>(
&'a self,
source_path: &'a PathUri,
destination_path: &'a PathUri,
options: CopyOptions,
sandbox: Option<&'a FileSystemSandboxContext>,
) -> ExecutorFileSystemFuture<'a, ()> {
self.inner
.copy(source_path, destination_path, options, sandbox)
}
}
#[tokio::test]
async fn loads_nearest_plugin_namespaces_without_reading_unused_sibling_manifests() {
let root = tempdir().expect("tempdir");
let standalone_skill = root.path().join("standalone/SKILL.md");
let outer_root = root.path().join("plugins/outer");
let outer_skill = outer_root.join("skills/deploy/SKILL.md");
let inner_root = outer_root.join("nested/inner");
let inner_skill = inner_root.join("skills/audit/SKILL.md");
let unused_root = root.path().join("plugins/unused");
for path in [&standalone_skill, &outer_skill, &inner_skill] {
fs::create_dir_all(path.parent().expect("skill parent")).expect("skill dir");
}
for (plugin_root, name) in [
(&outer_root, "outer"),
(&inner_root, "inner"),
(&unused_root, "unused"),
] {
fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("manifest dir");
fs::write(
plugin_root.join(".codex-plugin/plugin.json"),
format!(r#"{{"name":"{name}"}}"#),
)
.expect("manifest");
}
for (path, name) in [
(&standalone_skill, "standalone"),
(&outer_skill, "deploy"),
(&inner_skill, "audit"),
] {
fs::write(
path,
format!("---\nname: {name}\ndescription: {name} skill.\n---\n"),
)
.expect("skill");
}
let file_system =
RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate);
let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI");
let outcome = load_environment_skills_from_root(
&file_system,
&root_uri,
/*restriction_product*/ None,
)
.await;
assert_eq!(outcome.warnings, Vec::<String>::new());
assert_eq!(
outcome.skills,
vec![
EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(&inner_skill).unwrap(),
name: "inner:audit".to_string(),
description: "audit skill.".to_string(),
short_description: None,
dependencies: None,
policy: None,
},
EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(&outer_skill).unwrap(),
name: "outer:deploy".to_string(),
description: "deploy skill.".to_string(),
short_description: None,
dependencies: None,
policy: None,
},
EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(&standalone_skill).unwrap(),
name: "standalone".to_string(),
description: "standalone skill.".to_string(),
short_description: None,
dependencies: None,
policy: None,
},
]
);
let mut manifest_reads = file_system
.calls()
.read_files
.into_iter()
.filter(|path| path.basename().as_deref() == Some("plugin.json"))
.collect::<Vec<_>>();
manifest_reads.sort_by_key(ToString::to_string);
let mut expected_manifest_reads = [&outer_root, &inner_root]
.into_iter()
.map(|plugin_root| {
PathUri::from_host_native_path(plugin_root.join(".codex-plugin/plugin.json")).unwrap()
})
.collect::<Vec<_>>();
expected_manifest_reads.sort_by_key(ToString::to_string);
assert_eq!(manifest_reads, expected_manifest_reads);
}
#[tokio::test]
async fn reuses_walk_inventory_for_missing_skill_metadata() {
const SKILL_COUNT: usize = 66;
let root = tempdir().expect("tempdir");
let manifest_path = root.path().join(".codex-plugin/plugin.json");
fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir");
fs::write(&manifest_path, r#"{"name":"inventory"}"#).expect("manifest");
let mut skill_paths = Vec::new();
for index in 0..SKILL_COUNT {
let name = format!("skill-{index}");
let skill_path = root.path().join(&name).join("SKILL.md");
fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir");
fs::write(
&skill_path,
format!("---\nname: {name}\ndescription: {name} skill.\n---\n"),
)
.expect("skill");
skill_paths.push(skill_path);
}
let file_system =
RecordingFileSystem::new(LOCAL_FS.as_ref(), ManifestMetadataBehavior::Immediate);
let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI");
let outcome = load_environment_skills_from_root(
&file_system,
&root_uri,
/*restriction_product*/ None,
)
.await;
let mut expected_skills = skill_paths
.iter()
.enumerate()
.map(|(index, skill_path)| EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(),
name: format!("inventory:skill-{index}"),
description: format!("skill-{index} skill."),
short_description: None,
dependencies: None,
policy: None,
})
.collect::<Vec<_>>();
expected_skills.sort_by(|left, right| {
left.name.cmp(&right.name).then_with(|| {
left.path_to_skills_md
.to_string()
.cmp(&right.path_to_skills_md.to_string())
})
});
assert_eq!(outcome.skills, expected_skills);
assert_eq!(outcome.warnings, Vec::<String>::new());
let mut expected_read_files = skill_paths
.iter()
.map(|path| PathUri::from_host_native_path(path).unwrap())
.collect::<Vec<_>>();
let manifest_uri = PathUri::from_host_native_path(manifest_path).unwrap();
expected_read_files.push(manifest_uri.clone());
expected_read_files.sort_by_key(ToString::to_string);
assert_eq!(
file_system.calls(),
FileSystemCalls {
walks: 1,
read_files: expected_read_files,
metadata_files: vec![manifest_uri],
}
);
}
#[tokio::test]
async fn reads_skill_files_while_resolving_plugin_namespaces() {
let root = tempdir().expect("tempdir");
let manifest_path = root.path().join(".codex-plugin/plugin.json");
fs::create_dir_all(manifest_path.parent().expect("manifest parent")).expect("manifest dir");
fs::write(&manifest_path, r#"{"name":"parallel"}"#).expect("manifest");
let skill_path = root.path().join("demo/SKILL.md");
fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill dir");
fs::write(
&skill_path,
"---\nname: demo\ndescription: demo skill.\n---\n",
)
.expect("skill");
let file_system = RecordingFileSystem::new(
LOCAL_FS.as_ref(),
ManifestMetadataBehavior::WaitForSkillRead,
);
let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI");
let outcome = tokio::time::timeout(
Duration::from_secs(5),
load_environment_skills_from_root(
&file_system,
&root_uri,
/*restriction_product*/ None,
),
)
.await
.expect("skill reads should start before namespace resolution finishes");
assert_eq!(outcome.warnings, Vec::<String>::new());
assert_eq!(
outcome.skills,
vec![EnvironmentSkillMetadata {
path_to_skills_md: PathUri::from_host_native_path(skill_path).unwrap(),
name: "parallel:demo".to_string(),
description: "demo skill.".to_string(),
short_description: None,
dependencies: None,
policy: None,
}]
);
}
@@ -0,0 +1,151 @@
use std::fs;
use codex_core_skills::loader::load_environment_skills_from_root;
use codex_exec_server::CapabilityRootDiscoverRequest;
use codex_exec_server::CapabilityRootsDiscoverParams;
use codex_exec_server::LOCAL_FS;
use codex_exec_server::discover_capability_roots;
use codex_utils_path_uri::PathUri;
use pretty_assertions::assert_eq;
use tempfile::tempdir;
use super::load_environment_skills_from_discovery;
#[tokio::test]
async fn executor_bundle_parser_matches_the_existing_environment_loader() {
let root = tempdir().expect("tempdir");
let plugin_manifest = root.path().join(".codex-plugin/plugin.json");
let nested_manifest = root.path().join("nested/.claude-plugin/plugin.json");
let deploy_skill = root.path().join("skills/deploy/SKILL.md");
let deploy_metadata = root.path().join("skills/deploy/agents/openai.yaml");
let audit_skill = root.path().join("nested/skills/audit/SKILL.md");
for (path, contents) in [
(&plugin_manifest, r#"{"name":"demo"}"#),
(&nested_manifest, r#"{"name":"nested"}"#),
(
&deploy_skill,
"---\nname: deploy\ndescription: Deploy the service.\n---\n\nDeploy.\n",
),
(
&deploy_metadata,
"policy:\n allow_implicit_invocation: false\n",
),
(
&audit_skill,
"---\nname: audit\ndescription: Audit the service.\n---\n\nAudit.\n",
),
] {
fs::create_dir_all(path.parent().expect("test file parent")).expect("test directory");
fs::write(path, contents).expect("test file");
}
let root_uri = PathUri::from_host_native_path(root.path()).expect("root URI");
let existing = load_environment_skills_from_root(
LOCAL_FS.as_ref(),
&root_uri,
/*restriction_product*/ None,
)
.await;
let response = discover_capability_roots(
LOCAL_FS.as_ref(),
CapabilityRootsDiscoverParams {
roots: vec![CapabilityRootDiscoverRequest {
id: "demo@1".to_string(),
path: root_uri,
sandbox: None,
}],
},
)
.await
.expect("capability discovery");
let bundled = load_environment_skills_from_discovery(
response.roots.first().expect("discovered root"),
/*restriction_product*/ None,
);
assert_eq!(bundled.warnings, existing.warnings);
assert_eq!(
bundled
.skills
.iter()
.map(|skill| skill.metadata.clone())
.collect::<Vec<_>>(),
existing.skills
);
assert_eq!(
bundled
.skills
.iter()
.map(|skill| skill.instructions.as_str())
.collect::<Vec<_>>(),
vec![
"---\nname: deploy\ndescription: Deploy the service.\n---\n\nDeploy.\n",
"---\nname: audit\ndescription: Audit the service.\n---\n\nAudit.\n",
]
);
}
#[tokio::test]
async fn executor_bundle_preserves_parent_namespace_and_manifest_precedence() {
let plugin = tempdir().expect("tempdir");
for (relative_path, name) in [
(".codex-plugin/plugin.json", "codex-name"),
(".claude-plugin/plugin.json", "claude-name"),
(".cursor-plugin/plugin.json", "cursor-name"),
] {
let manifest = plugin.path().join(relative_path);
fs::create_dir_all(manifest.parent().expect("manifest parent"))
.expect("manifest directory");
fs::write(&manifest, format!(r#"{{"name":"{name}"}}"#)).expect("manifest");
}
let skills_root = plugin.path().join("skills");
let skill_path = skills_root.join("search/SKILL.md");
fs::create_dir_all(skill_path.parent().expect("skill parent")).expect("skill directory");
fs::write(
&skill_path,
"---\nname: search\ndescription: Search the project.\n---\n\nSearch.\n",
)
.expect("skill");
let root_uri = PathUri::from_host_native_path(&skills_root).expect("skills root URI");
let existing = load_environment_skills_from_root(
LOCAL_FS.as_ref(),
&root_uri,
/*restriction_product*/ None,
)
.await;
let response = discover_capability_roots(
LOCAL_FS.as_ref(),
CapabilityRootsDiscoverParams {
roots: vec![CapabilityRootDiscoverRequest {
id: "skills-only".to_string(),
path: root_uri,
sandbox: None,
}],
},
)
.await
.expect("capability discovery");
let discovery = response.roots.first().expect("discovered root");
let bundled =
load_environment_skills_from_discovery(discovery, /*restriction_product*/ None);
assert_eq!(discovery.plugin, None);
assert_eq!(discovery.namespace_manifests.len(), 1);
assert!(
discovery.namespace_manifests[0]
.path
.to_string()
.ends_with("/.codex-plugin/plugin.json")
);
assert_eq!(bundled.warnings, existing.warnings);
assert_eq!(
bundled
.skills
.iter()
.map(|skill| skill.metadata.clone())
.collect::<Vec<_>>(),
existing.skills
);
assert_eq!(bundled.skills[0].metadata.name, "codex-name:search");
}
+176
View File
@@ -0,0 +1,176 @@
use std::path::PathBuf;
use codex_protocol::protocol::Product;
use codex_skills::SkillDependencies;
use codex_skills::SkillParseError;
use codex_skills::SkillPolicy;
use codex_skills::SkillToolDependency;
use serde::Deserialize;
use super::MAX_DEPENDENCY_COMMAND_LEN;
use super::MAX_DEPENDENCY_DESCRIPTION_LEN;
use super::MAX_DEPENDENCY_TRANSPORT_LEN;
use super::MAX_DEPENDENCY_TYPE_LEN;
use super::MAX_DEPENDENCY_URL_LEN;
use super::MAX_DEPENDENCY_VALUE_LEN;
#[derive(Debug, Default, Deserialize)]
pub(super) struct SkillMetadataFile {
// Keep parsing the host-only interface fields so malformed interface metadata
// invalidates the file exactly as it does in the legacy loader.
#[serde(default, rename = "interface")]
_interface: Option<Interface>,
#[serde(default)]
pub(super) dependencies: Option<Dependencies>,
#[serde(default)]
pub(super) policy: Option<Policy>,
}
#[derive(Debug, Default, Deserialize)]
struct Interface {
#[serde(rename = "display_name")]
_display_name: Option<String>,
#[serde(rename = "short_description")]
_short_description: Option<String>,
#[serde(rename = "icon_small")]
_icon_small: Option<PathBuf>,
#[serde(rename = "icon_large")]
_icon_large: Option<PathBuf>,
#[serde(rename = "brand_color")]
_brand_color: Option<String>,
#[serde(rename = "default_prompt")]
_default_prompt: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
pub(super) struct Dependencies {
#[serde(default)]
tools: Vec<DependencyTool>,
}
#[derive(Debug, Deserialize)]
pub(super) struct Policy {
#[serde(default)]
allow_implicit_invocation: Option<bool>,
#[serde(default)]
products: Vec<Product>,
}
#[derive(Debug, Default, Deserialize)]
struct DependencyTool {
#[serde(rename = "type")]
kind: Option<String>,
value: Option<String>,
description: Option<String>,
transport: Option<String>,
command: Option<String>,
url: Option<String>,
}
pub(super) fn resolve_dependencies(
dependencies: Option<Dependencies>,
) -> Option<SkillDependencies> {
let dependencies = dependencies?;
let tools = dependencies
.tools
.into_iter()
.filter_map(resolve_dependency_tool)
.collect::<Vec<_>>();
if tools.is_empty() {
None
} else {
Some(SkillDependencies { tools })
}
}
pub(super) fn resolve_policy(policy: Option<Policy>) -> Option<SkillPolicy> {
policy.map(|policy| SkillPolicy {
allow_implicit_invocation: policy.allow_implicit_invocation,
products: policy.products,
})
}
fn resolve_dependency_tool(tool: DependencyTool) -> Option<SkillToolDependency> {
let r#type = resolve_required_str(
tool.kind,
MAX_DEPENDENCY_TYPE_LEN,
"dependencies.tools.type",
)?;
let value = resolve_required_str(
tool.value,
MAX_DEPENDENCY_VALUE_LEN,
"dependencies.tools.value",
)?;
let description = resolve_str(
tool.description,
MAX_DEPENDENCY_DESCRIPTION_LEN,
"dependencies.tools.description",
);
let transport = resolve_str(
tool.transport,
MAX_DEPENDENCY_TRANSPORT_LEN,
"dependencies.tools.transport",
);
let command = resolve_str(
tool.command,
MAX_DEPENDENCY_COMMAND_LEN,
"dependencies.tools.command",
);
let url = resolve_str(tool.url, MAX_DEPENDENCY_URL_LEN, "dependencies.tools.url");
Some(SkillToolDependency {
r#type,
value,
description,
transport,
command,
url,
})
}
pub(super) fn sanitize_single_line(raw: &str) -> String {
raw.split_whitespace().collect::<Vec<_>>().join(" ")
}
pub(super) fn validate_len(
value: &str,
max_len: usize,
field_name: &'static str,
) -> Result<(), SkillParseError> {
if value.is_empty() {
return Err(SkillParseError::MissingField(field_name));
}
if value.chars().count() > max_len {
return Err(SkillParseError::InvalidField {
field: field_name,
reason: format!("exceeds maximum length of {max_len} characters"),
});
}
Ok(())
}
fn resolve_str(value: Option<String>, max_len: usize, field: &'static str) -> Option<String> {
let value = value?;
let value = sanitize_single_line(&value);
if value.is_empty() {
tracing::warn!("ignoring {field}: value is empty");
return None;
}
if value.chars().count() > max_len {
tracing::warn!("ignoring {field}: exceeds maximum length of {max_len} characters");
return None;
}
Some(value)
}
fn resolve_required_str(
value: Option<String>,
max_len: usize,
field: &'static str,
) -> Option<String> {
let Some(value) = value else {
tracing::warn!("ignoring {field}: value is missing");
return None;
};
resolve_str(Some(value), max_len, field)
}
+22
View File
@@ -0,0 +1,22 @@
mod discovery;
mod environment;
mod metadata;
mod namespace;
pub(crate) use environment::load_environment_skills_from_discovery;
pub(crate) use environment::load_environment_skills_from_root;
pub(super) const SKILLS_FILENAME: &str = "SKILL.md";
pub(super) const SKILLS_METADATA_DIR: &str = "agents";
pub(super) const SKILLS_METADATA_FILENAME: &str = "openai.yaml";
pub(super) const MAX_NAME_LEN: usize = 64;
pub(super) const MAX_QUALIFIED_NAME_LEN: usize = 128;
pub(super) const MAX_DESCRIPTION_LEN: usize = 1024;
pub(super) const MAX_DEPENDENCY_TYPE_LEN: usize = MAX_NAME_LEN;
pub(super) const MAX_DEPENDENCY_TRANSPORT_LEN: usize = MAX_NAME_LEN;
pub(super) const MAX_DEPENDENCY_VALUE_LEN: usize = MAX_DESCRIPTION_LEN;
pub(super) const MAX_DEPENDENCY_DESCRIPTION_LEN: usize = MAX_DESCRIPTION_LEN;
pub(super) const MAX_DEPENDENCY_COMMAND_LEN: usize = MAX_DESCRIPTION_LEN;
pub(super) const MAX_DEPENDENCY_URL_LEN: usize = MAX_DESCRIPTION_LEN;
pub(super) const MAX_SCAN_DEPTH: usize = 6;
pub(super) const MAX_SKILLS_DIRS_PER_ROOT: usize = 2000;
+172
View File
@@ -0,0 +1,172 @@
use std::collections::HashMap;
use std::collections::HashSet;
use codex_exec_server::ExecutorFileSystem;
use codex_utils_path_uri::PathUri;
use codex_utils_plugins::plugin_namespace_for_root_uri;
use futures::StreamExt;
use super::discovery::MAX_CONCURRENT_SKILL_LOADS;
/// Resolves the namespace prefix applied to skill names during one skills scan.
///
/// A plugin namespace is the plugin name from the nearest valid plugin manifest
/// above a skill path. For example, a skill named `search` beneath a plugin named
/// `sample` is exposed as `sample:search`.
///
/// Resolving the namespace separately for every `SKILL.md` repeats the same
/// ancestor manifest probes for sibling skills. This resolver resolves relevant
/// roots once per scan, then selects the nearest matching root for each skill.
///
/// The deepest matching canonical symlink root or nested plugin root wins,
/// followed by the namespace inherited from the scanned skills root.
pub(crate) struct SkillNamespaceResolver {
inherited_namespace: ResolvedSkillNamespace,
nested_namespaces: Vec<(PathUri, ResolvedSkillNamespace)>,
}
impl SkillNamespaceResolver {
pub(crate) async fn discover(
fs: &dyn ExecutorFileSystem,
root: &PathUri,
skill_paths: &[PathUri],
plugin_roots: HashSet<PathUri>,
namespace_roots: HashSet<PathUri>,
) -> Self {
// Only probe plugin roots above loaded skills; unused siblings cannot affect names.
let mut skill_ancestors = HashSet::new();
for skill_path in skill_paths {
let mut ancestor = skill_path.parent();
while let Some(path) = ancestor {
skill_ancestors.insert(path.clone());
ancestor = path.parent();
}
}
let plugin_roots = plugin_roots
.into_iter()
.filter(|plugin_root| skill_ancestors.contains(plugin_root))
.collect::<HashSet<_>>();
// The scan root is already the fallback above if nothing else matches, exclude from the search.
let namespace_roots = namespace_roots
.into_iter()
.filter(|namespace_root| namespace_root != root)
.collect::<Vec<_>>();
let namespace_root_set = namespace_roots.iter().cloned().collect::<HashSet<_>>();
let plugin_roots = plugin_roots
.into_iter()
.filter(|plugin_root| plugin_root != root && !namespace_root_set.contains(plugin_root))
.collect::<Vec<_>>();
let lookup_roots = std::iter::once(root.clone())
.chain(namespace_roots.iter().cloned())
.collect::<Vec<_>>();
let mut pending_lookups = lookup_roots
.iter()
.cloned()
.map(|lookup_root| (lookup_root.clone(), lookup_root))
.collect::<Vec<_>>();
let mut direct_plugin_roots = plugin_roots.iter().cloned().collect::<HashSet<_>>();
let mut namespaces_by_root = HashMap::new();
let mut namespaces_by_lookup_root = HashMap::new();
while !pending_lookups.is_empty() {
let probe_roots = pending_lookups
.iter()
.map(|(_, ancestor)| ancestor.clone())
.chain(direct_plugin_roots.drain())
.filter(|ancestor| !namespaces_by_root.contains_key(ancestor))
.collect::<HashSet<_>>();
namespaces_by_root.extend(
futures::stream::iter(probe_roots)
.map(|manifest_root| async move {
let namespace = plugin_namespace_for_root_uri(fs, &manifest_root).await;
(manifest_root, namespace)
})
.buffered(MAX_CONCURRENT_SKILL_LOADS)
.collect::<HashMap<_, _>>()
.await,
);
let mut next_lookups = Vec::new();
for (lookup_root, ancestor) in pending_lookups {
match namespaces_by_root.get(&ancestor) {
Some(Some(namespace)) => {
namespaces_by_lookup_root.insert(lookup_root, Some(namespace.clone()));
}
Some(None) => match ancestor.parent() {
Some(parent) => next_lookups.push((lookup_root, parent)),
None => {
namespaces_by_lookup_root.insert(lookup_root, None);
}
},
None => unreachable!("pending namespace ancestor was not probed"),
}
}
pending_lookups = next_lookups;
}
// Ordinary descendants fall back to the nearest valid manifest at or above the scan root.
let inherited_namespace = namespaces_by_lookup_root
.get(root)
.and_then(Option::as_ref)
.cloned()
.map(ResolvedSkillNamespace::Plugin)
.unwrap_or(ResolvedSkillNamespace::Plain);
let namespace_lookups = namespace_roots.into_iter().map(|namespace_root| {
let namespace = namespaces_by_lookup_root
.get(&namespace_root)
.and_then(Option::as_ref)
.cloned()
.map(ResolvedSkillNamespace::Plugin)
.unwrap_or(ResolvedSkillNamespace::Plain);
(namespace_root, namespace)
});
// Invalid nested manifests are omitted, so the deepest remaining match wins.
let plugin_lookups = plugin_roots.into_iter().filter_map(|plugin_root| {
namespaces_by_root
.get(&plugin_root)
.and_then(Option::as_ref)
.cloned()
.map(|namespace| (plugin_root, ResolvedSkillNamespace::Plugin(namespace)))
});
let nested_namespaces = namespace_lookups.chain(plugin_lookups).collect();
Self {
inherited_namespace,
nested_namespaces,
}
}
pub(crate) fn for_skill(&self, root: &PathUri, path: &PathUri) -> &ResolvedSkillNamespace {
// Ancestor symlink targets cannot override skills still owned by the scan root.
let path_is_under_root = path.starts_with(root);
// The deepest matching path prefix is the nearest applicable namespace.
self.nested_namespaces
.iter()
.filter(|(namespace_root, _)| {
path.starts_with(namespace_root)
&& (!path_is_under_root || !root.starts_with(namespace_root))
})
.max_by_key(|(namespace_root, _)| namespace_root.ancestors().count())
.map(|(_, namespace)| namespace)
.unwrap_or(&self.inherited_namespace)
}
}
/// The completed namespace resolution for a skill root.
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) enum ResolvedSkillNamespace {
/// No plugin namespace applies to matching skills.
Plain,
/// Qualify matching skill names with this plugin namespace.
Plugin(String),
}
impl ResolvedSkillNamespace {
pub(crate) fn qualify(&self, base_name: &str) -> String {
match self {
Self::Plain => base_name.to_string(),
Self::Plugin(namespace) => format!("{namespace}:{base_name}"),
}
}
}
+2 -2
View File
@@ -1,7 +1,5 @@
use std::sync::Arc;
use codex_core_skills::loader::load_environment_skills_from_discovery;
use codex_core_skills::loader::load_environment_skills_from_root;
use codex_exec_server::EnvironmentManager;
use codex_exec_server::FileSystemSandboxContext;
use codex_protocol::capabilities::CapabilityRootLocation;
@@ -20,6 +18,8 @@ use crate::catalog::SkillReadResult;
use crate::catalog::SkillResourceId;
use crate::catalog::SkillSearchResult;
use crate::catalog::SkillSourceKind;
use crate::loader::load_environment_skills_from_discovery;
use crate::loader::load_environment_skills_from_root;
use crate::provider::MAX_SKILL_RESOURCE_CONTENT_BYTES;
use crate::provider::SkillListQuery;
use crate::provider::SkillProvider;
@@ -7,6 +7,7 @@ use std::sync::atomic::Ordering;
use codex_core_skills::HostSkillsSnapshot;
use codex_core_skills::loader::MAX_CONCURRENT_ROOT_SCANS;
use codex_core_skills::loader::SkillRoot;
use codex_core_skills::loader::load_environment_skills_from_root as load_legacy_environment_skills_from_root;
use codex_core_skills::loader::load_skills_from_roots;
use codex_exec_server::CopyOptions;
use codex_exec_server::CreateDirectoryOptions;
@@ -24,6 +25,7 @@ use codex_protocol::capabilities::SelectedCapabilityRoot;
use codex_protocol::models::PermissionProfile;
use codex_protocol::permissions::FileSystemSandboxPolicy;
use codex_protocol::permissions::NetworkSandboxPolicy;
use codex_protocol::protocol::Product;
use codex_protocol::protocol::SkillScope;
use codex_skills_extension::ExecutorSkillProvider;
use codex_skills_extension::catalog::SkillAuthority;
@@ -350,6 +352,413 @@ async fn selected_root_id_distinguishes_identical_executor_paths() {
std::fs::remove_dir_all(test_root).expect("remove skill directory");
}
#[tokio::test]
async fn executor_discovery_routes_produce_equivalent_catalog_metadata() {
let id = NEXT_TEST_ROOT_ID.fetch_add(1, Ordering::Relaxed);
let test_root = std::env::temp_dir().join(format!(
"codex-executor-skill-parity-{}-{id}",
std::process::id()
));
let plugin_manifest = test_root.join(".codex-plugin/plugin.json");
let deploy_skill = test_root.join("skills/deploy/SKILL.md");
let deploy_metadata = test_root.join("skills/deploy/agents/openai.yaml");
let excluded_skill = test_root.join("skills/excluded/SKILL.md");
let excluded_metadata = test_root.join("skills/excluded/agents/openai.yaml");
let repaired_skill = test_root.join("skills/repaired/SKILL.md");
let invalid_skill = test_root.join("skills/invalid/SKILL.md");
let invalid_metadata_skill = test_root.join("skills/invalid-metadata/SKILL.md");
let invalid_metadata = test_root.join("skills/invalid-metadata/agents/openai.yaml");
for path in [
&plugin_manifest,
&deploy_skill,
&deploy_metadata,
&excluded_skill,
&excluded_metadata,
&repaired_skill,
&invalid_skill,
&invalid_metadata_skill,
&invalid_metadata,
] {
std::fs::create_dir_all(path.parent().expect("test file parent"))
.expect("create test directory");
}
std::fs::write(&plugin_manifest, r#"{"name":"catalog"}"#).expect("write plugin manifest");
std::fs::write(
&deploy_skill,
"---\nname: deploy\ndescription: Deploy the service.\nmetadata:\n short-description: Deploy safely.\n---\n\nDeploy.\n",
)
.expect("write deploy skill");
std::fs::write(
&deploy_metadata,
"dependencies:\n tools:\n - type: mcp\n value: deployer\n description: Deployment server.\npolicy:\n allow_implicit_invocation: false\n products:\n - codex\n",
)
.expect("write deploy metadata");
std::fs::write(
&excluded_skill,
"---\nname: excluded\ndescription: Excluded skill.\n---\n",
)
.expect("write excluded skill");
std::fs::write(&excluded_metadata, "policy:\n products:\n - chatgpt\n")
.expect("write excluded metadata");
std::fs::write(
&repaired_skill,
"---\ndescription: Build for AWS: ECS\n---\n",
)
.expect("write repaired skill");
std::fs::write(&invalid_skill, "---\nname: invalid\n---\n").expect("write invalid skill");
std::fs::write(
&invalid_metadata_skill,
"---\nname: invalid-metadata\ndescription: Invalid optional metadata.\n---\n",
)
.expect("write invalid metadata skill");
std::fs::write(
&invalid_metadata,
"interface: []\ndependencies:\n tools:\n - type: mcp\n value: must-be-ignored\n",
)
.expect("write invalid metadata");
let manager = Arc::new(EnvironmentManager::default_for_tests());
let provider = ExecutorSkillProvider::new_with_restriction_product(
Arc::clone(&manager),
Some(Product::Codex),
);
let executor_roots = vec![SelectedCapabilityRoot {
id: "parity-root".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "local".to_string(),
path: PathUri::from_host_native_path(&test_root).expect("skill root URI"),
},
}];
let query = |executor_capability_discovery| SkillListQuery {
turn_id: "turn-1".to_string(),
executor_roots: executor_roots.clone(),
resolved_executor_roots: Vec::new(),
host_snapshot: None,
include_host_skills: false,
include_bundled_skills: true,
include_orchestrator_skills: false,
mcp_resources: None,
executor_capability_discovery,
};
let legacy_file_system = manager
.get_environment("local")
.expect("local environment")
.get_filesystem();
let legacy = load_legacy_environment_skills_from_root(
legacy_file_system.as_ref(),
&PathUri::from_host_native_path(&test_root).expect("skill root URI"),
Some(Product::Codex),
)
.await;
let direct = provider
.list(query(None))
.await
.expect("list directly discovered executor skills");
let discovery = ExecutorCapabilityDiscoveryCache::new(manager)
.snapshot(&executor_roots, &Default::default())
.await;
let bundled = provider
.list(query(Some(discovery)))
.await
.expect("list bundled executor skills");
assert_eq!(bundled.warnings, direct.warnings);
let comparable_entries = |catalog: &codex_skills_extension::catalog::SkillCatalog| {
catalog
.entries
.iter()
.map(|entry| {
(
entry.id.clone(),
entry.authority.clone(),
entry.name.clone(),
entry.description.clone(),
entry.short_description.clone(),
entry.display_path.clone(),
entry.dependencies.clone(),
entry.enabled,
entry.prompt_visible,
)
})
.collect::<Vec<_>>()
};
assert_eq!(comparable_entries(&bundled), comparable_entries(&direct));
assert_eq!(legacy.warnings, direct.warnings);
assert_eq!(direct.warnings.len(), 1);
assert!(direct.warnings[0].contains("missing field `description`"));
let legacy_metadata = legacy
.skills
.iter()
.map(|skill| {
(
skill.name.clone(),
skill.description.clone(),
skill.short_description.clone(),
skill.dependencies.clone(),
skill.allows_implicit_invocation(),
)
})
.collect::<Vec<_>>();
let catalog_metadata = direct
.entries
.iter()
.map(|entry| {
(
entry.name.clone(),
entry.description.clone(),
entry.short_description.clone(),
entry.dependencies.clone(),
entry.prompt_visible,
)
})
.collect::<Vec<_>>();
assert_eq!(catalog_metadata, legacy_metadata);
let entry = direct
.entries
.iter()
.find(|entry| entry.name == "catalog:deploy")
.expect("deploy skill");
assert_eq!(
entry
.dependencies
.as_ref()
.expect("skill dependencies")
.tools[0]
.value,
"deployer"
);
assert!(!entry.prompt_visible);
let repaired = direct
.entries
.iter()
.find(|entry| entry.name == "catalog:repaired")
.expect("repaired skill");
assert_eq!(repaired.description, "Build for AWS: ECS");
let invalid_metadata = direct
.entries
.iter()
.find(|entry| entry.name == "catalog:invalid-metadata")
.expect("invalid metadata skill");
assert_eq!(invalid_metadata.dependencies, None);
std::fs::remove_dir_all(test_root).expect("remove skill directory");
}
#[tokio::test]
async fn pre_discovered_executor_catalog_snapshot() {
let test_root = create_local_skill_root("snapshot").expect("create local skill root");
let manifest_dir = test_root.join(".codex-plugin");
let metadata_dir = test_root.join("skill/agents");
std::fs::create_dir_all(&manifest_dir).expect("create plugin manifest directory");
std::fs::create_dir_all(&metadata_dir).expect("create skill metadata directory");
std::fs::write(
manifest_dir.join("plugin.json"),
r#"{"name":"snapshot-plugin"}"#,
)
.expect("write plugin manifest");
std::fs::write(
metadata_dir.join("openai.yaml"),
"dependencies:\n tools:\n - type: mcp\n value: deployer\n description: Deployment server.\npolicy:\n allow_implicit_invocation: false\n",
)
.expect("write skill metadata");
let manager = Arc::new(EnvironmentManager::default_for_tests());
let provider = ExecutorSkillProvider::new_with_restriction_product(
Arc::clone(&manager),
/*restriction_product*/ None,
);
let executor_roots = vec![SelectedCapabilityRoot {
id: "snapshot-root".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "local".to_string(),
path: PathUri::from_host_native_path(&test_root).expect("skill root URI"),
},
}];
let executor_capability_discovery = ExecutorCapabilityDiscoveryCache::new(manager)
.snapshot(&executor_roots, &Default::default())
.await;
let catalog = provider
.list(SkillListQuery {
turn_id: "turn-1".to_string(),
executor_roots,
resolved_executor_roots: Vec::new(),
host_snapshot: None,
include_host_skills: false,
include_bundled_skills: true,
include_orchestrator_skills: false,
mcp_resources: None,
executor_capability_discovery: Some(executor_capability_discovery),
})
.await
.expect("list pre-discovered executor skills");
let mut snapshot = serde_json::json!({
"warnings": &catalog.warnings,
"entries": catalog
.entries
.iter()
.map(|entry| serde_json::json!({
"authority": {
"kind": entry.authority.kind.to_string(),
"id": &entry.authority.id,
},
"name": &entry.name,
"description": &entry.description,
"short_description": entry.short_description.as_deref(),
"dependencies": entry.dependencies.as_ref().map(|dependencies| {
dependencies.tools.iter().map(|tool| serde_json::json!({
"type": &tool.r#type,
"value": &tool.value,
"description": tool.description.as_deref(),
"transport": tool.transport.as_deref(),
"command": tool.command.as_deref(),
"url": tool.url.as_deref(),
})).collect::<Vec<_>>()
}),
"enabled": entry.enabled,
"prompt_visible": entry.prompt_visible,
}))
.collect::<Vec<_>>(),
});
snapshot.sort_all_objects();
let snapshot_file = codex_utils_cargo_bin::find_resource!(
"tests/snapshots/executor_file_system_authority__pre_discovered_executor_catalog.snap"
)
.expect("resolve catalog snapshot");
let snapshot_dir = snapshot_file
.parent()
.unwrap_or_else(|| panic!("snapshot file has no parent: {}", snapshot_file.display()));
let mut settings = insta::Settings::clone_current();
settings.set_snapshot_path(snapshot_dir);
settings.bind(|| {
insta::assert_snapshot!(
"pre_discovered_executor_catalog",
serde_json::to_string_pretty(&snapshot).expect("serialize catalog snapshot")
);
});
std::fs::remove_dir_all(test_root).expect("remove skill directory");
}
#[tokio::test]
async fn direct_executor_discovery_preserves_hidden_nested_and_probed_metadata() {
let id = NEXT_TEST_ROOT_ID.fetch_add(1, Ordering::Relaxed);
let test_root = std::env::temp_dir().join(format!(
"codex-executor-skill-discovery-{}-{id}",
std::process::id()
));
let outer_manifest = test_root.join(".codex-plugin/plugin.json");
let hidden_skill = test_root.join(".hidden/deploy/SKILL.md");
let hidden_metadata = test_root.join(".hidden/deploy/agents/openai.yaml");
let inner_manifest = test_root.join("nested/.codex-plugin/plugin.json");
let inner_skill = test_root.join("nested/skills/audit/SKILL.md");
for (path, contents) in [
(&outer_manifest, r#"{"name":"outer"}"#),
(
&hidden_skill,
"---\nname: hidden\ndescription: Hidden skill.\n---\n",
),
(&inner_manifest, r#"{"name":"inner"}"#),
(
&inner_skill,
"---\nname: audit\ndescription: Audit skill.\n---\n",
),
] {
std::fs::create_dir_all(path.parent().expect("test file parent"))
.expect("create test directory");
std::fs::write(path, contents).expect("write test file");
}
std::fs::create_dir_all(hidden_metadata.parent().expect("metadata parent"))
.expect("create metadata directory");
let metadata_contents = "policy:\n allow_implicit_invocation: false\n";
#[cfg(unix)]
{
use std::os::unix::fs::symlink;
let metadata_target = test_root.join("linked-openai.yaml");
std::fs::write(&metadata_target, metadata_contents).expect("write metadata target");
symlink(metadata_target, &hidden_metadata).expect("link metadata");
}
#[cfg(not(unix))]
std::fs::write(&hidden_metadata, metadata_contents).expect("write metadata");
let root_uri = PathUri::from_host_native_path(&test_root).expect("skill root URI");
let selected_root = SelectedCapabilityRoot {
id: "discovery-root".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "local".to_string(),
path: root_uri.clone(),
},
};
let manager = Arc::new(EnvironmentManager::default_for_tests());
let legacy_file_system = manager
.get_environment("local")
.expect("local environment")
.get_filesystem();
let legacy = load_legacy_environment_skills_from_root(
legacy_file_system.as_ref(),
&root_uri,
/*restriction_product*/ None,
)
.await;
let provider = ExecutorSkillProvider::new_with_restriction_product(
manager, /*restriction_product*/ None,
);
let direct = provider
.list(SkillListQuery {
turn_id: "turn-1".to_string(),
executor_roots: vec![selected_root],
resolved_executor_roots: Vec::new(),
host_snapshot: None,
include_host_skills: false,
include_bundled_skills: true,
include_orchestrator_skills: false,
mcp_resources: None,
executor_capability_discovery: None,
})
.await
.expect("list directly discovered executor skills");
assert_eq!(direct.warnings, legacy.warnings);
let legacy_metadata = legacy
.skills
.iter()
.map(|skill| {
(
skill.name.clone(),
skill.description.clone(),
skill.allows_implicit_invocation(),
)
})
.collect::<Vec<_>>();
let catalog_metadata = direct
.entries
.iter()
.map(|entry| {
(
entry.name.clone(),
entry.description.clone(),
entry.prompt_visible,
)
})
.collect::<Vec<_>>();
assert_eq!(catalog_metadata, legacy_metadata);
assert_eq!(
catalog_metadata,
vec![
("inner:audit".to_string(), "Audit skill.".to_string(), true),
(
"outer:hidden".to_string(),
"Hidden skill.".to_string(),
false,
),
]
);
std::fs::remove_dir_all(test_root).expect("remove skill directory");
}
#[tokio::test]
async fn high_level_discovery_reuses_materialized_skill_contents_for_reads() {
let test_root = create_local_skill_root("materialized").expect("create local skill root");
@@ -0,0 +1,30 @@
---
source: ext/skills/tests/executor_file_system_authority.rs
expression: "serde_json::to_string_pretty(&snapshot).expect(\"serialize catalog snapshot\")"
---
{
"entries": [
{
"authority": {
"id": "snapshot-root",
"kind": "executor"
},
"dependencies": [
{
"command": null,
"description": "Deployment server.",
"transport": null,
"type": "mcp",
"url": null,
"value": "deployer"
}
],
"description": "Synthetic executor skill.",
"enabled": true,
"name": "snapshot-plugin:synthetic",
"prompt_visible": false,
"short_description": null
}
],
"warnings": []
}
+8 -1
View File
@@ -1,5 +1,7 @@
use crate::config::OtelExporter;
use crate::metrics::Result;
use crate::metrics::names::API_CALL_COUNT_METRIC;
use crate::metrics::names::API_CALL_DURATION_METRIC;
use crate::metrics::names::TOOL_CALL_COUNT_METRIC;
use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
use crate::metrics::validation::validate_tag_key;
@@ -8,7 +10,12 @@ use opentelemetry_sdk::metrics::InMemoryMetricExporter;
use std::collections::BTreeMap;
use std::time::Duration;
const RUNTIME_ONLY_METRICS: &[&str] = &[TOOL_CALL_COUNT_METRIC, TOOL_CALL_DURATION_METRIC];
const RUNTIME_ONLY_METRICS: &[&str] = &[
API_CALL_COUNT_METRIC,
API_CALL_DURATION_METRIC,
TOOL_CALL_COUNT_METRIC,
TOOL_CALL_DURATION_METRIC,
];
#[derive(Clone, Debug)]
pub enum MetricsExporter {
+4
View File
@@ -462,6 +462,8 @@ mod shutdown_tests;
#[cfg(test)]
mod tests {
use super::*;
use crate::metrics::API_CALL_COUNT_METRIC;
use crate::metrics::API_CALL_DURATION_METRIC;
use crate::metrics::MetricsExporter;
use crate::metrics::TOOL_CALL_COUNT_METRIC;
use crate::metrics::TOOL_CALL_DURATION_METRIC;
@@ -544,6 +546,8 @@ mod tests {
config.exporter = MetricsExporter::InMemory(exporter.clone());
let metrics = MetricsClient::new(config)?;
metrics.counter(API_CALL_COUNT_METRIC, /*inc*/ 1, &[])?;
metrics.record_duration(API_CALL_DURATION_METRIC, Duration::from_millis(100), &[])?;
metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?;
metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?;
metrics.counter("codex.turns", /*inc*/ 1, &[])?;
@@ -186,6 +186,12 @@ fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> {
))?;
metrics.counter("codex.turns", /*inc*/ 1, &[("source", "test")])?;
metrics.counter("codex.api_request", /*inc*/ 1, &[("status", "200")])?;
metrics.record_duration(
"codex.api_request.duration_ms",
Duration::from_millis(100),
&[("status", "200")],
)?;
metrics.counter("codex.tool.call", /*inc*/ 1, &[("tool", "test")])?;
metrics.record_duration(
"codex.tool.call.duration_ms",
@@ -227,6 +233,16 @@ fn otlp_http_exporter_sends_metrics_to_collector() -> Result<()> {
"expected gauge not found; body prefix: {}",
&body.chars().take(2000).collect::<String>()
);
assert!(
body.contains("\"codex.api_request\""),
"expected API-request counter not found; body prefix: {}",
&body.chars().take(2000).collect::<String>()
);
assert!(
body.contains("\"codex.api_request.duration_ms\""),
"expected API-request duration not found; body prefix: {}",
&body.chars().take(2000).collect::<String>()
);
assert!(
body.contains("\"codex.tool.call\""),
"expected tool-call counter not found; body prefix: {}",
+5
View File
@@ -18,4 +18,9 @@ codex-protocol = { workspace = true }
codex-utils-absolute-path = { workspace = true }
codex-utils-path-uri = { workspace = true }
include_dir = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_yaml = { workspace = true }
thiserror = { workspace = true }
[dev-dependencies]
pretty_assertions = { workspace = true }
+4
View File
@@ -1,4 +1,5 @@
mod model;
mod parser;
pub use model::EnvironmentSkillMetadata;
pub use model::SkillConfigRule;
@@ -9,6 +10,9 @@ pub use model::SkillInterface;
pub use model::SkillMetadata;
pub use model::SkillPolicy;
pub use model::SkillToolDependency;
pub use parser::ParsedSkillFrontmatter;
pub use parser::SkillParseError;
pub use parser::parse_skill_frontmatter_metadata;
use codex_utils_absolute_path::AbsolutePathBuf;
use include_dir::Dir;
+225
View File
@@ -0,0 +1,225 @@
use serde::Deserialize;
use thiserror::Error;
const MAX_NAME_LEN: usize = 64;
#[derive(Debug, Deserialize)]
struct SkillFrontmatter {
#[serde(default)]
name: Option<String>,
#[serde(default)]
description: Option<String>,
#[serde(default)]
metadata: SkillFrontmatterMetadata,
}
#[derive(Debug, Default, Deserialize)]
struct SkillFrontmatterMetadata {
#[serde(default, rename = "short-description")]
short_description: Option<String>,
}
/// Validated metadata parsed from a `SKILL.md` frontmatter block.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ParsedSkillFrontmatter {
pub name: String,
pub description: String,
pub short_description: Option<String>,
}
/// Error produced while parsing or validating `SKILL.md` metadata.
#[derive(Debug, Error)]
pub enum SkillParseError {
#[error("missing YAML frontmatter delimited by ---")]
MissingFrontmatter,
#[error("invalid YAML: {0}")]
InvalidYaml(#[source] serde_yaml::Error),
#[error("missing field `{0}`")]
MissingField(&'static str),
#[error("invalid {field}: {reason}")]
InvalidField { field: &'static str, reason: String },
}
/// Parses and validates the metadata frontmatter from `SKILL.md` contents.
pub fn parse_skill_frontmatter_metadata(
contents: &str,
default_name: impl FnOnce() -> String,
) -> Result<ParsedSkillFrontmatter, SkillParseError> {
let frontmatter = extract_frontmatter(contents).ok_or(SkillParseError::MissingFrontmatter)?;
let parsed: SkillFrontmatter = match serde_yaml::from_str(&frontmatter) {
Ok(parsed) => Ok(parsed),
Err(original_error) => match repair_frontmatter_scalar_fields(&frontmatter) {
// Some third-party skills use prose like `description: Build for AWS: ECS`
// or `argument-hint: <duration: e.g. 7d>`. Keep the repair line-oriented
// so unrelated invalid YAML still surfaces.
Some(repaired_frontmatter) => {
serde_yaml::from_str(&repaired_frontmatter).map_err(|_| original_error)
}
None => Err(original_error),
},
}
.map_err(SkillParseError::InvalidYaml)?;
let name = parsed
.name
.as_deref()
.map(sanitize_single_line)
.filter(|value| !value.is_empty())
.unwrap_or_else(default_name);
let description = parsed
.description
.as_deref()
.map(sanitize_single_line)
.unwrap_or_default();
let short_description = parsed
.metadata
.short_description
.as_deref()
.map(sanitize_single_line)
.filter(|value| !value.is_empty());
validate_len(&name, MAX_NAME_LEN, "name")?;
if description.is_empty() {
return Err(SkillParseError::MissingField("description"));
}
Ok(ParsedSkillFrontmatter {
name,
description,
short_description,
})
}
fn sanitize_single_line(raw: &str) -> String {
raw.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn repair_frontmatter_scalar_fields(frontmatter: &str) -> Option<String> {
let mut changed = false;
let mut block_scalar_indent: Option<usize> = None;
let mut repaired_lines = Vec::new();
for line in frontmatter.lines() {
let indent = line
.chars()
.take_while(|character| *character == ' ')
.count();
if let Some(block_indent) = block_scalar_indent {
if line.trim().is_empty() || indent > block_indent {
repaired_lines.push(line.to_string());
continue;
}
block_scalar_indent = None;
}
let Some((key, value)) = line.split_once(':') else {
repaired_lines.push(line.to_string());
continue;
};
if key.trim().is_empty() || !value.chars().next().is_none_or(char::is_whitespace) {
repaired_lines.push(line.to_string());
continue;
}
let trimmed_start = value.trim_start();
let leading_whitespace = &value[..value.len() - trimmed_start.len()];
let mut scalar = trimmed_start;
let mut comment = "";
for (index, character) in trimmed_start.char_indices() {
if character == '#'
&& (index == 0
|| trimmed_start[..index]
.chars()
.next_back()
.is_some_and(char::is_whitespace))
{
let comment_start = trimmed_start[..index].trim_end().len();
scalar = &trimmed_start[..comment_start];
comment = &trimmed_start[comment_start..];
break;
}
}
let scalar = scalar.trim_end();
let Some(first_char) = scalar.chars().next() else {
repaired_lines.push(line.to_string());
continue;
};
if matches!(first_char, '|' | '>') {
block_scalar_indent = Some(indent);
repaired_lines.push(line.to_string());
continue;
}
if matches!(first_char, '\'' | '"') {
repaired_lines.push(line.to_string());
continue;
}
let mut has_colon_separator = false;
let mut chars = scalar.chars().peekable();
while let Some(character) = chars.next() {
if character == ':'
&& matches!(chars.peek(), Some(next_character) if next_character.is_whitespace())
{
has_colon_separator = true;
break;
}
}
let invalid_flow_like_scalar = matches!(first_char, '[' | '{' | '@' | '`')
&& serde_yaml::from_str::<serde_yaml::Value>(scalar).is_err();
if !has_colon_separator && !invalid_flow_like_scalar {
repaired_lines.push(line.to_string());
continue;
}
let quoted_scalar = format!("'{}'", scalar.replace('\'', "''"));
repaired_lines.push(format!(
"{key}:{leading_whitespace}{quoted_scalar}{comment}"
));
changed = true;
}
changed.then(|| repaired_lines.join("\n"))
}
fn validate_len(
value: &str,
max_len: usize,
field_name: &'static str,
) -> Result<(), SkillParseError> {
if value.is_empty() {
return Err(SkillParseError::MissingField(field_name));
}
if value.chars().count() > max_len {
return Err(SkillParseError::InvalidField {
field: field_name,
reason: format!("exceeds maximum length of {max_len} characters"),
});
}
Ok(())
}
fn extract_frontmatter(contents: &str) -> Option<String> {
let mut lines = contents.lines();
if !matches!(lines.next(), Some(line) if line.trim() == "---") {
return None;
}
let mut frontmatter_lines = Vec::new();
let mut found_closing = false;
for line in lines.by_ref() {
if line.trim() == "---" {
found_closing = true;
break;
}
frontmatter_lines.push(line);
}
if frontmatter_lines.is_empty() || !found_closing {
return None;
}
Some(frontmatter_lines.join("\n"))
}
#[cfg(test)]
#[path = "parser_tests.rs"]
mod tests;

Some files were not shown because too many files have changed in this diff Show More