From 1388e99674bf9c9e86c6bbc16417d1e6e5d3c18a Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Thu, 20 Nov 2025 16:36:29 -0800 Subject: [PATCH 1/3] fix flaky `tool_call_output_exceeds_limit_truncated_chars_limit` (#7043) I am suspecting this is flaky because of the wall time can become 0, 0.1, or 1. --- codex-rs/core/tests/suite/truncation.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index ed9e7cde3b..7d611f55fc 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -244,11 +244,16 @@ async fn tool_call_output_exceeds_limit_truncated_chars_limit() -> Result<()> { "expected truncated shell output to be plain text" ); - assert_eq!(output.len(), 9976); // ~10k characters - let truncated_pattern = r#"(?s)^Exit code: 0\nWall time: 0 seconds\nTotal output lines: 100000\nOutput:\n.*?…\d+ chars truncated….*$"#; + let truncated_pattern = r#"(?s)^Exit code: 0\nWall time: [0-9]+(?:\.[0-9]+)? seconds\nTotal output lines: 100000\nOutput:\n.*?…\d+ chars truncated….*$"#; assert_regex_match(truncated_pattern, &output); + let len = output.len(); + assert!( + (9_900..=10_000).contains(&len), + "expected ~10k chars after truncation, got {len}" + ); + Ok(()) } From 8e5f38c0f074646a613f589d6d78b16636a6d30d Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 20 Nov 2025 16:45:38 -0800 Subject: [PATCH 2/3] feat: waiting for an elicitation should not count against a shell tool timeout (#6973) Previously, we were running into an issue where we would run the `shell` tool call with a timeout of 10s, but it fired an elicitation asking for user approval, the time the user took to respond to the elicitation was counted agains the 10s timeout, so the `shell` tool call would fail with a timeout error unless the user is very fast! This PR addresses this issue by introducing a "stopwatch" abstraction that is used to manage the timeout. The idea is: - `Stopwatch::new()` is called with the _real_ timeout of the `shell` tool call. - `process_exec_tool_call()` is called with the `Cancellation` variant of `ExecExpiration` because it should not manage its own timeout in this case - the `Stopwatch` expiration is wired up to the `cancel_rx` passed to `process_exec_tool_call()` - when an elicitation for the `shell` tool call is received, the `Stopwatch` pauses - because it is possible for multiple elicitations to arrive concurrently, it keeps track of the number of "active pauses" and does not resume until that counter goes down to zero I verified that I can test the MCP server using `@modelcontextprotocol/inspector` and specify `git status` as the `command` with a timeout of 500ms and that the elicitation pops up and I have all the time in the world to respond whereas previous to this PR, that would not have been possible. --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/openai/codex/pull/6973). * #7005 * __->__ #6973 * #6972 --- codex-rs/Cargo.lock | 1 + codex-rs/core/src/exec.rs | 6 +- codex-rs/exec-server/Cargo.toml | 1 + codex-rs/exec-server/src/posix.rs | 1 + .../exec-server/src/posix/escalate_server.rs | 6 +- codex-rs/exec-server/src/posix/mcp.rs | 12 +- .../src/posix/mcp_escalation_policy.rs | 55 +++-- codex-rs/exec-server/src/posix/stopwatch.rs | 211 ++++++++++++++++++ 8 files changed, 268 insertions(+), 25 deletions(-) create mode 100644 codex-rs/exec-server/src/posix/stopwatch.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index fea573f8ac..9e365adac4 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1216,6 +1216,7 @@ dependencies = [ "socket2 0.6.0", "tempfile", "tokio", + "tokio-util", "tracing", "tracing-subscriber", ] diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 42576907e8..f45ecdce75 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -31,7 +31,7 @@ use crate::spawn::StdioPolicy; use crate::spawn::spawn_child_async; use crate::text_encoding::bytes_to_string_smart; -const DEFAULT_TIMEOUT_MS: u64 = 10_000; +pub const DEFAULT_EXEC_COMMAND_TIMEOUT_MS: u64 = 10_000; // Hardcode these since it does not seem worth including the libc crate just // for these. @@ -86,7 +86,7 @@ impl ExecExpiration { match self { ExecExpiration::Timeout(duration) => tokio::time::sleep(duration).await, ExecExpiration::DefaultTimeout => { - tokio::time::sleep(Duration::from_millis(DEFAULT_TIMEOUT_MS)).await + tokio::time::sleep(Duration::from_millis(DEFAULT_EXEC_COMMAND_TIMEOUT_MS)).await } ExecExpiration::Cancellation(cancel) => { cancel.cancelled().await; @@ -98,7 +98,7 @@ impl ExecExpiration { pub(crate) fn timeout_ms(&self) -> Option { match self { ExecExpiration::Timeout(duration) => Some(duration.as_millis() as u64), - ExecExpiration::DefaultTimeout => Some(DEFAULT_TIMEOUT_MS), + ExecExpiration::DefaultTimeout => Some(DEFAULT_EXEC_COMMAND_TIMEOUT_MS), ExecExpiration::Cancellation(_) => None, } } diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml index 54cead4118..24c13e0e25 100644 --- a/codex-rs/exec-server/Cargo.toml +++ b/codex-rs/exec-server/Cargo.toml @@ -49,6 +49,7 @@ tokio = { workspace = true, features = [ "rt-multi-thread", "signal", ] } +tokio-util = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } diff --git a/codex-rs/exec-server/src/posix.rs b/codex-rs/exec-server/src/posix.rs index ea0fab1a7e..b4dd0fbf40 100644 --- a/codex-rs/exec-server/src/posix.rs +++ b/codex-rs/exec-server/src/posix.rs @@ -71,6 +71,7 @@ mod escalation_policy; mod mcp; mod mcp_escalation_policy; mod socket; +mod stopwatch; /// Default value of --execve option relative to the current executable. /// Note this must match the name of the binary as specified in Cargo.toml. diff --git a/codex-rs/exec-server/src/posix/escalate_server.rs b/codex-rs/exec-server/src/posix/escalate_server.rs index a8620b0baf..784562f2ff 100644 --- a/codex-rs/exec-server/src/posix/escalate_server.rs +++ b/codex-rs/exec-server/src/posix/escalate_server.rs @@ -13,6 +13,7 @@ use codex_core::exec::process_exec_tool_call; use codex_core::get_platform_sandbox; use codex_core::protocol::SandboxPolicy; use tokio::process::Command; +use tokio_util::sync::CancellationToken; use crate::posix::escalate_protocol::BASH_EXEC_WRAPPER_ENV_VAR; use crate::posix::escalate_protocol::ESCALATE_SOCKET_ENV_VAR; @@ -24,6 +25,7 @@ use crate::posix::escalate_protocol::SuperExecResult; use crate::posix::escalation_policy::EscalationPolicy; use crate::posix::socket::AsyncDatagramSocket; use crate::posix::socket::AsyncSocket; +use codex_core::exec::ExecExpiration; pub(crate) struct EscalateServer { bash_path: PathBuf, @@ -48,7 +50,7 @@ impl EscalateServer { command: String, env: HashMap, workdir: PathBuf, - timeout_ms: Option, + cancel_rx: CancellationToken, ) -> anyhow::Result { let (escalate_server, escalate_client) = AsyncDatagramSocket::pair()?; let client_socket = escalate_client.into_inner(); @@ -79,7 +81,7 @@ impl EscalateServer { command, ], cwd: PathBuf::from(&workdir), - expiration: timeout_ms.into(), + expiration: ExecExpiration::Cancellation(cancel_rx), env, with_escalated_permissions: None, justification: None, diff --git a/codex-rs/exec-server/src/posix/mcp.rs b/codex-rs/exec-server/src/posix/mcp.rs index f5785dc5d0..b2f9b6de48 100644 --- a/codex-rs/exec-server/src/posix/mcp.rs +++ b/codex-rs/exec-server/src/posix/mcp.rs @@ -22,6 +22,7 @@ use crate::posix::escalate_server::EscalateServer; use crate::posix::escalate_server::{self}; use crate::posix::mcp_escalation_policy::ExecPolicy; use crate::posix::mcp_escalation_policy::McpEscalationPolicy; +use crate::posix::stopwatch::Stopwatch; /// Path to our patched bash. const CODEX_BASH_PATH_ENV_VAR: &str = "CODEX_BASH_PATH"; @@ -87,10 +88,17 @@ impl ExecTool { context: RequestContext, Parameters(params): Parameters, ) -> Result { + let effective_timeout = Duration::from_millis( + params + .timeout_ms + .unwrap_or(codex_core::exec::DEFAULT_EXEC_COMMAND_TIMEOUT_MS), + ); + let stopwatch = Stopwatch::new(effective_timeout); + let cancel_token = stopwatch.cancellation_token(); let escalate_server = EscalateServer::new( self.bash_path.clone(), self.execve_wrapper.clone(), - McpEscalationPolicy::new(self.policy, context), + McpEscalationPolicy::new(self.policy, context, stopwatch.clone()), ); let result = escalate_server .exec( @@ -98,7 +106,7 @@ impl ExecTool { // TODO: use ShellEnvironmentPolicy std::env::vars().collect(), PathBuf::from(¶ms.workdir), - params.timeout_ms, + cancel_token, ) .await .map_err(|e| McpError::internal_error(e.to_string(), None))?; diff --git a/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs index 069948ea06..9e059fdba5 100644 --- a/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs +++ b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs @@ -10,6 +10,7 @@ use rmcp::service::RequestContext; use crate::posix::escalate_protocol::EscalateAction; use crate::posix::escalation_policy::EscalationPolicy; +use crate::posix::stopwatch::Stopwatch; /// This is the policy which decides how to handle an exec() call. /// @@ -34,11 +35,20 @@ pub(crate) enum ExecPolicyOutcome { pub(crate) struct McpEscalationPolicy { policy: ExecPolicy, context: RequestContext, + stopwatch: Stopwatch, } impl McpEscalationPolicy { - pub(crate) fn new(policy: ExecPolicy, context: RequestContext) -> Self { - Self { policy, context } + pub(crate) fn new( + policy: ExecPolicy, + context: RequestContext, + stopwatch: Stopwatch, + ) -> Self { + Self { + policy, + context, + stopwatch, + } } async fn prompt( @@ -54,25 +64,34 @@ impl McpEscalationPolicy { } else { format!("{} {}", file.display(), args) }; - context - .peer - .create_elicitation(CreateElicitationRequestParam { - message: format!("Allow agent to run `{command}` in `{}`?", workdir.display()), - requested_schema: ElicitationSchema::builder() - .title("Execution Permission Request") - .optional_string_with("reason", |schema| { - schema.description("Optional reason for allowing or denying execution") + self.stopwatch + .pause_for(async { + context + .peer + .create_elicitation(CreateElicitationRequestParam { + message: format!( + "Allow agent to run `{command}` in `{}`?", + workdir.display() + ), + requested_schema: ElicitationSchema::builder() + .title("Execution Permission Request") + .optional_string_with("reason", |schema| { + schema.description( + "Optional reason for allowing or denying execution", + ) + }) + .build() + .map_err(|e| { + McpError::internal_error( + format!("failed to build elicitation schema: {e}"), + None, + ) + })?, }) - .build() - .map_err(|e| { - McpError::internal_error( - format!("failed to build elicitation schema: {e}"), - None, - ) - })?, + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) }) .await - .map_err(|e| McpError::internal_error(e.to_string(), None)) } } diff --git a/codex-rs/exec-server/src/posix/stopwatch.rs b/codex-rs/exec-server/src/posix/stopwatch.rs new file mode 100644 index 0000000000..de29a45685 --- /dev/null +++ b/codex-rs/exec-server/src/posix/stopwatch.rs @@ -0,0 +1,211 @@ +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; + +use tokio::sync::Mutex; +use tokio::sync::Notify; +use tokio_util::sync::CancellationToken; + +#[derive(Clone, Debug)] +pub(crate) struct Stopwatch { + limit: Duration, + inner: Arc>, + notify: Arc, +} + +#[derive(Debug)] +struct StopwatchState { + elapsed: Duration, + running_since: Option, + active_pauses: u32, +} + +impl Stopwatch { + pub(crate) fn new(limit: Duration) -> Self { + Self { + inner: Arc::new(Mutex::new(StopwatchState { + elapsed: Duration::ZERO, + running_since: Some(Instant::now()), + active_pauses: 0, + })), + notify: Arc::new(Notify::new()), + limit, + } + } + + pub(crate) fn cancellation_token(&self) -> CancellationToken { + let limit = self.limit; + let token = CancellationToken::new(); + let cancel = token.clone(); + let inner = Arc::clone(&self.inner); + let notify = Arc::clone(&self.notify); + tokio::spawn(async move { + loop { + let (remaining, running) = { + let guard = inner.lock().await; + let elapsed = guard.elapsed + + guard + .running_since + .map(|since| since.elapsed()) + .unwrap_or_default(); + if elapsed >= limit { + break; + } + (limit - elapsed, guard.running_since.is_some()) + }; + + if !running { + notify.notified().await; + continue; + } + + let sleep = tokio::time::sleep(remaining); + tokio::pin!(sleep); + tokio::select! { + _ = &mut sleep => { + break; + } + _ = notify.notified() => { + continue; + } + } + } + cancel.cancel(); + }); + token + } + + /// Runs `fut`, pausing the stopwatch while the future is pending. The clock + /// resumes automatically when the future completes. Nested/overlapping + /// calls are reference-counted so the stopwatch only resumes when every + /// pause is lifted. + pub(crate) async fn pause_for(&self, fut: F) -> T + where + F: Future, + { + self.pause().await; + let result = fut.await; + self.resume().await; + result + } + + async fn pause(&self) { + let mut guard = self.inner.lock().await; + guard.active_pauses += 1; + if guard.active_pauses == 1 + && let Some(since) = guard.running_since.take() + { + guard.elapsed += since.elapsed(); + self.notify.notify_waiters(); + } + } + + async fn resume(&self) { + let mut guard = self.inner.lock().await; + if guard.active_pauses == 0 { + return; + } + guard.active_pauses -= 1; + if guard.active_pauses == 0 && guard.running_since.is_none() { + guard.running_since = Some(Instant::now()); + self.notify.notify_waiters(); + } + } +} + +#[cfg(test)] +mod tests { + use super::Stopwatch; + use tokio::time::Duration; + use tokio::time::Instant; + use tokio::time::sleep; + use tokio::time::timeout; + + #[tokio::test] + async fn cancellation_receiver_fires_after_limit() { + let stopwatch = Stopwatch::new(Duration::from_millis(50)); + let token = stopwatch.cancellation_token(); + let start = Instant::now(); + token.cancelled().await; + assert!(start.elapsed() >= Duration::from_millis(50)); + } + + #[tokio::test] + async fn pause_prevents_timeout_until_resumed() { + let stopwatch = Stopwatch::new(Duration::from_millis(50)); + let token = stopwatch.cancellation_token(); + + let pause_handle = tokio::spawn({ + let stopwatch = stopwatch.clone(); + async move { + stopwatch + .pause_for(async { + sleep(Duration::from_millis(100)).await; + }) + .await; + } + }); + + assert!( + timeout(Duration::from_millis(30), token.cancelled()) + .await + .is_err() + ); + + pause_handle.await.expect("pause task should finish"); + + token.cancelled().await; + } + + #[tokio::test] + async fn overlapping_pauses_only_resume_once() { + let stopwatch = Stopwatch::new(Duration::from_millis(50)); + let token = stopwatch.cancellation_token(); + + // First pause. + let pause1 = { + let stopwatch = stopwatch.clone(); + tokio::spawn(async move { + stopwatch + .pause_for(async { + sleep(Duration::from_millis(80)).await; + }) + .await; + }) + }; + + // Overlapping pause that ends sooner. + let pause2 = { + let stopwatch = stopwatch.clone(); + tokio::spawn(async move { + stopwatch + .pause_for(async { + sleep(Duration::from_millis(30)).await; + }) + .await; + }) + }; + + // While both pauses are active, the cancellation should not fire. + assert!( + timeout(Duration::from_millis(40), token.cancelled()) + .await + .is_err() + ); + + pause2.await.expect("short pause should complete"); + + // Still paused because the long pause is active. + assert!( + timeout(Duration::from_millis(30), token.cancelled()) + .await + .is_err() + ); + + pause1.await.expect("long pause should complete"); + + // Now the stopwatch should resume and hit the limit shortly after. + token.cancelled().await; + } +} From 591f0ee2a0036eb046c41cdc24640b2944484731 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 20 Nov 2025 16:45:46 -0800 Subject: [PATCH 3/3] feat: codex-shell-tool-mcp --- .github/workflows/rust-release.yml | 14 +- .github/workflows/shell-tool-mcp.yml | 402 ++++++++++++++++++ shell-tool-mcp/README.md | 32 ++ shell-tool-mcp/bin/mcp-server.js | 262 ++++++++++++ shell-tool-mcp/package.json | 24 ++ .../patches/bash-exec-wrapper.patch | 24 ++ 6 files changed, 757 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/shell-tool-mcp.yml create mode 100644 shell-tool-mcp/README.md create mode 100644 shell-tool-mcp/bin/mcp-server.js create mode 100644 shell-tool-mcp/package.json create mode 100644 shell-tool-mcp/patches/bash-exec-wrapper.patch diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 6f27fbf543..5819c0a226 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -371,8 +371,20 @@ jobs: path: | codex-rs/dist/${{ matrix.target }}/* + shell-tool-mcp: + name: shell-tool-mcp + needs: tag-check + uses: ./.github/workflows/shell-tool-mcp.yml + with: + release-tag: ${{ github.ref_name }} + # We are not ready to publish yet. + publish: false + secrets: inherit + release: - needs: build + needs: + - build + - shell-tool-mcp name: release runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml new file mode 100644 index 0000000000..78ed5cb8f1 --- /dev/null +++ b/.github/workflows/shell-tool-mcp.yml @@ -0,0 +1,402 @@ +name: shell-tool-mcp + +on: + workflow_call: + inputs: + release-version: + description: Version to publish (x.y.z or x.y.z-alpha.N). Defaults to GITHUB_REF_NAME when it starts with rust-v. + required: false + type: string + release-tag: + description: Tag name to use when downloading release artifacts (defaults to rust-v). + required: false + type: string + publish: + description: Whether to publish to npm when the version is releasable. + required: false + default: true + type: boolean + +env: + NODE_VERSION: 22 + +jobs: + metadata: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.compute.outputs.version }} + release_tag: ${{ steps.compute.outputs.release_tag }} + should_publish: ${{ steps.compute.outputs.should_publish }} + npm_tag: ${{ steps.compute.outputs.npm_tag }} + steps: + - name: Compute version and tags + id: compute + run: | + set -euo pipefail + + version="${{ inputs.release-version }}" + release_tag="${{ inputs.release-tag }}" + + if [[ -z "$version" ]]; then + if [[ -n "$release_tag" && "$release_tag" =~ ^rust-v.+ ]]; then + version="${release_tag#rust-v}" + elif [[ "${GITHUB_REF_NAME:-}" =~ ^rust-v.+ ]]; then + version="${GITHUB_REF_NAME#rust-v}" + release_tag="${GITHUB_REF_NAME}" + else + echo "release-version is required when GITHUB_REF_NAME is not a rust-v tag." + exit 1 + fi + fi + + if [[ -z "$release_tag" ]]; then + release_tag="rust-v${version}" + fi + + npm_tag="" + should_publish="false" + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + should_publish="true" + elif [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+-alpha\.[0-9]+$ ]]; then + should_publish="true" + npm_tag="alpha" + fi + + echo "version=${version}" >> "$GITHUB_OUTPUT" + echo "release_tag=${release_tag}" >> "$GITHUB_OUTPUT" + echo "npm_tag=${npm_tag}" >> "$GITHUB_OUTPUT" + echo "should_publish=${should_publish}" >> "$GITHUB_OUTPUT" + + rust-binaries: + name: Build Rust - ${{ matrix.target }} + needs: metadata + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + defaults: + run: + working-directory: codex-rs + strategy: + fail-fast: false + matrix: + include: + - runner: macos-15-xlarge + target: aarch64-apple-darwin + - runner: macos-15-xlarge + target: x86_64-apple-darwin + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + install_musl: true + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + install_musl: true + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - uses: dtolnay/rust-toolchain@1.90 + with: + targets: ${{ matrix.target }} + + - if: ${{ matrix.install_musl }} + name: Install musl build dependencies + run: | + sudo apt-get update + sudo apt-get install -y musl-tools pkg-config + + - name: Build exec server binaries + run: cargo build --release --target ${{ matrix.target }} --bin codex-exec-mcp-server --bin codex-execve-wrapper + + - name: Stage exec server binaries + run: | + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}" + mkdir -p "$dest" + cp "target/${{ matrix.target }}/release/codex-exec-mcp-server" "$dest/" + cp "target/${{ matrix.target }}/release/codex-execve-wrapper" "$dest/" + + - uses: actions/upload-artifact@v4 + with: + name: shell-tool-mcp-rust-${{ matrix.target }} + path: artifacts/** + if-no-files-found: error + + bash-linux: + name: Build Bash (Linux) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: metadata + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + container: + image: ${{ matrix.image }} + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: ubuntu-24.04 + image: ubuntu:24.04 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: ubuntu-22.04 + image: ubuntu:22.04 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: ubuntu-20.04 + image: ubuntu:20.04 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: debian-12 + image: debian:12 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: debian-11 + image: debian:11 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + variant: centos-9 + image: quay.io/centos/centos:stream9 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: ubuntu-24.04 + image: arm64v8/ubuntu:24.04 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: ubuntu-22.04 + image: arm64v8/ubuntu:22.04 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: ubuntu-20.04 + image: arm64v8/ubuntu:20.04 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: debian-12 + image: arm64v8/debian:12 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: debian-11 + image: arm64v8/debian:11 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + variant: centos-9 + image: quay.io/centos/centos:stream9 + steps: + - name: Install build prerequisites + shell: bash + run: | + set -euo pipefail + if command -v apt-get >/dev/null 2>&1; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y git build-essential bison autoconf texinfo gettext + elif command -v dnf >/dev/null 2>&1; then + dnf install -y git gcc gcc-c++ make bison gettext + elif command -v yum >/dev/null 2>&1; then + yum install -y git gcc gcc-c++ make bison gettext + else + echo "Unsupported package manager in container" + exit 1 + fi + + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Build patched Bash + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/bminor/bash /tmp/bash + cd /tmp/bash + git fetch --depth 1 origin a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git checkout a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/bash-exec-wrapper.patch" + ./configure --without-bash-malloc + cores="$(command -v nproc >/dev/null 2>&1 && nproc || getconf _NPROCESSORS_ONLN)" + make -j"${cores}" + + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/bash/${{ matrix.variant }}" + mkdir -p "$dest" + cp bash "$dest/bash" + + - uses: actions/upload-artifact@v4 + with: + name: shell-tool-mcp-bash-${{ matrix.target }}-${{ matrix.variant }} + path: artifacts/** + if-no-files-found: error + + bash-darwin: + name: Build Bash (macOS) - ${{ matrix.variant }} - ${{ matrix.target }} + needs: metadata + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - runner: macos-15-xlarge + target: aarch64-apple-darwin + variant: macos-15 + - runner: macos-14 + target: aarch64-apple-darwin + variant: macos-14 + - runner: macos-13 + target: x86_64-apple-darwin + variant: macos-13 + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Build patched Bash + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/bminor/bash /tmp/bash + cd /tmp/bash + git fetch --depth 1 origin a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git checkout a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b + git apply "${GITHUB_WORKSPACE}/shell-tool-mcp/patches/bash-exec-wrapper.patch" + ./configure --without-bash-malloc + cores="$(getconf _NPROCESSORS_ONLN)" + make -j"${cores}" + + dest="${GITHUB_WORKSPACE}/artifacts/vendor/${{ matrix.target }}/bash/${{ matrix.variant }}" + mkdir -p "$dest" + cp bash "$dest/bash" + + - uses: actions/upload-artifact@v4 + with: + name: shell-tool-mcp-bash-${{ matrix.target }}-${{ matrix.variant }} + path: artifacts/** + if-no-files-found: error + + package: + name: Package npm module + needs: + - metadata + - rust-binaries + - bash-linux + - bash-darwin + runs-on: ubuntu-latest + env: + PACKAGE_VERSION: ${{ needs.metadata.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Assemble staging directory + id: staging + shell: bash + run: | + set -euo pipefail + staging="${STAGING_DIR}" + mkdir -p "$staging" "$staging/vendor" + rsync -av --exclude vendor shell-tool-mcp/ "$staging/" + + found_vendor="false" + shopt -s nullglob + for vendor_dir in artifacts/*/vendor; do + rsync -av "$vendor_dir/" "$staging/vendor/" + found_vendor="true" + done + if [[ "$found_vendor" == "false" ]]; then + echo "No vendor payloads were downloaded." + exit 1 + fi + + node - <<'NODE' + import fs from "node:fs"; + import path from "node:path"; + + const stagingDir = process.env.STAGING_DIR; + const version = process.env.PACKAGE_VERSION; + const pkgPath = path.join(stagingDir, "package.json"); + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8")); + pkg.version = version; + fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + "\n"); + NODE + + echo "dir=$staging" >> "$GITHUB_OUTPUT" + env: + STAGING_DIR: ${{ runner.temp }}/shell-tool-mcp + + - name: Ensure binaries are executable + run: | + set -euo pipefail + staging="${{ steps.staging.outputs.dir }}" + chmod +x \ + "$staging"/vendor/*/codex-exec-mcp-server \ + "$staging"/vendor/*/codex-execve-wrapper \ + "$staging"/vendor/*/bash/*/bash + + - name: Create npm tarball + shell: bash + run: | + set -euo pipefail + mkdir -p dist/npm + staging="${{ steps.staging.outputs.dir }}" + pack_info=$(cd "$staging" && npm pack --json --pack-destination "${GITHUB_WORKSPACE}/dist/npm") + filename=$(PACK_INFO="$pack_info" node -e 'const data = JSON.parse(process.env.PACK_INFO); console.log(data[0].filename);') + mv "dist/npm/${filename}" "dist/npm/codex-shell-tool-mcp-npm-${PACKAGE_VERSION}.tgz" + + - uses: actions/upload-artifact@v4 + with: + name: codex-shell-tool-mcp-npm + path: dist/npm/codex-shell-tool-mcp-npm-${{ env.PACKAGE_VERSION }}.tgz + if-no-files-found: error + + publish: + name: Publish npm package + needs: + - metadata + - package + if: ${{ inputs.publish && needs.metadata.outputs.should_publish == 'true' }} + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: https://registry.npmjs.org + scope: "@openai" + + - name: Update npm + run: npm install -g npm@latest + + - name: Download npm tarball + uses: actions/download-artifact@v4 + with: + name: codex-shell-tool-mcp-npm + path: dist/npm + + - name: Publish to npm + env: + NPM_TAG: ${{ needs.metadata.outputs.npm_tag }} + VERSION: ${{ needs.metadata.outputs.version }} + shell: bash + run: | + set -euo pipefail + tag_args=() + if [[ -n "${NPM_TAG}" ]]; then + tag_args+=(--tag "${NPM_TAG}") + fi + npm publish "dist/npm/codex-shell-tool-mcp-npm-${VERSION}.tgz" "${tag_args[@]}" diff --git a/shell-tool-mcp/README.md b/shell-tool-mcp/README.md new file mode 100644 index 0000000000..38518dae5e --- /dev/null +++ b/shell-tool-mcp/README.md @@ -0,0 +1,32 @@ +# @openai/codex-shell-tool-mcp + +This package wraps the `codex-exec-mcp-server` binary and its helpers so that the shell MCP can be invoked via `npx @openai/codex-shell-tool-mcp`. It bundles: + +- `codex-exec-mcp-server` and `codex-execve-wrapper` built for macOS (arm64, x64) and Linux (musl arm64, musl x64). +- A patched Bash that honors `BASH_EXEC_WRAPPER`, built for multiple glibc baselines (Ubuntu 24.04/22.04/20.04, Debian 12/11/10, CentOS-like 9/8/7) and macOS (15/14/13). +- A launcher (`bin/mcp-server.js`) that picks the correct binaries for the current `process.platform` / `process.arch`, wires `--execve` and `--bash`, and exports `CODEX_BASH_PATH` for the MCP. + +## Usage + +```bash +npx @openai/codex-shell-tool-mcp --help +``` + +The launcher selects a Rust target triple based on the host and chooses the closest Bash variant by inspecting `/etc/os-release` on Linux or the Darwin major version on macOS. You can override the bundled Bash by setting `CODEX_BASH_PATH` to an absolute path. + +## Patched Bash + +We carry a small patch to `execute_cmd.c` (see `patches/bash-exec-wrapper.patch`) that adds support for `BASH_EXEC_WRAPPER`. The original commit message is “add support for BASH_EXEC_WRAPPER” and the patch applies cleanly to `a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b` from https://github.com/bminor/bash. To rebuild manually: + +```bash +git clone https://github.com/bminor/bash +git checkout a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b +git apply /path/to/patches/bash-exec-wrapper.patch +./configure --without-bash-malloc +make -j"$(nproc)" +``` + +## Release workflow + +`.github/workflows/shell-tool-mcp.yml` builds the Rust binaries, compiles the patched Bash variants, assembles the `vendor/` tree, and creates `codex-shell-tool-mcp-npm-.tgz` for inclusion in the Rust GitHub Release. When the version is a stable or alpha tag, the workflow also publishes the tarball to npm using OIDC. The workflow is invoked from `rust-release.yml` so the package ships alongside other Codex artifacts. + diff --git a/shell-tool-mcp/bin/mcp-server.js b/shell-tool-mcp/bin/mcp-server.js new file mode 100644 index 0000000000..31f58db258 --- /dev/null +++ b/shell-tool-mcp/bin/mcp-server.js @@ -0,0 +1,262 @@ +#!/usr/bin/env node +// Launches the codex-exec-mcp-server binary bundled in this package. + +import { spawn } from "node:child_process"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const LINUX_BASH_VARIANTS = [ + { name: "ubuntu-24.04", ids: ["ubuntu"], versions: ["24.04"] }, + { name: "ubuntu-22.04", ids: ["ubuntu"], versions: ["22.04"] }, + { name: "ubuntu-20.04", ids: ["ubuntu"], versions: ["20.04"] }, + { name: "debian-12", ids: ["debian"], versions: ["12"] }, + { name: "debian-11", ids: ["debian"], versions: ["11"] }, + { name: "debian-10", ids: ["debian"], versions: ["10"] }, + { name: "centos-9", ids: ["centos", "rhel", "rocky", "almalinux"], versions: ["9"] }, + { name: "centos-8", ids: ["centos", "rhel", "rocky", "almalinux"], versions: ["8"] }, + { name: "centos-7", ids: ["centos", "rhel"], versions: ["7"] }, +]; + +const DARWIN_BASH_VARIANTS = [ + { name: "macos-15", minDarwin: 24 }, + { name: "macos-14", minDarwin: 23 }, + { name: "macos-13", minDarwin: 22 }, +]; + +function resolveTargetTriple(platform, arch) { + if (platform === "linux") { + if (arch === "x64") { + return "x86_64-unknown-linux-musl"; + } + if (arch === "arm64") { + return "aarch64-unknown-linux-musl"; + } + } else if (platform === "darwin") { + if (arch === "x64") { + return "x86_64-apple-darwin"; + } + if (arch === "arm64") { + return "aarch64-apple-darwin"; + } + } + throw new Error(`Unsupported platform: ${platform} (${arch})`); +} + +function parseOsRelease() { + try { + const contents = readFileSync("/etc/os-release", "utf8"); + const lines = contents.split("\n").filter(Boolean); + const info = {}; + for (const line of lines) { + const [rawKey, rawValue] = line.split("=", 2); + if (!rawKey || rawValue === undefined) { + continue; + } + const key = rawKey.toLowerCase(); + const value = rawValue.replace(/^"/, "").replace(/"$/, ""); + info[key] = value; + } + const idLike = (info.id_like || "") + .split(/\s+/) + .map((item) => item.trim().toLowerCase()) + .filter(Boolean); + return { + id: (info.id || "").toLowerCase(), + idLike, + versionId: info.version_id || "", + }; + } catch { + return { id: "", idLike: [], versionId: "" }; + } +} + +function variantExists(bashRoot, name) { + const candidate = path.join(bashRoot, name, "bash"); + return existsSync(candidate); +} + +function listAvailableVariants(bashRoot) { + try { + return readdirSync(bashRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .filter((name) => variantExists(bashRoot, name)); + } catch { + return []; + } +} + +function selectLinuxBash(bashRoot) { + const info = parseOsRelease(); + const versionId = info.versionId; + const candidates = []; + for (const variant of LINUX_BASH_VARIANTS) { + const matchesId = + variant.ids.includes(info.id) || + variant.ids.some((id) => info.idLike.includes(id)); + if (!matchesId) { + continue; + } + const matchesVersion = + versionId && + variant.versions.some((prefix) => versionId.startsWith(prefix)); + candidates.push({ variant, matchesVersion }); + } + + const pickVariant = (list) => + list.find(({ variant: candidate }) => variantExists(bashRoot, candidate.name)) + ?.variant; + + const preferred = pickVariant(candidates.filter((item) => item.matchesVersion)); + if (preferred) { + return { path: path.join(bashRoot, preferred.name, "bash"), variant: preferred.name }; + } + + const fallbackMatch = pickVariant(candidates); + if (fallbackMatch) { + return { path: path.join(bashRoot, fallbackMatch.name, "bash"), variant: fallbackMatch.name }; + } + + const available = pickVariant( + LINUX_BASH_VARIANTS.map((variant) => ({ variant, matchesVersion: false })), + ); + if (available) { + return { path: path.join(bashRoot, available.name, "bash"), variant: available.name }; + } + + const known = listAvailableVariants(bashRoot); + const detail = known.length + ? `Available variants: ${known.join(", ")}` + : "No bundled Bash binaries were found."; + throw new Error( + `Unable to select a Bash variant for ${info.id || "unknown"} ${versionId || ""}. ${detail}`, + ); +} + +function selectDarwinBash(bashRoot) { + const darwinMajor = Number.parseInt(os.release().split(".")[0] || "0", 10); + const pickVariant = (variantList) => + variantList.find((variant) => variantExists(bashRoot, variant.name)); + + const preferred = pickVariant( + DARWIN_BASH_VARIANTS.filter((variant) => darwinMajor >= variant.minDarwin), + ); + if (preferred) { + return { path: path.join(bashRoot, preferred.name, "bash"), variant: preferred.name }; + } + + const available = pickVariant(DARWIN_BASH_VARIANTS); + if (available) { + return { path: path.join(bashRoot, available.name, "bash"), variant: available.name }; + } + + const known = listAvailableVariants(bashRoot); + const detail = known.length + ? `Available variants: ${known.join(", ")}` + : "No bundled Bash binaries were found."; + throw new Error(`Unable to select a macOS Bash build (darwin ${darwinMajor}). ${detail}`); +} + +function resolveBashPath(targetRoot) { + const override = process.env.CODEX_BASH_PATH; + if (override) { + if (!existsSync(override)) { + throw new Error(`CODEX_BASH_PATH was set to ${override}, but it does not exist.`); + } + return { path: override, variant: "env" }; + } + + const bashRoot = path.join(targetRoot, "bash"); + if (!existsSync(bashRoot)) { + throw new Error(`Bundled Bash directory missing: ${bashRoot}`); + } + + if (process.platform === "linux") { + return selectLinuxBash(bashRoot); + } + if (process.platform === "darwin") { + return selectDarwinBash(bashRoot); + } + throw new Error(`Unsupported platform for Bash selection: ${process.platform}`); +} + +const ensurePathExists = (checkPath, label) => { + if (!existsSync(checkPath)) { + throw new Error(`Expected ${label} at ${checkPath}, but it was not found.`); + } +}; + +const targetTriple = resolveTargetTriple(process.platform, process.arch); +const vendorRoot = path.join(__dirname, "..", "vendor"); +const targetRoot = path.join(vendorRoot, targetTriple); +ensurePathExists(targetRoot, `vendor directory for ${targetTriple}`); + +const execveWrapperPath = path.join(targetRoot, "codex-execve-wrapper"); +ensurePathExists(execveWrapperPath, "execve wrapper"); + +const serverPath = path.join(targetRoot, "codex-exec-mcp-server"); +ensurePathExists(serverPath, "codex-exec-mcp-server"); + +const { path: bashPath, variant: bashVariant } = resolveBashPath(targetRoot); + +const childEnv = { + ...process.env, + CODEX_BASH_PATH: bashPath, +}; +if (bashVariant) { + childEnv.CODEX_BASH_VARIANT = bashVariant; +} + +const args = ["--execve", execveWrapperPath, "--bash", bashPath, ...process.argv.slice(2)]; +const child = spawn(serverPath, args, { + stdio: "inherit", + env: childEnv, +}); + +const forwardSignal = (signal) => { + if (child.killed) { + return; + } + try { + child.kill(signal); + } catch { + /* ignore */ + } +}; + +["SIGINT", "SIGTERM", "SIGHUP"].forEach((sig) => { + process.on(sig, () => forwardSignal(sig)); +}); + +child.on("error", (err) => { + // eslint-disable-next-line no-console + console.error(err); + process.exit(1); +}); + +const childResult = await new Promise((resolve) => { + child.on("exit", (code, signal) => { + if (signal) { + resolve({ type: "signal", signal }); + } else { + resolve({ type: "code", exitCode: code ?? 1 }); + } + }); +}); + +if (childResult.type === "signal") { + // This environment running under `node --test` may not allow rethrowing a signal. + // Wrap in a try to avoid masking the original termination reason. + try { + process.kill(process.pid, childResult.signal); + } catch { + process.exit(1); + } +} else { + process.exit(childResult.exitCode); +} diff --git a/shell-tool-mcp/package.json b/shell-tool-mcp/package.json new file mode 100644 index 0000000000..77fcc96711 --- /dev/null +++ b/shell-tool-mcp/package.json @@ -0,0 +1,24 @@ +{ + "name": "@openai/codex-shell-tool-mcp", + "version": "0.0.0-dev", + "description": "Codex MCP server for the shell tool with patched Bash and exec wrappers.", + "license": "Apache-2.0", + "type": "module", + "bin": { + "codex-shell-tool-mcp": "bin/mcp-server.js" + }, + "engines": { + "node": ">=18" + }, + "files": [ + "bin", + "vendor", + "patches", + "README.md" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/openai/codex.git", + "directory": "shell-tool-mcp" + } +} diff --git a/shell-tool-mcp/patches/bash-exec-wrapper.patch b/shell-tool-mcp/patches/bash-exec-wrapper.patch new file mode 100644 index 0000000000..6a7fedbb8f --- /dev/null +++ b/shell-tool-mcp/patches/bash-exec-wrapper.patch @@ -0,0 +1,24 @@ +diff --git a/execute_cmd.c b/execute_cmd.c +index 070f5119..d20ad2b9 100644 +--- a/execute_cmd.c ++++ b/execute_cmd.c +@@ -6129,6 +6129,19 @@ shell_execve (char *command, char **args, char **env) + char sample[HASH_BANG_BUFSIZ]; + size_t larray; + ++ char* exec_wrapper = getenv("BASH_EXEC_WRAPPER"); ++ if (exec_wrapper && *exec_wrapper && !whitespace (*exec_wrapper)) ++ { ++ char *orig_command = command; ++ ++ larray = strvec_len (args); ++ ++ memmove (args + 2, args, (++larray) * sizeof (char *)); ++ args[0] = exec_wrapper; ++ args[1] = orig_command; ++ command = exec_wrapper; ++ } ++ + SETOSTYPE (0); /* Some systems use for USG/POSIX semantics */ + execve (command, args, env); + i = errno; /* error from execve() */