mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Report HTTP MCP startup problems clearly in codex exec (#62906)
## Why Low-level transport errors can clutter stderr and include endpoint URLs with credentials. An optional HTTP MCP server can also still be starting when its tools are omitted from a turn. ## What changed - Warn once per Streamable HTTP connection when startup fails or the server is omitted from the turn because startup is still pending. Include the server name and a short reason without the raw transport error. - Filter redundant MCP worker errors from default `codex exec` stderr and preserve caller control through `RUST_LOG`. - Add server context to errors when startup blocks a tool call or resource operation, when a tool call finds a disconnected server, or when HTTP session recovery fails. ## Testing Add regression coverage for startup warnings, credential omission, explicit `RUST_LOG` diagnostics, and an optional server that becomes ready in time to be included in the turn. GitOrigin-RevId: 1fd012a5e73f205c3d8735adde395c36672c205c
This commit is contained in:
@@ -1529,14 +1529,19 @@ impl McpConnectionSet {
|
||||
view.connection
|
||||
.client()
|
||||
.await
|
||||
.inspect_err(|_| tracing::error!(server = %server_key, tool = ?tool, "MCP tool call failed during startup"))
|
||||
.context("failed to get client")?
|
||||
} else {
|
||||
let client = view
|
||||
.connection
|
||||
.client
|
||||
.ready_client()
|
||||
.ok_or_else(|| anyhow!("MCP server '{server_key}' is not connected"))?;
|
||||
.ok_or_else(|| {
|
||||
tracing::error!(server = %server_key, tool = ?tool, "MCP tool call failed: server is not connected");
|
||||
anyhow!("MCP server '{server_key}' is not connected")
|
||||
})?;
|
||||
if client.client.is_closed().await {
|
||||
tracing::error!(server = %server_key, tool = ?tool, "MCP tool call failed: server is not connected");
|
||||
bail!("MCP server '{server_key}' is not connected");
|
||||
}
|
||||
client
|
||||
|
||||
@@ -231,6 +231,9 @@ impl McpConnectionSet {
|
||||
.connection
|
||||
.client()
|
||||
.await
|
||||
.inspect_err(
|
||||
|_| tracing::error!(server = %key, "MCP resource operation failed during startup"),
|
||||
)
|
||||
.context("failed to get client")?;
|
||||
Ok((client, view.tool_timeout))
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ use tracing::Instrument;
|
||||
use tracing::instrument;
|
||||
use tracing::trace;
|
||||
use tracing::trace_span;
|
||||
use tracing::warn;
|
||||
|
||||
use super::McpConnectionSet;
|
||||
use super::catalog_telemetry::emit_binding_catalog;
|
||||
@@ -412,6 +413,11 @@ impl McpConnectionSet {
|
||||
&& !(accepts_cached_catalog
|
||||
&& self.required_servers.binary_search(server_name).is_ok())
|
||||
{
|
||||
if let Some(warning_emitted) = &view.connection.client.startup_warning_emitted
|
||||
&& !warning_emitted.swap(true, Ordering::Relaxed)
|
||||
{
|
||||
warn!(target: "codex_mcp_startup", server_name = ?server_name.name(), reason = "startup still pending; omitted from this turn", "MCP server unavailable");
|
||||
}
|
||||
return None;
|
||||
}
|
||||
let Ok(mut client) = view.connection.client().await else {
|
||||
@@ -590,6 +596,9 @@ impl McpConnectionSet {
|
||||
.connection
|
||||
.client()
|
||||
.await
|
||||
.inspect_err(
|
||||
|_| tracing::error!(server = ?server_name, "MCP tool call failed during startup"),
|
||||
)
|
||||
.with_context(|| format!("Could not start MCP server `{server_name}`"))?;
|
||||
client.tool_timeout = view.tool_timeout;
|
||||
let snapshot = client.tool_catalog.read(Arc::new).await;
|
||||
|
||||
@@ -645,6 +645,7 @@ pub(crate) async fn create_ready_async_managed_client(tools: Vec<ToolInfo>) -> A
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -932,6 +933,7 @@ fn create_gated_async_managed_client(
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete,
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -1006,6 +1008,7 @@ pub(crate) async fn create_test_manager_with_ready_apps_client(
|
||||
codex_apps_tools_cache_context: Some(cache_context),
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -1063,6 +1066,7 @@ fn create_test_manager_with_failed_apps_startup(
|
||||
codex_apps_tools_cache_context: Some(cache_context),
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: Some(Arc::new(CodexAppsStartupReconnect::new(reconnect_factory))),
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -2627,6 +2631,7 @@ async fn list_all_tools_uses_shared_codex_apps_cache_while_client_is_pending() {
|
||||
codex_apps_tools_cache_context: Some(cache_context),
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -2706,6 +2711,7 @@ async fn capture_binding_preserves_ready_client_catalog_and_shares_prepared_meta
|
||||
codex_apps_tools_cache_context: Some(cache_context),
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -3163,6 +3169,7 @@ async fn list_available_server_infos_uses_cache_while_client_is_pending() {
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -3262,6 +3269,7 @@ async fn capture_binding_exposes_cached_tools_before_startup() {
|
||||
codex_apps_tools_cache_context: Some(cache_context),
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete,
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -3371,6 +3379,7 @@ async fn capture_binding_skips_pending_optional_servers_after_configured_shared_
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(AtomicBool::new(false)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -3662,6 +3671,7 @@ async fn capture_binding_shares_optional_startup_grace_across_connection_sets()
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: Some(cache_context.clone()),
|
||||
startup_complete: Arc::new(AtomicBool::new(false)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -4105,6 +4115,47 @@ async fn capture_binding_resolves_concurrently_and_rechecks_cached_clients() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn capture_binding_does_not_warn_for_optional_server_ready_before_catalog_capture() {
|
||||
let (mut optional, optional_started, release_optional) = create_gated_async_managed_client(
|
||||
create_test_managed_client(vec![create_test_tool("optional", "ready_tool")]).await,
|
||||
);
|
||||
let warning_emitted = Arc::new(AtomicBool::new(false));
|
||||
optional.startup_warning_emitted = Some(Arc::clone(&warning_emitted));
|
||||
let (required, required_started, release_required) =
|
||||
create_gated_async_managed_client(create_test_managed_client(vec![]).await);
|
||||
let mut manager = McpConnectionSet::new_uninitialized(
|
||||
&Constrained::allow_any(AskForApproval::OnRequest),
|
||||
&Constrained::allow_any(PermissionProfile::default()),
|
||||
/*prefix_mcp_tool_names*/ true,
|
||||
);
|
||||
manager.insert_test_client("optional", optional);
|
||||
manager.insert_test_client("required", required);
|
||||
manager.required_servers = vec![McpServerKey::unscoped("required")];
|
||||
let manager = Arc::new(manager);
|
||||
let manager_for_startup = Arc::clone(&manager);
|
||||
let startup = tokio::spawn(async move {
|
||||
manager_for_startup
|
||||
.wait_for_server_startup(&McpServerKey::unscoped("optional"))
|
||||
.await
|
||||
});
|
||||
optional_started.await.expect("optional startup began");
|
||||
let binding = capture_binding(&manager);
|
||||
tokio::pin!(binding);
|
||||
assert!(futures::poll!(&mut binding).is_pending());
|
||||
required_started.await.expect("required startup began");
|
||||
tokio::time::advance(crate::mcp::DEFAULT_OPTIONAL_MCP_STARTUP_GRACE).await;
|
||||
assert!(futures::poll!(&mut binding).is_pending());
|
||||
release_optional.send(()).expect("release optional startup");
|
||||
assert!(startup.await.expect("optional startup task"));
|
||||
release_required.send(()).expect("release required startup");
|
||||
assert_eq!(
|
||||
model_tool_names(binding.await.tools()),
|
||||
HashSet::from([ToolName::namespaced("mcp__optional", "ready_tool")])
|
||||
);
|
||||
assert!(!warning_emitted.load(Ordering::Relaxed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_all_tools_applies_legacy_mcp_prefix_by_default() {
|
||||
let managed_client =
|
||||
@@ -4374,6 +4425,7 @@ async fn list_all_tools_blocks_while_client_is_pending_without_cached_tools() {
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -4434,6 +4486,7 @@ async fn shutdown_cancels_pending_tool_listing() {
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token,
|
||||
@@ -4483,6 +4536,7 @@ async fn shutdown_continues_after_caller_is_aborted() {
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -4538,6 +4592,7 @@ async fn list_all_tools_does_not_block_when_shared_codex_apps_cache_is_empty() {
|
||||
codex_apps_tools_cache_context: Some(cache_context),
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -4595,6 +4650,7 @@ async fn list_all_tools_uses_shared_codex_apps_cache_when_client_startup_fails()
|
||||
codex_apps_tools_cache_context: Some(cache_context),
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete,
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -6312,6 +6368,7 @@ async fn reconciliation_reuses_connection_without_relisting_regular_tools() -> a
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
@@ -7160,6 +7217,7 @@ async fn reconciliation_replaces_closed_connections() -> anyhow::Result<()> {
|
||||
codex_apps_tools_cache_context: None,
|
||||
tool_catalog_cache_context: None,
|
||||
startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)),
|
||||
startup_warning_emitted: None,
|
||||
connection_slot: None,
|
||||
startup_reconnect: None,
|
||||
cancel_token: CancellationToken::new(),
|
||||
|
||||
@@ -299,6 +299,7 @@ struct ManagedClientStartup {
|
||||
catalog_item_limit: usize,
|
||||
cancel_token: CancellationToken,
|
||||
startup_complete: Arc<AtomicBool>,
|
||||
startup_warning_emitted: Option<Arc<AtomicBool>>,
|
||||
server_capabilities: Arc<StdMutex<Option<serde_json::Value>>>,
|
||||
}
|
||||
|
||||
@@ -335,6 +336,7 @@ impl ManagedClientStartup {
|
||||
catalog_item_limit,
|
||||
cancel_token,
|
||||
startup_complete,
|
||||
startup_warning_emitted,
|
||||
server_capabilities,
|
||||
} = self.clone();
|
||||
let server_name = server_key.name().to_owned();
|
||||
@@ -437,9 +439,46 @@ impl ManagedClientStartup {
|
||||
Err(StartupOutcomeError::Cancelled)
|
||||
}
|
||||
};
|
||||
// Log once per startup attempt, including discovery without startup notifications.
|
||||
if let Err(StartupOutcomeError::Failed { error, .. }) = &outcome {
|
||||
warn!(server_name, %error, "MCP server startup failed");
|
||||
// HTTP startup and the optional grace deadline share one warning gate.
|
||||
if let Err(StartupOutcomeError::Failed {
|
||||
error: detail,
|
||||
is_authentication_required,
|
||||
..
|
||||
}) = &outcome
|
||||
{
|
||||
if let Some(warning_emitted) = startup_warning_emitted {
|
||||
let status = detail
|
||||
.split_once("unexpected server response: HTTP ")
|
||||
.and_then(|(_, response)| response.split_once(':'))
|
||||
.and_then(|(status, _)| status.parse::<u16>().ok());
|
||||
let reason = if *is_authentication_required {
|
||||
"authentication required; sign in to the MCP server"
|
||||
} else if status == Some(403) {
|
||||
"HTTP 403 Forbidden"
|
||||
} else if status == Some(404) {
|
||||
"HTTP 404 Not Found"
|
||||
} else if status.is_some() {
|
||||
"HTTP response error"
|
||||
} else if detail.contains("certificate") {
|
||||
"TLS certificate error"
|
||||
} else if detail.contains("dns error") {
|
||||
"DNS lookup failed"
|
||||
} else if detail.contains("Connection refused") {
|
||||
"connection refused"
|
||||
} else if detail.contains("http/request failed") {
|
||||
"HTTP request failed"
|
||||
} else if detail.contains("timed out") {
|
||||
"startup timed out"
|
||||
} else {
|
||||
"initialization failed"
|
||||
};
|
||||
// Transport errors may contain credentials in the endpoint URL.
|
||||
if !warning_emitted.swap(true, Ordering::Relaxed) {
|
||||
warn!(target: "codex_mcp_startup", server_name = ?server_name, reason, "MCP server unavailable");
|
||||
}
|
||||
} else {
|
||||
warn!(server_name, error = %detail, "MCP server startup failed");
|
||||
}
|
||||
}
|
||||
if outcome.is_ok()
|
||||
&& let Some(refresh_start) = refresh_start
|
||||
@@ -472,6 +511,7 @@ pub(crate) struct AsyncManagedClient {
|
||||
pub(crate) codex_apps_tools_cache_context: Option<ConnectorRuntimeContext<ToolInfo>>,
|
||||
pub(crate) tool_catalog_cache_context: Option<McpToolCatalogCacheContext>,
|
||||
pub(crate) startup_complete: Arc<AtomicBool>,
|
||||
pub(crate) startup_warning_emitted: Option<Arc<AtomicBool>>,
|
||||
pub(crate) startup_reconnect: Option<Arc<CodexAppsStartupReconnect>>,
|
||||
pub(crate) connection_slot: Option<Arc<ConnectionSlot>>,
|
||||
pub(crate) cancel_token: CancellationToken,
|
||||
@@ -519,6 +559,11 @@ impl AsyncManagedClient {
|
||||
None
|
||||
};
|
||||
let startup_complete = Arc::new(AtomicBool::new(false));
|
||||
let startup_warning_emitted = matches!(
|
||||
server.config().transport,
|
||||
McpServerTransportConfig::StreamableHttp { .. }
|
||||
)
|
||||
.then(|| Arc::new(AtomicBool::new(false)));
|
||||
let server_capabilities = Arc::new(StdMutex::new(None));
|
||||
let ema_auth = ema_auth
|
||||
.filter(|_| matches!(server.config().auth, McpServerAuth::EmaAuth))
|
||||
@@ -565,6 +610,7 @@ impl AsyncManagedClient {
|
||||
catalog_item_limit,
|
||||
cancel_token: cancel_token.clone(),
|
||||
startup_complete: Arc::clone(&startup_complete),
|
||||
startup_warning_emitted: startup_warning_emitted.clone(),
|
||||
server_capabilities: Arc::clone(&server_capabilities),
|
||||
});
|
||||
let (client, connection_slot) = if let Some((pool, key)) = connection_pool {
|
||||
@@ -632,6 +678,7 @@ impl AsyncManagedClient {
|
||||
codex_apps_tools_cache_context,
|
||||
tool_catalog_cache_context: advertised_catalog,
|
||||
startup_complete,
|
||||
startup_warning_emitted,
|
||||
startup_reconnect,
|
||||
connection_slot,
|
||||
cancel_token,
|
||||
|
||||
@@ -11,6 +11,7 @@ mod event_processor;
|
||||
mod event_processor_with_human_output;
|
||||
pub(crate) mod event_processor_with_jsonl_output;
|
||||
pub(crate) mod exec_events;
|
||||
mod mcp_log_filter;
|
||||
mod remote;
|
||||
mod worktree;
|
||||
|
||||
@@ -187,7 +188,8 @@ use crate::cli::Command as ExecCommand;
|
||||
use crate::event_processor::EventProcessor;
|
||||
|
||||
const DEFAULT_ANALYTICS_ENABLED: bool = true;
|
||||
const EXEC_DEFAULT_LOG_FILTER: &str = "error,opentelemetry_sdk=off,opentelemetry_otlp=off";
|
||||
const EXEC_DEFAULT_LOG_FILTER: &str =
|
||||
"error,codex_mcp_startup=warn,opentelemetry_sdk=off,opentelemetry_otlp=off";
|
||||
|
||||
enum InitialOperation {
|
||||
ForkOnly,
|
||||
@@ -267,12 +269,16 @@ fn exec_root_span() -> tracing::Span {
|
||||
)
|
||||
}
|
||||
|
||||
fn exec_stderr_env_filter() -> EnvFilter {
|
||||
fn exec_stderr_env_filter() -> (EnvFilter, Option<mcp_log_filter::McpWorkerFilter>) {
|
||||
// OTEL export is best-effort; keep exporter self-diagnostics out of
|
||||
// headless command output unless the caller opts in with RUST_LOG.
|
||||
EnvFilter::try_from_default_env()
|
||||
.or_else(|_| EnvFilter::try_new(EXEC_DEFAULT_LOG_FILTER))
|
||||
.unwrap_or_else(|_| EnvFilter::new("error"))
|
||||
match EnvFilter::try_from_default_env() {
|
||||
Ok(filter) => (filter, None),
|
||||
Err(_) => (
|
||||
EnvFilter::try_new(EXEC_DEFAULT_LOG_FILTER).unwrap_or_else(|_| EnvFilter::new("error")),
|
||||
Some(mcp_log_filter::McpWorkerFilter),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result<()> {
|
||||
@@ -365,10 +371,12 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result
|
||||
supports_color::on_cached(Stream::Stderr).is_some(),
|
||||
),
|
||||
};
|
||||
let (stderr_env_filter, mcp_worker_filter) = exec_stderr_env_filter();
|
||||
let fmt_layer = tracing_subscriber::fmt::layer()
|
||||
.with_ansi(stderr_with_ansi)
|
||||
.with_writer(std::io::stderr)
|
||||
.with_filter(exec_stderr_env_filter());
|
||||
.with_filter(mcp_worker_filter)
|
||||
.with_filter(stderr_env_filter);
|
||||
|
||||
let sandbox_mode = if dangerously_bypass_approvals_and_sandbox {
|
||||
Some(SandboxMode::DangerFullAccess)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
//! Filters redundant HTTP MCP worker failures from default exec stderr output.
|
||||
//! Startup and tool-call owners report these failures with server attribution.
|
||||
|
||||
use std::fmt::Debug;
|
||||
use tracing::Event;
|
||||
use tracing::Metadata;
|
||||
use tracing::Subscriber;
|
||||
use tracing::field::Field;
|
||||
use tracing::field::Visit;
|
||||
use tracing_subscriber::layer::Context;
|
||||
use tracing_subscriber::layer::Filter;
|
||||
|
||||
pub(super) struct McpWorkerFilter;
|
||||
|
||||
impl<S: Subscriber> Filter<S> for McpWorkerFilter {
|
||||
fn enabled(&self, _: &Metadata<'_>, _: &Context<'_, S>) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn event_enabled(&self, event: &Event<'_>, _: &Context<'_, S>) -> bool {
|
||||
if event.metadata().target() != "rmcp::transport::worker" {
|
||||
return true;
|
||||
}
|
||||
let mut message = WorkerMessage::default();
|
||||
event.record(&mut message);
|
||||
!message.redundant_http_failure
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct WorkerMessage {
|
||||
redundant_http_failure: bool,
|
||||
}
|
||||
|
||||
impl Visit for WorkerMessage {
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn Debug) {
|
||||
if field.name() == "message" {
|
||||
let message = format!("{value:?}");
|
||||
// The SDK uses Display for notification errors, which omits the adapter variant.
|
||||
self.redundant_http_failure = message
|
||||
.strip_prefix("worker quit with fatal: Transport channel closed, when ")
|
||||
.is_some_and(|error| {
|
||||
error.starts_with("Client(HandshakeHttpRequest(")
|
||||
|| error.starts_with("UnexpectedServerResponse(")
|
||||
})
|
||||
|| (message.starts_with("worker quit with fatal: ")
|
||||
&& message.ends_with(", when send initialized notification"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,148 @@ use core_test_support::responses;
|
||||
use core_test_support::test_codex_exec::test_codex_exec;
|
||||
use predicates::str::contains;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
use std::fs;
|
||||
use std::net::TcpListener;
|
||||
use wiremock::Mock;
|
||||
use wiremock::Request;
|
||||
use wiremock::ResponseTemplate;
|
||||
use wiremock::matchers::body_partial_json;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::path;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn optional_http_mcp_failure_is_reported_once_without_endpoint_credentials()
|
||||
-> anyhow::Result<()> {
|
||||
let test = test_codex_exec();
|
||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
let address = listener.local_addr()?;
|
||||
drop(listener);
|
||||
let mcp = responses::start_mock_server().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/mcp"))
|
||||
.and(body_partial_json(json!({"method": "initialize"})))
|
||||
.respond_with(|request: &Request| {
|
||||
let body: Value = request.body_json().unwrap();
|
||||
ResponseTemplate::new(200).set_body_json(json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": body["id"],
|
||||
"result": {
|
||||
"protocolVersion": "2025-06-18",
|
||||
"capabilities": {},
|
||||
"serverInfo": {"name": "test", "version": "1"}
|
||||
}
|
||||
}))
|
||||
})
|
||||
.mount(&mcp)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/mcp"))
|
||||
.and(body_partial_json(
|
||||
json!({"method": "notifications/initialized"}),
|
||||
))
|
||||
.respond_with(ResponseTemplate::new(307).insert_header(
|
||||
"location",
|
||||
format!("http://{address}/mcp?apiKey=TEST_SECRET"),
|
||||
))
|
||||
.expect(1..)
|
||||
.mount(&mcp)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/status"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(503).set_body_string("Auth required; upstream HTTP 404"),
|
||||
)
|
||||
.mount(&mcp)
|
||||
.await;
|
||||
fs::write(
|
||||
test.home_path().join("config.toml"),
|
||||
format!(
|
||||
"mcp_optional_startup_grace_ms = 0\n\n[mcp_servers.deadhttp]\nurl = 'http://{address}/mcp?apiKey=TEST_SECRET'\n\n[mcp_servers.notification]\nurl = '{0}/mcp?apiKey=TEST_SECRET'\n\n[mcp_servers.status]\nurl = '{0}/status'\n",
|
||||
mcp.uri(),
|
||||
),
|
||||
)?;
|
||||
let server = responses::start_mock_server().await;
|
||||
responses::mount_sse_once(&server, responses::sse_completed("resp_1")).await;
|
||||
let result = test
|
||||
.cmd_with_server(&server)
|
||||
.env_remove("RUST_LOG")
|
||||
.args(["--ephemeral", "--skip-git-repo-check", "hello"])
|
||||
.assert()
|
||||
.success();
|
||||
let stderr = std::str::from_utf8(&result.get_output().stderr)?;
|
||||
let mut warnings = stderr
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
line.split_once("codex_mcp_startup: ")
|
||||
.map(|(_, warning)| warning)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
warnings.sort_unstable();
|
||||
insta::assert_snapshot!(warnings.join("\n"), @r#"
|
||||
MCP server unavailable server_name="deadhttp" reason="HTTP request failed"
|
||||
MCP server unavailable server_name="notification" reason="initialization failed"
|
||||
MCP server unavailable server_name="status" reason="HTTP response error"
|
||||
"#);
|
||||
assert!(!stderr.contains("rmcp::transport::worker"));
|
||||
assert!(!stderr.contains("TEST_SECRET"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn optional_http_mcp_pending_at_grace_is_reported() -> anyhow::Result<()> {
|
||||
let test = test_codex_exec();
|
||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
let address = listener.local_addr()?;
|
||||
drop(listener);
|
||||
fs::write(
|
||||
test.home_path().join("config.toml"),
|
||||
format!(
|
||||
"mcp_optional_startup_grace_ms = 20\n[mcp_servers.deadhttp]\nurl = 'http://{address}/mcp'\n"
|
||||
),
|
||||
)?;
|
||||
let server = responses::start_mock_server().await;
|
||||
responses::mount_sse_once(&server, responses::sse_completed("resp_1")).await;
|
||||
let result = test
|
||||
.cmd_with_server(&server)
|
||||
.env_remove("RUST_LOG")
|
||||
.args(["--ephemeral", "--skip-git-repo-check", "hello"])
|
||||
.assert()
|
||||
.success();
|
||||
let stderr = std::str::from_utf8(&result.get_output().stderr)?;
|
||||
let warnings = stderr
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
line.split_once("codex_mcp_startup: ")
|
||||
.map(|(_, warning)| warning)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
insta::assert_snapshot!(warnings.join("\n"), @"MCP server unavailable server_name=\"deadhttp\" reason=\"startup still pending; omitted from this turn\"");
|
||||
assert!(!stderr.contains("rmcp::transport::worker"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn explicit_rust_log_preserves_http_mcp_worker_diagnostics() -> anyhow::Result<()> {
|
||||
let test = test_codex_exec();
|
||||
let listener = TcpListener::bind("127.0.0.1:0")?;
|
||||
let address = listener.local_addr()?;
|
||||
drop(listener);
|
||||
fs::write(
|
||||
test.home_path().join("config.toml"),
|
||||
format!("[mcp_servers.deadhttp]\nurl = 'http://{address}/mcp'\n"),
|
||||
)?;
|
||||
let server = responses::start_mock_server().await;
|
||||
responses::mount_sse_once(&server, responses::sse_completed("resp_1")).await;
|
||||
test.cmd_with_server(&server)
|
||||
.env("RUST_LOG", "rmcp::transport::worker=trace")
|
||||
.args(["--ephemeral", "--skip-git-repo-check", "hello"])
|
||||
.assert()
|
||||
.success()
|
||||
.stderr(contains("worker quit with fatal:"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn exits_non_zero_when_required_mcp_server_fails_to_initialize() -> anyhow::Result<()> {
|
||||
|
||||
@@ -114,6 +114,8 @@ pub(crate) enum StreamableHttpClientAdapterError {
|
||||
SessionExpired404,
|
||||
#[error(transparent)]
|
||||
HttpRequest(#[from] ExecServerError),
|
||||
#[error(transparent)]
|
||||
HandshakeHttpRequest(ExecServerError),
|
||||
#[error("invalid HTTP header: {0}")]
|
||||
Header(String),
|
||||
#[error("MCP response body exceeds {maximum_bytes} bytes")]
|
||||
@@ -221,11 +223,11 @@ impl StreamableHttpClient for StreamableHttpClientAdapter {
|
||||
} else {
|
||||
self.redirect_policy(&headers)
|
||||
};
|
||||
let timeout_ms = if matches!(
|
||||
let is_handshake_request = matches!(
|
||||
mcp_method.as_deref(),
|
||||
Some("initialize" | "notifications/initialized")
|
||||
) || mcp_method.as_deref() == Some(DiscoverRequestMethod::VALUE)
|
||||
{
|
||||
) || is_discovery_request;
|
||||
let timeout_ms = if is_handshake_request {
|
||||
self.initialize_deadline
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
@@ -291,9 +293,12 @@ impl StreamableHttpClient for StreamableHttpClientAdapter {
|
||||
has_session_id,
|
||||
has_authorization_header,
|
||||
);
|
||||
return Err(StreamableHttpError::Client(
|
||||
StreamableHttpClientAdapterError::from(error),
|
||||
));
|
||||
let error = if is_handshake_request {
|
||||
StreamableHttpClientAdapterError::HandshakeHttpRequest(error)
|
||||
} else {
|
||||
StreamableHttpClientAdapterError::HttpRequest(error)
|
||||
};
|
||||
return Err(StreamableHttpError::Client(error));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1863,7 +1863,13 @@ impl RmcpClient {
|
||||
{
|
||||
Ok(result) => Ok(result),
|
||||
Err(error) if Self::is_session_expired_404(&error) => {
|
||||
self.reinitialize_after_session_expiry(&service).await?;
|
||||
self.reinitialize_after_session_expiry(&service)
|
||||
.await
|
||||
.inspect_err(|_| {
|
||||
if let TransportRecipe::StreamableHttp { server_name, .. } = &self.transport_recipe {
|
||||
tracing::error!(server = ?server_name, operation = label, "MCP operation failed during session recovery");
|
||||
}
|
||||
})?;
|
||||
let recovered_service = self.service().await?;
|
||||
Self::run_service_operation_with_transient_retries(
|
||||
recovered_service,
|
||||
|
||||
@@ -77,9 +77,10 @@ fn transport_error_requires_authentication(error: &DynamicTransportError) -> boo
|
||||
.is_some_and(|error| match error {
|
||||
StreamableHttpError::AuthRequired(_) => true,
|
||||
StreamableHttpError::Auth(auth_error) => auth_error_requires_authentication(auth_error),
|
||||
StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest(error)) => {
|
||||
exec_server_error_requires_authentication(error)
|
||||
}
|
||||
StreamableHttpError::Client(
|
||||
StreamableHttpClientAdapterError::HttpRequest(error)
|
||||
| StreamableHttpClientAdapterError::HandshakeHttpRequest(error),
|
||||
) => exec_server_error_requires_authentication(error),
|
||||
_ => false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -155,17 +155,20 @@ impl RmcpClient {
|
||||
StreamableHttpError::Client(StreamableHttpClientAdapterError::Auth(
|
||||
codex_api::AuthError::Transient(_),
|
||||
)) => true,
|
||||
StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest(
|
||||
ExecServerError::HttpRequest(_),
|
||||
)) => true,
|
||||
StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest(
|
||||
ExecServerError::Server { code, message },
|
||||
)) => {
|
||||
*code == JSON_RPC_INTERNAL_ERROR_CODE && message.starts_with("http/request failed:")
|
||||
}
|
||||
StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest(
|
||||
ExecServerError::Protocol(message),
|
||||
)) => message.starts_with("http response stream `") && message.contains("` failed:"),
|
||||
StreamableHttpError::Client(
|
||||
StreamableHttpClientAdapterError::HttpRequest(error)
|
||||
| StreamableHttpClientAdapterError::HandshakeHttpRequest(error),
|
||||
) => match error {
|
||||
ExecServerError::HttpRequest(_) => true,
|
||||
ExecServerError::Server { code, message } => {
|
||||
*code == JSON_RPC_INTERNAL_ERROR_CODE
|
||||
&& message.starts_with("http/request failed:")
|
||||
}
|
||||
ExecServerError::Protocol(message) => {
|
||||
message.starts_with("http response stream `") && message.contains("` failed:")
|
||||
}
|
||||
_ => false,
|
||||
},
|
||||
StreamableHttpError::UnexpectedServerResponse(message) => {
|
||||
is_retryable_unexpected_server_response(message.as_ref())
|
||||
}
|
||||
|
||||
@@ -111,9 +111,9 @@ fn retryable_initialize_error(context: &'static str) -> rmcp::service::ClientIni
|
||||
"streamable_http",
|
||||
TypeId::of::<()>(),
|
||||
Box::new(StreamableHttpError::Client(
|
||||
StreamableHttpClientAdapterError::HttpRequest(ExecServerError::HttpRequest(
|
||||
"error sending request for url".to_string(),
|
||||
)),
|
||||
StreamableHttpClientAdapterError::HandshakeHttpRequest(
|
||||
ExecServerError::HttpRequest("error sending request for url".to_string()),
|
||||
),
|
||||
)),
|
||||
),
|
||||
context: context.into(),
|
||||
|
||||
Reference in New Issue
Block a user