Collect metrics from plugin shell commands (#38252)

## What changed

- Provide matching local plugin commands with a sandbox-writable temporary output file through `CODEX_PLUGIN_METRICS_OUTPUT` when analytics is enabled.
- Validate successful command output against the plugin's `analytics.yaml` declaration, including measurement names, enum dimensions, finite values, duplicate rows, and size limits, before publishing analytics events.
- Keep the output path reserved from user overrides and clean up the temporary file after execution.

## Testing

- Cover output validation, limits, cleanup, sandbox permissions, environment handling, and path replacement.
- Verify measurement collection through both classic and zsh-fork shell runtimes.

GitOrigin-RevId: 88af0f87dc2f207fcbcca6af498f5c940d79349d

codex-rs/core/src/plugins/metrics.rs

@@ -0,0 +1,32 @@
+use crate::session::session::Session;
+use crate::session::turn_context::TurnContext;
+use codex_analytics::PluginMeasurementsInput;
+use codex_core_plugins::PluginMetricsSidecar;
+
+/// Finishes a metrics sidecar and publishes any valid rows.
+pub(crate) fn finish_and_track_measurements(
+    metrics_sidecar: Option<PluginMetricsSidecar>,
+    exit_code: i32,
+    session: &Session,
+    turn: &TurnContext,
+    item_id: &str,
+) {
+    let Some(metrics_sidecar) = metrics_sidecar else {
+        return;
+    };
+    let Some(batch) = metrics_sidecar.finish(exit_code) else {
+        return;
+    };
+    session
+        .services
+        .analytics_events_client
+        .track_plugin_measurements(PluginMeasurementsInput {
+            thread_id: session.thread_id().to_string(),
+            turn_id: turn.sub_id.clone(),
+            item_id: item_id.to_string(),
+            plugin_id: batch.plugin_id,
+            execution_id: batch.execution_id,
+            operation: batch.operation,
+            rows: batch.rows,
+        });
+}

codex-rs/core/src/plugins/mod.rs

@@ -1,6 +1,7 @@
 mod discoverable;
 mod injection;
 mod mentions;
+pub(crate) mod metrics;
 mod render;
 #[cfg(test)]
 #[path = "skill_snapshot_tests.rs"]

codex-rs/core/src/session/turn_context.rs

@@ -4,6 +4,7 @@ use crate::exec_policy::AllowPrefixRules;
 use crate::shell_snapshot::ShellSnapshotFile;
 use crate::tools::sandboxing::executor_windows_sandbox_level;
 use codex_core_plugins::PluginCommandAttribution;
+use codex_core_plugins::ResolvedPluginMetricsOperation;
 use codex_core_plugins::TrustedPluginRoots;
 use codex_exec_server::ExecutorFileSystem;
 use codex_file_system::FileSystemSandboxContext;
@@ -242,6 +243,16 @@ impl TurnContext {
         }
     }
 
+    pub(crate) fn plugin_metrics_operation_for_command(
+        &self,
+        command: &[String],
+        cwd: &AbsolutePathBuf,
+    ) -> Option<ResolvedPluginMetricsOperation> {
+        self.extension_data
+            .get::<TrustedPluginRoots>()?
+            .resolve_metrics_operation(command, cwd)
+    }
+
     pub(crate) fn permission_profile(&self) -> PermissionProfile {
         self.config.permissions.effective_permission_profile()
     }

codex-rs/core/src/tools/handlers/shell.rs

@@ -24,6 +24,7 @@ use crate::tools::runtimes::shell::ShellRequest;
 use crate::tools::runtimes::shell::ShellRuntime;
 use crate::tools::runtimes::shell::ShellRuntimeBackend;
 use crate::tools::sandboxing::ToolCtx;
+use codex_core_plugins::strip_output_env;
 use codex_protocol::models::AdditionalPermissionProfile;
 use codex_protocol::protocol::ExecCommandSource;
 use codex_tools::ToolName;
@@ -80,12 +81,15 @@ async fn run_exec_like(args: RunExecLikeArgs) -> Result<FunctionToolOutput, Func
 
     let fs = turn_environment.environment.get_filesystem();
 
-    let explicit_env_overrides = turn
+    let mut explicit_env_overrides = turn
         .config
         .permissions
         .shell_environment_policy
         .r#set
         .clone();
+    let mut env = exec_params.env.clone();
+    strip_output_env(&mut env);
+    strip_output_env(&mut explicit_env_overrides);
     let exec_permission_approvals_enabled =
         session.features().enabled(Feature::ExecPermissionApprovals);
     let requested_additional_permissions = additional_permissions.clone();
@@ -199,7 +203,7 @@ async fn run_exec_like(args: RunExecLikeArgs) -> Result<FunctionToolOutput, Func
         cwd: exec_params.cwd.clone(),
         timeout_ms: exec_params.expiration.timeout_ms(),
         cancellation_token,
-        env: exec_params.env.clone(),
+        env,
         explicit_env_overrides,
         network: exec_params.network.clone(),
         sandbox_permissions: effective_additional_permissions.sandbox_permissions,

codex-rs/core/src/tools/runtimes/mod.rs

@@ -12,6 +12,7 @@ use crate::shell::Shell;
 use crate::shell::ShellType;
 use crate::tools::sandboxing::ToolError;
 use codex_apply_patch::CODEX_APPLY_PATCH_PRESERVE_LINE_ENDINGS_ENV_VAR;
+use codex_core_plugins::PLUGIN_METRICS_OUTPUT_ENV_VAR;
 #[cfg(unix)]
 use codex_install_context::InstallContext;
 #[cfg(target_os = "macos")]
@@ -274,6 +275,7 @@ pub(crate) fn maybe_wrap_shell_lc_with_snapshot(
         CODEX_THREAD_ID_ENV_VAR,
         CODEX_PERMISSION_PROFILE_ENV_VAR,
         CODEX_APPLY_PATCH_PRESERVE_LINE_ENDINGS_ENV_VAR,
+        PLUGIN_METRICS_OUTPUT_ENV_VAR,
     ] {
         if let Some(value) = env.get(key) {
             override_env.insert(key.to_string(), value.clone());
@@ -285,6 +287,7 @@ pub(crate) fn maybe_wrap_shell_lc_with_snapshot(
         &[
             CODEX_PERMISSION_PROFILE_ENV_VAR,
             CODEX_APPLY_PATCH_PRESERVE_LINE_ENDINGS_ENV_VAR,
+            PLUGIN_METRICS_OUTPUT_ENV_VAR,
         ],
     );
     let (proxy_captures, proxy_exports) = build_proxy_env_exports();

codex-rs/core/src/tools/runtimes/mod_tests.rs

@@ -662,6 +662,50 @@ fn maybe_wrap_shell_lc_with_snapshot_restores_apply_patch_rollout_state() {
     assert_eq!(output.stdout, b"");
 }
 
+#[test]
+fn maybe_wrap_shell_lc_with_snapshot_restores_reserved_metrics_output_env() {
+    let dir = tempdir().expect("create temp dir");
+    let snapshot_path = dir.path().join("snapshot.sh");
+    std::fs::write(
+        &snapshot_path,
+        "# Snapshot file\nexport CODEX_PLUGIN_METRICS_OUTPUT='/stale/path'\n",
+    )
+    .expect("write snapshot");
+    let (session_shell, shell_snapshot) =
+        shell_with_snapshot(ShellType::Bash, "/bin/bash", snapshot_path.abs());
+    let command = vec![
+        "/bin/bash".to_string(),
+        "-lc".to_string(),
+        "printf '%s' \"${CODEX_PLUGIN_METRICS_OUTPUT-unset}\"".to_string(),
+    ];
+
+    for (live_value, expected) in [(None, "unset"), (Some("/private/path"), "/private/path")] {
+        let env = live_value
+            .map(|value| {
+                HashMap::from([(PLUGIN_METRICS_OUTPUT_ENV_VAR.to_string(), value.to_string())])
+            })
+            .unwrap_or_default();
+        let rewritten = maybe_wrap_shell_lc_with_snapshot(
+            &command,
+            &session_shell,
+            Some(&shell_snapshot),
+            &HashMap::new(),
+            &env,
+            &RuntimePathPrepends::default(),
+        );
+        let mut process = Command::new(&rewritten[0]);
+        process.args(&rewritten[1..]);
+        match live_value {
+            Some(value) => process.env(PLUGIN_METRICS_OUTPUT_ENV_VAR, value),
+            None => process.env_remove(PLUGIN_METRICS_OUTPUT_ENV_VAR),
+        };
+        let output = process.output().expect("run rewritten command");
+
+        assert!(output.status.success(), "command failed: {output:?}");
+        assert_eq!(String::from_utf8_lossy(&output.stdout), expected);
+    }
+}
+
 #[test]
 fn maybe_wrap_shell_lc_with_snapshot_restores_proxy_env_from_process_env() {
     let dir = tempdir().expect("create temp dir");

codex-rs/core/src/tools/runtimes/shell.rs

@@ -10,6 +10,7 @@ pub(crate) mod zsh_fork_backend;
 
 use crate::exec::ExecCapturePolicy;
 use crate::guardian::GuardianNetworkAccessTrigger;
+use crate::plugins::metrics::finish_and_track_measurements;
 use crate::sandboxing::ExecOptions;
 use crate::sandboxing::SandboxPermissions;
 use crate::sandboxing::execute_env;
@@ -35,10 +36,12 @@ use crate::tools::sandboxing::ToolError;
 use crate::tools::sandboxing::ToolRuntime;
 use crate::tools::sandboxing::managed_network_for_sandbox_permissions;
 use crate::tools::sandboxing::sandbox_permissions_preserving_denied_reads;
+use codex_core_plugins::PluginMetricsSidecar;
 use codex_network_proxy::NetworkProxy;
 use codex_protocol::exec_output::ExecToolCallOutput;
 use codex_protocol::models::AdditionalPermissionProfile;
 use codex_sandboxing::SandboxablePreference;
+use codex_sandboxing::policy_transforms::merge_permission_profiles;
 use codex_shell_command::powershell::prefix_powershell_script_with_utf8;
 use codex_utils_absolute_path::AbsolutePathBuf;
 use codex_utils_path_uri::PathUri;
@@ -207,8 +210,20 @@ impl ToolRuntime<ShellRequest, ExecToolCallOutput> for ShellRuntime {
         );
         let managed_network =
             managed_network_for_sandbox_permissions(req.network.as_ref(), sandbox_permissions);
-        let env = exec_env_for_sandbox_permissions(&req.env, sandbox_permissions);
+        let mut env = exec_env_for_sandbox_permissions(&req.env, sandbox_permissions);
         let explicit_env_overrides = req.explicit_env_overrides.clone();
+        let metrics_sidecar = (!req.turn_environment.environment.is_remote()
+            && ctx.session.services.analytics_events_client.is_enabled())
+        .then(|| {
+            ctx.step_context
+                .turn
+                .plugin_metrics_operation_for_command(&req.command, &req.cwd)
+        })
+        .flatten()
+        .and_then(PluginMetricsSidecar::create);
+        if let Some(sidecar) = metrics_sidecar.as_ref() {
+            sidecar.install_output_env(&mut env);
+        }
         #[cfg(unix)]
         let (env, runtime_path_prepends) = {
             let mut env = env;
@@ -246,39 +261,66 @@ impl ToolRuntime<ShellRequest, ExecToolCallOutput> for ShellRuntime {
             command
         };
 
-        if self.backend == ShellRuntimeBackend::ShellCommandZshFork {
-            match zsh_fork_backend::maybe_run_shell_command(req, attempt, ctx, &command).await? {
-                Some(out) => return Ok(out),
+        let zsh_fork_output = if self.backend == ShellRuntimeBackend::ShellCommandZshFork {
+            match zsh_fork_backend::maybe_run_shell_command(
+                req,
+                attempt,
+                ctx,
+                &command,
+                metrics_sidecar.as_ref(),
+            )
+            .await?
+            {
+                Some(out) => Some(out),
                 None => {
                     tracing::warn!(
                         "ZshFork backend specified, but conditions for using it were not met, falling back to normal execution",
                     );
+                    None
                 }
             }
-        }
-
-        let command =
-            build_sandbox_command(&command, &req.cwd, &env, req.additional_permissions.clone())?;
-        let mut expiration: crate::exec::ExecExpiration = req.timeout_ms.into();
-        expiration = expiration.with_cancellation(req.cancellation_token.clone());
-        if let Some(cancellation) = attempt.network_denial_cancellation_token.clone() {
-            expiration = expiration.with_cancellation(cancellation);
-        }
-        let options = ExecOptions {
-            expiration,
-            capture_policy: ExecCapturePolicy::ShellTool,
+        } else {
+            None
         };
-        let env = attempt
-            .env_for(
-                command,
-                options,
-                managed_network,
-                Some(&req.turn_environment.environment_id),
-            )
-            .map_err(ToolError::Codex)?;
-        let out = execute_env(env, Self::stdout_stream(ctx))
-            .await
-            .map_err(ToolError::Codex)?;
+        let out = if let Some(out) = zsh_fork_output {
+            out
+        } else {
+            let sidecar_permissions = metrics_sidecar
+                .as_ref()
+                .map(PluginMetricsSidecar::additional_permissions);
+            let additional_permissions = merge_permission_profiles(
+                req.additional_permissions.as_ref(),
+                sidecar_permissions.as_ref(),
+            );
+            let command = build_sandbox_command(&command, &req.cwd, &env, additional_permissions)?;
+            let mut expiration: crate::exec::ExecExpiration = req.timeout_ms.into();
+            expiration = expiration.with_cancellation(req.cancellation_token.clone());
+            if let Some(cancellation) = attempt.network_denial_cancellation_token.clone() {
+                expiration = expiration.with_cancellation(cancellation);
+            }
+            let options = ExecOptions {
+                expiration,
+                capture_policy: ExecCapturePolicy::ShellTool,
+            };
+            let env = attempt
+                .env_for(
+                    command,
+                    options,
+                    managed_network,
+                    Some(&req.turn_environment.environment_id),
+                )
+                .map_err(ToolError::Codex)?;
+            execute_env(env, Self::stdout_stream(ctx))
+                .await
+                .map_err(ToolError::Codex)?
+        };
+        finish_and_track_measurements(
+            metrics_sidecar,
+            out.exit_code,
+            &ctx.session,
+            &ctx.step_context.turn,
+            &ctx.call_id,
+        );
         Ok(out)
     }
 }

codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs

@@ -19,6 +19,7 @@ use crate::tools::sandboxing::ToolError;
 use crate::tools::sandboxing::managed_network_for_sandbox_permissions;
 use crate::tools::sandboxing::sandbox_permissions_preserving_denied_reads;
 use crate::tools::sandboxing::unsandboxed_execution_allowed;
+use codex_core_plugins::PluginMetricsSidecar;
 use codex_execpolicy::Decision;
 use codex_execpolicy::Evaluation;
 use codex_execpolicy::MatchOptions;
@@ -41,6 +42,7 @@ use codex_sandboxing::SandboxManager;
 use codex_sandboxing::SandboxTransformRequest;
 use codex_sandboxing::SandboxType;
 use codex_sandboxing::SandboxablePreference;
+use codex_sandboxing::policy_transforms::merge_permission_profiles;
 use codex_sandboxing::record_filesystem_sandbox_violation;
 use codex_shell_command::bash::parse_shell_lc_plain_commands;
 use codex_shell_command::bash::parse_shell_lc_single_command_prefix;
@@ -103,6 +105,7 @@ pub(super) async fn try_run_zsh_fork(
     attempt: &SandboxAttempt<'_>,
     ctx: &ToolCtx,
     command: &[String],
+    metrics_sidecar: Option<&PluginMetricsSidecar>,
 ) -> Result<Option<ExecToolCallOutput>, ToolError> {
     let Some(shell_zsh_path) = ctx.session.services.shell_zsh_path.as_ref() else {
         tracing::warn!("ZshFork backend specified, but shell_zsh_path is not configured.");
@@ -127,9 +130,16 @@ pub(super) async fn try_run_zsh_fork(
         ..req.clone()
     };
     let mut env = exec_env_for_sandbox_permissions(&req.env, req.sandbox_permissions);
+    if let Some(sidecar) = metrics_sidecar {
+        sidecar.install_output_env(&mut env);
+    }
     prepend_zsh_fork_bin_to_path(&mut env, shell_zsh_path);
-    let command =
-        build_sandbox_command(command, &req.cwd, &env, req.additional_permissions.clone())?;
+    let sidecar_permissions = metrics_sidecar.map(PluginMetricsSidecar::additional_permissions);
+    let additional_permissions = merge_permission_profiles(
+        req.additional_permissions.as_ref(),
+        sidecar_permissions.as_ref(),
+    );
+    let command = build_sandbox_command(command, &req.cwd, &env, additional_permissions)?;
     let options = ExecOptions {
         expiration: req.timeout_ms.into(),
         capture_policy: ExecCapturePolicy::ShellTool,

codex-rs/core/src/tools/runtimes/shell/zsh_fork_backend.rs

@@ -5,6 +5,7 @@ use crate::tools::sandboxing::SandboxAttempt;
 use crate::tools::sandboxing::ToolCtx;
 use crate::tools::sandboxing::ToolError;
 use crate::unified_exec::SpawnLifecycleHandle;
+use codex_core_plugins::PluginMetricsSidecar;
 use codex_protocol::exec_output::ExecToolCallOutput;
 use codex_tools::ZshForkConfig;
 
@@ -23,8 +24,9 @@ pub(crate) async fn maybe_run_shell_command(
     attempt: &SandboxAttempt<'_>,
     ctx: &ToolCtx,
     command: &[String],
+    metrics_sidecar: Option<&PluginMetricsSidecar>,
 ) -> Result<Option<ExecToolCallOutput>, ToolError> {
-    imp::maybe_run_shell_command(req, attempt, ctx, command).await
+    imp::maybe_run_shell_command(req, attempt, ctx, command, metrics_sidecar).await
 }
 
 /// Prepares unified exec to launch through the zsh-fork backend when the
@@ -76,8 +78,9 @@ mod imp {
         attempt: &SandboxAttempt<'_>,
         ctx: &ToolCtx,
         command: &[String],
+        metrics_sidecar: Option<&PluginMetricsSidecar>,
     ) -> Result<Option<ExecToolCallOutput>, ToolError> {
-        unix_escalation::try_run_zsh_fork(req, attempt, ctx, command).await
+        unix_escalation::try_run_zsh_fork(req, attempt, ctx, command, metrics_sidecar).await
     }
 
     pub(super) async fn maybe_prepare_unified_exec(
@@ -118,8 +121,9 @@ mod imp {
         attempt: &SandboxAttempt<'_>,
         ctx: &ToolCtx,
         command: &[String],
+        metrics_sidecar: Option<&PluginMetricsSidecar>,
     ) -> Result<Option<ExecToolCallOutput>, ToolError> {
-        let _ = (req, attempt, ctx, command);
+        let _ = (req, attempt, ctx, command, metrics_sidecar);
         Ok(None)
     }