Refactor execpolicy fallback evaluation (#7544)

## Refactor of the `execpolicy` crate

To illustrate why we need this refactor, consider an agent attempting to
run `apple | rm -rf ./`. Suppose `apple` is allowed by `execpolicy`.
Before this PR, `execpolicy` would consider `apple` and `pear` and only
render one rule match: `Allow`. We would skip any heuristics checks on
`rm -rf ./` and immediately approve `apple | rm -rf ./` to run.

To fix this, we now thread a `fallback` evaluation function into
`execpolicy` that runs when no `execpolicy` rules match a given command.
In our example, we would run `fallback` on `rm -rf ./` and prevent
`apple | rm -rf ./` from being run without approval.

codex-rs/execpolicy/README.md

@@ -30,32 +30,24 @@ codex execpolicy check --policy path/to/policy.codexpolicy git status
 cargo run -p codex-execpolicy -- check --policy path/to/policy.codexpolicy git status
 ```
 - Example outcomes:
-  - Match: `{"match": { ... "decision": "allow" ... }}`
-  - No match: `{"noMatch": {}}`
+  - Match: `{"matchedRules":[{...}],"decision":"allow"}`
+  - No match: `{"matchedRules":[]}`
 
-## Response shapes
-- Match:
+## Response shape
 ```json
 {
-  "match": {
-    "decision": "allow|prompt|forbidden",
-    "matchedRules": [
-      {
-        "prefixRuleMatch": {
-          "matchedPrefix": ["<token>", "..."],
-          "decision": "allow|prompt|forbidden"
-        }
+  "matchedRules": [
+    {
+      "prefixRuleMatch": {
+        "matchedPrefix": ["<token>", "..."],
+        "decision": "allow|prompt|forbidden"
       }
-    ]
-  }
+    }
+  ],
+  "decision": "allow|prompt|forbidden"
 }
 ```
-
-- No match:
-```json
-{"noMatch": {}}
-```
-
+- When no rules match, `matchedRules` is an empty array and `decision` is omitted.
 - `matchedRules` lists every rule whose prefix matched the command; `matchedPrefix` is the exact prefix that matched.
 - The effective `decision` is the strictest severity across all matches (`forbidden` > `prompt` > `allow`).
 

codex-rs/execpolicy/src/execpolicycheck.rs

@@ -4,10 +4,12 @@ use std::path::PathBuf;
 use anyhow::Context;
 use anyhow::Result;
 use clap::Parser;
+use serde::Serialize;
 
-use crate::Evaluation;
+use crate::Decision;
 use crate::Policy;
 use crate::PolicyParser;
+use crate::RuleMatch;
 
 /// Arguments for evaluating a command against one or more execpolicy files.
 #[derive(Debug, Parser, Clone)]
@@ -34,20 +36,25 @@ impl ExecPolicyCheckCommand {
     /// Load the policies for this command, evaluate the command, and render JSON output.
     pub fn run(&self) -> Result<()> {
         let policy = load_policies(&self.policies)?;
-        let evaluation = policy.check(&self.command);
+        let matched_rules = policy.matches_for_command(&self.command, None);
 
-        let json = format_evaluation_json(&evaluation, self.pretty)?;
+        let json = format_matches_json(&matched_rules, self.pretty)?;
         println!("{json}");
 
         Ok(())
     }
 }
 
-pub fn format_evaluation_json(evaluation: &Evaluation, pretty: bool) -> Result<String> {
+pub fn format_matches_json(matched_rules: &[RuleMatch], pretty: bool) -> Result<String> {
+    let output = ExecPolicyCheckOutput {
+        matched_rules,
+        decision: matched_rules.iter().map(RuleMatch::decision).max(),
+    };
+
     if pretty {
-        serde_json::to_string_pretty(evaluation).map_err(Into::into)
+        serde_json::to_string_pretty(&output).map_err(Into::into)
     } else {
-        serde_json::to_string(evaluation).map_err(Into::into)
+        serde_json::to_string(&output).map_err(Into::into)
     }
 }
 
@@ -65,3 +72,12 @@ pub fn load_policies(policy_paths: &[PathBuf]) -> Result<Policy> {
 
     Ok(parser.build())
 }
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ExecPolicyCheckOutput<'a> {
+    #[serde(rename = "matchedRules")]
+    matched_rules: &'a [RuleMatch],
+    #[serde(skip_serializing_if = "Option::is_none")]
+    decision: Option<Decision>,
+}

codex-rs/execpolicy/src/main.rs

@@ -1,6 +1,6 @@
 use anyhow::Result;
 use clap::Parser;
-use codex_execpolicy::ExecPolicyCheckCommand;
+use codex_execpolicy::execpolicycheck::ExecPolicyCheckCommand;
 
 /// CLI for evaluating exec policies
 #[derive(Parser)]
@@ -13,10 +13,6 @@ enum Cli {
 fn main() -> Result<()> {
     let cli = Cli::parse();
     match cli {
-        Cli::Check(cmd) => cmd_check(cmd),
+        Cli::Check(cmd) => cmd.run(),
     }
 }
-
-fn cmd_check(cmd: ExecPolicyCheckCommand) -> Result<()> {
-    cmd.run()
-}

codex-rs/execpolicy/src/policy.rs

@@ -11,6 +11,8 @@ use serde::Deserialize;
 use serde::Serialize;
 use std::sync::Arc;
 
+type HeuristicsFallback<'a> = Option<&'a dyn Fn(&[String]) -> Decision>;
+
 #[derive(Clone, Debug)]
 pub struct Policy {
     rules_by_program: MultiMap<String, RuleRef>,
@@ -50,62 +52,84 @@ impl Policy {
         Ok(())
     }
 
-    pub fn check(&self, cmd: &[String]) -> Evaluation {
-        let rules = match cmd.first() {
-            Some(first) => match self.rules_by_program.get_vec(first) {
-                Some(rules) => rules,
-                None => return Evaluation::NoMatch {},
-            },
-            None => return Evaluation::NoMatch {},
-        };
-
-        let matched_rules: Vec<RuleMatch> =
-            rules.iter().filter_map(|rule| rule.matches(cmd)).collect();
-        match matched_rules.iter().map(RuleMatch::decision).max() {
-            Some(decision) => Evaluation::Match {
-                decision,
-                matched_rules,
-            },
-            None => Evaluation::NoMatch {},
-        }
+    pub fn check<F>(&self, cmd: &[String], heuristics_fallback: &F) -> Evaluation
+    where
+        F: Fn(&[String]) -> Decision,
+    {
+        let matched_rules = self.matches_for_command(cmd, Some(heuristics_fallback));
+        Evaluation::from_matches(matched_rules)
     }
 
-    pub fn check_multiple<Commands>(&self, commands: Commands) -> Evaluation
+    pub fn check_multiple<Commands, F>(
+        &self,
+        commands: Commands,
+        heuristics_fallback: &F,
+    ) -> Evaluation
     where
         Commands: IntoIterator,
         Commands::Item: AsRef<[String]>,
+        F: Fn(&[String]) -> Decision,
     {
         let matched_rules: Vec<RuleMatch> = commands
             .into_iter()
-            .flat_map(|command| match self.check(command.as_ref()) {
-                Evaluation::Match { matched_rules, .. } => matched_rules,
-                Evaluation::NoMatch { .. } => Vec::new(),
+            .flat_map(|command| {
+                self.matches_for_command(command.as_ref(), Some(heuristics_fallback))
             })
             .collect();
 
-        match matched_rules.iter().map(RuleMatch::decision).max() {
-            Some(decision) => Evaluation::Match {
-                decision,
-                matched_rules,
-            },
-            None => Evaluation::NoMatch {},
+        Evaluation::from_matches(matched_rules)
+    }
+
+    pub fn matches_for_command(
+        &self,
+        cmd: &[String],
+        heuristics_fallback: HeuristicsFallback<'_>,
+    ) -> Vec<RuleMatch> {
+        let mut matched_rules: Vec<RuleMatch> = match cmd.first() {
+            Some(first) => self
+                .rules_by_program
+                .get_vec(first)
+                .map(|rules| rules.iter().filter_map(|rule| rule.matches(cmd)).collect())
+                .unwrap_or_default(),
+            None => Vec::new(),
+        };
+
+        if let (true, Some(heuristics_fallback)) = (matched_rules.is_empty(), heuristics_fallback) {
+            matched_rules.push(RuleMatch::HeuristicsRuleMatch {
+                command: cmd.to_vec(),
+                decision: heuristics_fallback(cmd),
+            });
         }
+
+        matched_rules
     }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub enum Evaluation {
-    NoMatch {},
-    Match {
-        decision: Decision,
-        #[serde(rename = "matchedRules")]
-        matched_rules: Vec<RuleMatch>,
-    },
+pub struct Evaluation {
+    pub decision: Decision,
+    #[serde(rename = "matchedRules")]
+    pub matched_rules: Vec<RuleMatch>,
 }
 
 impl Evaluation {
     pub fn is_match(&self) -> bool {
-        matches!(self, Self::Match { .. })
+        self.matched_rules
+            .iter()
+            .any(|rule_match| !matches!(rule_match, RuleMatch::HeuristicsRuleMatch { .. }))
+    }
+
+    fn from_matches(matched_rules: Vec<RuleMatch>) -> Self {
+        let decision = matched_rules
+            .iter()
+            .map(RuleMatch::decision)
+            .max()
+            .unwrap_or(Decision::Allow);
+
+        Self {
+            decision,
+            matched_rules,
+        }
     }
 }

codex-rs/execpolicy/src/rule.rs

@@ -64,12 +64,17 @@ pub enum RuleMatch {
         matched_prefix: Vec<String>,
         decision: Decision,
     },
+    HeuristicsRuleMatch {
+        command: Vec<String>,
+        decision: Decision,
+    },
 }
 
 impl RuleMatch {
     pub fn decision(&self) -> Decision {
         match self {
             Self::PrefixRuleMatch { decision, .. } => *decision,
+            Self::HeuristicsRuleMatch { decision, .. } => *decision,
         }
     }
 }

codex-rs/execpolicy/tests/basic.rs

@@ -19,6 +19,14 @@ fn tokens(cmd: &[&str]) -> Vec<String> {
     cmd.iter().map(std::string::ToString::to_string).collect()
 }
 
+fn allow_all(_: &[String]) -> Decision {
+    Decision::Allow
+}
+
+fn prompt_all(_: &[String]) -> Decision {
+    Decision::Prompt
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 enum RuleSnapshot {
     Prefix(PrefixRule),
@@ -49,9 +57,9 @@ prefix_rule(
     parser.parse("test.codexpolicy", policy_src)?;
     let policy = parser.build();
     let cmd = tokens(&["git", "status"]);
-    let evaluation = policy.check(&cmd);
+    let evaluation = policy.check(&cmd, &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Allow,
             matched_rules: vec![RuleMatch::PrefixRuleMatch {
                 matched_prefix: tokens(&["git", "status"]),
@@ -80,9 +88,9 @@ fn add_prefix_rule_extends_policy() -> Result<()> {
         rules
     );
 
-    let evaluation = policy.check(&tokens(&["ls", "-l", "/tmp"]));
+    let evaluation = policy.check(&tokens(&["ls", "-l", "/tmp"]), &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Prompt,
             matched_rules: vec![RuleMatch::PrefixRuleMatch {
                 matched_prefix: tokens(&["ls", "-l"]),
@@ -146,9 +154,9 @@ prefix_rule(
         git_rules
     );
 
-    let status_eval = policy.check(&tokens(&["git", "status"]));
+    let status_eval = policy.check(&tokens(&["git", "status"]), &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Prompt,
             matched_rules: vec![RuleMatch::PrefixRuleMatch {
                 matched_prefix: tokens(&["git"]),
@@ -158,9 +166,9 @@ prefix_rule(
         status_eval
     );
 
-    let commit_eval = policy.check(&tokens(&["git", "commit", "-m", "hi"]));
+    let commit_eval = policy.check(&tokens(&["git", "commit", "-m", "hi"]), &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Forbidden,
             matched_rules: vec![
                 RuleMatch::PrefixRuleMatch {
@@ -217,9 +225,9 @@ prefix_rule(
         sh_rules
     );
 
-    let bash_eval = policy.check(&tokens(&["bash", "-c", "echo", "hi"]));
+    let bash_eval = policy.check(&tokens(&["bash", "-c", "echo", "hi"]), &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Allow,
             matched_rules: vec![RuleMatch::PrefixRuleMatch {
                 matched_prefix: tokens(&["bash", "-c"]),
@@ -229,9 +237,9 @@ prefix_rule(
         bash_eval
     );
 
-    let sh_eval = policy.check(&tokens(&["sh", "-l", "echo", "hi"]));
+    let sh_eval = policy.check(&tokens(&["sh", "-l", "echo", "hi"]), &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Allow,
             matched_rules: vec![RuleMatch::PrefixRuleMatch {
                 matched_prefix: tokens(&["sh", "-l"]),
@@ -273,9 +281,9 @@ prefix_rule(
         rules
     );
 
-    let npm_i = policy.check(&tokens(&["npm", "i", "--legacy-peer-deps"]));
+    let npm_i = policy.check(&tokens(&["npm", "i", "--legacy-peer-deps"]), &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Allow,
             matched_rules: vec![RuleMatch::PrefixRuleMatch {
                 matched_prefix: tokens(&["npm", "i", "--legacy-peer-deps"]),
@@ -285,9 +293,12 @@ prefix_rule(
         npm_i
     );
 
-    let npm_install = policy.check(&tokens(&["npm", "install", "--no-save", "leftpad"]));
+    let npm_install = policy.check(
+        &tokens(&["npm", "install", "--no-save", "leftpad"]),
+        &allow_all,
+    );
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Allow,
             matched_rules: vec![RuleMatch::PrefixRuleMatch {
                 matched_prefix: tokens(&["npm", "install", "--no-save"]),
@@ -314,9 +325,9 @@ prefix_rule(
     let mut parser = PolicyParser::new();
     parser.parse("test.codexpolicy", policy_src)?;
     let policy = parser.build();
-    let match_eval = policy.check(&tokens(&["git", "status"]));
+    let match_eval = policy.check(&tokens(&["git", "status"]), &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Allow,
             matched_rules: vec![RuleMatch::PrefixRuleMatch {
                 matched_prefix: tokens(&["git", "status"]),
@@ -326,13 +337,20 @@ prefix_rule(
         match_eval
     );
 
-    let no_match_eval = policy.check(&tokens(&[
-        "git",
-        "--config",
-        "color.status=always",
-        "status",
-    ]));
-    assert_eq!(Evaluation::NoMatch {}, no_match_eval);
+    let no_match_eval = policy.check(
+        &tokens(&["git", "--config", "color.status=always", "status"]),
+        &allow_all,
+    );
+    assert_eq!(
+        Evaluation {
+            decision: Decision::Allow,
+            matched_rules: vec![RuleMatch::HeuristicsRuleMatch {
+                command: tokens(&["git", "--config", "color.status=always", "status",]),
+                decision: Decision::Allow,
+            }],
+        },
+        no_match_eval
+    );
     Ok(())
 }
 
@@ -352,9 +370,9 @@ prefix_rule(
     parser.parse("test.codexpolicy", policy_src)?;
     let policy = parser.build();
 
-    let commit = policy.check(&tokens(&["git", "commit", "-m", "hi"]));
+    let commit = policy.check(&tokens(&["git", "commit", "-m", "hi"]), &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Forbidden,
             matched_rules: vec![
                 RuleMatch::PrefixRuleMatch {
@@ -393,9 +411,9 @@ prefix_rule(
         tokens(&["git", "commit", "-m", "hi"]),
     ];
 
-    let evaluation = policy.check_multiple(&commands);
+    let evaluation = policy.check_multiple(&commands, &allow_all);
     assert_eq!(
-        Evaluation::Match {
+        Evaluation {
             decision: Decision::Forbidden,
             matched_rules: vec![
                 RuleMatch::PrefixRuleMatch {
@@ -416,3 +434,21 @@ prefix_rule(
     );
     Ok(())
 }
+
+#[test]
+fn heuristics_match_is_returned_when_no_policy_matches() {
+    let policy = Policy::empty();
+    let command = tokens(&["python"]);
+
+    let evaluation = policy.check(&command, &prompt_all);
+    assert_eq!(
+        Evaluation {
+            decision: Decision::Prompt,
+            matched_rules: vec![RuleMatch::HeuristicsRuleMatch {
+                command,
+                decision: Decision::Prompt,
+            }],
+        },
+        evaluation
+    );
+}