Warn when project instructions exceed the configured byte budget (#59445)

## Why

Project instructions could be truncated or skipped when the byte budget ran out without a user-facing warning.

## What changed

- Report truncated files and skipped files or environments, with guidance to trim instructions or increase `project_doc_max_bytes`.
- Warn when combined instructions exceed the configured budget, clarifying that global and thread instructions contribute to the total but the limit applies only to project files.
- Refresh cached project instructions when `project_doc_max_bytes` changes.

## Testing

Add warning snapshots for truncated files, skipped files and environments, and combined instructions exceeding the budget.

GitOrigin-RevId: 121eebb8ac8891decada2d028c194aa687815466

codex-rs/core/src/agents_md.rs

@@ -48,26 +48,32 @@ pub const LOCAL_AGENTS_MD_FILENAME: &str = "AGENTS.override.md";
 /// concatenated with the following separator.
 const AGENTS_MD_SEPARATOR: &str = "\n\n--- project-doc ---\n\n";
 
+const PROJECT_DOC_BUDGET_GUIDANCE: &str = "Trim your project instructions to fit the budget. If you need to keep them all, increase `project_doc_max_bytes`.";
+
 // Metadata probes are cheap and the exec-server transport already bounds total in-flight calls.
 // This covers typical project hierarchies in one remote round trip without monopolizing that
 // transport when independent startup discovery runs concurrently.
 const MAX_CONCURRENT_ANCESTOR_PROBES: usize = 256;
 
-/// Loads project AGENTS.md content and combines it with host-provided user
-/// instructions.
+/// Loads repository instructions across the selected environments.
 pub(crate) async fn load_project_instructions(
     config: &Config,
-    user_instructions: Option<Instructions>,
     environments: &TurnEnvironmentSnapshot,
-) -> io::Result<Option<LoadedAgentsMd>> {
-    let mut loaded = LoadedAgentsMd::from_user_instructions(user_instructions);
-    if config.active_project.is_untrusted() {
-        return Ok((!loaded.is_empty()).then_some(loaded));
+    warnings: &mut Vec<String>,
+) -> io::Result<LoadedAgentsMd> {
+    let mut loaded = LoadedAgentsMd::default();
+    if config.active_project.is_untrusted() || config.project_doc_max_bytes == 0 {
+        return Ok(loaded);
     }
 
     let mut remaining = config.project_doc_max_bytes;
     for turn_environment in environments.turn_environments() {
         if remaining == 0 {
+            warnings.push(format!(
+                "The {}-byte project instruction budget was exhausted before checking environment `{}` or any later environments. {PROJECT_DOC_BUDGET_GUIDANCE}",
+                config.project_doc_max_bytes,
+                turn_environment.selection.environment_id
+            ));
             break;
         }
 
@@ -84,6 +90,7 @@ pub(crate) async fn load_project_instructions(
             turn_environment.cwd(),
             remaining,
             sandbox.as_ref(),
+            warnings,
         )
         .await
         {
@@ -112,7 +119,7 @@ pub(crate) async fn load_project_instructions(
         }
     }
 
-    Ok((!loaded.is_empty()).then_some(loaded))
+    Ok(loaded)
 }
 
 /// Attempt to locate and load AGENTS.md documentation.
@@ -129,21 +136,19 @@ async fn read_agents_md(
     cwd: &PathUri,
     max_total: usize,
     sandbox: Option<&FileSystemSandboxContext>,
+    warnings: &mut Vec<String>,
 ) -> io::Result<Option<LoadedAgentsMd>> {
-    if max_total == 0 {
-        return Ok(None);
-    }
-
     let paths = agents_md_paths(config, cwd, fs, sandbox).await?;
-    if paths.is_empty() {
-        return Ok(None);
-    }
-
-    let mut remaining: u64 = max_total as u64;
+    let mut remaining = max_total;
     let mut loaded = LoadedAgentsMd::default();
 
     for p in paths {
         if remaining == 0 {
+            warnings.push(format!(
+                "Skipped project instructions in environment `{environment_id}` starting at `{}` because the {}-byte budget was exhausted. {PROJECT_DOC_BUDGET_GUIDANCE}",
+                p.inferred_native_path_string(),
+                config.project_doc_max_bytes
+            ));
             break;
         }
 
@@ -152,12 +157,14 @@ async fn read_agents_md(
             Err(err) if err.kind() == io::ErrorKind::NotFound => continue,
             Err(err) => return Err(err),
         };
-        let size = data.len() as u64;
-        if size > remaining {
-            data.truncate(remaining as usize);
-        }
-
+        let size = data.len();
         if size > remaining {
+            data.truncate(remaining);
+            warnings.push(format!(
+                "Loaded only {remaining} of {size} bytes from `{}` in environment `{environment_id}` under the {}-byte project instruction budget. {PROJECT_DOC_BUDGET_GUIDANCE}",
+                p.inferred_native_path_string(),
+                config.project_doc_max_bytes
+            ));
             tracing::warn!(
                 path = %p,
                 remaining_bytes = remaining,
@@ -175,7 +182,7 @@ async fn read_agents_md(
                     cwd: cwd.clone(),
                 },
             });
-            remaining = remaining.saturating_sub(data.len() as u64);
+            remaining -= data.len();
         }
     }
 
@@ -325,15 +332,6 @@ impl LoadedAgentsMd {
         }
     }
 
-    fn from_user_instructions(user_instructions: Option<Instructions>) -> Self {
-        Self {
-            user_instructions: user_instructions
-                .filter(|instructions| !instructions.text.trim().is_empty()),
-            thread_instructions: None,
-            entries: Vec::new(),
-        }
-    }
-
     pub(crate) fn with_instructions(
         mut self,
         user_instructions: Option<Instructions>,

codex-rs/core/src/agents_md_manager.rs

@@ -44,6 +44,7 @@ struct AgentsMdState {
 #[derive(Default)]
 struct AgentsMdCache {
     selections: Option<Vec<TurnEnvironmentSelection>>,
+    project_doc_max_bytes: usize,
     active_project_trust_level: Option<TrustLevel>,
     loaded: Option<Arc<LoadedAgentsMd>>,
 }
@@ -87,7 +88,8 @@ impl AgentsMdManager {
         let (mut instructions, cached, refresh_repository) = {
             let mut state = self.state.lock().await;
             let refresh_repository = state.cache.selections.as_ref() != Some(&selections)
-                || state.cache.active_project_trust_level != active_project_trust_level;
+                || state.cache.active_project_trust_level != active_project_trust_level
+                || state.cache.project_doc_max_bytes != config.project_doc_max_bytes;
             if refresh_repository {
                 // Tightened read permissions must not leave inaccessible instructions visible,
                 // even if discovery fails or the caller cancels the refresh.
@@ -127,18 +129,28 @@ impl AgentsMdManager {
             let loaded = if let Some(project) = &instructions.project_snapshot {
                 project.as_ref().clone()
             } else if refresh_repository {
-                load_project_instructions(config, /*user_instructions*/ None, environments)
-                    .await?
-                    .unwrap_or_default()
+                load_project_instructions(config, environments, &mut warnings).await?
             } else {
                 cached.as_deref().cloned().unwrap_or_default()
             }
             .with_instructions(instructions.user.clone(), instructions.thread.clone())
             .map(Arc::new);
+            let budget = config.project_doc_max_bytes;
+            if budget > 0
+                && let Some(loaded) = &loaded
+            {
+                let bytes = loaded.text().len();
+                if bytes > budget {
+                    warnings.push(format!(
+                        "Combined user instructions use {bytes} bytes, above the {budget}-byte `project_doc_max_bytes` setting. The total includes global and thread instructions. The limit applies only to project files."
+                    ));
+                }
+            }
             let mut state = self.state.lock().await;
             state.instructions = instructions;
             state.cache = AgentsMdCache {
                 selections: Some(selections),
+                project_doc_max_bytes: budget,
                 active_project_trust_level,
                 loaded: loaded.clone(),
             };
@@ -165,7 +177,8 @@ impl AgentsMdManager {
             .collect::<Vec<_>>();
         let state = self.state.lock().await;
         (state.cache.selections.as_ref() == Some(&selections)
-            && state.cache.active_project_trust_level == config.active_project.trust_level)
+            && state.cache.active_project_trust_level == config.active_project.trust_level
+            && state.cache.project_doc_max_bytes == config.project_doc_max_bytes)
             .then(|| state.cache.loaded.clone().unwrap_or_default())
     }
 

codex-rs/core/src/agents_md_tests.rs

@@ -341,13 +341,13 @@ async fn get_user_instructions(config: &TestConfig) -> Option<String> {
 async fn load_agents_md(config: &TestConfig) -> Option<LoadedAgentsMd> {
     let environments = resolved_local_environments([("local", config.config.cwd.clone())]);
 
-    load_project_instructions(
-        &config.config,
-        config.user_instructions.clone(),
-        &environments,
-    )
-    .await
-    .expect("project instructions should load")
+    load_project_instructions(config, &environments, &mut Vec::new())
+        .await
+        .expect("project instructions should load")
+        .with_instructions(
+            config.user_instructions.clone(),
+            /*thread_instructions*/ None,
+        )
 }
 
 async fn agents_md_paths(config: &TestConfig) -> std::io::Result<Vec<PathUri>> {
@@ -675,12 +675,15 @@ async fn doc_larger_than_limit_is_truncated() {
     let huge = "A".repeat(LIMIT * 2); // 2 KiB
     fs::write(tmp.path().join("AGENTS.md"), &huge).unwrap();
 
-    let res = get_user_instructions(&make_config(&tmp, LIMIT, /*instructions*/ None).await)
+    let config = make_config(&tmp, LIMIT, /*instructions*/ None).await;
+    let environments = resolved_local_environments([("local", config.cwd.clone())]);
+    let mut warnings = Vec::new();
+    let loaded = load_project_instructions(&config, &environments, &mut warnings)
         .await
-        .expect("doc expected");
-
-    assert_eq!(res.len(), LIMIT, "doc should be truncated to LIMIT bytes");
-    assert_eq!(res, huge[..LIMIT]);
+        .expect("project instructions should load");
+    assert_eq!(loaded.text(), huge[..LIMIT]);
+    let path = config.cwd.join("AGENTS.md");
+    insta::assert_snapshot!(warnings.join("\n").replace(path.to_string_lossy().as_ref(), "AGENTS.md"), @"Loaded only 1024 of 2048 bytes from `AGENTS.md` in environment `local` under the 1024-byte project instruction budget. Trim your project instructions to fit the budget. If you need to keep them all, increase `project_doc_max_bytes`.");
 }
 
 #[tokio::test]
@@ -742,6 +745,7 @@ async fn read_agents_md_loads_cwd_instructions_when_parent_markers_are_unreadabl
         &PathUri::from_abs_path(&cwd),
         config.project_doc_max_bytes,
         /*sandbox*/ None,
+        &mut Vec::new(),
     )
     .await
     .expect("unreadable parent markers should not hide cwd instructions")
@@ -769,6 +773,7 @@ async fn read_agents_md_propagates_read_errors() {
         &PathUri::from_abs_path(&cwd),
         config.project_doc_max_bytes,
         /*sandbox*/ None,
+        &mut Vec::new(),
     )
     .await
     .expect_err("read error");
@@ -795,6 +800,7 @@ async fn read_agents_md_ignores_files_removed_after_discovery() {
         &PathUri::from_abs_path(&cwd),
         config.project_doc_max_bytes,
         /*sandbox*/ None,
+        &mut Vec::new(),
     )
     .await
     .expect("removed file is recoverable");
@@ -1147,12 +1153,14 @@ async fn multiple_environment_docs_use_labeled_layout_and_preserve_source_order(
         ("primary", config.cwd.clone()),
         ("secondary", secondary.abs()),
     ]);
-    let user_instructions = config.user_instructions.clone();
-
-    let loaded = load_project_instructions(&config.config, user_instructions, &environments)
+    let loaded = load_project_instructions(&config, &environments, &mut Vec::new())
         .await
         .expect("project instructions should load")
-        .expect("instructions expected");
+        .with_instructions(
+            config.user_instructions.clone(),
+            /*thread_instructions*/ None,
+        )
+        .expect("global instructions");
     let inner = format!(
         r#"global instructions
 
@@ -1211,12 +1219,14 @@ async fn secondary_only_project_doc_uses_single_contributor_layout() {
         ("primary", config.cwd.clone()),
         ("secondary", secondary.abs()),
     ]);
-    let user_instructions = config.user_instructions.clone();
-
-    let loaded = load_project_instructions(&config.config, user_instructions, &environments)
+    let loaded = load_project_instructions(&config, &environments, &mut Vec::new())
         .await
         .expect("project instructions should load")
-        .expect("instructions expected");
+        .with_instructions(
+            config.user_instructions.clone(),
+            /*thread_instructions*/ None,
+        )
+        .expect("global instructions");
     let inner = format!("global instructions{AGENTS_MD_SEPARATOR}secondary doc");
 
     assert_eq!(loaded.legacy_text(), inner);
@@ -1241,12 +1251,14 @@ async fn primary_only_project_doc_preserves_legacy_layout_with_multiple_bound_en
         ("primary", config.cwd.clone()),
         ("secondary", secondary.abs()),
     ]);
-    let user_instructions = config.user_instructions.clone();
-
-    let loaded = load_project_instructions(&config.config, user_instructions, &environments)
+    let loaded = load_project_instructions(&config, &environments, &mut Vec::new())
         .await
         .expect("project instructions should load")
-        .expect("instructions expected");
+        .with_instructions(
+            config.user_instructions.clone(),
+            /*thread_instructions*/ None,
+        )
+        .expect("global instructions");
     let inner = format!("global instructions{AGENTS_MD_SEPARATOR}primary doc");
 
     assert_eq!(loaded.legacy_text(), inner);
@@ -1272,12 +1284,9 @@ async fn project_doc_byte_limit_is_shared_across_environments() {
         ("primary", config.cwd.clone()),
         ("secondary", secondary.abs()),
     ]);
-    let user_instructions = config.user_instructions.clone();
-
-    let loaded = load_project_instructions(&config.config, user_instructions, &environments)
+    let loaded = load_project_instructions(&config, &environments, &mut Vec::new())
         .await
-        .expect("project instructions should load")
-        .expect("instructions expected");
+        .expect("project instructions should load");
 
     assert_eq!(
         loaded.text(),
@@ -1290,7 +1299,7 @@ async fn project_doc_byte_limit_is_shared_across_environments() {
 }
 
 #[tokio::test]
-async fn full_primary_environment_budget_excludes_later_environment_docs() {
+async fn full_primary_environment_budget_skips_later_files_and_environments() {
     const LIMIT: usize = 8;
     let primary = tempfile::tempdir().expect("primary tempdir");
     let secondary = tempfile::tempdir().expect("secondary tempdir");
@@ -1299,29 +1308,23 @@ async fn full_primary_environment_budget_excludes_later_environment_docs() {
     fs::write(primary.path().join("AGENTS.md"), &primary_doc).unwrap();
     fs::write(secondary.path().join("AGENTS.md"), &secondary_doc).unwrap();
     let config = make_config(&primary, LIMIT, /*instructions*/ None).await;
-    let environments = resolved_local_environments([
-        ("primary", config.cwd.clone()),
-        ("secondary", secondary.abs()),
-    ]);
+    fs::create_dir(primary.path().join(".git")).unwrap();
+    let nested = config.cwd.join("nested");
+    fs::create_dir(&nested).unwrap();
+    let skipped = nested.join("AGENTS.md");
+    fs::write(&skipped, "nested instructions").unwrap();
+    let environments =
+        resolved_local_environments([("primary", nested), ("secondary", secondary.abs())]);
 
-    let loaded = load_project_instructions(
-        &config.config,
-        /*user_instructions*/ None,
-        &environments,
-    )
-    .await
-    .expect("project instructions should load")
-    .expect("instructions expected");
-    let project_bytes = loaded
-        .entries
-        .iter()
-        .filter(|entry| matches!(&entry.provenance, InstructionProvenance::Project { .. }))
-        .map(|entry| entry.contents.len())
-        .sum::<usize>();
-
-    assert_eq!(project_bytes, LIMIT);
-    assert!(loaded.text().contains(&primary_doc));
-    assert!(!loaded.text().contains(&secondary_doc));
+    let mut warnings = Vec::new();
+    let loaded = load_project_instructions(&config, &environments, &mut warnings)
+        .await
+        .expect("project instructions should load");
+    insta::assert_snapshot!(warnings.join("\n").replace(skipped.to_string_lossy().as_ref(), "nested/AGENTS.md"), @r"
+    Skipped project instructions in environment `primary` starting at `nested/AGENTS.md` because the 8-byte budget was exhausted. Trim your project instructions to fit the budget. If you need to keep them all, increase `project_doc_max_bytes`.
+    The 8-byte project instruction budget was exhausted before checking environment `secondary` or any later environments. Trim your project instructions to fit the budget. If you need to keep them all, increase `project_doc_max_bytes`.
+    ");
+    assert_eq!(loaded.text(), primary_doc);
 }
 
 #[tokio::test]
@@ -1336,14 +1339,9 @@ async fn secondary_environment_invalid_utf8_does_not_suppress_other_docs() {
         ("secondary", secondary.abs()),
     ]);
 
-    let loaded = load_project_instructions(
-        &config.config,
-        /*user_instructions*/ None,
-        &environments,
-    )
-    .await
-    .expect("project instructions should load")
-    .expect("instructions expected");
+    let loaded = load_project_instructions(&config, &environments, &mut Vec::new())
+        .await
+        .expect("project instructions should load");
 
     assert!(loaded.text().contains("primary doc"));
     assert!(loaded.text().contains("secondary\u{FFFD}doc"));

codex-rs/core/tests/suite/agents_md.rs

@@ -1168,6 +1168,14 @@ async fn thread_provider_composes_and_clears_only_its_instructions() -> Result<(
     )
     .await;
     let fixture = ThreadInstructionsFixture::new(&server).await?;
+    let warning = core_test_support::wait_for_event_match(&fixture.thread, |event| match event {
+        EventMsg::Warning(warning) if warning.message.contains("project_doc_max_bytes") => {
+            Some(warning.message.clone())
+        }
+        _ => None,
+    })
+    .await;
+    insta::assert_snapshot!(warning, @"Combined user instructions use 86 bytes, above the 20-byte `project_doc_max_bytes` setting. The total includes global and thread instructions. The limit applies only to project files.");
     let sources = vec![
         PathUri::from_abs_path(&fixture.test.config.codex_home.join(GLOBAL_AGENTS_FILENAME)),
         fixture.test.workspace_path_uri(GLOBAL_AGENTS_FILENAME)?,