diff --git a/codex-rs/core/src/agents_md.rs b/codex-rs/core/src/agents_md.rs index a3438e49b0..08983d72ff 100644 --- a/codex-rs/core/src/agents_md.rs +++ b/codex-rs/core/src/agents_md.rs @@ -48,26 +48,32 @@ pub const LOCAL_AGENTS_MD_FILENAME: &str = "AGENTS.override.md"; /// concatenated with the following separator. const AGENTS_MD_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; +const PROJECT_DOC_BUDGET_GUIDANCE: &str = "Trim your project instructions to fit the budget. If you need to keep them all, increase `project_doc_max_bytes`."; + // Metadata probes are cheap and the exec-server transport already bounds total in-flight calls. // This covers typical project hierarchies in one remote round trip without monopolizing that // transport when independent startup discovery runs concurrently. const MAX_CONCURRENT_ANCESTOR_PROBES: usize = 256; -/// Loads project AGENTS.md content and combines it with host-provided user -/// instructions. +/// Loads repository instructions across the selected environments. pub(crate) async fn load_project_instructions( config: &Config, - user_instructions: Option, environments: &TurnEnvironmentSnapshot, -) -> io::Result> { - let mut loaded = LoadedAgentsMd::from_user_instructions(user_instructions); - if config.active_project.is_untrusted() { - return Ok((!loaded.is_empty()).then_some(loaded)); + warnings: &mut Vec, +) -> io::Result { + let mut loaded = LoadedAgentsMd::default(); + if config.active_project.is_untrusted() || config.project_doc_max_bytes == 0 { + return Ok(loaded); } let mut remaining = config.project_doc_max_bytes; for turn_environment in environments.turn_environments() { if remaining == 0 { + warnings.push(format!( + "The {}-byte project instruction budget was exhausted before checking environment `{}` or any later environments. {PROJECT_DOC_BUDGET_GUIDANCE}", + config.project_doc_max_bytes, + turn_environment.selection.environment_id + )); break; } @@ -84,6 +90,7 @@ pub(crate) async fn load_project_instructions( turn_environment.cwd(), remaining, sandbox.as_ref(), + warnings, ) .await { @@ -112,7 +119,7 @@ pub(crate) async fn load_project_instructions( } } - Ok((!loaded.is_empty()).then_some(loaded)) + Ok(loaded) } /// Attempt to locate and load AGENTS.md documentation. @@ -129,21 +136,19 @@ async fn read_agents_md( cwd: &PathUri, max_total: usize, sandbox: Option<&FileSystemSandboxContext>, + warnings: &mut Vec, ) -> io::Result> { - if max_total == 0 { - return Ok(None); - } - let paths = agents_md_paths(config, cwd, fs, sandbox).await?; - if paths.is_empty() { - return Ok(None); - } - - let mut remaining: u64 = max_total as u64; + let mut remaining = max_total; let mut loaded = LoadedAgentsMd::default(); for p in paths { if remaining == 0 { + warnings.push(format!( + "Skipped project instructions in environment `{environment_id}` starting at `{}` because the {}-byte budget was exhausted. {PROJECT_DOC_BUDGET_GUIDANCE}", + p.inferred_native_path_string(), + config.project_doc_max_bytes + )); break; } @@ -152,12 +157,14 @@ async fn read_agents_md( Err(err) if err.kind() == io::ErrorKind::NotFound => continue, Err(err) => return Err(err), }; - let size = data.len() as u64; - if size > remaining { - data.truncate(remaining as usize); - } - + let size = data.len(); if size > remaining { + data.truncate(remaining); + warnings.push(format!( + "Loaded only {remaining} of {size} bytes from `{}` in environment `{environment_id}` under the {}-byte project instruction budget. {PROJECT_DOC_BUDGET_GUIDANCE}", + p.inferred_native_path_string(), + config.project_doc_max_bytes + )); tracing::warn!( path = %p, remaining_bytes = remaining, @@ -175,7 +182,7 @@ async fn read_agents_md( cwd: cwd.clone(), }, }); - remaining = remaining.saturating_sub(data.len() as u64); + remaining -= data.len(); } } @@ -325,15 +332,6 @@ impl LoadedAgentsMd { } } - fn from_user_instructions(user_instructions: Option) -> Self { - Self { - user_instructions: user_instructions - .filter(|instructions| !instructions.text.trim().is_empty()), - thread_instructions: None, - entries: Vec::new(), - } - } - pub(crate) fn with_instructions( mut self, user_instructions: Option, diff --git a/codex-rs/core/src/agents_md_manager.rs b/codex-rs/core/src/agents_md_manager.rs index ee80cebce5..ff38ca573e 100644 --- a/codex-rs/core/src/agents_md_manager.rs +++ b/codex-rs/core/src/agents_md_manager.rs @@ -44,6 +44,7 @@ struct AgentsMdState { #[derive(Default)] struct AgentsMdCache { selections: Option>, + project_doc_max_bytes: usize, active_project_trust_level: Option, loaded: Option>, } @@ -87,7 +88,8 @@ impl AgentsMdManager { let (mut instructions, cached, refresh_repository) = { let mut state = self.state.lock().await; let refresh_repository = state.cache.selections.as_ref() != Some(&selections) - || state.cache.active_project_trust_level != active_project_trust_level; + || state.cache.active_project_trust_level != active_project_trust_level + || state.cache.project_doc_max_bytes != config.project_doc_max_bytes; if refresh_repository { // Tightened read permissions must not leave inaccessible instructions visible, // even if discovery fails or the caller cancels the refresh. @@ -127,18 +129,28 @@ impl AgentsMdManager { let loaded = if let Some(project) = &instructions.project_snapshot { project.as_ref().clone() } else if refresh_repository { - load_project_instructions(config, /*user_instructions*/ None, environments) - .await? - .unwrap_or_default() + load_project_instructions(config, environments, &mut warnings).await? } else { cached.as_deref().cloned().unwrap_or_default() } .with_instructions(instructions.user.clone(), instructions.thread.clone()) .map(Arc::new); + let budget = config.project_doc_max_bytes; + if budget > 0 + && let Some(loaded) = &loaded + { + let bytes = loaded.text().len(); + if bytes > budget { + warnings.push(format!( + "Combined user instructions use {bytes} bytes, above the {budget}-byte `project_doc_max_bytes` setting. The total includes global and thread instructions. The limit applies only to project files." + )); + } + } let mut state = self.state.lock().await; state.instructions = instructions; state.cache = AgentsMdCache { selections: Some(selections), + project_doc_max_bytes: budget, active_project_trust_level, loaded: loaded.clone(), }; @@ -165,7 +177,8 @@ impl AgentsMdManager { .collect::>(); let state = self.state.lock().await; (state.cache.selections.as_ref() == Some(&selections) - && state.cache.active_project_trust_level == config.active_project.trust_level) + && state.cache.active_project_trust_level == config.active_project.trust_level + && state.cache.project_doc_max_bytes == config.project_doc_max_bytes) .then(|| state.cache.loaded.clone().unwrap_or_default()) } diff --git a/codex-rs/core/src/agents_md_tests.rs b/codex-rs/core/src/agents_md_tests.rs index aad04d540c..354c5d21a5 100644 --- a/codex-rs/core/src/agents_md_tests.rs +++ b/codex-rs/core/src/agents_md_tests.rs @@ -341,13 +341,13 @@ async fn get_user_instructions(config: &TestConfig) -> Option { async fn load_agents_md(config: &TestConfig) -> Option { let environments = resolved_local_environments([("local", config.config.cwd.clone())]); - load_project_instructions( - &config.config, - config.user_instructions.clone(), - &environments, - ) - .await - .expect("project instructions should load") + load_project_instructions(config, &environments, &mut Vec::new()) + .await + .expect("project instructions should load") + .with_instructions( + config.user_instructions.clone(), + /*thread_instructions*/ None, + ) } async fn agents_md_paths(config: &TestConfig) -> std::io::Result> { @@ -675,12 +675,15 @@ async fn doc_larger_than_limit_is_truncated() { let huge = "A".repeat(LIMIT * 2); // 2 KiB fs::write(tmp.path().join("AGENTS.md"), &huge).unwrap(); - let res = get_user_instructions(&make_config(&tmp, LIMIT, /*instructions*/ None).await) + let config = make_config(&tmp, LIMIT, /*instructions*/ None).await; + let environments = resolved_local_environments([("local", config.cwd.clone())]); + let mut warnings = Vec::new(); + let loaded = load_project_instructions(&config, &environments, &mut warnings) .await - .expect("doc expected"); - - assert_eq!(res.len(), LIMIT, "doc should be truncated to LIMIT bytes"); - assert_eq!(res, huge[..LIMIT]); + .expect("project instructions should load"); + assert_eq!(loaded.text(), huge[..LIMIT]); + let path = config.cwd.join("AGENTS.md"); + insta::assert_snapshot!(warnings.join("\n").replace(path.to_string_lossy().as_ref(), "AGENTS.md"), @"Loaded only 1024 of 2048 bytes from `AGENTS.md` in environment `local` under the 1024-byte project instruction budget. Trim your project instructions to fit the budget. If you need to keep them all, increase `project_doc_max_bytes`."); } #[tokio::test] @@ -742,6 +745,7 @@ async fn read_agents_md_loads_cwd_instructions_when_parent_markers_are_unreadabl &PathUri::from_abs_path(&cwd), config.project_doc_max_bytes, /*sandbox*/ None, + &mut Vec::new(), ) .await .expect("unreadable parent markers should not hide cwd instructions") @@ -769,6 +773,7 @@ async fn read_agents_md_propagates_read_errors() { &PathUri::from_abs_path(&cwd), config.project_doc_max_bytes, /*sandbox*/ None, + &mut Vec::new(), ) .await .expect_err("read error"); @@ -795,6 +800,7 @@ async fn read_agents_md_ignores_files_removed_after_discovery() { &PathUri::from_abs_path(&cwd), config.project_doc_max_bytes, /*sandbox*/ None, + &mut Vec::new(), ) .await .expect("removed file is recoverable"); @@ -1147,12 +1153,14 @@ async fn multiple_environment_docs_use_labeled_layout_and_preserve_source_order( ("primary", config.cwd.clone()), ("secondary", secondary.abs()), ]); - let user_instructions = config.user_instructions.clone(); - - let loaded = load_project_instructions(&config.config, user_instructions, &environments) + let loaded = load_project_instructions(&config, &environments, &mut Vec::new()) .await .expect("project instructions should load") - .expect("instructions expected"); + .with_instructions( + config.user_instructions.clone(), + /*thread_instructions*/ None, + ) + .expect("global instructions"); let inner = format!( r#"global instructions @@ -1211,12 +1219,14 @@ async fn secondary_only_project_doc_uses_single_contributor_layout() { ("primary", config.cwd.clone()), ("secondary", secondary.abs()), ]); - let user_instructions = config.user_instructions.clone(); - - let loaded = load_project_instructions(&config.config, user_instructions, &environments) + let loaded = load_project_instructions(&config, &environments, &mut Vec::new()) .await .expect("project instructions should load") - .expect("instructions expected"); + .with_instructions( + config.user_instructions.clone(), + /*thread_instructions*/ None, + ) + .expect("global instructions"); let inner = format!("global instructions{AGENTS_MD_SEPARATOR}secondary doc"); assert_eq!(loaded.legacy_text(), inner); @@ -1241,12 +1251,14 @@ async fn primary_only_project_doc_preserves_legacy_layout_with_multiple_bound_en ("primary", config.cwd.clone()), ("secondary", secondary.abs()), ]); - let user_instructions = config.user_instructions.clone(); - - let loaded = load_project_instructions(&config.config, user_instructions, &environments) + let loaded = load_project_instructions(&config, &environments, &mut Vec::new()) .await .expect("project instructions should load") - .expect("instructions expected"); + .with_instructions( + config.user_instructions.clone(), + /*thread_instructions*/ None, + ) + .expect("global instructions"); let inner = format!("global instructions{AGENTS_MD_SEPARATOR}primary doc"); assert_eq!(loaded.legacy_text(), inner); @@ -1272,12 +1284,9 @@ async fn project_doc_byte_limit_is_shared_across_environments() { ("primary", config.cwd.clone()), ("secondary", secondary.abs()), ]); - let user_instructions = config.user_instructions.clone(); - - let loaded = load_project_instructions(&config.config, user_instructions, &environments) + let loaded = load_project_instructions(&config, &environments, &mut Vec::new()) .await - .expect("project instructions should load") - .expect("instructions expected"); + .expect("project instructions should load"); assert_eq!( loaded.text(), @@ -1290,7 +1299,7 @@ async fn project_doc_byte_limit_is_shared_across_environments() { } #[tokio::test] -async fn full_primary_environment_budget_excludes_later_environment_docs() { +async fn full_primary_environment_budget_skips_later_files_and_environments() { const LIMIT: usize = 8; let primary = tempfile::tempdir().expect("primary tempdir"); let secondary = tempfile::tempdir().expect("secondary tempdir"); @@ -1299,29 +1308,23 @@ async fn full_primary_environment_budget_excludes_later_environment_docs() { fs::write(primary.path().join("AGENTS.md"), &primary_doc).unwrap(); fs::write(secondary.path().join("AGENTS.md"), &secondary_doc).unwrap(); let config = make_config(&primary, LIMIT, /*instructions*/ None).await; - let environments = resolved_local_environments([ - ("primary", config.cwd.clone()), - ("secondary", secondary.abs()), - ]); + fs::create_dir(primary.path().join(".git")).unwrap(); + let nested = config.cwd.join("nested"); + fs::create_dir(&nested).unwrap(); + let skipped = nested.join("AGENTS.md"); + fs::write(&skipped, "nested instructions").unwrap(); + let environments = + resolved_local_environments([("primary", nested), ("secondary", secondary.abs())]); - let loaded = load_project_instructions( - &config.config, - /*user_instructions*/ None, - &environments, - ) - .await - .expect("project instructions should load") - .expect("instructions expected"); - let project_bytes = loaded - .entries - .iter() - .filter(|entry| matches!(&entry.provenance, InstructionProvenance::Project { .. })) - .map(|entry| entry.contents.len()) - .sum::(); - - assert_eq!(project_bytes, LIMIT); - assert!(loaded.text().contains(&primary_doc)); - assert!(!loaded.text().contains(&secondary_doc)); + let mut warnings = Vec::new(); + let loaded = load_project_instructions(&config, &environments, &mut warnings) + .await + .expect("project instructions should load"); + insta::assert_snapshot!(warnings.join("\n").replace(skipped.to_string_lossy().as_ref(), "nested/AGENTS.md"), @r" + Skipped project instructions in environment `primary` starting at `nested/AGENTS.md` because the 8-byte budget was exhausted. Trim your project instructions to fit the budget. If you need to keep them all, increase `project_doc_max_bytes`. + The 8-byte project instruction budget was exhausted before checking environment `secondary` or any later environments. Trim your project instructions to fit the budget. If you need to keep them all, increase `project_doc_max_bytes`. + "); + assert_eq!(loaded.text(), primary_doc); } #[tokio::test] @@ -1336,14 +1339,9 @@ async fn secondary_environment_invalid_utf8_does_not_suppress_other_docs() { ("secondary", secondary.abs()), ]); - let loaded = load_project_instructions( - &config.config, - /*user_instructions*/ None, - &environments, - ) - .await - .expect("project instructions should load") - .expect("instructions expected"); + let loaded = load_project_instructions(&config, &environments, &mut Vec::new()) + .await + .expect("project instructions should load"); assert!(loaded.text().contains("primary doc")); assert!(loaded.text().contains("secondary\u{FFFD}doc")); diff --git a/codex-rs/core/tests/suite/agents_md.rs b/codex-rs/core/tests/suite/agents_md.rs index 729c704c1e..fb0fdde340 100644 --- a/codex-rs/core/tests/suite/agents_md.rs +++ b/codex-rs/core/tests/suite/agents_md.rs @@ -1168,6 +1168,14 @@ async fn thread_provider_composes_and_clears_only_its_instructions() -> Result<( ) .await; let fixture = ThreadInstructionsFixture::new(&server).await?; + let warning = core_test_support::wait_for_event_match(&fixture.thread, |event| match event { + EventMsg::Warning(warning) if warning.message.contains("project_doc_max_bytes") => { + Some(warning.message.clone()) + } + _ => None, + }) + .await; + insta::assert_snapshot!(warning, @"Combined user instructions use 86 bytes, above the 20-byte `project_doc_max_bytes` setting. The total includes global and thread instructions. The limit applies only to project files."); let sources = vec![ PathUri::from_abs_path(&fixture.test.config.codex_home.join(GLOBAL_AGENTS_FILENAME)), fixture.test.workspace_path_uri(GLOBAL_AGENTS_FILENAME)?,