From 799e50412e6a5af88c77c1e071f3e1245936a6ed Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 21 Apr 2026 17:28:58 -0700 Subject: [PATCH 1/4] sandboxing: materialize cwd-relative permission globs (#18867) ## Why #18275 anchors session-scoped `:cwd` and `:project_roots` grants to the request cwd before recording them for reuse. Relative deny glob entries need the same treatment. Without anchoring, a stored session permission can keep a pattern such as `**/*.env` relative, then reinterpret that deny against a later turn cwd. That makes the persisted profile depend on the cwd at reuse time instead of the cwd that was reviewed and approved. ## What changed `intersect_permission_profiles` now materializes retained `FileSystemPath::GlobPattern` entries against the request cwd, matching the existing materialization for cwd-sensitive special paths. Materialized accepted grants are now deduplicated before deny retention runs. This keeps the sticky-grant preapproval shape stable when a repeated request is merged with the stored grant and both `:cwd = write` and the materialized absolute cwd write are present. The preapproval check compares against the same materialized form, so a later request for the same cwd-relative deny glob still matches the stored anchored grant instead of re-prompting or rejecting. Tests cover both the storage path and the preapproval path: a session-scoped `:cwd = write` grant with `**/*.env = none` is stored with both the cwd write and deny glob anchored to the original request cwd, cannot be reused from a later cwd, and remains preapproved when re-requested from the original cwd after merging with the stored grant. ## Verification - `cargo test -p codex-sandboxing policy_transforms` - `cargo test -p codex-core --lib relative_deny_glob_grants_remain_preapproved_after_materialization` - `cargo clippy -p codex-sandboxing --tests -- -D clippy::redundant_clone` - `cargo clippy -p codex-core --lib -- -D clippy::redundant_clone` --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/openai/codex/pull/18867). * #18288 * #18287 * #18286 * #18285 * #18284 * #18283 * #18282 * #18281 * #18280 * #18279 * #18278 * #18277 * #18276 * __->__ #18867 --- codex-rs/core/src/tools/handlers/mod.rs | 63 +++++++- codex-rs/sandboxing/src/policy_transforms.rs | 41 +++--- .../sandboxing/src/policy_transforms_tests.rs | 139 +++++++++++++++++- 3 files changed, 220 insertions(+), 23 deletions(-) diff --git a/codex-rs/core/src/tools/handlers/mod.rs b/codex-rs/core/src/tools/handlers/mod.rs index 0217cab33a..af4178f71f 100644 --- a/codex-rs/core/src/tools/handlers/mod.rs +++ b/codex-rs/core/src/tools/handlers/mod.rs @@ -193,8 +193,7 @@ pub(super) async fn apply_granted_turn_permissions( ); let permissions_preapproved = match (effective_permissions.as_ref(), granted_permissions) { (Some(effective_permissions), Some(granted_permissions)) => { - intersect_permission_profiles(effective_permissions.clone(), granted_permissions, cwd) - == *effective_permissions + permissions_are_preapproved(effective_permissions, granted_permissions, cwd) } _ => false, }; @@ -213,17 +212,38 @@ pub(super) async fn apply_granted_turn_permissions( } } +fn permissions_are_preapproved( + effective_permissions: &PermissionProfile, + granted_permissions: PermissionProfile, + cwd: &Path, +) -> bool { + let materialized_effective_permissions = intersect_permission_profiles( + effective_permissions.clone(), + effective_permissions.clone(), + cwd, + ); + intersect_permission_profiles(effective_permissions.clone(), granted_permissions, cwd) + == materialized_effective_permissions +} + #[cfg(test)] mod tests { use super::EffectiveAdditionalPermissions; use super::implicit_granted_permissions; use super::normalize_and_validate_additional_permissions; + use super::permissions_are_preapproved; use crate::sandboxing::SandboxPermissions; use codex_protocol::models::FileSystemPermissions; use codex_protocol::models::NetworkPermissions; use codex_protocol::models::PermissionProfile; + use codex_protocol::permissions::FileSystemAccessMode; + use codex_protocol::permissions::FileSystemPath; + use codex_protocol::permissions::FileSystemSandboxEntry; + use codex_protocol::permissions::FileSystemSpecialPath; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::GranularApprovalConfig; + use codex_sandboxing::policy_transforms::intersect_permission_profiles; + use codex_sandboxing::policy_transforms::merge_permission_profiles; use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; use tempfile::tempdir; @@ -326,4 +346,43 @@ mod tests { assert_eq!(implicit_permissions, None); } + + #[test] + fn relative_deny_glob_grants_remain_preapproved_after_materialization() { + let cwd = tempdir().expect("tempdir"); + let requested_permissions = PermissionProfile { + file_system: Some(FileSystemPermissions { + entries: vec![ + FileSystemSandboxEntry { + path: FileSystemPath::Special { + value: FileSystemSpecialPath::CurrentWorkingDirectory, + }, + access: FileSystemAccessMode::Write, + }, + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: "**/*.env".to_string(), + }, + access: FileSystemAccessMode::None, + }, + ], + glob_scan_max_depth: None, + }), + ..Default::default() + }; + let stored_grant = intersect_permission_profiles( + requested_permissions.clone(), + requested_permissions.clone(), + cwd.path(), + ); + let effective_permissions = + merge_permission_profiles(Some(&requested_permissions), Some(&stored_grant)) + .expect("merged permissions"); + + assert!(permissions_are_preapproved( + &effective_permissions, + stored_grant, + cwd.path(), + )); + } } diff --git a/codex-rs/sandboxing/src/policy_transforms.rs b/codex-rs/sandboxing/src/policy_transforms.rs index 134ac84f15..9b5e444d7c 100644 --- a/codex-rs/sandboxing/src/policy_transforms.rs +++ b/codex-rs/sandboxing/src/policy_transforms.rs @@ -159,20 +159,21 @@ pub fn intersect_permission_profiles( let requested_policy = FileSystemSandboxPolicy::restricted(requested_file_system.entries.clone()); let requested_read_deny_matcher = ReadDenyMatcher::new(&requested_policy, cwd); - let accepted_entries: Vec<_> = granted_file_system - .entries - .iter() - .filter(|entry| { - granted_file_system_entry_within_request( - &requested_file_system, - &requested_policy, - requested_read_deny_matcher.as_ref(), - entry, - cwd, - ) - }) - .map(|entry| materialize_cwd_dependent_entry(entry, cwd)) - .collect(); + let mut accepted_entries = Vec::new(); + for entry in granted_file_system.entries.iter().filter(|entry| { + granted_file_system_entry_within_request( + &requested_file_system, + &requested_policy, + requested_read_deny_matcher.as_ref(), + entry, + cwd, + ) + }) { + let entry = materialize_cwd_dependent_entry(entry, cwd); + if !accepted_entries.contains(&entry) { + accepted_entries.push(entry); + } + } let mut entries = accepted_entries.clone(); let requested_retained_deny_entries = retain_constraining_deny_entries( &requested_file_system.entries, @@ -383,9 +384,15 @@ fn materialize_cwd_dependent_entry( access: entry.access, }) .unwrap_or_else(|| entry.clone()), - FileSystemPath::Path { .. } - | FileSystemPath::GlobPattern { .. } - | FileSystemPath::Special { .. } => entry.clone(), + FileSystemPath::GlobPattern { pattern } => FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: AbsolutePathBuf::resolve_path_against_base(pattern, cwd) + .to_string_lossy() + .into_owned(), + }, + access: entry.access, + }, + FileSystemPath::Path { .. } | FileSystemPath::Special { .. } => entry.clone(), } } diff --git a/codex-rs/sandboxing/src/policy_transforms_tests.rs b/codex-rs/sandboxing/src/policy_transforms_tests.rs index e34158bc99..38b4bd5f02 100644 --- a/codex-rs/sandboxing/src/policy_transforms_tests.rs +++ b/codex-rs/sandboxing/src/policy_transforms_tests.rs @@ -377,6 +377,42 @@ fn intersect_permission_profiles_materializes_cwd_grant_for_reuse() { ); } +#[test] +fn intersect_permission_profiles_deduplicates_materialized_grants() { + let temp_dir = TempDir::new().expect("create temp dir"); + let cwd = + AbsolutePathBuf::from_absolute_path(temp_dir.path().join("cwd")).expect("absolute cwd"); + let permissions = PermissionProfile { + file_system: Some(FileSystemPermissions { + entries: vec![ + FileSystemSandboxEntry { + path: FileSystemPath::Special { + value: FileSystemSpecialPath::CurrentWorkingDirectory, + }, + access: FileSystemAccessMode::Write, + }, + FileSystemSandboxEntry { + path: FileSystemPath::Path { path: cwd.clone() }, + access: FileSystemAccessMode::Write, + }, + ], + glob_scan_max_depth: None, + }), + ..Default::default() + }; + + assert_eq!( + intersect_permission_profiles(permissions.clone(), permissions, cwd.as_path()), + PermissionProfile { + file_system: Some(FileSystemPermissions::from_read_write_roots( + /*read*/ None, + Some(vec![cwd]), + )), + ..Default::default() + } + ); +} + #[test] fn intersect_permission_profiles_materializes_cwd_deny_entries() { let temp_dir = TempDir::new().expect("create temp dir"); @@ -509,6 +545,75 @@ fn intersect_permission_profiles_rejects_concrete_grants_matched_by_requested_de ); } +#[test] +fn intersect_permission_profiles_materializes_relative_deny_globs_for_reuse() { + let temp_dir = TempDir::new().expect("create temp dir"); + let request_cwd = AbsolutePathBuf::from_absolute_path(temp_dir.path().join("request-cwd")) + .expect("absolute request cwd"); + let later_cwd = AbsolutePathBuf::from_absolute_path(temp_dir.path().join("later-cwd")) + .expect("absolute later cwd"); + let cwd_write = FileSystemSandboxEntry { + path: FileSystemPath::Special { + value: FileSystemSpecialPath::CurrentWorkingDirectory, + }, + access: FileSystemAccessMode::Write, + }; + let deny_env_files = FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: "**/*.env".to_string(), + }, + access: FileSystemAccessMode::None, + }; + let permissions = PermissionProfile { + file_system: Some(FileSystemPermissions { + entries: vec![cwd_write, deny_env_files], + glob_scan_max_depth: std::num::NonZeroUsize::new(2), + }), + ..Default::default() + }; + + let intersected = + intersect_permission_profiles(permissions.clone(), permissions, request_cwd.as_path()); + + assert_eq!( + intersected, + PermissionProfile { + file_system: Some(FileSystemPermissions { + entries: vec![ + FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: request_cwd.clone(), + }, + access: FileSystemAccessMode::Write, + }, + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: request_cwd.join("**/*.env").to_string_lossy().into_owned(), + }, + access: FileSystemAccessMode::None, + }, + ], + glob_scan_max_depth: std::num::NonZeroUsize::new(2), + }), + ..Default::default() + } + ); + assert_eq!( + intersect_permission_profiles( + PermissionProfile { + file_system: Some(FileSystemPermissions::from_read_write_roots( + /*read*/ None, + Some(vec![later_cwd.join("token.env")]), + )), + ..Default::default() + }, + intersected, + later_cwd.as_path(), + ), + PermissionProfile::default() + ); +} + #[test] fn intersect_permission_profiles_drops_broader_cwd_grant_for_requested_child_path() { let temp_dir = TempDir::new().expect("create temp dir"); @@ -567,7 +672,7 @@ fn intersect_permission_profiles_uses_granted_bounded_glob_scan_depth() { }; let granted = PermissionProfile { file_system: Some(FileSystemPermissions { - entries: vec![root_write.clone(), deny_env_files.clone()], + entries: vec![root_write.clone(), deny_env_files], glob_scan_max_depth: std::num::NonZeroUsize::new(4), }), ..Default::default() @@ -577,7 +682,20 @@ fn intersect_permission_profiles_uses_granted_bounded_glob_scan_depth() { intersect_permission_profiles(requested, granted, cwd.as_path()), PermissionProfile { file_system: Some(FileSystemPermissions { - entries: vec![root_write, deny_env_files], + entries: vec![ + root_write, + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: AbsolutePathBuf::resolve_path_against_base( + "**/*.env", + cwd.as_path() + ) + .to_string_lossy() + .into_owned(), + }, + access: FileSystemAccessMode::None, + }, + ], glob_scan_max_depth: std::num::NonZeroUsize::new(4), }), ..Default::default() @@ -609,7 +727,7 @@ fn intersect_permission_profiles_uses_granted_unbounded_glob_scan_depth() { }; let granted = PermissionProfile { file_system: Some(FileSystemPermissions { - entries: vec![root_write.clone(), deny_env_files.clone()], + entries: vec![root_write.clone(), deny_env_files], glob_scan_max_depth: None, }), ..Default::default() @@ -619,7 +737,20 @@ fn intersect_permission_profiles_uses_granted_unbounded_glob_scan_depth() { intersect_permission_profiles(requested, granted, cwd.as_path()), PermissionProfile { file_system: Some(FileSystemPermissions { - entries: vec![root_write, deny_env_files], + entries: vec![ + root_write, + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: AbsolutePathBuf::resolve_path_against_base( + "**/*.env", + cwd.as_path() + ) + .to_string_lossy() + .into_owned(), + }, + access: FileSystemAccessMode::None, + }, + ], glob_scan_max_depth: None, }), ..Default::default() From 6368f506b70fe43b92208ab796c7207c7a1dfb26 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 21 Apr 2026 17:32:46 -0700 Subject: [PATCH 2/4] fix: windows snapshot for external_agent_config_migration::tests::prompt_snapshot did not match windows output (#18915) Fix a snapshot test that is failing on Windows, but is currently missed by Bazel due to https://github.com/openai/codex/pull/18913. We see this failing on Cargo builds on Windows, though. This Bazel vs. Cargo inconsistency explains why https://github.com/openai/codex/pull/18768 did not fix the Cargo Windows build. --- ..._tests__external_agent_config_migration_prompt_windows.snap | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt_windows.snap b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt_windows.snap index 3c438fe458..44f41784e6 100644 --- a/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt_windows.snap +++ b/codex-rs/tui/src/snapshots/codex_tui__external_agent_config_migration__tests__external_agent_config_migration_prompt_windows.snap @@ -2,6 +2,7 @@ source: tui/src/external_agent_config_migration.rs expression: rendered --- + > External agent config detected We found settings from another agent that you can add to this project. Select what to import @@ -9,7 +10,7 @@ expression: rendered [x] Migrate /Users/alex/.claude/settings.json into /Users/alex/.codex/con… Project: C:\workspace\project - [x] Migrate enabled plugins from .claude\settings.json (4 marketplaces, 6… + [x] Migrate enabled plugins from .claude/settings.json (4 marketplaces, 6… • acme-tools: deployer, formatter, +1 more • team-marketplace: asana • debug: sample From 72baf32fd3cf2738eafe02a77744eb4c4a575f03 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 21 Apr 2026 17:43:06 -0700 Subject: [PATCH 3/4] fix build_prompt_input_includes_context_and_user_message --- codex-rs/core/src/prompt_debug.rs | 59 ------------------ codex-rs/core/tests/suite/mod.rs | 1 + .../core/tests/suite/prompt_debug_tests.rs | 60 +++++++++++++++++++ 3 files changed, 61 insertions(+), 59 deletions(-) create mode 100644 codex-rs/core/tests/suite/prompt_debug_tests.rs diff --git a/codex-rs/core/src/prompt_debug.rs b/codex-rs/core/src/prompt_debug.rs index 1f62c2b088..b6c66410a1 100644 --- a/codex-rs/core/src/prompt_debug.rs +++ b/codex-rs/core/src/prompt_debug.rs @@ -98,62 +98,3 @@ pub(crate) async fn build_prompt_input_from_session( Ok(prompt.get_formatted_input()) } - -#[cfg(test)] -mod tests { - use codex_protocol::models::ContentItem; - use codex_protocol::models::ResponseItem; - use codex_protocol::user_input::UserInput; - use codex_utils_absolute_path::AbsolutePathBuf; - use pretty_assertions::assert_eq; - - use crate::config::test_config; - - use super::build_prompt_input; - - #[tokio::test] - async fn build_prompt_input_includes_context_and_user_message() { - let codex_home = tempfile::tempdir().expect("create codex home"); - let cwd = tempfile::tempdir().expect("create cwd"); - let mut config = test_config().await; - config.codex_home = - AbsolutePathBuf::from_absolute_path(codex_home.path()).expect("codex home is absolute"); - config.cwd = AbsolutePathBuf::try_from(cwd.path().to_path_buf()).expect("absolute cwd"); - config.user_instructions = Some("Project-specific test instructions".to_string()); - - let input = build_prompt_input( - config, - vec![UserInput::Text { - text: "hello from debug prompt".to_string(), - text_elements: Vec::new(), - }], - ) - .await - .expect("build prompt input"); - - let expected_user_message = ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "hello from debug prompt".to_string(), - }], - end_turn: None, - phase: None, - }; - assert_eq!(input.last(), Some(&expected_user_message)); - assert!(input.iter().any(|item| { - let ResponseItem::Message { content, .. } = item else { - return false; - }; - - content.iter().any(|content_item| { - let (ContentItem::InputText { text } | ContentItem::OutputText { text }) = - content_item - else { - return false; - }; - text.contains("Project-specific test instructions") - }) - })); - } -} diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 9f8389c45e..6c4cb1981f 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -67,6 +67,7 @@ mod personality; mod personality_migration; mod plugins; mod prompt_caching; +mod prompt_debug_tests; mod quota_exceeded; mod realtime_conversation; mod remote_env; diff --git a/codex-rs/core/tests/suite/prompt_debug_tests.rs b/codex-rs/core/tests/suite/prompt_debug_tests.rs new file mode 100644 index 0000000000..1221560bb3 --- /dev/null +++ b/codex-rs/core/tests/suite/prompt_debug_tests.rs @@ -0,0 +1,60 @@ +use anyhow::Result; +use codex_core::build_prompt_input; +use codex_core::config::ConfigBuilder; +use codex_core::config::ConfigOverrides; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use codex_protocol::user_input::UserInput; +use pretty_assertions::assert_eq; +use tempfile::TempDir; + +#[tokio::test] +async fn build_prompt_input_includes_context_and_user_message() -> Result<()> { + let codex_home = TempDir::new()?; + let cwd = TempDir::new()?; + let mut config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .harness_overrides(ConfigOverrides { + cwd: Some(cwd.path().to_path_buf()), + codex_self_exe: Some(std::env::current_exe()?), + ..ConfigOverrides::default() + }) + .build() + .await?; + config.user_instructions = Some("Project-specific test instructions".to_string()); + + let input = build_prompt_input( + config, + vec![UserInput::Text { + text: "hello from debug prompt".to_string(), + text_elements: Vec::new(), + }], + ) + .await?; + + let expected_user_message = ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "hello from debug prompt".to_string(), + }], + end_turn: None, + phase: None, + }; + assert_eq!(input.last(), Some(&expected_user_message)); + assert!(input.iter().any(|item| { + let ResponseItem::Message { content, .. } = item else { + return false; + }; + + content.iter().any(|content_item| { + let (ContentItem::InputText { text } | ContentItem::OutputText { text }) = content_item + else { + return false; + }; + text.contains("Project-specific test instructions") + }) + })); + + Ok(()) +} From 69103a7180ac210831517f78163aae9a94f3786c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 21 Apr 2026 17:43:06 -0700 Subject: [PATCH 4/4] bazel: run wrapped Rust unit test shards --- defs.bzl | 8 ++-- workspace_root_test_launcher.bat.tpl | 72 ++++++++++++++++++++++++++++ workspace_root_test_launcher.sh.tpl | 66 +++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 5 deletions(-) diff --git a/defs.bzl b/defs.bzl index 53114a5773..d1e38ea91b 100644 --- a/defs.bzl +++ b/defs.bzl @@ -255,10 +255,9 @@ def codex_rust_crate( unit_test_name = name + "-unit-tests" unit_test_binary = name + "-unit-tests-bin" unit_test_shard_count = _test_shard_count(test_shard_counts, unit_test_name) - unit_test_binary_kwargs = {} - if unit_test_shard_count: - unit_test_binary_kwargs["experimental_enable_sharding"] = True - + # Shard at the workspace_root_test layer. rules_rust's sharding wrapper + # expects to run from its own runfiles cwd, while workspace_root_test + # deliberately changes cwd so Insta sees Cargo-like snapshot paths. rust_test( name = unit_test_binary, crate = name, @@ -277,7 +276,6 @@ def codex_rust_crate( rustc_env = rustc_env, data = test_data_extra, tags = test_tags + ["manual"], - **unit_test_binary_kwargs ) unit_test_kwargs = {} diff --git a/workspace_root_test_launcher.bat.tpl b/workspace_root_test_launcher.bat.tpl index 6b7222cca6..f3efe51826 100644 --- a/workspace_root_test_launcher.bat.tpl +++ b/workspace_root_test_launcher.bat.tpl @@ -12,9 +12,81 @@ if errorlevel 1 exit /b 1 set "INSTA_WORKSPACE_ROOT=%workspace_root%" cd /d "%workspace_root%" || exit /b 1 + +set "TOTAL_SHARDS=%RULES_RUST_TEST_TOTAL_SHARDS%" +if not defined TOTAL_SHARDS set "TOTAL_SHARDS=%TEST_TOTAL_SHARDS%" +if defined TOTAL_SHARDS if not "%TOTAL_SHARDS%"=="0" ( + call :run_sharded_libtest %* + exit /b !ERRORLEVEL! +) + "%test_bin%" %* exit /b %ERRORLEVEL% +:run_sharded_libtest +if defined TEST_SHARD_STATUS_FILE if defined TEST_TOTAL_SHARDS if not "%TEST_TOTAL_SHARDS%"=="0" ( + type nul > "%TEST_SHARD_STATUS_FILE%" +) + +if not "%~1"=="" ( + "%test_bin%" %* + exit /b !ERRORLEVEL! +) + +set "SHARD_INDEX=%RULES_RUST_TEST_SHARD_INDEX%" +if not defined SHARD_INDEX set "SHARD_INDEX=%TEST_SHARD_INDEX%" +if not defined SHARD_INDEX ( + >&2 echo TEST_SHARD_INDEX or RULES_RUST_TEST_SHARD_INDEX must be set when sharding is enabled + exit /b 1 +) + +set "TEMP_ROOT=%TEST_TMPDIR%" +if not defined TEMP_ROOT set "TEMP_ROOT=%TEMP%" +if not defined TEMP_ROOT set "TEMP_ROOT=." +:CREATE_TEMP_DIR +set "TEMP_DIR=%TEMP_ROOT%\workspace_root_test_sharding_!RANDOM!_!RANDOM!_!RANDOM!" +mkdir "!TEMP_DIR!" 2>nul +if errorlevel 1 goto :CREATE_TEMP_DIR +set "TEMP_LIST=!TEMP_DIR!\list.txt" +set "TEMP_SHARD_LIST=!TEMP_DIR!\shard.txt" + +"%test_bin%" --list --format terse > "!TEMP_LIST!" +if errorlevel 1 ( + rmdir /s /q "!TEMP_DIR!" 2>nul + exit /b 1 +) + +powershell.exe -NoProfile -ExecutionPolicy Bypass -Command ^ + "$ErrorActionPreference = 'Stop';" ^ + "$tests = @(Get-Content -LiteralPath $env:TEMP_LIST | Where-Object { $_.EndsWith(': test') } | ForEach-Object { $_.Substring(0, $_.Length - 6) });" ^ + "[Array]::Sort($tests, [StringComparer]::Ordinal);" ^ + "$totalShards = [uint32]$env:TOTAL_SHARDS; $shardIndex = [uint32]$env:SHARD_INDEX;" ^ + "$fnvPrime = [uint64]16777619; $u32Mask = [uint64]4294967295;" ^ + "foreach ($test in $tests) { $hash = [uint32]2166136261; foreach ($byte in [Text.Encoding]::UTF8.GetBytes($test)) { $hash = [uint32](([uint64]($hash -bxor $byte) * $fnvPrime) -band $u32Mask) }; if (($hash %% $totalShards) -eq $shardIndex) { $test } }" ^ + > "!TEMP_SHARD_LIST!" +if errorlevel 1 ( + rmdir /s /q "!TEMP_DIR!" 2>nul + exit /b 1 +) + +set SHARD_TESTS= +for /f "usebackq delims=" %%T in ("!TEMP_SHARD_LIST!") do ( + if "!SHARD_TESTS!"=="" ( + set SHARD_TESTS=%%T + ) else ( + set SHARD_TESTS=!SHARD_TESTS! %%T + ) +) + +rmdir /s /q "!TEMP_DIR!" 2>nul + +if "!SHARD_TESTS!"=="" ( + exit /b 0 +) + +"%test_bin%" !SHARD_TESTS! --exact +exit /b !ERRORLEVEL! + :resolve_runfile setlocal EnableExtensions EnableDelayedExpansion set "logical_path=%~2" diff --git a/workspace_root_test_launcher.sh.tpl b/workspace_root_test_launcher.sh.tpl index 1409e5672a..1ba752506b 100644 --- a/workspace_root_test_launcher.sh.tpl +++ b/workspace_root_test_launcher.sh.tpl @@ -48,6 +48,72 @@ workspace_root_marker="$(resolve_runfile "__WORKSPACE_ROOT_MARKER__")" workspace_root="$(dirname "$(dirname "$(dirname "${workspace_root_marker}")")")" test_bin="$(resolve_runfile "__TEST_BIN__")" +test_shard_index() { + local test_name="$1" + # FNV-1a 32-bit hash. Keep this stable so adding one test does not reshuffle + # unrelated tests between shards. + local hash=2166136261 + local byte + local char + local i + local LC_ALL=C + + for ((i = 0; i < ${#test_name}; i++)); do + char="${test_name:i:1}" + printf -v byte "%d" "'$char" + hash=$(( ((hash ^ byte) * 16777619) & 0xffffffff )) + done + + echo $(( hash % TOTAL_SHARDS )) +} + +run_sharded_libtest() { + if [[ -n "${TEST_SHARD_STATUS_FILE:-}" && "${TEST_TOTAL_SHARDS:-0}" != "0" ]]; then + touch "${TEST_SHARD_STATUS_FILE}" + fi + + # Extra libtest args are usually ad-hoc local filters. Preserve those exactly + # rather than combining them with generated exact filters. + if [[ $# -gt 0 ]]; then + exec "${test_bin}" "$@" + fi + + if [[ -z "${SHARD_INDEX}" ]]; then + echo "TEST_SHARD_INDEX or RULES_RUST_TEST_SHARD_INDEX must be set when sharding is enabled" >&2 + exit 1 + fi + + local list_output + local test_list + list_output="$("${test_bin}" --list --format terse)" + test_list="$(printf '%s\n' "${list_output}" | grep ': test$' | sed 's/: test$//' | LC_ALL=C sort || true)" + + if [[ -z "${test_list}" ]]; then + exit 0 + fi + + local shard_tests=() + local test_name + while IFS= read -r test_name; do + if (( $(test_shard_index "${test_name}") == SHARD_INDEX )); then + shard_tests+=("${test_name}") + fi + done <<< "${test_list}" + + if [[ ${#shard_tests[@]} -eq 0 ]]; then + exit 0 + fi + + exec "${test_bin}" "${shard_tests[@]}" --exact +} + export INSTA_WORKSPACE_ROOT="${workspace_root}" cd "${workspace_root}" + +TOTAL_SHARDS="${RULES_RUST_TEST_TOTAL_SHARDS:-${TEST_TOTAL_SHARDS:-}}" +SHARD_INDEX="${RULES_RUST_TEST_SHARD_INDEX:-${TEST_SHARD_INDEX:-}}" +if [[ -n "${TOTAL_SHARDS}" && "${TOTAL_SHARDS}" != "0" ]]; then + run_sharded_libtest "$@" +fi + exec "${test_bin}" "$@"