From 14ff9c3a43ebf9bf543cf5f7c45cff2bfea2e512 Mon Sep 17 00:00:00 2001 From: rafael-oai Date: Thu, 8 Oct 2026 20:53:29 +0000 Subject: [PATCH] Preserve UTF-8 byte semantics in network domain matching (#52277) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Domain wildcards need to retain the byte-based behavior of `globset`. Counting Unicode scalar values changes how `?` matches non-ASCII hosts. ## What changed Match domain patterns against UTF-8 bytes so `?` consumes exactly one byte. For example, `api??.com` matches `apié.com`. Match hosts through borrowed byte slices, avoiding the per-match character-vector allocation. Document ASCII case insensitivity and the absence of IDNA conversion: hosts supplied as Punycode require patterns using their Punycode spelling. ## Testing Compare Unicode wildcard cases against `globset` for both allowlists and denylists, including multibyte characters, combining marks, Punycode, and case sensitivity. Update the exhaustive dynamic-programming comparison and retry-boundary tests to operate on bytes. GitOrigin-RevId: fae5b3b1ae60b469cd61a44c5c6e245730adc8ba --- codex-rs/network-proxy/README.md | 5 +- codex-rs/network-proxy/src/domain_matcher.rs | 23 +++---- .../network-proxy/src/domain_matcher_tests.rs | 61 +++++++++++-------- 3 files changed, 53 insertions(+), 36 deletions(-) diff --git a/codex-rs/network-proxy/README.md b/codex-rs/network-proxy/README.md index 88d677f16b..bfee26f55d 100644 --- a/codex-rs/network-proxy/README.md +++ b/codex-rs/network-proxy/README.md @@ -241,9 +241,12 @@ what it can reasonably guarantee. - Allowlist-first policy: if `domains` has no `allow` entries, requests are blocked until an allowlist is configured. - Domain patterns: exact hosts are supported, `*.example.com` matches subdomains only, and `**.example.com` matches the apex plus subdomains; the global `*` wildcard is only accepted when explicitly enabled for allowlist compilation and is otherwise rejected. -- Within a domain pattern, `?` matches exactly one character, including a dot. For example, +- Within a domain pattern, `?` matches exactly one byte, including a dot. For example, `api?.example.com` matches `api1.example.com`, but not `api.example.com` or `api12.example.com`. It can be combined with `*`, `*.`, and `**.` in both allow and deny entries. + Matching uses UTF-8 bytes with ASCII case insensitivity; it does not convert between Unicode + domain names and IDNA/Punycode. Internationalized domains arriving as Punycode must be matched + using their Punycode spelling. - Deny wins: `domains` entries marked `deny` always override the allowlist. - Local/private network protection: when `allow_local_binding = false`, the proxy blocks loopback and common private/link-local ranges. Explicit allowlisting of local IP literals (or `localhost`) diff --git a/codex-rs/network-proxy/src/domain_matcher.rs b/codex-rs/network-proxy/src/domain_matcher.rs index 3fcd3b90d8..bf71f8a276 100644 --- a/codex-rs/network-proxy/src/domain_matcher.rs +++ b/codex-rs/network-proxy/src/domain_matcher.rs @@ -1,6 +1,7 @@ -//! Network-domain wildcards over Unicode scalar values, independent of filesystem semantics. +//! Network-domain wildcards over UTF-8 bytes, independent of filesystem semantics. //! Construction validates and normalizes patterns and expands domain prefixes. -//! Callers normalize candidate hosts before matching. +//! Callers normalize candidate hosts before matching. No IDNA conversion is performed. +//! Byte matching preserves the historical globset wildcard semantics. use anyhow::Result; use anyhow::bail; @@ -18,7 +19,7 @@ pub(crate) enum GlobalWildcard { #[derive(Clone)] pub struct DomainPatternSet { - patterns: Arc<[Vec]>, + patterns: Arc<[Vec]>, } impl DomainPatternSet { @@ -65,14 +66,14 @@ impl DomainPatternSet { // - "example.com": match the exact host // - "*.example.com": match any subdomain (not the apex) // - "**.example.com": match the apex and any subdomain - // - "api?.example.com": match exactly one Unicode scalar value after "api" + // - "api?.example.com": match exactly one byte after "api" // - "*": match every host when explicitly enabled for allowlist compilation for candidate in expand_domain_pattern(&pattern) { if !seen.insert(candidate.clone()) { continue; } ensure!(!candidate.is_empty(), "domain pattern is empty"); - compiled.push(candidate.chars().collect()); + compiled.push(candidate.into_bytes()); } } Ok(Self { @@ -81,8 +82,8 @@ impl DomainPatternSet { } pub fn is_match(&self, host: impl AsRef) -> bool { - let host: Vec = host.as_ref().chars().collect(); - self.patterns.iter().any(|pattern| matches(pattern, &host)) + let host = host.as_ref().as_bytes(); + self.patterns.iter().any(|pattern| matches(pattern, host)) } } @@ -186,16 +187,16 @@ fn expand_domain_pattern(pattern: &str) -> Vec { // Only the latest star needs retrying: it can absorb any extension that an earlier star could. // Its retry position advances monotonically, bounding work by O(pattern length * host length). -fn matches(pattern: &[char], host: &[char]) -> bool { +fn matches(pattern: &[u8], host: &[u8]) -> bool { let (mut p, mut h) = (0, 0); let mut star = None; while h < host.len() { match pattern.get(p) { - Some('*') => { + Some(b'*') => { star = Some((p + 1, h)); p += 1; } - Some(c) if *c == '?' || c.eq_ignore_ascii_case(&host[h]) => { + Some(c) if *c == b'?' || c.eq_ignore_ascii_case(&host[h]) => { p += 1; h += 1; } @@ -209,7 +210,7 @@ fn matches(pattern: &[char], host: &[char]) -> bool { }, } } - pattern[p..].iter().all(|c| *c == '*') + pattern[p..].iter().all(|c| *c == b'*') } #[cfg(test)] diff --git a/codex-rs/network-proxy/src/domain_matcher_tests.rs b/codex-rs/network-proxy/src/domain_matcher_tests.rs index c38562b07d..83c0eec3bf 100644 --- a/codex-rs/network-proxy/src/domain_matcher_tests.rs +++ b/codex-rs/network-proxy/src/domain_matcher_tests.rs @@ -5,19 +5,36 @@ use crate::policy::normalize_host; use pretty_assertions::assert_eq; #[test] -fn unicode_wildcards_count_scalars_not_bytes_or_graphemes() { +fn unicode_wildcards_preserve_utf8_byte_matching() { for (pattern, host, expected) in [ - ("api?.com", "apié.com", true), - ("api?.com", "api😀.com", true), - ("api??.com", "apié.com", false), - ("api?.com", "apie\u{301}.com", false), - ("api??.com", "apie\u{301}.com", true), - ("*.例?.com", "a.例子.com", true), - ("*.例?.com", "例子.com", false), - ("**.例?.com", "例子.com", true), + ("api?.com", "api1.com", true), + ("api??.com", "apié.com", true), + ("api????.com", "api😀.com", true), + ("api???.com", "apie\u{301}.com", true), + ("api?*.com", "apié.com", true), + ("*.例???.com", "a.例子.com", true), + ("*.例???.com", "例子.com", false), + ("**.例???.com", "例子.com", true), + ("büch?.de", "büch1.de", true), + ("bücher.de", "bücher.de", true), + ("bücher.de", "xn--bcher-kva.de", false), + ("xn--*.de", "xn--bcher-kva.de", true), ("É.com", "é.com", false), ("É.COM", "É.com", true), ] { + let mut legacy = globset::GlobSetBuilder::new(); + for expanded in expand_domain_pattern(&normalize_pattern(pattern)) { + legacy.add( + globset::GlobBuilder::new(&expanded) + .case_insensitive(true) + .build() + .unwrap(), + ); + } + assert_eq!( + legacy.build().unwrap().is_match(normalize_host(host)), + expected + ); for compile in [compile_allowlist, compile_denylist] { assert_eq!( compile(&[pattern.to_string()]) @@ -105,7 +122,7 @@ fn bracketed_patterns_preserve_port_and_trailing_dot_normalization() { #[test] fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() { - fn words(alphabet: &[char], max_len: usize) -> Vec> { + fn words(alphabet: &[u8], max_len: usize) -> Vec> { let mut all = vec![vec![]]; let mut level = vec![vec![]]; for _ in 0..max_len { @@ -123,23 +140,23 @@ fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() { } all } - fn reference(pattern: &[char], host: &[char]) -> bool { + fn reference(pattern: &[u8], host: &[u8]) -> bool { let mut rows = vec![vec![false; host.len() + 1]; pattern.len() + 1]; rows[0][0] = true; for (p, c) in pattern.iter().enumerate() { - rows[p + 1][0] = *c == '*' && rows[p][0]; + rows[p + 1][0] = *c == b'*' && rows[p][0]; for (h, candidate) in host.iter().enumerate() { - rows[p + 1][h + 1] = if *c == '*' { + rows[p + 1][h + 1] = if *c == b'*' { rows[p][h + 1] || rows[p + 1][h] } else { - rows[p][h] && (*c == '?' || c.eq_ignore_ascii_case(candidate)) + rows[p][h] && (*c == b'?' || c.eq_ignore_ascii_case(candidate)) }; } } rows[pattern.len()][host.len()] } - for pattern in words(&['a', 'é', '?', '*'], /*max_len*/ 4) { - for host in words(&['a', 'é'], /*max_len*/ 5) { + for pattern in words(&[b'a', 0xc3, 0xa9, b'?', b'*'], /*max_len*/ 4) { + for host in words(&[b'a', 0xc3, 0xa9], /*max_len*/ 5) { assert_eq!( matches(&pattern, &host), reference(&pattern, &host), @@ -151,17 +168,13 @@ fn matcher_agrees_with_independent_dynamic_program_for_short_inputs() { #[test] fn adversarial_retries_and_empty_boundaries() { - let pattern: Vec = format!("*{}b", "a".repeat(1024)).chars().collect(); - let host = vec!['a'; 4096]; + let pattern = format!("*{}b", "a".repeat(1024)).into_bytes(); + let host = vec![b'a'; 4096]; assert!(!matches(&pattern, &host)); - let pattern: Vec = format!("{}b", "*a".repeat(1024)).chars().collect(); + let pattern = format!("{}b", "*a".repeat(1024)).into_bytes(); assert!(!matches(&pattern, &host)); assert_eq!( - ( - matches(&[], &[]), - matches(&['*'], &[]), - matches(&['?'], &[]) - ), + (matches(&[], &[]), matches(b"*", &[]), matches(b"?", &[])), (true, true, false) ); assert!(!compile_allowlist(&[]).unwrap().is_match("host"));