From f65d3ca707dce1e7cad546e81eebdb6c2b4bd733 Mon Sep 17 00:00:00 2001 From: Alexandre van Beurden <1949482+alexvbrdn@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:36:45 +0200 Subject: [PATCH] big optimisations --- CHANGELOG.md | 9 +- Cargo.lock | 176 ++++++--- Cargo.toml | 2 +- src/fast_automaton/analyze/cardinality.rs | 12 +- src/fast_automaton/analyze/mod.rs | 78 ++-- src/fast_automaton/builder.rs | 17 +- src/fast_automaton/condition/converter.rs | 33 +- .../condition/fast_bit_vec/mod.rs | 29 ++ src/fast_automaton/condition/mod.rs | 50 ++- src/fast_automaton/convert/to_regex/mod.rs | 1 - .../to_regex/state_elimination/builder.rs | 26 +- .../to_regex/state_elimination/eliminate.rs | 185 +++++---- .../convert/to_regex/state_elimination/mod.rs | 141 ++++++- src/fast_automaton/generate.rs | 352 ++++++++---------- src/fast_automaton/mod.rs | 41 +- src/fast_automaton/operation/concat.rs | 45 ++- src/fast_automaton/operation/determinize.rs | 57 +-- src/fast_automaton/operation/intersection.rs | 140 +++++-- src/fast_automaton/operation/minimize.rs | 8 +- src/fast_automaton/operation/mod.rs | 14 + src/fast_automaton/operation/union.rs | 33 +- src/fast_automaton/spanning_set/mod.rs | 275 +++++++++++++- src/lib.rs | 72 ++-- src/regex/analyze/mod.rs | 1 - src/regex/analyze/number_of_states.rs | 22 +- src/regex/builder.rs | 15 +- src/regex/mod.rs | 89 +++-- src/regex/operation/concat.rs | 21 ++ src/regex/operation/repeat.rs | 2 - 29 files changed, 1318 insertions(+), 628 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 519142b..339faea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,10 +58,11 @@ below. language in. `Exhaustive` is the previous behaviour: shortest strings first, one path expanded in full before the next. `Sampled` covers every shape the automaton holds before asking any of them for a second string, - and picks representative characters (`a`, `0`, `A`, ` `, ...) spread over - each range, so `.*abc.*` yields `abc`, `abc `, `aabc`, `0abc`, ... instead - of a million variations of `abc\u{0}`. It stays deterministic and pages - with `offset` the same way. + so `.*abc.*` yields `abc`, `abc\u{0}`, `\u{0}abc`, ... instead of a + million variations of `abc\u{0}`. Within a shape, characters come in the + same ascending order `Exhaustive` uses — the order chooses which strings + come first, never the characters they are made of. It stays deterministic + and pages with `offset` the same way. - `GenerationOptions`, what `generate_strings`/`iter_strings` may generate: the order, plus an optional charset (`with_charset(CharRange)`) that keeps generation to a set of characters. Only strings made entirely of them come diff --git a/Cargo.lock b/Cargo.lock index 2e6d146..a398ab9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "anes" version = "0.1.6" @@ -75,6 +84,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -110,9 +129,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.2" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", ] @@ -135,25 +154,24 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "criterion" -version = "0.5.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ + "alloca", "anes", "cast", "ciborium", "clap", "criterion-plot", - "is-terminal", "itertools", "num-traits", - "once_cell", "oorandom", + "page_size", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "walkdir", @@ -161,9 +179,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", "itertools", @@ -202,9 +220,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "equivalent" @@ -224,9 +242,15 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fnv" @@ -236,21 +260,21 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-core", "futures-task", @@ -298,12 +322,6 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - [[package]] name = "indexmap" version = "2.14.0" @@ -320,22 +338,11 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de8eb46ec263fe6dd2dea011ecc4cfbf23ef16be6ec5984ac1b75d8edad19d6" -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - [[package]] name = "itertools" -version = "0.10.5" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -359,9 +366,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "linux-raw-sys" @@ -396,6 +403,16 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -441,9 +458,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -475,9 +492,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -649,9 +666,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -659,29 +676,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -690,6 +707,12 @@ dependencies = [ "zmij", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "slab" version = "0.4.12" @@ -707,6 +730,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -749,7 +783,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -839,7 +873,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -862,6 +896,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -871,6 +921,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" @@ -894,22 +950,22 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 87db791..958b154 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ default = ["parallel"] parallel = ["dep:rayon"] [dev-dependencies] -criterion = { version = "0.5", features = ["html_reports"] } +criterion = { version = "0.8", features = ["html_reports"] } proptest = "1" regex = "1.10.3" diff --git a/src/fast_automaton/analyze/cardinality.rs b/src/fast_automaton/analyze/cardinality.rs index eb8fc18..1bf13e4 100644 --- a/src/fast_automaton/analyze/cardinality.rs +++ b/src/fast_automaton/analyze/cardinality.rs @@ -62,12 +62,12 @@ impl FastAutomaton { continue; } let condition_cardinality = condition.cardinality(&self.spanning_set)?; - if let Some(distance) = current_distance.checked_mul(condition_cardinality) - && let Some(new_distance) = - distances.get(to_state).unwrap_or(&0).checked_add(distance) - { - distances.insert(*to_state, new_distance); - continue; + if let Some(distance) = current_distance.checked_mul(condition_cardinality) { + let slot = distances.entry(*to_state).or_insert(0); + if let Some(new_distance) = slot.checked_add(distance) { + *slot = new_distance; + continue; + } } return Ok(Cardinality::BigInteger); diff --git a/src/fast_automaton/analyze/mod.rs b/src/fast_automaton/analyze/mod.rs index feeb9aa..69d8fe5 100644 --- a/src/fast_automaton/analyze/mod.rs +++ b/src/fast_automaton/analyze/mod.rs @@ -1,5 +1,3 @@ -use std::hash::BuildHasherDefault; - use crate::{cardinality::Cardinality, error::EngineError}; use super::*; @@ -75,7 +73,7 @@ impl FastAutomaton { if cond.is_empty() { continue; } - covered = covered.union(cond); + covered.union_with(cond); if visited.insert(*to) { worklist.push_back(*to); } @@ -158,34 +156,27 @@ impl FastAutomaton { /// This is co-reachability; note it is *not* the set of states reachable /// from the start state. pub fn live_states(&self) -> IntSet { - let mut states_map: IntMap> = - IntMap::with_capacity_and_hasher(self.transitions.len(), BuildHasherDefault::default()); - for from_state in self.states() { - for (condition, to_state) in self.transitions_from(from_state) { - if condition.is_empty() { - continue; - } - match states_map.entry(*to_state) { - Entry::Occupied(mut o) => o.get_mut().insert(from_state), - Entry::Vacant(v) => { - let mut new_states = IntSet::default(); - new_states.insert(from_state); - v.insert(new_states); - true - } - }; - } - } - + // Reverse BFS over the maintained `transitions_in` adjacency. + // `transitions_in` doesn't filter empty-condition edges + // (constructible via the public `add_transition`), so each edge's + // condition is checked on traversal — the lookup on `transitions` + // also makes tombstoned predecessors fall out naturally. let mut worklist = VecDeque::from_iter(self.accept_states.iter().cloned()); let mut live = self.accept_states.clone(); while let Some(live_state) = worklist.pop_front() { - if let Some(states) = states_map.get(&live_state) { - for state in states { - if !live.contains(state) { - live.insert(*state); - worklist.push_back(*state); - } + let Some(predecessors) = self.transitions_in.get(&live_state) else { + continue; + }; + for &from_state in predecessors { + if live.contains(&from_state) { + continue; + } + let takeable = self + .condition(from_state, live_state) + .is_some_and(|condition| !condition.is_empty()); + if takeable { + live.insert(from_state); + worklist.push_back(from_state); } } } @@ -203,11 +194,10 @@ impl FastAutomaton { /// spanning set with an empty rest this is exactly the spanning ranges, so /// well-formed automata are unaffected.) pub fn spanning_bases(&self) -> Result, EngineError> { - self.spanning_set - .spanning_ranges_with_rest() - .iter() - .map(|range| Condition::from_range(range, &self.spanning_set)) - .collect() + // Base `i` is by construction exactly bit `i` of a condition. + Ok((0..self.spanning_set.spanning_ranges_with_rest_len()) + .map(|i| Condition::single_base(i, &self.spanning_set)) + .collect()) } } @@ -242,4 +232,26 @@ mod tests { Ok(()) } + + // An empty-condition transition (constructible via the public + // `add_transition`) can't be taken, so a state whose only path to an + // accept state goes through one is dead. `live_states` walks + // `transitions_in`, which records such edges — it must check the + // condition instead of trusting the adjacency. + #[test] + fn live_states_ignores_empty_condition_edges() { + use crate::fast_automaton::condition::Condition; + + let mut a = FastAutomaton::new_empty(); + let s1 = a.new_state(); + let s2 = a.new_state(); + a.add_transition(0, s1, &Condition::total(a.spanning_set())); + a.add_transition(s2, s1, &Condition::empty(a.spanning_set())); + a.accept(s1); + + let live = a.live_states(); + assert!(live.contains(&0)); + assert!(live.contains(&s1)); + assert!(!live.contains(&s2)); + } } diff --git a/src/fast_automaton/builder.rs b/src/fast_automaton/builder.rs index 359444e..7330548 100644 --- a/src/fast_automaton/builder.rs +++ b/src/fast_automaton/builder.rs @@ -159,7 +159,7 @@ impl FastAutomaton { .insert(from_state); match self.transitions[from_state].entry(to_state) { Entry::Occupied(mut o) => { - o.insert(o.get().union(new_cond)); + o.get_mut().union_with(new_cond); } Entry::Vacant(v) => { v.insert(new_cond.clone()); @@ -307,7 +307,7 @@ impl FastAutomaton { .insert(from_state); match self.transitions[from_state].entry(state) { Entry::Occupied(mut o) => { - o.insert(o.get().union(&cond)); + o.get_mut().union_with(&cond); } Entry::Vacant(v) => { v.insert(cond); @@ -441,14 +441,11 @@ impl FastAutomaton { return Ok(()); } let condition_converter = ConditionConverter::new(&self.spanning_set, new_spanning_set)?; - for &from_state in &self.states_vec() { - for to_state in self.direct_states_vec(from_state) { - match self.transitions[from_state].entry(to_state) { - Entry::Occupied(mut o) => { - o.insert(condition_converter.convert(o.get())?); - } - Entry::Vacant(_) => {} - }; + // Removed states keep a cleared transition map (see `remove_state`), + // so every stored condition can be converted in place directly. + for transitions in self.transitions.iter_mut() { + for condition in transitions.values_mut() { + *condition = condition_converter.convert(condition)?; } } self.spanning_set = new_spanning_set.clone(); diff --git a/src/fast_automaton/condition/converter.rs b/src/fast_automaton/condition/converter.rs index bb0e210..624ba4d 100644 --- a/src/fast_automaton/condition/converter.rs +++ b/src/fast_automaton/condition/converter.rs @@ -1,6 +1,4 @@ -use ahash::HashMapExt; - -use crate::{IntMap, error::EngineError, fast_automaton::spanning_set::SpanningSet}; +use crate::{CharRange, error::EngineError, fast_automaton::spanning_set::SpanningSet}; use super::Condition; @@ -24,28 +22,25 @@ impl<'a, 'b> ConditionConverter<'a, 'b> { from_spanning_set: &'a SpanningSet, to_spanning_set: &'b SpanningSet, ) -> Result { - let mut to_base_map = - IntMap::with_capacity(to_spanning_set.spanning_ranges_with_rest_len()); - for (i, base) in to_spanning_set + // Each target base maps to at most one source base; consumed entries + // are marked `None` so later source bases skip them. + let mut to_bases: Vec> = to_spanning_set .spanning_ranges_with_rest() - .into_iter() - .enumerate() - { - to_base_map.insert(i, base); - } + .map(Some) + .collect(); let mut equivalence_map: Vec> = - Vec::with_capacity(from_spanning_set.number_of_spanning_ranges() + 1); - for from_base in from_spanning_set.spanning_ranges_with_rest().iter() { + Vec::with_capacity(from_spanning_set.spanning_ranges_with_rest_len()); + for from_base in from_spanning_set.spanning_ranges_with_rest() { let mut index = Vec::with_capacity(1); - for (i, to_base) in &to_base_map { - if from_base == to_base || from_base.has_intersection(to_base) { - index.push(*i); + for (i, slot) in to_bases.iter_mut().enumerate() { + if let Some(to_base) = slot + && (from_base == *to_base || from_base.has_intersection(to_base)) + { + index.push(i); + *slot = None; } } - index.iter().for_each(|i| { - to_base_map.remove(i); - }); equivalence_map.push(index); } diff --git a/src/fast_automaton/condition/fast_bit_vec/mod.rs b/src/fast_automaton/condition/fast_bit_vec/mod.rs index d8b8ec3..481b157 100644 --- a/src/fast_automaton/condition/fast_bit_vec/mod.rs +++ b/src/fast_automaton/condition/fast_bit_vec/mod.rs @@ -112,6 +112,14 @@ impl FastBitVec { } } + #[inline] + pub fn difference(&mut self, other: &Self) { + self.assert_same_len(other); + for (a, b) in self.bits.iter_mut().zip(&other.bits) { + *a &= !b; + } + } + #[inline] pub fn has_intersection(&self, other: &Self) -> bool { self.assert_same_len(other); @@ -149,4 +157,25 @@ impl FastBitVec { } bits } + + /// Iterates the indices of the set bits in ascending order, word-wise + /// (no allocation). + #[inline] + pub fn iter_set_bits(&self) -> impl Iterator + '_ { + self.bits + .iter() + .enumerate() + .flat_map(|(word_index, &word)| { + let mut remaining = word; + std::iter::from_fn(move || { + if remaining == 0 { + None + } else { + let bit = remaining.trailing_zeros() as usize; + remaining &= remaining - 1; + Some(word_index * 64 + bit) + } + }) + }) + } } diff --git a/src/fast_automaton/condition/mod.rs b/src/fast_automaton/condition/mod.rs index 0d4d794..3269d8a 100644 --- a/src/fast_automaton/condition/mod.rs +++ b/src/fast_automaton/condition/mod.rs @@ -65,7 +65,7 @@ impl Condition { let mut cond = Self::empty(spanning_set); - for (i, base) in spanning_set.spanning_ranges_with_rest().iter().enumerate() { + for (i, base) in spanning_set.spanning_ranges_with_rest().enumerate() { if range.contains_all(base) { cond.0.set(i, true); } @@ -78,6 +78,16 @@ impl Condition { Ok(cond) } + /// Returns the condition whose only set bit is base `i` of `spanning_set` + /// (bit `i` corresponds to `spanning_ranges_with_rest()`'s element `i`, + /// the rest range first when present). + #[inline] + pub(crate) fn single_base(i: usize, spanning_set: &SpanningSet) -> Self { + let mut cond = Self::empty(spanning_set); + cond.0.set(i, true); + cond + } + /// Converts this `Condition` back to the [`CharRange`] it represents, /// evaluated against `spanning_set`. /// @@ -94,7 +104,7 @@ impl Condition { let mut range = CharRange::empty(); - for (i, base) in spanning_set.spanning_ranges_with_rest().iter().enumerate() { + for (i, base) in spanning_set.spanning_ranges_with_rest().enumerate() { if self.0.get(i) { range = range.union(base); } @@ -112,6 +122,13 @@ impl Condition { new_cond } + /// Unions `other` into `self` in place (bitwise OR). Both must share the + /// same spanning set. + #[inline] + pub fn union_with(&mut self, other: &Condition) { + self.0.union(&other.0); + } + /// Returns the condition matching characters in both `self` and `other` /// (bitwise AND). Both must share the same spanning set. #[inline] @@ -135,8 +152,7 @@ impl Condition { #[inline] pub fn difference(&self, other: &Condition) -> Self { let mut new_cond = self.clone(); - let subtrahend = other.complement(); - new_cond.0.intersection(&subtrahend.0); + new_cond.0.difference(&other.0); new_cond } @@ -200,9 +216,24 @@ impl Condition { /// Returns the number of characters the condition matches, evaluated /// against `spanning_set`. + /// + /// Returns [`EngineError::IncompatibleSpanningSet`] if this condition's + /// bit width does not match `spanning_set`. #[inline] pub fn cardinality(&self, spanning_set: &SpanningSet) -> Result { - Ok(self.to_range(spanning_set)?.get_cardinality()) + if self.0.len() != spanning_set.spanning_ranges_with_rest_len() { + return Err(EngineError::IncompatibleSpanningSet); + } + + // The bases are disjoint, so the cardinality of their union is the + // sum of their cardinalities. + let mut cardinality = 0u32; + for (i, base) in spanning_set.spanning_ranges_with_rest().enumerate() { + if self.0.get(i) { + cardinality += base.get_cardinality(); + } + } + Ok(cardinality) } /// Returns the condition as a vector of bits, one per range of the spanning @@ -211,6 +242,13 @@ impl Condition { pub fn binary_representation(&self) -> Vec { self.0.bits() } + + /// Iterates the indices of the set bits (i.e., the bases the condition + /// covers) in ascending order, without allocating. + #[inline] + pub(crate) fn iter_set_bits(&self) -> impl Iterator + '_ { + self.0.iter_set_bits() + } } #[cfg(test)] @@ -288,14 +326,12 @@ mod tests { fn test_empty_total() -> Result<(), String> { let spanning_set = spanning_set(); let empty = Condition::empty(&spanning_set); - //println!("{empty}"); assert!(empty.is_empty()); assert_eq!( vec![false, false, false, false], empty.binary_representation() ); let total = Condition::total(&spanning_set); - //println!("{total}"); assert!(total.is_total()); assert_eq!(vec![true, true, true, true], total.binary_representation()); diff --git a/src/fast_automaton/convert/to_regex/mod.rs b/src/fast_automaton/convert/to_regex/mod.rs index 768b325..a44e8d1 100644 --- a/src/fast_automaton/convert/to_regex/mod.rs +++ b/src/fast_automaton/convert/to_regex/mod.rs @@ -70,7 +70,6 @@ mod tests { assert!(input_automaton.equivalent(&output_automaton).unwrap()); let input_automaton = input_automaton.determinize().unwrap(); - //input_automaton.to_dot(); let output_regex = input_automaton.to_regex().unwrap(); println!("OUT (deterministic) : {}", output_regex); diff --git a/src/fast_automaton/convert/to_regex/state_elimination/builder.rs b/src/fast_automaton/convert/to_regex/state_elimination/builder.rs index ab4eb6f..568e9ae 100644 --- a/src/fast_automaton/convert/to_regex/state_elimination/builder.rs +++ b/src/fast_automaton/convert/to_regex/state_elimination/builder.rs @@ -79,10 +79,10 @@ impl Gnfa { } fn new_state(&mut self) -> usize { - if let Some(new_state) = self.removed_states.clone().iter().next() { - self.removed_states.remove(new_state); - self.transitions_in.insert(*new_state, IntSet::new()); - *new_state + if let Some(&new_state) = self.removed_states.iter().next() { + self.removed_states.remove(&new_state); + self.transitions_in.insert(new_state, IntSet::new()); + new_state } else { self.transitions.push(IntMap::default()); self.transitions_in @@ -136,7 +136,11 @@ impl Gnfa { "Can not remove the state {state}, it is still used as start state or accept state." ); } - self.transitions_in.remove(&state); + // Snapshot the exact predecessor and successor sets before the maps + // below are cleared; only their entries need updating. + let predecessors = self.transitions_in.remove(&state).unwrap_or_default(); + let successors: Vec = self.transitions[state].keys().copied().collect(); + if self.transitions.len() - 1 == state { self.transitions.remove(state); @@ -153,11 +157,15 @@ impl Gnfa { self.removed_states.insert(state); } - for transitions in self.transitions.iter_mut() { - transitions.remove(&state); + for &from_state in &predecessors { + if let Some(transitions) = self.transitions.get_mut(from_state) { + transitions.remove(&state); + } } - for transitions in self.transitions_in.values_mut() { - transitions.remove(&state); + for to_state in successors { + if let Some(transitions_in) = self.transitions_in.get_mut(&to_state) { + transitions_in.remove(&state); + } } } } diff --git a/src/fast_automaton/convert/to_regex/state_elimination/eliminate.rs b/src/fast_automaton/convert/to_regex/state_elimination/eliminate.rs index 351a6b5..92d49cf 100644 --- a/src/fast_automaton/convert/to_regex/state_elimination/eliminate.rs +++ b/src/fast_automaton/convert/to_regex/state_elimination/eliminate.rs @@ -7,76 +7,98 @@ impl Gnfa { } let execution_profile = crate::execution_profile::ExecutionProfile::get(); - while let Some(state) = self.get_next_state_to_eliminate() { - execution_profile.assert_not_timed_out()?; - self.eliminate_state(state); - } - - Ok(self - .get_transition(self.start_state, self.accept_state) - .cloned() - .unwrap_or(RegularExpression::new_empty_string())) - } - - fn get_next_state_to_eliminate(&self) -> Option { - let states: Vec = self - .all_states_iter() - .filter(|&s| s != self.start_state && s != self.accept_state) - .collect(); - let score_state = |state: usize| -> Option<(u128, usize)> { - let preds = self.transitions_to_vec(state); - let succs = self.transitions_from_vec(state); + // Cached elimination scores, indexed by state. Ids are stable during + // elimination (no state is created; tombstone compaction only pops + // trailing, already-removed entries), and `None` marks + // non-candidates: start, accept, and eliminated states. + // + // A state's score depends only on its own in/out edges, and + // eliminating `k` only touches edges incident to k's predecessors + // and successors — so exactly those need re-scoring each round. + let mut scores: Vec> = vec![None; self.transitions.len()]; + for state in self.all_states_iter() { + if state != self.start_state && state != self.accept_state { + scores[state] = Some(self.score_state(state)); + } + } - let in_deg = preds.len() as u128; - let out_deg = succs.len() as u128; + loop { + execution_profile.assert_not_timed_out()?; - if in_deg == 0 || out_deg == 0 { - let score = (state as u128) & 0xFF; - return Some((score, state)); + // Lowest score wins; `<=` keeps the last minimal state on ties + // (the order `convert_reference` pins). + let mut best: Option<(u128, usize)> = None; + for (state, &score) in scores.iter().enumerate() { + if let Some(score) = score + && best.is_none_or(|(best_score, _)| score <= best_score) + { + best = Some((score, state)); + } + } + let Some((_, k)) = best else { + break; + }; + + scores[k] = None; + let (predecessors, successors) = self.eliminate_state(k); + for state in predecessors.into_iter().chain(successors) { + if scores[state].is_some() { + scores[state] = Some(self.score_state(state)); + } } + } - let mut score: u128 = in_deg * out_deg; + // Moved out, not cloned: the `Gnfa` is discarded right after. + Ok(self + .transitions + .get_mut(self.start_state) + .and_then(|transitions| transitions.remove(&self.accept_state)) + .unwrap_or_else(RegularExpression::new_empty_string)) + } - if self.has_self_loop(state) { - score = score + (score >> 1); - } + /// The elimination score. It reads only `state`'s own degrees and label + /// complexities, which is what makes the cached, neighbors-only + /// re-scoring in [`convert`](Self::convert) sound. + fn score_state(&self, state: usize) -> u128 { + let mut in_deg: u128 = 0; + let mut label_cost: u128 = 0; + for (_, regex) in self.transitions_to_iter(state) { + in_deg += 1; + label_cost += regex.evaluate_complexity() as u128; + } + let mut out_deg: u128 = 0; + for (regex, _) in self.transitions_from_iter(state) { + out_deg += 1; + label_cost += regex.evaluate_complexity() as u128; + } - let mut label_cost: u128 = 0; + if in_deg == 0 || out_deg == 0 { + return (state as u128) & 0xFF; + } - for (_, regex) in &preds { - label_cost += regex.evaluate_complexity() as u128; - } - for (regex, _) in &succs { - label_cost += regex.evaluate_complexity() as u128; - } - if let Some(re) = self.get_transition(state, state) { - label_cost += (re.evaluate_complexity() as u128) * 2; - } + let mut score: u128 = in_deg * out_deg; - score = score.saturating_add(label_cost); + if self.has_self_loop(state) { + score = score + (score >> 1); + } - let tie = (state as u128) & 0xFFFF; - Some((score.saturating_add(tie), state)) - }; + if let Some(re) = self.get_transition(state, state) { + label_cost += (re.evaluate_complexity() as u128) * 2; + } - #[cfg(feature = "parallel")] - let best = states - .into_par_iter() - .filter_map(score_state) - .reduce_with(|a, b| if a.0 < b.0 { a } else { b }); - #[cfg(not(feature = "parallel"))] - let best = states - .into_iter() - .filter_map(score_state) - .reduce(|a, b| if a.0 < b.0 { a } else { b }); + score = score.saturating_add(label_cost); - best.map(|(_, state)| state) + let tie = (state as u128) & 0xFFFF; + score.saturating_add(tie) } - fn eliminate_state(&mut self, k: usize) { + /// Bridges every predecessor to every successor and removes `k`, + /// returning those predecessors and successors (`k` excluded): the only + /// states whose elimination scores the operation changed. + fn eliminate_state(&mut self, k: usize) -> (Vec, Vec) { if self.removed_states.contains(&k) { - return; + return (vec![], vec![]); } let in_states = self @@ -93,29 +115,66 @@ impl Gnfa { .filter(|&s| s != k) .collect::>(); - for p in in_states { + // The k→k self-loop star is the same for every (p, q) pair, and + // bridging never touches the (k, k) edge, so build it once. + let star = self + .get_transition(k, k) + .map(|self_loop| self_loop.repeat(0, None)); + + for &p in &in_states { for &q in &out_states { - self.bridge(p, k, q); + self.bridge(p, k, q, star.as_ref()); } } self.remove_state(k); + + (in_states, out_states) } - fn bridge(&mut self, p: usize, k: usize, q: usize) { + fn bridge(&mut self, p: usize, k: usize, q: usize, star: Option<&RegularExpression>) { let rpk = self.get_transition(p, k); - let rkk = self.get_transition(k, k); let rkq = self.get_transition(k, q); if let (Some(rpk), Some(rkq)) = (rpk, rkq) { let mut regex = rpk.clone(); - if let Some(rkk) = rkk { - //regex = RegularExpression::Concat(VecDeque::from_iter(vec![regex, RegularExpression::Repetition(Box::new(rkk.clone()), 0, None)])); - regex = regex.concat(&rkk.repeat(0, None), true); + if let Some(star) = star { + regex = regex.concat(star, true); } - //regex = RegularExpression::Concat(VecDeque::from_iter(vec![regex, rkq.clone()])); regex = regex.concat(rkq, true); self.add_transition(p, q, regex); } } } + +#[cfg(test)] +impl Gnfa { + /// [`convert`](Self::convert) with the score cache disabled: every + /// candidate is re-scored from scratch each round, with the serial fold + /// (last minimal state wins on ties) the cached version replaced. The + /// oracle proving the cache never yields a stale score — i.e. the + /// produced pattern is identical to the pre-cache implementation's. + pub(super) fn convert_reference(&mut self) -> Result { + if self.empty { + return Ok(RegularExpression::new_empty()); + } + + loop { + let best = self + .all_states_iter() + .filter(|&s| s != self.start_state && s != self.accept_state) + .map(|state| (self.score_state(state), state)) + .reduce(|a, b| if a.0 < b.0 { a } else { b }); + let Some((_, state)) = best else { + break; + }; + self.eliminate_state(state); + } + + Ok(self + .transitions + .get_mut(self.start_state) + .and_then(|transitions| transitions.remove(&self.accept_state)) + .unwrap_or_else(RegularExpression::new_empty_string)) + } +} diff --git a/src/fast_automaton/convert/to_regex/state_elimination/mod.rs b/src/fast_automaton/convert/to_regex/state_elimination/mod.rs index c0d60c6..dcf43f9 100644 --- a/src/fast_automaton/convert/to_regex/state_elimination/mod.rs +++ b/src/fast_automaton/convert/to_regex/state_elimination/mod.rs @@ -28,7 +28,7 @@ impl Display for Gnfa { writeln!(sb, "\tinitial [shape=plaintext,label=\"\"];")?; writeln!(sb, "\tinitial -> {from_state}")?; } - for (regex, to_state) in self.transitions_from_vec(from_state) { + for (regex, to_state) in self.transitions_from_iter(from_state) { writeln!(sb, "\t{from_state} -> {to_state} [label=\"{regex}\"]")?; } } @@ -46,26 +46,32 @@ impl Gnfa { (0..self.transitions.len()).filter(|s| !self.removed_states.contains(s)) } - fn transitions_to_vec(&self, state: State) -> Vec<(State, RegularExpression)> { - let mut in_transitions = vec![]; - for from_state in self.transitions_in.get(&state).unwrap_or(&IntSet::new()) { - for (condition, to_state) in self.transitions_from_vec(*from_state) { - if to_state == state { - in_transitions.push((*from_state, condition)); - break; - } - } - } - in_transitions + /// Incoming transitions by reference: `transitions_in` gives the exact + /// predecessor set, so each edge label is one direct map lookup. + fn transitions_to_iter( + &self, + state: State, + ) -> impl Iterator { + self.transitions_in + .get(&state) + .into_iter() + .flatten() + .filter_map(move |&from_state| { + self.get_transition(from_state, state) + .map(|regex| (from_state, regex)) + }) } + /// Outgoing transitions by reference. #[inline] - fn transitions_from_vec(&self, state: State) -> Vec<(RegularExpression, State)> { + fn transitions_from_iter( + &self, + state: State, + ) -> impl Iterator { self.transitions[state] .iter() - .map(|(s, c)| (c.clone(), *s)) - .filter(|s| !self.removed_states.contains(&s.1)) - .collect() + .filter(|(s, _)| !self.removed_states.contains(*s)) + .map(|(s, c)| (c, *s)) } #[inline] @@ -84,6 +90,109 @@ pub(super) fn convert_to_regex( #[cfg(test)] mod tests { use super::*; + use crate::CharRange; + use proptest::prelude::*; + use regex_charclass::char::Char; + + /// Every produced pattern must be byte-identical to what full + /// re-scoring produces: the score cache must never go stale. + fn assert_same_pattern_as_reference(automaton: &FastAutomaton) { + let incremental = Gnfa::from_automaton(automaton).unwrap().convert().unwrap(); + let reference = Gnfa::from_automaton(automaton) + .unwrap() + .convert_reference() + .unwrap(); + assert_eq!( + incremental.to_string(), + reference.to_string(), + "incremental scoring changed the elimination order" + ); + } + + #[test] + fn incremental_scoring_matches_full_rescoring() { + let patterns = [ + "abc", + ".*de", + "(a*ba*)*", + ".*u(ab|de)", + "a(bcfe|bcdg|mkv)*(abc){2,3}", + "(aad|ads|a)*abc.*def(x|q)", + "(a|b)*a(a|b){3}", + "[a-z]{1,6}", + "(ab|xy){2}", + "x*|(xxx)*|y", + "a{0,3}b{2}(c|d)?", + ]; + for pattern in patterns { + let automaton = RegularExpression::new(pattern) + .unwrap() + .to_automaton() + .unwrap(); + assert_same_pattern_as_reference(&automaton); + assert_same_pattern_as_reference(&automaton.determinize().unwrap()); + } + } + + /// A small palette of character ranges for random automata. + fn palette(index: usize) -> CharRange { + let bounds = [ + ('a', 'a'), + ('b', 'b'), + ('c', 'c'), + ('a', 'c'), + ('b', 'd'), + ('x', 'z'), + ('\u{0}', '\u{10FFFF}'), + ]; + let (low, high) = bounds[index % bounds.len()]; + CharRange::new_from_range(Char::new(low)..=Char::new(high)) + } + + fn arb_automaton() -> impl Strategy { + ( + 2usize..7, + proptest::collection::vec((0usize..6, 0usize..6, 0usize..7), 1..15), + 0u8..=255, + ) + .prop_map(|(number_of_states, edges, accept_mask)| { + let mut automaton = FastAutomaton::new_empty(); + for _ in 1..number_of_states { + automaton.new_state(); + } + for (from, to, range) in edges { + automaton + .add_transition_from_range( + from % number_of_states, + to % number_of_states, + &palette(range), + ) + .unwrap(); + } + for state in 0..number_of_states { + if accept_mask & (1 << state) != 0 { + automaton.accept(state); + } + } + automaton + }) + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(256))] + + #[test] + fn incremental_scoring_matches_full_rescoring_on_random_automata( + automaton in arb_automaton() + ) { + let incremental = Gnfa::from_automaton(&automaton).unwrap().convert().unwrap(); + let reference = Gnfa::from_automaton(&automaton) + .unwrap() + .convert_reference() + .unwrap(); + prop_assert_eq!(incremental.to_string(), reference.to_string()); + } + } #[test] fn test_state_elimination() -> Result<(), String> { diff --git a/src/fast_automaton/generate.rs b/src/fast_automaton/generate.rs index ac42751..2d9e317 100644 --- a/src/fast_automaton/generate.rs +++ b/src/fast_automaton/generate.rs @@ -22,9 +22,8 @@ pub enum GenerationOrder { #[default] Exhaustive, /// Samples the language instead of sweeping it: a few strings per path - /// before moving to the next one, with representative characters (`a`, - /// `0`, `A`, ` `, ...) spread over each range rather than always its first - /// one. `.*abc.*` yields `abc`, `0abc`, `aabc`, `abc0`, ... — the shapes + /// before moving to the next one, so `.*abc.*` yields `abc` and a string + /// for each of the other shapes (`abc\u{0}`, `\u{0}abc`, ...) — the shapes /// the pattern allows, instead of a million variations of one of them, /// which is what makes it usable to derive test cases. /// @@ -33,6 +32,12 @@ pub enum GenerationOrder { /// the number of shapes is spent entirely on distinct shapes, and only a /// larger one starts varying the characters within them. /// + /// Within a path, characters come in the same ascending order + /// [`Exhaustive`](Self::Exhaustive) uses: the order chooses which strings + /// come first, never the characters they are made of. To generate from + /// specific characters, restrict generation with + /// [`GenerationOptions::with_charset`]. + /// /// Deterministic, and pages with `offset` like [`Exhaustive`](Self::Exhaustive). /// Each pass takes twice as many strings per path as the previous one, so /// a finite language is still enumerated in full given a large enough @@ -40,27 +45,6 @@ pub enum GenerationOrder { Sampled, } -/// The characters a sampled string reaches for first, in order of preference: -/// one per kind of character a range is usually built from, so that an early -/// sample lands on a letter, a digit or a space rather than on `\u{0}`. The -/// list is rotated by position, so neighbouring characters of a sample differ. -const SAMPLE_CHARS: [char; 10] = [ - 'a', - '0', - 'A', - ' ', - '_', - '~', - '\n', - '\u{e9}', - '\u{4e2d}', - '\u{1f600}', -]; - -/// The block of code points `char` cannot hold: [`Char`] values skip it, so -/// scalar values have to be shifted down past it to be counted. -const SURROGATES: Range = 0xD800..0xE000; - /// The most strings to reserve room for up front. `limit` is caller-controlled /// and huge values (up to `usize::MAX`) are legitimate ways to ask for /// everything, so it cannot size the allocation on its own; past this hint the @@ -363,9 +347,9 @@ fn resolve_ranges<'a>( for state in automaton.states() { for (cond, _) in automaton.transitions_from(state) { - if range_ids.contains_key(cond) { + let std::collections::hash_map::Entry::Vacant(vacant) = range_ids.entry(cond) else { continue; - } + }; let range = cond.to_range(&automaton.spanning_set)?; let range = match charset { Some(charset) => range.intersection(charset), @@ -377,7 +361,7 @@ fn resolve_ranges<'a>( range_pool.push(range); Some((range_pool.len() - 1) as u32) }; - range_ids.insert(cond, id); + vacant.insert(id); } } @@ -703,8 +687,9 @@ impl Emitter { } /// Emits the combinations of `ranges` whose index falls inside `window`, - /// spread over the ranges by the sampling permutation. Returns how many of - /// them the window covered, the ones `offset` skipped included. + /// in the ascending order [`emit_all`](Self::emit_all) walks them in. + /// Returns how many of them the window covered, the ones `offset` skipped + /// included. fn emit_sampled( &mut self, ranges: &[&CharRange], @@ -726,25 +711,13 @@ impl Emitter { return Ok(covered); } - // The permutation needs `index * multiplier` to stay within `u128`; - // beyond that the mixed-radix digits alone give enough variety, since - // consecutive indices already differ from their first character on. - let scramble = total_combinations - .filter(|&total| total <= u64::MAX as u128) - .map(|total| (scramble_multiplier(total), total)); - let first = window.start.min(bound) + self.offset; self.offset = 0; for index in first..window.end.min(bound) { self.execution_profile.assert_not_timed_out()?; - let combination = match scramble { - Some((multiplier, total)) => (index as u128 * multiplier) % total, - None => index as u128, - }; - - let string = sample_string(ranges, &range_lengths, combination)?; + let string = sample_string(ranges, &range_lengths, index as u128)?; self.strings.insert(string); if self.is_full() { @@ -757,60 +730,28 @@ impl Emitter { } /// Builds the combination of `ranges` at index `combination`, read as a -/// mixed-radix number whose least significant digit is the first character: -/// consecutive combinations then differ from their first character on, instead -/// of only in their last one. +/// mixed-radix number whose least significant digit is the last character, +/// each digit indexing its range in ascending order: combinations come out in +/// the lexicographic order [`Emitter::emit_all`] walks them in. +/// +/// The mapping is a bijection over the combinations, which is what keeps the +/// sampled strings distinct and `offset` exact. fn sample_string( ranges: &[&CharRange], range_lengths: &[u128], mut combination: u128, ) -> Result { - let mut string = String::with_capacity(ranges.len()); + let mut chars = Vec::with_capacity(ranges.len()); - for (position, (&range, &length)) in ranges.iter().zip(range_lengths).enumerate() { + for (&range, &length) in ranges.iter().zip(range_lengths).rev() { let index = (combination % length) as u32; combination /= length; - string.push(sample_char(range, position, index)?.to_char()); + let ch = char_at(range, index).ok_or(EngineError::InvalidCharacterInRegex)?; + chars.push(ch.to_char()); } - Ok(string) -} - -/// Returns the character `range` holds at `index` in sampling order: the -/// [`SAMPLE_CHARS`] it contains first, rotated by `position` so that adjacent -/// characters of a sample differ, then the rest of the range in order. -/// -/// The mapping is a permutation of the range, which is what keeps the sampled -/// strings distinct and `offset` exact. -fn sample_char(range: &CharRange, position: usize, index: u32) -> Result { - let mut sampled = [0u32; SAMPLE_CHARS.len()]; - let mut count = 0; - - for rotation in 0..SAMPLE_CHARS.len() { - let ch = Char::new(SAMPLE_CHARS[(position + rotation) % SAMPLE_CHARS.len()]); - if !range.contains(ch) { - continue; - } - if count as u32 == index { - return Ok(ch); - } - sampled[count] = ordinal_of(range, ch); - count += 1; - } - - // Past the sample characters: take the `index - count`-th character of the - // range that is not one of them, so none is handed out twice. - let sampled = &mut sampled[..count]; - sampled.sort_unstable(); - let mut ordinal = index - count as u32; - for &taken in sampled.iter() { - if taken > ordinal { - break; - } - ordinal += 1; - } - - char_at(range, ordinal).ok_or(EngineError::InvalidCharacterInRegex) + chars.reverse(); + Ok(chars.into_iter().collect()) } /// A cursor over the characters a [`CharRange`] holds, in order, opened at an @@ -871,94 +812,14 @@ impl<'a> RangeCursor<'a> { } } -/// The `(low, high)` scalar bounds of the intervals `range` is made of. -fn intervals(range: &CharRange) -> impl Iterator + '_ { - range - .0 - .chunks_exact(2) - .map(|bounds| (scalar(bounds[0]), scalar(bounds[1]))) -} - -/// The number of characters `range` holds before `target`, which it contains. -fn ordinal_of(range: &CharRange, target: Char) -> u32 { - // An absent target would underflow `target - low` below, silently in - // release builds. - debug_assert!(range.contains(target), "{target} is not in {range}"); - - let target = scalar(target); - let mut ordinal = 0; - - for (low, high) in intervals(range) { - if target <= high { - return ordinal + (target - low); - } - ordinal += high - low + 1; - } - - ordinal -} - /// The character `range` holds at `ordinal`, `None` past its cardinality. fn char_at(range: &CharRange, ordinal: u32) -> Option { RangeCursor::new(range, ordinal).next() } -/// The index of `ch` among all the characters, the surrogate block excluded. -#[inline] -fn scalar(ch: Char) -> u32 { - let code = ch.to_u32(); - if code >= SURROGATES.end { - code - (SURROGATES.end - SURROGATES.start) - } else { - code - } -} - -/// The inverse of [`scalar`]. -#[inline] -fn from_scalar(index: u32) -> Option { - Char::from_u32(if index >= SURROGATES.start { - index + (SURROGATES.end - SURROGATES.start) - } else { - index - }) -} - -/// A stride coprime with `total`, close to its golden-ratio fraction, so that -/// consecutive sample indices land far apart in the combination space instead -/// of walking it in order. Being coprime keeps `index * stride % total` a -/// permutation, so no combination comes up twice. -fn scramble_multiplier(total: u128) -> u128 { - if total < 3 { - return 1; - } - - let mut multiplier = ((total as f64) * 0.618_033_988_749_895) as u128; - multiplier = multiplier.clamp(1, total - 1); - - for _ in 0..64 { - if gcd(multiplier, total) == 1 { - return multiplier; - } - multiplier += 1; - if multiplier >= total { - multiplier = 1; - } - } - - 1 -} - -fn gcd(mut a: u128, mut b: u128) -> u128 { - while b != 0 { - (a, b) = (b, a % b); - } - a -} - #[cfg(test)] mod tests { - use super::{GenerationOptions, GenerationOrder, RangeCursor, char_at, sample_char}; + use super::{GenerationOptions, GenerationOrder, RangeCursor, char_at}; use crate::CharRange; use crate::cardinality::Cardinality; use crate::{fast_automaton::FastAutomaton, regex::RegularExpression}; @@ -1128,49 +989,156 @@ mod tests { assert_eq!(expected, sampled); } - /// The sample characters are what a range is reached for first, so a - /// pattern made of wide ranges samples as readable text rather than as - /// control characters. + /// The order chooses which strings come first, never the characters they + /// are made of: a sampled string reaches for the same low end of each + /// range the exhaustive order starts from, and a charset is how specific + /// characters are asked for. #[test] - fn test_generate_strings_sampled_prefers_representative_characters() { + fn test_generate_strings_sampled_uses_the_same_characters_as_exhaustive() { let automaton = automaton_of(".{3}"); let sampled = automaton .generate_strings(1, 0, GenerationOrder::Sampled) .unwrap(); + let exhaustive = automaton + .generate_strings(1, 0, GenerationOrder::Exhaustive) + .unwrap(); + assert_eq!(exhaustive, sampled); - assert_eq!(vec!["a0A".to_string()], sampled); + let lowercase = CharRange::new_from_range_char('a'..='z'); + let options = GenerationOptions::from(GenerationOrder::Sampled).with_charset(lowercase); + assert_eq!( + vec!["aaa".to_string()], + automaton.generate_strings(1, 0, options).unwrap() + ); } - /// Sampling hands out each character of a range exactly once: anything else - /// and a path would repeat a string, or `offset` would drift off its page. + /// A single-path language has only one shape, so there is nothing for the + /// sampled order to spread over: within a path the characters come in the + /// lexicographic order the exhaustive order walks. #[test] - fn test_sample_char_is_a_permutation_of_the_range() { - let ranges = [ - CharRange::new_from_range_char('a'..='z'), - // Several intervals, one of them straddling the surrogate hole. - CharRange::new_from_ranges(&[ - AnyRange::from(Char::new('0')..=Char::new('9')), - AnyRange::from(Char::new('\u{d7fe}')..=Char::new('\u{e001}')), - ]), - // Past the hole, around one of the sample characters. - CharRange::new_from_range_char('\u{1f5ff}'..='\u{1f601}'), + fn test_generate_strings_sampled_matches_exhaustive_on_a_single_path() { + let automaton = automaton_of("[a-z]{2}"); + + let exhaustive = automaton + .generate_strings(10, 0, GenerationOrder::Exhaustive) + .unwrap(); + let sampled = automaton + .generate_strings(10, 0, GenerationOrder::Sampled) + .unwrap(); + + assert_eq!(exhaustive, sampled); + } + + /// The strongest form of "the order chooses which strings come first, + /// never the characters": on a finite language, sampled and exhaustive + /// enumerate exactly the same set — including through nondeterministic + /// automata, multi-interval charsets, and ranges straddling the surrogate + /// hole. + #[test] + fn test_generate_strings_sampled_and_exhaustive_agree_as_sets() { + let multi_interval = CharRange::new_from_ranges(&[ + AnyRange::from(Char::new('x')..=Char::new('z')), + AnyRange::from(Char::new('0')..=Char::new('1')), + ]); + let surrogate_straddle = CharRange::new_from_range_char('\u{d7fe}'..='\u{e001}'); + + let cases: Vec<(&str, Option)> = vec![ + ("(a|bc){0,3}", None), + // Nondeterministic: "ab" is reachable through both branches. + ("(ab|a)b{0,2}", None), + ("[a-e]{0,2}[0-3]?", None), + (".{0,2}", Some(multi_interval)), + (".", Some(surrogate_straddle)), ]; - for range in ranges { - let expected: Vec = range.iter().map(|ch| ch.to_char()).collect(); + for (pattern, charset) in cases { + for determinize in [false, true] { + let automaton = if determinize { + automaton_of(pattern).determinize().unwrap().into_owned() + } else { + automaton_of(pattern) + }; + + let options = |order| match &charset { + Some(charset) => GenerationOptions::from(order).with_charset(charset.clone()), + None => GenerationOptions::from(order), + }; + + let mut exhaustive = automaton + .generate_strings(100_000, 0, options(GenerationOrder::Exhaustive)) + .unwrap(); + let mut sampled = automaton + .generate_strings(100_000, 0, options(GenerationOrder::Sampled)) + .unwrap(); - for position in 0..3 { - let mut sampled: Vec = (0..range.get_cardinality()) - .map(|index| sample_char(&range, position, index).unwrap().to_char()) - .collect(); - sampled.sort_unstable(); + exhaustive.sort(); + sampled.sort(); + assert_eq!( + exhaustive, sampled, + "{pattern:?} (determinized: {determinize})" + ); + } + } + } + + /// Sampled pages stay consistent at every chunk size, not only at the + /// window boundaries: an offset landing mid-window or mid-path continues + /// exactly where the previous page stopped. + #[test] + fn test_generate_strings_sampled_pages_at_any_boundary() { + for pattern in ["(a|bc){0,3}", "[a-c]{1,3}", "(x|yy)(0|11)?"] { + // Deterministic and minimal, so pages are exactly disjoint. + let mut automaton = automaton_of(pattern).determinize().unwrap().into_owned(); + automaton.minimize().unwrap(); + + let bulk = automaton + .generate_strings(60, 0, GenerationOrder::Sampled) + .unwrap(); + + for chunk in 1..=7usize { + let mut paged = vec![]; + loop { + let page = automaton + .generate_strings(chunk, paged.len(), GenerationOrder::Sampled) + .unwrap(); + if page.is_empty() { + break; + } + paged.extend(page); + if paged.len() >= bulk.len() { + break; + } + } + paged.truncate(bulk.len()); - assert_eq!(expected, sampled, "range {range}, position {position}"); + assert_eq!(bulk, paged, "{pattern:?} at chunk size {chunk}"); } } } + /// A path holding more combinations than `u128` fits still samples the + /// ascending sequence: the decode consumes the index from the last + /// position, so the positions it never reaches keep their range's first + /// character. + #[test] + fn test_generate_strings_sampled_orders_huge_paths() { + let automaton = automaton_of(".{40}"); + + let sampled = automaton + .generate_strings(3, 0, GenerationOrder::Sampled) + .unwrap(); + + assert_eq!( + vec![ + "\u{0}".repeat(40), + format!("{}\u{1}", "\u{0}".repeat(39)), + format!("{}\u{2}", "\u{0}".repeat(39)), + ], + sampled + ); + } + /// A charset rules out whole paths, not single characters: a path that /// needs a ruled-out character is dropped even when it only needs it /// several transitions in, and the strings around it still come out. diff --git a/src/fast_automaton/mod.rs b/src/fast_automaton/mod.rs index 8e04de1..0b45db0 100644 --- a/src/fast_automaton/mod.rs +++ b/src/fast_automaton/mod.rs @@ -28,6 +28,34 @@ pub mod spanning_set; pub use generate::{GenerationOptions, GenerationOrder}; +/// The block of code points `char` cannot hold: [`regex_charclass::char::Char`] +/// values skip it, so scalar values have to be shifted down past it to be +/// counted. +const SURROGATES: std::ops::Range = 0xD800..0xE000; + +/// The index of `ch` among all the characters, the surrogate block excluded. +/// Consecutive scalars are consecutive `Char`s, so `+ 1` arithmetic in scalar +/// space cannot land inside the surrogate hole. +#[inline] +fn scalar(ch: regex_charclass::char::Char) -> u32 { + let code = ch.to_u32(); + if code >= SURROGATES.end { + code - (SURROGATES.end - SURROGATES.start) + } else { + code + } +} + +/// The inverse of [`scalar`]. +#[inline] +fn from_scalar(index: u32) -> Option { + regex_charclass::char::Char::from_u32(if index >= SURROGATES.start { + index + (SURROGATES.end - SURROGATES.start) + } else { + index + }) +} + /// Represents a finite-state automaton. #[derive(Clone, Debug, PartialEq, Eq)] #[must_use = "non-`_mut` operations return a new automaton"] @@ -99,10 +127,7 @@ impl FastAutomaton { /// Returns the number of transitions to the provided state. #[inline] pub fn in_degree(&self, state: State) -> usize { - self.transitions_in - .get(&state) - .unwrap_or(&IntSet::new()) - .len() + self.transitions_in.get(&state).map_or(0, IntSet::len) } /// Returns the number of transitions from the provided state. @@ -146,14 +171,14 @@ impl FastAutomaton { /// Returns a vector of transitions to the given state. pub fn transitions_to_vec(&self, state: State) -> Vec<(State, Condition)> { - // Direct `(from, state)` lookups: scanning each predecessor's whole - // out-list made this O(predecessors × out-degree), and `minimize` - // builds its inverse-transition table through here. if !self.has_state(state) { return vec![]; } + let Some(predecessors) = self.transitions_in.get(&state) else { + return vec![]; + }; let mut in_transitions = vec![]; - for from_state in self.transitions_in.get(&state).unwrap_or(&IntSet::new()) { + for from_state in predecessors { if !self.has_state(*from_state) { continue; } diff --git a/src/fast_automaton/operation/concat.rs b/src/fast_automaton/operation/concat.rs index 60d773f..804ad84 100644 --- a/src/fast_automaton/operation/concat.rs +++ b/src/fast_automaton/operation/concat.rs @@ -99,9 +99,20 @@ impl FastAutomaton { self.concat_state_count_nondegenerate(other, force_no_merge) })?; - let new_spanning_set = &self.spanning_set.merge(&other.spanning_set); - self.apply_new_spanning_set(new_spanning_set)?; - let condition_converter = ConditionConverter::new(&other.spanning_set, new_spanning_set)?; + // Equal spanning sets (the dominant case: `repeat_mut` concatenates + // the same operand over and over) skip the quadratic merge and treat + // every condition conversion as identity. + let new_spanning_set; + let condition_converter = if self.spanning_set == other.spanning_set { + None + } else { + new_spanning_set = self.spanning_set.merge(&other.spanning_set); + self.apply_new_spanning_set(&new_spanning_set)?; + Some(ConditionConverter::new( + &other.spanning_set, + &new_spanning_set, + )?) + }; let mut new_states: IntMap = IntMap::with_capacity_and_hasher( other.number_of_states(), @@ -135,44 +146,54 @@ impl FastAutomaton { } } + // `other`'s start maps to all of `self`'s accept states (when merging); + // every other state maps to exactly one. Borrowing a slice for both + // shapes keeps the per-transition loop allocation-free. for from_state in other.states() { - let new_from_states = match new_states.entry(from_state) { + let single_from; + let new_from_states: &[usize] = match new_states.entry(from_state) { Entry::Occupied(o) => { - vec![*o.get()] + single_from = [*o.get()]; + &single_from } Entry::Vacant(v) => { if from_state == other.start_state { - accept_states.clone() + &accept_states } else { let new_state = self.new_state(); if other.accept_states.contains(&from_state) { self.accept(new_state); } v.insert(new_state); - vec![new_state] + single_from = [new_state]; + &single_from } } }; for (condition, to_state) in other.transitions_from(from_state) { - let new_to_states = match new_states.entry(*to_state) { + let single_to; + let new_to_states: &[usize] = match new_states.entry(*to_state) { Entry::Occupied(o) => { - vec![*o.get()] + single_to = [*o.get()]; + &single_to } Entry::Vacant(v) => { if *to_state == other.start_state { - accept_states.clone() + &accept_states } else { let new_state = self.new_state(); if other.accept_states.contains(to_state) { self.accept(new_state); } v.insert(new_state); - vec![new_state] + single_to = [new_state]; + &single_to } } }; - let projected_condition = condition_converter.convert(condition)?; + let projected_condition = + convert_condition(condition_converter.as_ref(), condition)?; for new_from_state in new_from_states.iter() { for new_to_state in new_to_states.iter() { self.add_transition(*new_from_state, *new_to_state, &projected_condition); diff --git a/src/fast_automaton/operation/determinize.rs b/src/fast_automaton/operation/determinize.rs index 7e7ebe4..82786fa 100644 --- a/src/fast_automaton/operation/determinize.rs +++ b/src/fast_automaton/operation/determinize.rs @@ -46,7 +46,11 @@ impl FastAutomaton { worklist.push_back((initial_state.clone(), new_automaton.start_state)); new_states.insert(initial_state, new_automaton.start_state); - let mut new_states_to_add = BitSet::new(); + // Per-base successor subsets, reused across popped subsets. Base `b` + // of the spanning set is exactly bit `b` of a condition, so one sweep + // over the subset's transitions distributes each target into the + // bases its condition covers. + let mut base_targets: Vec = vec![BitSet::new(); bases.len()]; while let Some((states, r)) = worklist.pop_front() { execution_profile.assert_not_timed_out()?; execution_profile.assert_max_number_of_states(new_states.len())?; @@ -55,33 +59,35 @@ impl FastAutomaton { new_automaton.accept(r); } - for base in &bases { - for from_state in &states { - for (cond, to_state) in self.transitions_from(from_state) { - if cond.has_intersection(base) { - new_states_to_add.insert(*to_state); - } + for from_state in &states { + for (cond, to_state) in self.transitions_from(from_state) { + for base_index in cond.iter_set_bits() { + base_targets[base_index].insert(*to_state); } } - if !new_states_to_add.is_empty() { - match new_states.entry(new_states_to_add.clone()) { - Entry::Occupied(o) => { - let q = *o.get(); - - new_states_to_add.clear(); - - new_automaton.add_transition(r, q, base); - } - Entry::Vacant(v) => { - let new_q = new_automaton.new_state(); - v.insert(new_q); - - let new_states = std::mem::take(&mut new_states_to_add); - worklist.push_back((new_states, new_q)); + } - new_automaton.add_transition(r, new_q, base); - } - }; + // Base index order keeps the resulting state numbering + // deterministic. + for (targets, base) in base_targets.iter_mut().zip(&bases) { + if targets.is_empty() { + continue; + } + // Once the construction converges, the subset usually + // already exists: look it up first so the hit path pays + // no `BitSet` clone (the entry API would need an owned + // key), and only clone-free-insert on a miss. + if let Some(&q) = new_states.get(targets) { + targets.clear(); + + new_automaton.add_transition(r, q, base); + } else { + let new_q = new_automaton.new_state(); + let subset = std::mem::take(targets); + new_states.insert(subset.clone(), new_q); + worklist.push_back((subset, new_q)); + + new_automaton.add_transition(r, new_q, base); } } } @@ -160,7 +166,6 @@ mod tests { deterministic_automaton.number_of_states() ); assert!(deterministic_automaton.is_deterministic()); - //deterministic_automaton.print_dot(); assert!( automaton .difference(&deterministic_automaton) diff --git a/src/fast_automaton/operation/intersection.rs b/src/fast_automaton/operation/intersection.rs index 7ce4bfe..f03c112 100644 --- a/src/fast_automaton/operation/intersection.rs +++ b/src/fast_automaton/operation/intersection.rs @@ -74,12 +74,34 @@ impl FastAutomaton { } let execution_profile = ExecutionProfile::get(); - let new_spanning_set = self.spanning_set.merge(&other.spanning_set); - - let condition_converter_self_to_new = - ConditionConverter::new(&self.spanning_set, &new_spanning_set)?; - let condition_converter_other_to_new = - ConditionConverter::new(&other.spanning_set, &new_spanning_set)?; + // Equal spanning sets (the dominant case in operation chains) need no + // merge and no condition projection at all. + let same_spanning_set = self.spanning_set == other.spanning_set; + let new_spanning_set = if same_spanning_set { + self.spanning_set.clone() + } else { + self.spanning_set.merge(&other.spanning_set) + }; + + let condition_converter_self_to_new = if same_spanning_set { + None + } else { + Some(ConditionConverter::new( + &self.spanning_set, + &new_spanning_set, + )?) + }; + let condition_converter_other_to_new = if same_spanning_set { + None + } else { + Some(ConditionConverter::new( + &other.spanning_set, + &new_spanning_set, + )?) + }; + + let mut projected_self: IntMap> = IntMap::default(); + let mut projected_other: IntMap> = IntMap::default(); let mut new_automaton = FastAutomaton::new_empty(); let mut worklist = @@ -103,22 +125,28 @@ impl FastAutomaton { new_automaton.accept(p.0); } - let transitions_1 = - self.get_projected_transitions(p.1, &condition_converter_self_to_new)?; - let transitions_2 = - other.get_projected_transitions(p.2, &condition_converter_other_to_new)?; + let transitions_1 = self.projected_transitions( + &mut projected_self, + p.1, + condition_converter_self_to_new.as_ref(), + )?; + let transitions_2 = other.projected_transitions( + &mut projected_other, + p.2, + condition_converter_other_to_new.as_ref(), + )?; for (condition_1, n1) in transitions_1 { - for (condition_2, n2) in &transitions_2 { + for (condition_2, n2) in transitions_2 { let intersection = condition_1.intersection(condition_2); if intersection.is_empty() { continue; } - let k = (n1, *n2); + let k = (*n1, *n2); let r = match new_states.get(&k) { Some(new_r) => *new_r, None => { - let new_r = (new_automaton.new_state(), n1, *n2); + let new_r = (new_automaton.new_state(), *n1, *n2); worklist.push_back(new_r); new_states.insert(k, new_r); new_r @@ -143,12 +171,32 @@ impl FastAutomaton { } let execution_profile = ExecutionProfile::get(); - let new_spanning_set = self.spanning_set.merge(&other.spanning_set); - - let condition_converter_self_to_new = - ConditionConverter::new(&self.spanning_set, &new_spanning_set)?; - let condition_converter_other_to_new = - ConditionConverter::new(&other.spanning_set, &new_spanning_set)?; + let same_spanning_set = self.spanning_set == other.spanning_set; + let new_spanning_set = if same_spanning_set { + self.spanning_set.clone() + } else { + self.spanning_set.merge(&other.spanning_set) + }; + + let condition_converter_self_to_new = if same_spanning_set { + None + } else { + Some(ConditionConverter::new( + &self.spanning_set, + &new_spanning_set, + )?) + }; + let condition_converter_other_to_new = if same_spanning_set { + None + } else { + Some(ConditionConverter::new( + &other.spanning_set, + &new_spanning_set, + )?) + }; + + let mut projected_self: IntMap> = IntMap::default(); + let mut projected_other: IntMap> = IntMap::default(); let mut new_automaton = FastAutomaton::new_empty(); let mut worklist = @@ -172,22 +220,28 @@ impl FastAutomaton { return Ok(true); } - let transitions_1 = - self.get_projected_transitions(p.1, &condition_converter_self_to_new)?; - let transitions_2 = - other.get_projected_transitions(p.2, &condition_converter_other_to_new)?; + let transitions_1 = self.projected_transitions( + &mut projected_self, + p.1, + condition_converter_self_to_new.as_ref(), + )?; + let transitions_2 = other.projected_transitions( + &mut projected_other, + p.2, + condition_converter_other_to_new.as_ref(), + )?; for (condition_1, n1) in transitions_1 { - for (condition_2, n2) in &transitions_2 { + for (condition_2, n2) in transitions_2 { let intersection = condition_1.intersection(condition_2); if intersection.is_empty() { continue; } - let k = (n1, *n2); + let k = (*n1, *n2); let r = match new_states.get(&k) { Some(new_r) => *new_r, None => { - let new_r = (new_automaton.new_state(), n1, *n2); + let new_r = (new_automaton.new_state(), *n1, *n2); worklist.push_back(new_r); new_states.insert(k, new_r); new_r @@ -200,20 +254,30 @@ impl FastAutomaton { Ok(false) } - fn get_projected_transitions( + /// Returns `state`'s outgoing transitions projected on the operation's + /// spanning set (`condition_converter` is `None` when both inputs already + /// share it), memoized in `cache`: a component state participates in up + /// to |other| product pairs, and projecting it once instead of once per + /// pair keeps the product construction's inner loop allocation-free. + fn projected_transitions<'m>( &self, + cache: &'m mut IntMap>, state: State, - condition_converter: &ConditionConverter, - ) -> Result, EngineError> { - let transitions_1: Result, EngineError> = self - .transitions_from(state) - .map(|(c, &s)| match condition_converter.convert(c) { - Ok(condition) => Ok((condition, s)), - Err(err) => Err(err), - }) - .collect(); - - transitions_1 + condition_converter: Option<&ConditionConverter>, + ) -> Result<&'m Vec<(Condition, State)>, EngineError> { + match cache.entry(state) { + Entry::Occupied(o) => Ok(o.into_mut()), + Entry::Vacant(v) => { + let transitions: Result, EngineError> = self + .transitions_from(state) + .map(|(c, &s)| match condition_converter { + Some(converter) => converter.convert(c).map(|c| (c, s)), + None => Ok((c.clone(), s)), + }) + .collect(); + Ok(v.insert(transitions?)) + } + } } } diff --git a/src/fast_automaton/operation/minimize.rs b/src/fast_automaton/operation/minimize.rs index 4a383ba..f33f555 100644 --- a/src/fast_automaton/operation/minimize.rs +++ b/src/fast_automaton/operation/minimize.rs @@ -62,10 +62,12 @@ impl FastAutomaton { let bases = self.spanning_bases()?; + // One forward sweep with direct indexing; going through + // `transitions_to_vec` per state would pay two hash lookups per edge. let mut inverse_transitions: Vec> = vec![Vec::new(); max_states]; - for to_state in self.states() { - for (from_state, condition) in self.transitions_to_vec(to_state) { - inverse_transitions[to_state].push((from_state, condition)); + for from_state in self.states() { + for (condition, &to_state) in self.transitions_from(from_state) { + inverse_transitions[to_state].push((from_state, condition.clone())); } } diff --git a/src/fast_automaton/operation/mod.rs b/src/fast_automaton/operation/mod.rs index f303f6e..de8dcc8 100644 --- a/src/fast_automaton/operation/mod.rs +++ b/src/fast_automaton/operation/mod.rs @@ -1,6 +1,20 @@ use std::cmp; use super::*; +use condition::converter::ConditionConverter; + +/// Projects `condition` through `converter`, or borrows it unchanged when no +/// projection is needed (`None`: both operands already share the spanning +/// set, the dominant case in operation chains). +fn convert_condition<'c>( + converter: Option<&ConditionConverter<'_, '_>>, + condition: &'c Condition, +) -> Result, EngineError> { + Ok(match converter { + Some(converter) => std::borrow::Cow::Owned(converter.convert(condition)?), + None => std::borrow::Cow::Borrowed(condition), + }) +} mod concat; mod determinize; diff --git a/src/fast_automaton/operation/union.rs b/src/fast_automaton/operation/union.rs index 248ede1..284807c 100644 --- a/src/fast_automaton/operation/union.rs +++ b/src/fast_automaton/operation/union.rs @@ -84,7 +84,7 @@ impl FastAutomaton { &mut self, other: &FastAutomaton, new_states: &mut IntMap, - condition_converter: &ConditionConverter, + condition_converter: Option<&ConditionConverter>, ) -> Result, EngineError> { let mut imcomplete_states = IntSet::with_capacity(other.out_degree(other.start_state) + 1); // If `other` accepts the empty string we must make the union's *entry* @@ -116,9 +116,9 @@ impl FastAutomaton { new_states.insert(other.start_state, new_state); imcomplete_states.insert(new_state); - for (cond, other_to_state) in other.transitions_from_vec(other.start_state) { - let cond = condition_converter.convert(&cond)?; - let to_state = match new_states.entry(other_to_state) { + for (cond, other_to_state) in other.transitions_from(other.start_state) { + let cond = convert_condition(condition_converter, cond)?; + let to_state = match new_states.entry(*other_to_state) { Entry::Occupied(o) => *o.get(), Entry::Vacant(v) => { let new_state = self.new_state(); @@ -234,9 +234,19 @@ impl FastAutomaton { self.union_state_count_nondegenerate(other) })?; - let new_spanning_set = &self.spanning_set.merge(&other.spanning_set); - self.apply_new_spanning_set(new_spanning_set)?; - let condition_converter = ConditionConverter::new(&other.spanning_set, new_spanning_set)?; + // Equal spanning sets (the dominant case in `union_all` folds) skip + // the quadratic merge and treat every condition conversion as identity. + let new_spanning_set; + let condition_converter = if self.spanning_set == other.spanning_set { + None + } else { + new_spanning_set = self.spanning_set.merge(&other.spanning_set); + self.apply_new_spanning_set(&new_spanning_set)?; + Some(ConditionConverter::new( + &other.spanning_set, + &new_spanning_set, + )?) + }; let mut new_states: IntMap = IntMap::with_capacity_and_hasher( other.number_of_states(), @@ -244,7 +254,7 @@ impl FastAutomaton { ); let imcomplete_states = - self.prepare_start_states(other, &mut new_states, &condition_converter)?; + self.prepare_start_states(other, &mut new_states, condition_converter.as_ref())?; self.prepare_accept_states(other, &mut new_states, &imcomplete_states); for from_state in other.states() { @@ -257,7 +267,7 @@ impl FastAutomaton { } }; for (condition, to_state) in other.transitions_from(from_state) { - let new_condition = condition_converter.convert(condition)?; + let new_condition = convert_condition(condition_converter.as_ref(), condition)?; let new_to_state = match new_states.entry(*to_state) { Entry::Occupied(o) => *o.get(), Entry::Vacant(v) => { @@ -309,7 +319,7 @@ impl FastAutomaton { // Track which 'other' states are already mapped in the start phase // so we don't double-count them when calculating accept state savings. - let mut mapped_other_states = std::collections::HashSet::new(); + let mut mapped_other_states = IntSet::new(); mapped_other_states.insert(other.start_state); if other_in != 0 { @@ -321,8 +331,7 @@ impl FastAutomaton { // --- 2. Accept States Math --- // Gather self's accept states. If other.start_state is accepted, // it virtually triggers self.accept(self.start_state) early. - let mut self_accepts: std::collections::HashSet = - self.accept_states.iter().cloned().collect(); + let mut self_accepts: IntSet = self.accept_states.iter().cloned().collect(); if other.is_accepted(other.start_state) { self_accepts.insert(self.start_state); diff --git a/src/fast_automaton/spanning_set/mod.rs b/src/fast_automaton/spanning_set/mod.rs index dda6178..d6629a5 100644 --- a/src/fast_automaton/spanning_set/mod.rs +++ b/src/fast_automaton/spanning_set/mod.rs @@ -1,9 +1,24 @@ use std::slice::Iter; -use ahash::AHashSet; +use ahash::AHashMap; +use regex_charclass::irange::RangeSet; +use super::{from_scalar, scalar}; use crate::CharRange; +/// Converts merged, ascending scalar segments (see [`scalar`](super::scalar)) +/// back into a [`CharRange`]. The segments are disjoint and non-adjacent, so +/// the flat bound list is already the canonical representation the set +/// operations produce. +fn segments_to_range(segments: &[(u32, u32)]) -> CharRange { + let mut bounds = Vec::with_capacity(segments.len() * 2); + for &(start, end) in segments { + bounds.push(from_scalar(start).expect("segment bounds are valid scalars")); + bounds.push(from_scalar(end).expect("segment bounds are valid scalars")); + } + RangeSet(bounds) +} + /// A set of [`CharRange`] that spans all transitions of a [`crate::FastAutomaton`]. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SpanningSet(Vec, CharRange); @@ -49,14 +64,10 @@ impl SpanningSet { } } - pub(crate) fn spanning_ranges_with_rest(&self) -> Vec { - if self.1.is_empty() { - self.0.clone() - } else { - let mut elements = vec![self.1.clone()]; - elements.extend(self.0.clone()); - elements - } + pub(crate) fn spanning_ranges_with_rest(&self) -> impl Iterator { + std::iter::once(&self.1) + .filter(|rest| !rest.is_empty()) + .chain(self.0.iter()) } /// Returns an iterator over the explicit (non-rest) ranges in the spanning set. @@ -89,7 +100,168 @@ impl SpanningSet { } /// Compute a new minimal spanning set for the provided ranges. + /// + /// The bases are the *atoms* of the inputs: the classes of characters + /// covered by exactly the same subset of input ranges. They are computed + /// with one endpoint sweep over the surrogate-free scalar space, + /// O(E log E) in the total number of input intervals. pub fn compute_spanning_set(ranges: &[CharRange]) -> Self { + let mut inputs: Vec<&CharRange> = ranges.iter().filter(|r| !r.is_empty()).collect(); + inputs.sort_unstable(); + inputs.dedup(); + + // The two dominant real-world calls: an empty automaton, and + // `add_transition_from_range` extending the set by a single range. + match inputs.as_slice() { + [] => return SpanningSet(vec![], CharRange::total()), + [input] => return SpanningSet(vec![(*input).clone()], input.complement()), + _ => {} + } + + // Interval [lo, hi] becomes the half-open [scalar(lo), scalar(hi)+1). + let interval_count: usize = inputs.iter().map(|input| input.0.len() / 2).sum(); + let mut events: Vec = Vec::with_capacity(interval_count * 2); + for (input_index, input) in inputs.iter().enumerate() { + for pair in input.0.chunks_exact(2) { + events.push(Event { + position: scalar(pair[0]), + input_index: input_index as u32, + is_start: true, + }); + events.push(Event { + position: scalar(pair[1]) + 1, + input_index: input_index as u32, + is_start: false, + }); + } + } + events.sort_unstable_by_key(|event| event.position); + + let (mut spanning_ranges, covered) = if inputs.len() <= 64 { + Self::sweep_small(&events) + } else { + Self::sweep_wide(inputs.len(), &events) + }; + spanning_ranges.sort_unstable(); + + SpanningSet(spanning_ranges, segments_to_range(&covered).complement()) + } + + /// The sweep for at most 64 inputs (virtually every call): the signature + /// fits one `u64`, so tagged segments are grouped into atoms with a plain + /// sort instead of a hash map. + fn sweep_small(events: &[Event]) -> (Vec, Vec<(u32, u32)>) { + let mut active = 0u64; + let mut tagged: Vec<(u64, u32, u32)> = Vec::with_capacity(events.len()); + let mut covered: Vec<(u32, u32)> = Vec::new(); + + let mut i = 0; + let mut previous_position = 0u32; + while i < events.len() { + let position = events[i].position; + if active != 0 && previous_position < position { + let segment = (previous_position, position - 1); + push_merging_adjacent(&mut covered, segment); + tagged.push((active, segment.0, segment.1)); + } + while i < events.len() && events[i].position == position { + active ^= 1 << events[i].input_index; + i += 1; + } + previous_position = position; + } + + // Group by signature; the sweep emitted segments in ascending + // position order, which the (signature, position) sort preserves + // within each group, so adjacent segments merge as in `sweep_wide`. + tagged.sort_unstable(); + let mut atoms: Vec = Vec::new(); + let mut i = 0; + while i < tagged.len() { + let signature = tagged[i].0; + let mut segments: Vec<(u32, u32)> = Vec::new(); + while i < tagged.len() && tagged[i].0 == signature { + push_merging_adjacent(&mut segments, (tagged[i].1, tagged[i].2)); + i += 1; + } + atoms.push(segments_to_range(&segments)); + } + (atoms, covered) + } + + /// The sweep for more than 64 inputs: the signature is a bit vector and + /// atoms are grouped through a hash map. + fn sweep_wide(input_count: usize, events: &[Event]) -> (Vec, Vec<(u32, u32)>) { + let mut active = vec![0u64; input_count.div_ceil(64)]; + let mut active_count = 0usize; + let mut atoms: AHashMap, Vec<(u32, u32)>> = AHashMap::new(); + let mut covered: Vec<(u32, u32)> = Vec::new(); + + let mut i = 0; + let mut previous_position = 0u32; + while i < events.len() { + let position = events[i].position; + if active_count > 0 && previous_position < position { + let segment = (previous_position, position - 1); + push_merging_adjacent(&mut covered, segment); + if let Some(segments) = atoms.get_mut(&active) { + push_merging_adjacent(segments, segment); + } else { + atoms.insert(active.clone(), vec![segment]); + } + } + while i < events.len() && events[i].position == position { + let event = &events[i]; + let (word, bit) = ((event.input_index / 64) as usize, event.input_index % 64); + if event.is_start { + active[word] |= 1 << bit; + active_count += 1; + } else { + active[word] &= !(1 << bit); + active_count -= 1; + } + i += 1; + } + previous_position = position; + } + + ( + atoms + .values() + .map(|segments| segments_to_range(segments)) + .collect(), + covered, + ) + } +} + +/// A boundary of one input interval during the sweep: the covering-input +/// signature is constant between consecutive event positions. +struct Event { + position: u32, + input_index: u32, + is_start: bool, +} + +/// Appends `segment` to an ascending segment list, extending the last entry +/// instead when they touch. +#[inline] +fn push_merging_adjacent(segments: &mut Vec<(u32, u32)>, segment: (u32, u32)) { + match segments.last_mut() { + Some(last) if last.1 + 1 == segment.0 => last.1 = segment.1, + _ => segments.push(segment), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ahash::AHashSet; + use proptest::prelude::*; + + /// The pairwise intersection-splitting fixpoint `compute_spanning_set` + /// replaced, kept as the oracle the sweep is fuzzed against. + fn compute_spanning_set_reference(ranges: &[CharRange]) -> SpanningSet { let mut spanning_ranges: Vec = ranges.to_vec(); spanning_ranges.sort_unstable(); spanning_ranges.dedup(); @@ -120,7 +292,7 @@ impl SpanningSet { new_spanning_ranges.insert(set); } } - spanning_ranges = new_spanning_ranges.iter().cloned().collect(); + spanning_ranges = new_spanning_ranges.drain().collect(); } spanning_ranges.sort_unstable(); @@ -132,4 +304,87 @@ impl SpanningSet { SpanningSet(spanning_ranges, total.complement()) } + + /// A random `CharRange` in canonical representation (built by unioning + /// single intervals, which merges overlaps and adjacency) — the form + /// every real caller passes in. + fn arb_char_range() -> impl Strategy { + proptest::collection::vec((0u32..=0x10F7FF, 0u32..=0x10F7FF), 1..4).prop_map(|pairs| { + let mut range = CharRange::empty(); + for (a, b) in pairs { + let (low, high) = if a <= b { (a, b) } else { (b, a) }; + let interval = CharRange::new_from_range( + from_scalar(low).unwrap()..=from_scalar(high).unwrap(), + ); + range = range.union(&interval); + } + range + }) + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn sweep_matches_the_fixpoint_reference( + inputs in proptest::collection::vec(arb_char_range(), 0..6) + ) { + let expected = compute_spanning_set_reference(&inputs); + let actual = SpanningSet::compute_spanning_set(&inputs); + prop_assert_eq!(expected, actual); + } + } + + // More than 64 inputs exercises the wide (bit-vector signature) sweep; + // the wide ranges force overlaps across many inputs at once. + #[test] + fn sweep_wide_matches_the_reference() { + let single = |low: u32, high: u32| { + CharRange::new_from_range(from_scalar(low).unwrap()..=from_scalar(high).unwrap()) + }; + + let mut inputs = Vec::new(); + for i in 0..70u32 { + inputs.push(single(i * 10, i * 10 + 5)); + } + inputs.push(single(3, 400)); + inputs.push(single(250, 699)); + + assert_eq!( + compute_spanning_set_reference(&inputs), + SpanningSet::compute_spanning_set(&inputs) + ); + } + + // Inputs whose intervals touch the extremes and the surrogate hole. + #[test] + fn sweep_handles_boundary_ranges() { + use regex_charclass::char::Char; + + let cases: Vec> = vec![ + vec![], + vec![CharRange::empty()], + vec![CharRange::total()], + vec![CharRange::total(), CharRange::total()], + vec![ + CharRange::new_from_range(Char::new('\u{0}')..=Char::new('\u{D7FF}')), + CharRange::new_from_range(Char::new('\u{E000}')..=Char::new('\u{10FFFF}')), + ], + vec![ + CharRange::new_from_range(Char::new('\u{D000}')..=Char::new('\u{F000}')), + CharRange::new_from_range(Char::new('\u{E000}')..=Char::new('\u{E000}')), + ], + vec![ + CharRange::new_from_range(Char::new('\u{10FFFF}')..=Char::new('\u{10FFFF}')), + CharRange::new_from_range(Char::new('\u{0}')..=Char::new('\u{0}')), + ], + ]; + for inputs in cases { + assert_eq!( + compute_spanning_set_reference(&inputs), + SpanningSet::compute_spanning_set(&inputs), + "inputs: {inputs:?}" + ); + } + } } diff --git a/src/lib.rs b/src/lib.rs index 36a7319..a9998cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -135,52 +135,48 @@ pub type CharRange = RangeSet; /// use regexsolver::error::EngineError; /// use regexsolver::fast_automaton::GenerationOrder; /// -/// fn main() -> Result<(), EngineError> { -/// // Create terms from regex -/// let t1 = Term::from_pattern("abc.*")?; -/// let t2 = Term::from_pattern(".*xyz")?; +/// // Create terms from regex +/// let t1 = Term::from_pattern("abc.*")?; +/// let t2 = Term::from_pattern(".*xyz")?; /// -/// // Concatenate -/// let concat = t1.concat(&[t2])?; -/// assert_eq!(concat.to_pattern()?, "abc.*xyz"); +/// // Concatenate +/// let concat = t1.concat(&[t2])?; +/// assert_eq!(concat.to_pattern()?, "abc.*xyz"); /// -/// // Union -/// let union = t1.union(&[Term::from_pattern("fgh")?])?; -/// assert_eq!(union.to_pattern()?, "(abc.*|fgh)"); +/// // Union +/// let union = t1.union(&[Term::from_pattern("fgh")?])?; +/// assert_eq!(union.to_pattern()?, "(abc.*|fgh)"); /// -/// // Intersection -/// let inter = Term::from_pattern("(ab|xy){2}")? -/// .intersection(&[Term::from_pattern(".*xy")?])?; -/// assert_eq!(inter.to_pattern()?, "(ab|xy)xy"); +/// // Intersection +/// let inter = Term::from_pattern("(ab|xy){2}")? +/// .intersection(&[Term::from_pattern(".*xy")?])?; +/// assert_eq!(inter.to_pattern()?, "(ab|xy)xy"); /// -/// // Difference -/// let diff = Term::from_pattern("a*")? -/// .difference(&Term::from_pattern("")?)?; -/// assert_eq!(diff.to_pattern()?, "a+"); +/// // Difference +/// let diff = Term::from_pattern("a*")? +/// .difference(&Term::from_pattern("")?)?; +/// assert_eq!(diff.to_pattern()?, "a+"); /// -/// // Repetition -/// let rep = Term::from_pattern("abc")? -/// .repeat(2..=4)?; -/// assert_eq!(rep.to_pattern()?, "(abc){2,4}"); +/// // Repetition +/// let rep = Term::from_pattern("abc")? +/// .repeat(2..=4)?; +/// assert_eq!(rep.to_pattern()?, "(abc){2,4}"); /// -/// // Analyze -/// assert_eq!(rep.length(), (Some(6), Some(12))); -/// assert!(!rep.is_empty()?); +/// // Analyze +/// assert_eq!(rep.length(), (Some(6), Some(12))); +/// assert!(!rep.is_empty()?); /// -/// // Generate examples -/// let samples = Term::from_pattern("(x|y){1,3}")? -/// .generate_strings(5, 0, GenerationOrder::Sampled)?; -/// println!("Some matches: {:?}", samples); +/// // Generate examples +/// let samples = Term::from_pattern("(x|y){1,3}")? +/// .generate_strings(5, 0, GenerationOrder::Sampled)?; +/// println!("Some matches: {:?}", samples); /// -/// // Equivalence & subset -/// let a = Term::from_pattern("a+")?; -/// let b = Term::from_pattern("a*")?; -/// assert!(!a.equivalent(&b)?); -/// assert!(a.subset(&b)?); -/// -/// Ok(()) -/// } -/// # main(); +/// // Equivalence & subset +/// let a = Term::from_pattern("a+")?; +/// let b = Term::from_pattern("a*")?; +/// assert!(!a.equivalent(&b)?); +/// assert!(a.subset(&b)?); +/// # Ok::<(), EngineError>(()) /// ``` /// /// To put constraint and limitation on the execution of operations please refer to [`ExecutionProfile`]. diff --git a/src/regex/analyze/mod.rs b/src/regex/analyze/mod.rs index cc854b1..bc17051 100644 --- a/src/regex/analyze/mod.rs +++ b/src/regex/analyze/mod.rs @@ -159,7 +159,6 @@ mod tests { let (min, max_opt) = regex.length(); let automaton = regex.to_automaton().unwrap(); - //automaton.to_dot(); let (min_automaton_opt, max_automaton_opt) = automaton.length(); diff --git a/src/regex/analyze/number_of_states.rs b/src/regex/analyze/number_of_states.rs index 5f33538..ccb9ea9 100644 --- a/src/regex/analyze/number_of_states.rs +++ b/src/regex/analyze/number_of_states.rs @@ -258,15 +258,18 @@ impl AbstractNFAMetadata { return_number_of_states = return_number_of_states.saturating_add(1); } + // A looping start (incoming edges) makes the union materialize + // the start's direct successors before accept states are merged, + // and an accept among those successors never merges (e.g. `a*a`, + // whose accept hangs directly off the looping start). Withhold the + // saving when an accept may sit there: an upper bound may + // overshoot, but never undershoot. + let nfa_accept_beside_looping_start = + nfa_start_state_not_mergeable && nfa.accept_adjacent_to_start; + if !self_accepted_not_mergeable && !nfa_accepted_not_mergeable - // A looping start (incoming edges) makes the union materialize - // the start's direct successors before accept states are merged, - // and an accept among those successors never merges (e.g. `a*a`, - // whose accept hangs directly off the looping start). Withhold the - // saving when an accept may sit there: an upper bound may - // overshoot, but never undershoot. - && !(nfa_start_state_not_mergeable && nfa.accept_adjacent_to_start) + && !nfa_accept_beside_looping_start && !self.accepted.is_empty() && !nfa.accepted.is_empty() && self.number_of_states > 1 @@ -351,8 +354,7 @@ mod tests { #[test] fn test_number_of_states_in_nfa() -> Result<(), String> { - //TODO: - //assert_number_of_states_in_nfa("(ab|c)+"); + assert_number_of_states_in_nfa("(ab|c)+"); assert_number_of_states_in_nfa("A+"); assert_number_of_states_in_nfa("B*"); assert_number_of_states_in_nfa("([ab]*a)"); @@ -526,8 +528,6 @@ mod tests { println!("{}", regex); let regex = RegularExpression::new(regex).unwrap(); - //regex.to_automaton().unwrap().to_dot(); - let number_of_states_in_nfa = regex.get_number_of_states_in_nfa(); let automaton = regex.to_automaton().unwrap(); diff --git a/src/regex/builder.rs b/src/regex/builder.rs index 22d4712..12b2dda 100644 --- a/src/regex/builder.rs +++ b/src/regex/builder.rs @@ -107,17 +107,20 @@ impl RegularExpression { match hir.kind() { HirKind::Empty => Ok(RegularExpression::new_empty_string()), HirKind::Literal(literal) => { - let mut regex_concat = RegularExpression::new_empty_string(); - if let Ok(string) = String::from_utf8(literal.0.clone().into_vec()) { + if let Ok(string) = std::str::from_utf8(&literal.0) { + let mut elements = VecDeque::new(); for char in string.chars() { - regex_concat = regex_concat.concat( - &RegularExpression::Character(CharRange::new_from_range( + RegularExpression::push_concat_element( + &mut elements, + RegularExpression::Character(CharRange::new_from_range( Char::new(char)..=Char::new(char), )), - true, ); } - Ok(regex_concat) + Ok(match elements.len() { + 1 => elements.pop_front().expect("len() == 1"), + _ => RegularExpression::Concat(elements), + }) } else { Err(EngineError::InvalidCharacterInRegex) } diff --git a/src/regex/mod.rs b/src/regex/mod.rs index 15cc542..7a79c5e 100644 --- a/src/regex/mod.rs +++ b/src/regex/mod.rs @@ -50,63 +50,56 @@ pub enum RegularExpression { impl Display for RegularExpression { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let str = match self { + match self { RegularExpression::Character(range) => { if range.is_empty() { return write!(f, "[]"); } - range.to_regex() + write!(f, "{}", range.to_regex()) } RegularExpression::Repetition(regular_expression, min, max_opt) => { - let regex_part = regular_expression.to_string(); - let multiplicator_part; + if RegularExpression::quantifier_needs_parens(regular_expression) { + write!(f, "({regular_expression})")?; + } else { + write!(f, "{regular_expression}")?; + } if *min == 0 && max_opt.is_none() { - multiplicator_part = String::from("*"); + write!(f, "*") } else if *min == 1 && max_opt.is_none() { - multiplicator_part = String::from("+"); - } else if *min == 0 && max_opt.is_some() && max_opt.unwrap() == 1 { - multiplicator_part = String::from("?"); + write!(f, "+") + } else if *min == 0 && *max_opt == Some(1) { + write!(f, "?") } else if let Some(max) = max_opt { if max == min { - multiplicator_part = format!("{{{max}}}"); + write!(f, "{{{max}}}") } else { - multiplicator_part = format!("{{{min},{max}}}"); + write!(f, "{{{min},{max}}}") } } else { - multiplicator_part = format!("{{{min},}}"); - } - if RegularExpression::quantifier_needs_parens(regular_expression) { - format!("({regex_part}){multiplicator_part}") - } else { - format!("{regex_part}{multiplicator_part}") + write!(f, "{{{min},}}") } } RegularExpression::Concat(concat) => { - let mut sb = String::new(); for regex in concat.iter() { - sb.push_str(regex.to_string().as_str()); + write!(f, "{regex}")?; } - sb + Ok(()) } - RegularExpression::Alternation(alternation) => { - if alternation.is_empty() { - return write!(f, "[]"); - } - let mut sb = String::new(); - for i in 0..alternation.len() { - sb.push_str(alternation[i].to_string().as_str()); - if i != alternation.len() - 1 { - sb.push('|'); + RegularExpression::Alternation(alternation) => match alternation.as_slice() { + [] => write!(f, "[]"), + [single] => write!(f, "{single}"), + _ => { + write!(f, "(")?; + for (i, regex) in alternation.iter().enumerate() { + if i != 0 { + write!(f, "|")?; + } + write!(f, "{regex}")?; } + write!(f, ")") } - if alternation.len() == 1 { - sb - } else { - format!("({sb})") - } - } - }; - write!(f, "{str}") + }, + } } } @@ -205,8 +198,24 @@ impl RegularExpression { /// Converts the regular expression to an equivalent [`FastAutomaton`]. #[tracing::instrument(level = "trace", skip_all)] pub fn to_automaton(&self) -> Result { + // Both whole-tree checks run once here: a subtree can never nest + // deeper than the tree it came from, and the execution profile's + // thread-locals don't change mid-conversion. self.assert_depth_within_limit()?; - ExecutionProfile::get().assert_max_number_of_states(self.get_number_of_states_in_nfa())?; + self.to_automaton_inner(&ExecutionProfile::get()) + } + + fn to_automaton_inner( + &self, + execution_profile: &ExecutionProfile, + ) -> Result { + // The per-node state estimate is load-bearing (a subtree like the + // inner of `big{0,0}` can exceed the budget even when the root's + // estimate doesn't), but is only worth its O(subtree) walk when a + // state limit is actually configured. + if execution_profile.limits_number_of_states() { + execution_profile.assert_max_number_of_states(self.get_number_of_states_in_nfa())?; + } match self { RegularExpression::Character(range) => Ok(FastAutomaton::new_from_range(range)), @@ -218,21 +227,21 @@ impl RegularExpression { { return Err(EngineError::InvalidRepetitionBounds(*min, *max)); } - let mut automaton = regular_expression.to_automaton()?; + let mut automaton = regular_expression.to_automaton_inner(execution_profile)?; automaton.repeat_mut(*min, *max_opt)?; Ok(automaton) } RegularExpression::Concat(concat) => { let mut concats = Vec::with_capacity(concat.len()); for c in concat.iter() { - concats.push(c.to_automaton()?); + concats.push(c.to_automaton_inner(execution_profile)?); } FastAutomaton::concat_all(&concats) } RegularExpression::Alternation(alternation) => { let mut alternates = Vec::with_capacity(alternation.len()); for c in alternation.iter() { - alternates.push(c.to_automaton()?); + alternates.push(c.to_automaton_inner(execution_profile)?); } FastAutomaton::union_all(&alternates) } diff --git a/src/regex/operation/concat.rs b/src/regex/operation/concat.rs index 9cabd54..b127bea 100644 --- a/src/regex/operation/concat.rs +++ b/src/regex/operation/concat.rs @@ -167,6 +167,27 @@ impl RegularExpression { } } + /// Appends `that` to the fold accumulator `elements`, merging with the + /// back element when possible: the in-place equivalent of + /// `acc = acc.concat(&that, true)` for a char-by-char fold. + /// + /// `that` must be neither the empty language nor the empty string (both + /// are handled by `concat`'s degenerate checks, not here), and a merge + /// between the whole accumulated `Concat` and a single element is + /// structurally impossible, so only the back element needs checking. + pub(crate) fn push_concat_element( + elements: &mut VecDeque, + that: RegularExpression, + ) { + if let Some(back) = elements.back() + && let Some(merged) = Self::opconcat_can_be_merged(back, &that) + { + *elements.back_mut().expect("back() was Some") = merged; + return; + } + elements.push_back(that); + } + /// Merges the bounds of two adjacent repetitions of the same expression, /// `r{a,b}r{c,d}` → `r{a+c,b+d}`. Returns `None` ("cannot be merged", /// falling back to plain concatenation) when an addition would overflow. diff --git a/src/regex/operation/repeat.rs b/src/regex/operation/repeat.rs index 1c9f4be..21855bf 100644 --- a/src/regex/operation/repeat.rs +++ b/src/regex/operation/repeat.rs @@ -370,8 +370,6 @@ mod tests { let repeat = repeat.to_automaton().unwrap(); - //repeat.to_dot(); - let result = got.to_automaton().unwrap(); assert!(repeat.equivalent(&result).unwrap());