diff --git a/Cargo.lock b/Cargo.lock index 9efa18c..532f354 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1232,7 +1232,6 @@ version = "0.2.0" dependencies = [ "clap", "codspeed-criterion-compat", - "glob", "memchr", "onig", "onig_sys", diff --git a/Cargo.toml b/Cargo.toml index 604c9bf..11033bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,6 @@ path = "src/lib.rs" [dependencies] clap = { version = "4.5", features = ["wrap_help", "cargo", "color"] } -glob = "0.3.1" memchr = "2.7.2" onig = { version = "~6.5.1", default-features = false } onig_sys = { version = "69.9.3", default-features = false } diff --git a/src/glob.rs b/src/glob.rs new file mode 100644 index 0000000..17a9cb3 --- /dev/null +++ b/src/glob.rs @@ -0,0 +1,324 @@ +// This file is part of the uutils grep package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +//! fnmatch-like glob handling. Translates glob patterns to Onig +//! regular expressions, to make `[:class:]` work correctly. + +use crate::matcher::{OnigRegex, find_bracket_subexpr_end}; +use onig::{RegexOptions, Syntax}; +use std::ffi::OsStr; +use std::path::{Path, is_separator}; +use std::str::Chars; + +const SEPARATOR: &str = if cfg!(windows) { r"[/\\]" } else { "/" }; + +/// Include and exclude patterns, in command-line order. The last matching +/// pattern decides; otherwise, skip only if the first pattern is an include. +#[derive(Default)] +pub struct GlobSet { + patterns: Vec<(bool, Option)>, +} + +impl GlobSet { + pub fn new() -> Self { + Self::default() + } + + pub fn with_capacity(capacity: usize) -> Self { + Self { + patterns: Vec::with_capacity(capacity), + } + } + + /// Like GNU, an invalid pattern such as `[z-a]` matches nothing. + pub(crate) fn add(&mut self, include: bool, pattern: &str) { + let regex = translate(pattern).and_then(|regex| { + OnigRegex::compile( + ®ex, + Syntax::perl_ng(), + RegexOptions::REGEX_OPTION_MULTILINE, + ) + .ok() + }); + self.patterns.push((include, regex)); + } + + /// Match a recursive entry's basename, not its path suffixes. + pub fn skips(&self, name: &OsStr) -> bool { + self.skips_if(name, |regex, name| regex.match_at(name, 0, None).is_some()) + } + + /// Match the whole path or a nonempty suffix after a run of separators. + pub fn skips_command_line_file(&self, path: &Path) -> bool { + self.skips_if(path.as_os_str(), |regex, name| { + regex.search(name, 0, None).is_some() + }) + } + + fn skips_if(&self, name: &OsStr, matches: impl Fn(&OnigRegex, &[u8]) -> bool) -> bool { + if self.patterns.is_empty() { + return false; + } + + let name = name.as_encoded_bytes(); + match self + .patterns + .iter() + .rev() + .find(|(_, pattern)| pattern.as_ref().is_some_and(|p| matches(p, name))) + { + Some((include, _)) => !include, + None => self.patterns[0].0, + } + } +} + +fn translate(pattern: &str) -> Option { + // Exclude empty suffixes and positions inside a run of separators, but + // allow any pattern (including a leading separator) at the start of input. + let mut out = format!(r"(?:\A|(?<={SEPARATOR})(?!{SEPARATOR}|\z))"); + let mut chars = pattern.chars(); + while let Some(c) = chars.next() { + match c { + '?' => out.push('.'), + '*' => { + out.push_str(".*"); + while chars.as_str().starts_with('*') { + chars.next(); + } + } + '\\' => push_literal(chars.next().unwrap_or('\\'), &mut out), + '[' => translate_class(&mut chars, &mut out)?, + _ if is_separator(c) => out.push_str(SEPARATOR), + _ => push_literal(c, &mut out), + } + } + out.push_str(r"\z"); + Some(out) +} + +fn push_literal(c: char, out: &mut String) { + if r"\.^$|?*+()[]{}-".contains(c) { + out.push('\\'); + } + out.push(c); +} + +fn translate_class(chars: &mut Chars<'_>, out: &mut String) -> Option<()> { + let start = out.len(); + let mut separator = false; + let mut body = chars.clone(); + out.push('['); + if body.as_str().starts_with(['!', '^']) { + body.next(); + out.push('^'); + } + let mut first = true; + while let Some(c) = body.next() { + match c { + ']' if !first => { + if separator { + out.push_str(r"\\"); + } + out.push(']'); + *chars = body; + return Some(()); + } + '\\' => push_literal(body.next().unwrap_or('\\'), out), + '[' => { + let rest = body.as_str(); + if let Some(&delimiter @ (b':' | b'.' | b'=')) = rest.as_bytes().first() + && let Some(end) = find_bracket_subexpr_end(rest.as_bytes(), 1, delimiter) + { + if delimiter == b':' { + out.push('['); + out.push_str(&rest[..end]); + } else { + // `[.a.]` and `[=a=]` stand for the character itself. + let mut inner = rest[1..end - 2].chars(); + let (Some(c), None) = (inner.next(), inner.next()) else { + return None; + }; + push_literal(c, out); + } + body = rest[end..].chars(); + } else { + out.push_str(r"\["); + } + } + '/' if cfg!(windows) => { + out.push('/'); + separator = true; + } + _ => out.push(c), + } + first = false; + } + + // An unclosed bracket is literal; leave the remaining input unconsumed. + out.truncate(start); + out.push_str(r"\["); + Some(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn separator_matches_platform() { + let regex = OnigRegex::compile( + &format!(r"\A{SEPARATOR}\z"), + Syntax::perl_ng(), + RegexOptions::REGEX_OPTION_NONE, + ) + .unwrap_or_else(|err| panic!("{}", err.message)); + for c in ['/', '\\', ':', 'a'] { + assert_eq!( + regex.match_at(c.to_string().as_bytes(), 0, None).is_some(), + is_separator(c), + "{c:?}", + ); + } + } + + fn excluding(pattern: &str) -> GlobSet { + let mut globs = GlobSet::new(); + globs.add(false, pattern); + globs + } + + #[test] + fn gnu_wildcards() { + let cases: &[(&str, &[&str], &[&str])] = &[ + ("*", &["", ".hidden", "a\nb"], &[]), + ("**.txt", &["a.txt", "t/a.txt"], &["a.log"]), + ("***.txt", &["a.txt", "t/a.txt"], &["a.log"]), + ("t/**/a.txt", &["t/x/a.txt", "t/x/y/a.txt"], &["t/a.txt"]), + ("t/*", &["t/a.txt", "t/x/a.txt"], &["x/a.txt"]), + ("q?x", &["q/x", "qax", "q\nx"], &["qx", "qaax"]), + (r"a\*.txt", &["a*.txt"], &["abc.txt"]), + (r"a\?.txt", &["a?.txt"], &["ab.txt"]), + (r"a\[b]", &["a[b]"], &["ab"]), + (r"a\\b", &[r"a\b"], &["ab"]), + (r"\a", &["a"], &[r"\a", "\x07"]), + ("a\\", &["a\\"], &["a"]), + (".^$+(){}|", &[".^$+(){}|"], &[""]), + ]; + assert_cases(cases); + assert_eq!(translate("***.txt"), translate("*.txt")); + } + + #[test] + fn gnu_brackets() { + let cases: &[(&str, &[&str], &[&str])] = &[ + ("[ab]", &["a", "b"], &["[ab]", "c"]), + ("[!a]", &["b", "!"], &["a"]), + ("[^a]", &["b", "^"], &["a"]), + ("[", &["["], &["a"]), + ("[]", &["[]"], &["]"]), + ("[!]", &["[!]"], &["!"]), + ("[abc", &["[abc"], &["a"]), + ("[]]", &["]"], &["[", "[]]"]), + ("[!]]", &["[", "a"], &["]"]), + ("[[]", &["["], &["]"]), + ("[a-z]", &["a", "m", "z"], &["A", "0"]), + ("[-a]", &["-", "a"], &["z"]), + ("[a-]", &["-", "a"], &["z"]), + ("[a&&b]", &["a", "&", "b"], &["c"]), + ("[[:digit:]]", &["0", "9"], &["a", "]"]), + ("[![:digit:]]", &["a", "\n"], &["9"]), + ("[[:alpha:]]", &["a", "\u{e9}"], &["9"]), + (r"[\]]", &["]"], &[r"\", r"\]"]), + (r"[\d]", &["d"], &["9"]), + (r"[a\-z]", &["a", "-", "z"], &["m"]), + ("[[.a.]]", &["a"], &[".]", "[.a.]"]), + ("[[=a=]]", &["a"], &["=]", "[=a=]"]), + ("[[.-.]]", &["-"], &["a"]), + ]; + assert_cases(cases); + } + + fn assert_cases(cases: &[(&str, &[&str], &[&str])]) { + for &(pattern, matching, nonmatching) in cases { + let globs = excluding(pattern); + for (names, expected) in [(matching, true), (nonmatching, false)] { + for name in names { + assert_eq!( + globs.skips(OsStr::new(name)), + expected, + "{pattern:?}, {name:?}", + ); + } + } + } + } + + #[test] + fn invalid_patterns_match_nothing() { + for pattern in ["[z-a]", "[[:notaclass:]]", "[[.ab.]]"] { + for name in ["a", "z", "ab", pattern] { + assert!(!excluding(pattern).skips(OsStr::new(name)), "{pattern:?}"); + } + } + } + + #[test] + fn suffix_boundaries() { + let globs = excluding("a.txt"); + for name in ["a.txt", "t/a.txt", "t//a.txt", "./t/a.txt"] { + assert!(globs.skips_command_line_file(Path::new(name)), "{name}"); + } + for name in ["data.txt", "t/data.txt", "t/a.txt\n", "a.txt/"] { + assert!(!globs.skips_command_line_file(Path::new(name)), "{name}"); + } + assert!(!globs.skips(OsStr::new("t/a.txt"))); + assert!(!excluding("/a.txt").skips_command_line_file(Path::new("t//a.txt"))); + assert!(!excluding("").skips_command_line_file(Path::new("t/"))); + } + + #[cfg(windows)] + #[test] + fn windows_path_separators() { + let globs = excluding("t/*.txt"); + assert!(globs.skips_command_line_file(Path::new(r"t\a.txt"))); + assert!(excluding("a.txt").skips_command_line_file(Path::new(r"t\\a.txt"))); + assert!(excluding("[/]").skips(OsStr::new(r"\"))); + assert!(excluding("[/-0].txt").skips(OsStr::new("0.txt"))); + } + + #[test] + fn last_matching_pattern_wins_across_suffixes() { + for include_last in [false, true] { + let mut globs = GlobSet::new(); + globs.add(!include_last, "t/*"); + globs.add(include_last, "a.txt"); + assert_eq!( + globs.skips_command_line_file(Path::new("t/a.txt")), + !include_last, + ); + assert_eq!(globs.skips(OsStr::new("unmatched")), !include_last); + } + } + + #[test] + fn empty_set_keeps_files() { + let globs = GlobSet::new(); + assert!(!globs.skips(OsStr::new("a"))); + assert!(!globs.skips_command_line_file(Path::new("t/a"))); + } + + #[cfg(unix)] + #[test] + fn non_utf8_names_match() { + use std::os::unix::ffi::OsStrExt; + + let name = OsStr::from_bytes(b"t\xff/a.txt"); + for pattern in ["*.txt", "a.txt", "t?/a.txt"] { + assert!(excluding(pattern).skips_command_line_file(Path::new(name))); + } + assert!(excluding("*.txt").skips(OsStr::from_bytes(b"\xff.txt"))); + } +} diff --git a/src/lib.rs b/src/lib.rs index 7fd4174..950629f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ #[doc(hidden)] pub mod context_buffer; +mod glob; #[doc(hidden)] pub mod line_buffer; #[doc(hidden)] @@ -12,12 +13,13 @@ pub mod matcher; mod output; mod searcher; +use crate::glob::GlobSet; use crate::line_buffer::LineBuffer; use crate::matcher::Matcher; use crate::output::OutputWriter; use crate::searcher::Searcher; use clap::{Arg, ArgAction, Command}; -use std::ffi::{OsStr, OsString}; +use std::ffi::OsString; use std::io::{IsTerminal as _, Read}; use std::path::Path; use uucore::error::{ExitCode, UError, UResult, USimpleError, strip_errno}; @@ -78,14 +80,6 @@ pub struct ColorConfig<'a> { pub no_erase: bool, } -/// Include and exclude patterns, in command-line order. As in GNU grep, the -/// last pattern that matches a name decides. If none matches, the name is -/// skipped only when the first pattern is an include. -#[doc(hidden)] -pub struct GlobSet { - patterns: Vec<(bool, glob::Pattern)>, -} - #[doc(hidden)] pub struct Config<'a> { // Searcher @@ -365,12 +359,12 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { globs.sort_by_key(|(i, _, _)| *i); let mut file_globs = GlobSet::with_capacity(globs.len()); for (_, include, pattern) in &globs { - file_globs.add(*include, pattern)?; + file_globs.add(*include, pattern); } let exclude_dir_globs = { let mut patterns = GlobSet::with_capacity(exclude_dir.len()); for pattern in exclude_dir { - patterns.add(false, pattern)?; + patterns.add(false, pattern); } patterns }; @@ -963,47 +957,6 @@ fn expand_num_shorthand(args: impl Iterator) -> Vec { out } -impl Default for GlobSet { - fn default() -> Self { - Self::new() - } -} - -impl GlobSet { - /// Create an empty GlobSet. - pub fn new() -> Self { - Self { - patterns: Vec::new(), - } - } - - pub fn with_capacity(capacity: usize) -> Self { - Self { - patterns: Vec::with_capacity(capacity), - } - } - - fn add(&mut self, include: bool, pattern: &str) -> UResult<()> { - let compiled_pattern = glob::Pattern::new(pattern) - .map_err(|e| USimpleError::new(2, format!("invalid glob '{pattern}': {e}")))?; - self.patterns.push((include, compiled_pattern)); - Ok(()) - } - - pub fn skips(&self, name: &OsStr) -> bool { - let last_match = name.to_str().and_then(|name| { - self.patterns - .iter() - .rev() - .find(|(_, pattern)| pattern.matches(name)) - }); - match last_match { - Some((include, _)) => !include, - None => self.patterns.first().is_some_and(|(include, _)| *include), - } - } -} - fn is_color_value(value: &str) -> bool { value.bytes().all(|b| b.is_ascii_digit() || b == b';') } diff --git a/src/matcher.rs b/src/matcher.rs index dac786b..bcbecbe 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -372,7 +372,7 @@ impl CompiledPattern { } } -struct OnigRegex { +pub(crate) struct OnigRegex { raw: onig_sys::OnigRegex, } @@ -384,7 +384,11 @@ unsafe impl Send for OnigRegex {} unsafe impl Sync for OnigRegex {} impl OnigRegex { - fn compile(pattern: &str, syntax: &Syntax, options: RegexOptions) -> Result { + pub(crate) fn compile( + pattern: &str, + syntax: &Syntax, + options: RegexOptions, + ) -> Result { let pattern = pattern.as_bytes(); let mut raw = null_mut(); let mut error = onig_sys::OnigErrorInfo { @@ -427,7 +431,12 @@ impl OnigRegex { } } - fn search(&self, line: &[u8], offset: usize, region: Option<&mut Region>) -> Option { + pub(crate) fn search( + &self, + line: &[u8], + offset: usize, + region: Option<&mut Region>, + ) -> Option { debug_assert!(offset <= line.len()); // SAFETY: `offset` is bounded by `line.len()`, all byte pointers are // derived from `line`, and `region_ptr` preserves `onig::Region`'s @@ -448,7 +457,12 @@ impl OnigRegex { onig_match_result(result) } - fn match_at(&self, line: &[u8], offset: usize, region: Option<&mut Region>) -> Option { + pub(crate) fn match_at( + &self, + line: &[u8], + offset: usize, + region: Option<&mut Region>, + ) -> Option { debug_assert!(offset <= line.len()); // SAFETY: `offset` is bounded by `line.len()`, all byte pointers are // derived from `line`, and `region_ptr` preserves `onig::Region`'s @@ -476,9 +490,9 @@ impl Drop for OnigRegex { } } -struct OnigError { +pub(crate) struct OnigError { code: i32, - message: String, + pub(crate) message: String, } impl OnigError { @@ -610,7 +624,11 @@ fn scan_bracket(pattern: &[u8], start: usize) -> (bool, usize) { /// Index just past the `:]`, `.]` or `=]` closing a `[: [. [=` subexpression /// whose body starts at `start`. -fn find_bracket_subexpr_end(pattern: &[u8], start: usize, delimiter: u8) -> Option { +pub(crate) fn find_bracket_subexpr_end( + pattern: &[u8], + start: usize, + delimiter: u8, +) -> Option { (start..pattern.len().saturating_sub(1)) .find(|&i| pattern[i] == delimiter && pattern[i + 1] == b']') .map(|i| i + 2) diff --git a/src/searcher.rs b/src/searcher.rs index 64c4c40..c91e1c6 100644 --- a/src/searcher.rs +++ b/src/searcher.rs @@ -77,6 +77,11 @@ impl<'a> Searcher<'a> { if self.config.device_mode == DeviceMode::Skip && Self::is_special_file(path) { return ControlFlow::Continue(()); } + // Files named on the command line are filtered by --include and + // --exclude too. A missing file is still opened, to report it. + if self.config.file_globs.skips_command_line_file(path) && path.exists() { + return ControlFlow::Continue(()); + } self.process_file(lb, path) } } diff --git a/tests/test_grep.rs b/tests/test_grep.rs index 83a472f..74f6847 100644 --- a/tests/test_grep.rs +++ b/tests/test_grep.rs @@ -1639,6 +1639,100 @@ fn recursive_include_exclude_last_match_wins() { .no_stdout(); } +#[test] +fn include_exclude_apply_to_command_line_files() { + let (scene, _) = ucmd(); + build_tree(&scene); + + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--exclude=*.log", "grep", "tree/a.txt", "tree/b.log"]) + .succeeds() + .stdout_is("tree/a.txt:grep me\n"); + + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["-l", "--include=*.log", "grep", "tree/a.txt", "tree/b.log"]) + .succeeds() + .stdout_is("tree/b.log\n"); + + // The pattern can match the whole name or any part after a '/'. + for exclude in ["--exclude=tree/*", "--exclude=sub/c.txt", "--exclude=c.txt"] { + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&[exclude, "grep", "tree/sub/c.txt"]) + .fails_with_code(1) + .no_stdout(); + } + + #[cfg(windows)] + { + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--exclude=c.txt", "grep", r"tree\sub\c.txt"]) + .fails_with_code(1) + .no_stdout(); + } + + // A missing file is still reported. + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--exclude=missing", "grep", "missing"]) + .fails_with_code(2) + .no_stdout() + .stderr_contains("missing: "); +} + +#[test] +fn include_exclude_regex_globs() { + let (scene, _) = ucmd(); + build_tree(&scene); + + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--exclude=tree/[ab].?xt", "grep", "tree/a.txt"]) + .fails_with_code(1) + .no_output(); + + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--include=tree/**/c.txt", "grep", "tree/sub/c.txt"]) + .succeeds() + .stdout_is("grep me\n") + .no_stderr(); + + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--include=tree/**/a.txt", "grep", "tree/a.txt"]) + .fails_with_code(1) + .no_output(); + + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&[ + "--include=***.[[:alpha:]][[:alpha:]][[:alpha:]]", + "grep", + "tree/a.txt", + ]) + .succeeds() + .stdout_is("grep me\n") + .no_stderr(); + + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--exclude=tree/*", "--include=a.txt", "grep", "tree/a.txt"]) + .succeeds() + .stdout_is("grep me\n") + .no_stderr(); + + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--include=a.txt", "--exclude=tree/*", "grep", "tree/a.txt"]) + .fails_with_code(1) + .no_output(); + + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--exclude=[[:notaclass:]]", "grep", "tree/a.txt"]) + .succeeds() + .stdout_is("grep me\n") + .no_stderr(); + + scene.fixtures.write("[abc", "grep me\n"); + let mut c = scene.cmd(env!("CARGO_BIN_EXE_grep")); + c.args(&["--exclude=[abc", "grep", "[abc"]) + .fails_with_code(1) + .no_output(); +} + #[test] fn recursive_exclude_from_file() { let (scene, mut c) = ucmd();