diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 83ca2d1..c226a37 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - component: [sqlite-compress, reglite] + component: [sqlite-compress, reglite, matcher] steps: - name: Checkout repository uses: actions/checkout@v6.0.2 @@ -64,7 +64,7 @@ jobs: strategy: fail-fast: false matrix: - component: [sqlite-compress, reglite] + component: [sqlite-compress, reglite, matcher] steps: - name: Checkout repository uses: actions/checkout@v6.0.2 @@ -98,4 +98,4 @@ jobs: log_level: debug install_args: --env ${{ env.MISE_ENV }} - name: Run Miri - run: mise run ${{ matrix.component }}:miri \ No newline at end of file + run: mise run ${{ matrix.component }}:miri diff --git a/.mise/tasks/matcher/build b/.mise/tasks/matcher/build new file mode 100755 index 0000000..d16c531 --- /dev/null +++ b/.mise/tasks/matcher/build @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +#MISE description="Build the project" +cargo build -p matcher \ No newline at end of file diff --git a/.mise/tasks/matcher/clippy b/.mise/tasks/matcher/clippy new file mode 100755 index 0000000..3f20524 --- /dev/null +++ b/.mise/tasks/matcher/clippy @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +#MISE description="Lint with clippy" + +cargo clippy -p matcher --all-targets -- -D warnings diff --git a/.mise/tasks/matcher/test b/.mise/tasks/matcher/test new file mode 100755 index 0000000..5dff5a2 --- /dev/null +++ b/.mise/tasks/matcher/test @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +#MISE description="Run tests" +cargo test -p matcher -- --test-threads=1 diff --git a/.mise/tasks/matcher/validate b/.mise/tasks/matcher/validate new file mode 100755 index 0000000..bfdeaf8 --- /dev/null +++ b/.mise/tasks/matcher/validate @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +#MISE description="Run all checks (fmt, clippy, test)" +#MISE depends=["fmt:dev", "matcher:clippy", "matcher:test"] \ No newline at end of file diff --git a/.mise/tasks/sqlite-compress/clippy b/.mise/tasks/sqlite-compress/clippy index 1d9659f..d721d67 100755 --- a/.mise/tasks/sqlite-compress/clippy +++ b/.mise/tasks/sqlite-compress/clippy @@ -1,4 +1,4 @@ #!/usr/bin/env bash #MISE description="Lint with clippy" -cargo clippy -p sqlite-compress --all-targets --no-default-features --features loadable_extension -- -D warnings +cargo clippy -p sqlite-compress --all-targets --no-default-features --features loadable_extension,like -- -D warnings diff --git a/.mise/tasks/sqlite-compress/test b/.mise/tasks/sqlite-compress/test index 12f2954..29b0714 100755 --- a/.mise/tasks/sqlite-compress/test +++ b/.mise/tasks/sqlite-compress/test @@ -1,3 +1,3 @@ #!/usr/bin/env bash #MISE description="Run tests" -cargo test -p sqlite-compress -- --test-threads=1 +cargo test -p sqlite-compress --features like -- --test-threads=1 diff --git a/Cargo.lock b/Cargo.lock index e71655d..e178e41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,14 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "matcher" +version = "0.1.0" +dependencies = [ + "memchr", + "thiserror", +] + [[package]] name = "memchr" version = "2.8.3" @@ -232,6 +240,7 @@ name = "sqlite-compress" version = "0.1.0" dependencies = [ "libsqlite3-sys", + "matcher", "parking_lot", "sqlite-ffi", "sqlite-ffi-macros", diff --git a/Cargo.toml b/Cargo.toml index 101dfd0..c9eedaa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,8 @@ members = [ "sqlite-compress", "reglite", "crates/sqlite-ffi", - "crates/sqlite-ffi-macros", + "crates/sqlite-ffi-macros", + "crates/matcher", ] resolver = "2" @@ -11,4 +12,5 @@ resolver = "2" thiserror = "2.0" libsqlite3-sys = { version = "=0.30.1", features = ["bundled"], package = "libsqlite3-sys" } sqlite-ffi = { version = "0.1.0", path = "crates/sqlite-ffi", default-features = false } -sqlite-ffi-macros = { version = "0.1.0", path = "crates/sqlite-ffi-macros" } \ No newline at end of file +sqlite-ffi-macros = { version = "0.1.0", path = "crates/sqlite-ffi-macros" } +matcher = { version = "0.1.0", path = "crates/matcher" } \ No newline at end of file diff --git a/crates/matcher/Cargo.toml b/crates/matcher/Cargo.toml new file mode 100644 index 0000000..777cbeb --- /dev/null +++ b/crates/matcher/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "matcher" +authors = ["Caido Labs Inc. "] +repository = "https://github.com/caido/sqlite-rs" +license = "MIT" +version = "0.1.0" +edition = "2024" + +[dependencies] +memchr = { version = "2.8.3", default-features = false, features = ["alloc"] } +thiserror = { workspace = true } diff --git a/crates/matcher/src/errors.rs b/crates/matcher/src/errors.rs new file mode 100644 index 0000000..935cf4f --- /dev/null +++ b/crates/matcher/src/errors.rs @@ -0,0 +1,9 @@ +use std::io; + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum MatcherError { + #[error(transparent)] + Io(#[from] io::Error), +} diff --git a/crates/matcher/src/lib.rs b/crates/matcher/src/lib.rs new file mode 100644 index 0000000..c756394 --- /dev/null +++ b/crates/matcher/src/lib.rs @@ -0,0 +1,6 @@ +mod errors; +mod matcher; +mod utils; + +pub use errors::MatcherError; +pub use matcher::Matcher; diff --git a/crates/matcher/src/matcher.rs b/crates/matcher/src/matcher.rs new file mode 100644 index 0000000..4d701fd --- /dev/null +++ b/crates/matcher/src/matcher.rs @@ -0,0 +1,328 @@ +use std::io::{self, Read}; + +use memchr::memmem::Finder; + +use crate::{ + errors::MatcherError, + utils::{Segment, compile_segments}, +}; + +const STREAM_CHUNK: usize = 16 * 1024; + +/// Compiled `LIKE` pattern, cached across rows through SQLite auxdata. +pub enum Matcher { + /// No `_`: lowercase literals that must appear in order. + Ordered(Vec>), + /// At least one `_`: segments matched over the buffered stream. + WithAny(Vec), +} + +impl Matcher { + pub fn compile(pattern: &[u8]) -> Self { + let segments = compile_segments(pattern); + if segments.iter().any(|s| matches!(s, Segment::Any)) { + return Self::WithAny(segments); + } + + let finders = segments + .into_iter() + .filter_map(|s| match s { + Segment::Literal(f) => Some(*f), + _ => None, + }) + .collect(); + Self::Ordered(finders) + } + + pub fn find_in(&self, reader: R, skip: usize) -> Result { + match self { + Self::Ordered(finders) => fast_path_ordered_segments(reader, finders, skip), + Self::WithAny(segments) => slow_path_with_any(reader, segments, skip), + } + } +} + +/// Finds each literal after the end of the previous one, over a byte stream. +/// +/// Leftmost matching is optimal for `%`-separated literals, so no +/// backtracking is needed. The window keeps `len - 1` bytes of the current +/// literal's prefix so matches spanning chunks are found. +fn fast_path_ordered_segments( + mut reader: R, + finders: &[Finder<'static>], + mut skip: usize, +) -> Result { + let Some(longest) = finders.iter().map(|f| f.needle().len()).max() else { + return Ok(true); + }; + + let mut buf = vec![0u8; longest - 1 + STREAM_CHUNK]; + let mut kept = 0; + let mut current = 0; + + loop { + let n = reader.read(&mut buf[kept..kept + STREAM_CHUNK])?; + if n == 0 { + return Ok(false); + } + let mut end = kept + n; + + if skip > 0 { + if skip >= n { + skip -= n; + continue; + } + buf.copy_within(kept + skip..end, kept); + end -= skip; + skip = 0; + } + + buf[kept..end].make_ascii_lowercase(); + + let mut pos = 0; + while let Some(i) = finders[current].find(&buf[pos..end]) { + pos += i + finders[current].needle().len(); + current += 1; + if current == finders.len() { + return Ok(true); + } + } + + let keep = (finders[current].needle().len() - 1).min(end - pos); + buf.copy_within(end - keep..end, 0); + kept = keep; + } +} + +/// Matches each `%`-separated group at its leftmost position, using a bounded +/// buffer that retains only bytes needed by the current group. +/// +/// The leftmost match of a group leaves the most room for the next ones, so +/// backtracking never crosses a `Skip`: on mismatch, only the group's start +/// moves forward, to the next hit of its first literal. +fn slow_path_with_any( + mut reader: R, + segments: &[Segment], + mut skip: usize, +) -> Result { + let mut buf = Vec::new(); + let mut eof = false; + let mut group_start = 0; + let mut pos = 0; + + loop { + let group_end = segments[group_start..] + .iter() + .position(|s| matches!(s, Segment::Skip)) + .map_or(segments.len(), |i| group_start + i); + + match find_group(&segments[group_start..group_end], &buf, pos, eof) { + Group::Found(end) => { + if group_end == segments.len() { + return Ok(true); + } + group_start = group_end + 1; + pos = end; + } + Group::Missing => return Ok(false), + Group::Pending(from) => { + buf.drain(..from); + pos = 0; + eof = !read_more(&mut reader, &mut buf, &mut skip)?; + } + } + } +} + +/// Appends the next bytes past `skip`, lowercased; returns `false` at EOF. +fn read_more(reader: &mut R, buf: &mut Vec, skip: &mut usize) -> io::Result { + loop { + let len = buf.len(); + buf.resize(len + STREAM_CHUNK, 0); + let n = reader.read(&mut buf[len..])?; + let dropped = n.min(*skip); + *skip -= dropped; + buf.copy_within(len + dropped..len + n, len); + buf.truncate(len + n - dropped); + + if n == 0 { + return Ok(false); + } + if n > dropped { + buf[len..].make_ascii_lowercase(); + return Ok(true); + } + } +} + +enum Group { + /// Matched, ending at this offset. + Found(usize), + /// No match starts before this offset, more input is needed past it. + Pending(usize), + Missing, +} + +/// Finds the leftmost match of a `Skip`-free group starting at or after `from`. +fn find_group(group: &[Segment], hay: &[u8], mut from: usize, eof: bool) -> Group { + loop { + let start = match &group[0] { + Segment::Literal(f) => match f.find(&hay[from..]) { + Some(i) => from + i, + None if eof => return Group::Missing, + None => { + let tail = hay.len().saturating_sub(f.needle().len() - 1); + return Group::Pending(tail.max(from)); + } + }, + _ if from < hay.len() => from, + _ if eof => return Group::Missing, + _ => return Group::Pending(from), + }; + + match match_at(group, hay, start, eof) { + Step::Match(end) => return Group::Found(end), + Step::Mismatch => from = start + 1, + Step::NeedMore => return Group::Pending(start), + } + } +} + +enum Step { + Match(usize), + Mismatch, + NeedMore, +} + +/// Matches a `Skip`-free group anchored at `pos`. +fn match_at(group: &[Segment], hay: &[u8], mut pos: usize, eof: bool) -> Step { + for segment in group { + let rest = &hay[pos..]; + match segment { + Segment::Literal(f) => { + let needle = f.needle(); + if rest.len() < needle.len() { + return if !eof && needle.starts_with(rest) { + Step::NeedMore + } else { + Step::Mismatch + }; + } + if !rest.starts_with(needle) { + return Step::Mismatch; + } + pos += needle.len(); + } + Segment::Any => match any_len(rest, eof) { + Some(0) => return Step::Mismatch, + Some(n) => pos += n, + None => return Step::NeedMore, + }, + Segment::Skip => unreachable!("groups are split on Skip"), + } + } + Step::Match(pos) +} + +/// Bytes taken by one `_` at the start of `hay`: a whole UTF-8 character, or +/// a single byte when the sequence is invalid. `None` when more input could +/// change the answer. +fn any_len(hay: &[u8], eof: bool) -> Option { + let Some(&lead) = hay.first() else { + return if eof { Some(0) } else { None }; + }; + let width = match lead { + 0xC2..=0xDF => 2, + 0xE0..=0xEF => 3, + 0xF0..=0xF4 => 4, + _ => return Some(1), + }; + match std::str::from_utf8(&hay[..width.min(hay.len())]) { + Ok(_) => Some(width), + Err(e) if e.error_len().is_none() && !eof => None, + Err(_) => Some(1), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn matches(pattern: &str, haystack: &[u8]) -> bool { + Matcher::compile(pattern.as_bytes()) + .find_in(haystack, 0) + .unwrap() + } + + /// Feeds one byte per `read` to exercise matches spanning chunks. + struct Trickle<'a>(&'a [u8]); + + impl Read for Trickle<'_> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let Some((&b, rest)) = self.0.split_first() else { + return Ok(0); + }; + buf[0] = b; + self.0 = rest; + Ok(1) + } + } + + #[test] + fn literals_are_case_insensitive_and_ordered() { + assert!(matches("hello", b"say HeLLo world")); + assert!(matches("say%WORLD", b"say hello world")); + assert!(!matches("world%say", b"say hello world")); + assert!(matches("", b"anything")); + assert!(matches("%%", b"")); + assert!(!matches("x", b"")); + } + + #[test] + fn underscore_matches_one_character() { + assert!(matches("h_llo", b"xx HALLO yy")); + assert!(matches("a_c", "aéc".as_bytes())); + assert!(!matches("a__c", "aéc".as_bytes())); + assert!(matches("a_c", b"a\xFFc")); + assert!(matches("a_c", b"a\xE2c")); + assert!(!matches("a_c", b"ac")); + assert!(matches("_", b"z")); + assert!(!matches("_", b"")); + assert!(matches("a_%z", b"ab----Z")); + assert!(matches("%_%", b"z")); + assert!(!matches("a_", b"a")); + } + + #[test] + fn backtracks_to_next_literal_hit() { + assert!(matches("test_oui", b"testXtestYoui")); + assert!(matches("te%test_oui", b"te---testX---te---testYoui")); + assert!(!matches("te%test_oui", b"te---testX---testYnon")); + assert!(matches("a_c", b"axxa_c")); + assert!(matches("aa_b", b"aaaxb")); + assert!(matches("_%z", b"..z")); + assert!(!matches("_%z", b"z")); + assert!(matches("_b", b"aab")); + } + + #[test] + fn matches_span_chunks() { + for pattern in ["needle%tail", "ne_dle%tail"] { + let m = Matcher::compile(pattern.as_bytes()); + assert!(m.find_in(Trickle(b"..NEEDLE..TAIL.."), 0).unwrap()); + assert!(!m.find_in(Trickle(b"..TAIL..NEEDLE.."), 0).unwrap()); + } + let m = Matcher::compile(b"a_c"); + assert!(m.find_in(Trickle("xaéc".as_bytes()), 0).unwrap()); + assert!(!m.find_in(Trickle("xaé".as_bytes()), 0).unwrap()); + } + + #[test] + fn offset_skips_prefix() { + for pattern in ["abc", "a_c"] { + let m = Matcher::compile(pattern.as_bytes()); + assert!(m.find_in(&b"abc---abc"[..], 3).unwrap()); + assert!(!m.find_in(&b"abc------"[..], 1).unwrap()); + } + } +} diff --git a/crates/matcher/src/utils.rs b/crates/matcher/src/utils.rs new file mode 100644 index 0000000..b82f337 --- /dev/null +++ b/crates/matcher/src/utils.rs @@ -0,0 +1,111 @@ +use memchr::memmem::Finder; + +/// One piece of a compiled `LIKE` pattern. +pub enum Segment { + /// ASCII-lowercased bytes that must appear verbatim. + Literal(Box>), + /// `%`: any run of bytes, possibly empty. + Skip, + /// `_`: one UTF-8 character, or one byte when the sequence is invalid. + Any, +} + +/// Splits a `LIKE` pattern into segments. +/// +/// Runs of `%` collapse into one `Skip`, and leading or trailing `Skip`s are +/// dropped since matching is unanchored on both ends. +pub(crate) fn compile_segments(pattern: &[u8]) -> Vec { + let mut segments = Vec::new(); + let mut cur = Vec::new(); + let mut i = 0; + while i < pattern.len() { + match pattern[i] { + b'\\' if i + 1 < pattern.len() => { + cur.push(pattern[i + 1]); + i += 2; + } + b'%' => { + push_literal(&mut segments, &mut cur); + if !segments.is_empty() && !matches!(segments.last(), Some(Segment::Skip)) { + segments.push(Segment::Skip); + } + i += 1; + } + b'_' => { + push_literal(&mut segments, &mut cur); + segments.push(Segment::Any); + i += 1; + } + b => { + cur.push(b); + i += 1; + } + } + } + push_literal(&mut segments, &mut cur); + if matches!(segments.last(), Some(Segment::Skip)) { + segments.pop(); + } + segments +} + +fn push_literal(segments: &mut Vec, cur: &mut Vec) { + if cur.is_empty() { + return; + } + cur.make_ascii_lowercase(); + segments.push(Segment::Literal(Box::new(Finder::new(cur).into_owned()))); + cur.clear(); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn segs(p: &str) -> Vec { + compile_segments(p.as_bytes()) + .iter() + .map(|s| match s { + Segment::Literal(f) => String::from_utf8_lossy(f.needle()).into_owned(), + Segment::Skip => "".into(), + Segment::Any => "".into(), + }) + .collect() + } + + #[test] + fn splits_on_wildcards() { + assert_eq!( + segs("te%test_oui"), + ["te", "", "test", "", "oui"] + ); + assert_eq!(segs("a_%z"), ["a", "", "", "z"]); + assert_eq!(segs("__"), ["", ""]); + } + + #[test] + fn normalizes_skips() { + assert_eq!(segs(""), [] as [&str; 0]); + assert_eq!(segs("%%"), [] as [&str; 0]); + assert_eq!(segs("%abc%"), ["abc"]); + assert_eq!(segs("a%%b"), ["a", "", "b"]); + assert_eq!(segs("_abc_%"), ["", "abc", ""]); + } + + #[test] + fn lowercases_ascii_literals() { + assert_eq!(segs("SaY%WÉrld"), ["say", "", "wÉrld"]); + } + + #[test] + fn unescapes() { + assert_eq!(segs(r"foo\%bar"), ["foo%bar"]); + assert_eq!(segs(r"a\%b%c"), ["a%b", "", "c"]); + assert_eq!(segs(r"\_"), ["_"]); + assert_eq!(segs(r"\\"), ["\\"]); + assert_eq!(segs(r"\\%x"), ["\\", "", "x"]); + assert_eq!(segs(r"\\_"), ["\\", ""]); + assert_eq!(segs(r"a\xb"), ["axb"]); + assert_eq!(segs(r"trail\"), ["trail\\"]); + } +} diff --git a/crates/sqlite-ffi/src/blob.rs b/crates/sqlite-ffi/src/blob.rs new file mode 100644 index 0000000..cf68a36 --- /dev/null +++ b/crates/sqlite-ffi/src/blob.rs @@ -0,0 +1,147 @@ +use std::{ + ffi::{CString, c_char, c_int, c_void}, + io::{self, Read}, + ptr::NonNull, +}; + +use libsqlite3_sys::{ + SQLITE_OK, sqlite3, sqlite3_blob, sqlite3_blob_bytes, sqlite3_blob_close, sqlite3_blob_open, + sqlite3_blob_read, +}; + +use crate::{Connection, error::SqliteError}; + +pub struct SqliteBlob<'a> { + db: &'a NonNull, + handle: NonNull, +} + +impl SqliteBlob<'_> { + pub fn len(&self) -> usize { + unsafe { blob_bytes_raw(self.handle.as_ptr()) as usize } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<(), SqliteError> { + let n = i32::try_from(buf.len()) + .map_err(|_| SqliteError::Message("read length exceeds i32".into()))?; + + let offset = i32::try_from(offset) + .map_err(|_| SqliteError::Message("read offset exceeds i32".into()))?; + + let rc = unsafe { blob_read_raw(self.handle.as_ptr(), buf.as_mut_ptr().cast(), n, offset) }; + + if rc != SQLITE_OK { + return Err(unsafe { SqliteError::from_db(self.db.as_ptr(), "sqlite3_blob_read", rc) }); + } + + Ok(()) + } + + pub fn reader(&self, start: usize) -> BlobReader<'_> { + BlobReader::new(self, start) + } +} + +pub struct BlobReader<'a> { + blob: &'a SqliteBlob<'a>, + pos: usize, + end: usize, +} + +impl<'a> BlobReader<'a> { + fn new(blob: &'a SqliteBlob<'a>, start: usize) -> Self { + let end = blob.len(); + Self { + blob, + pos: start.min(end), + end, + } + } +} + +impl Read for BlobReader<'_> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let remaining = self.end.saturating_sub(self.pos); + if remaining == 0 || buf.is_empty() { + return Ok(0); + } + let n = buf.len().min(remaining); + self.blob + .read_at(self.pos, &mut buf[..n]) + .map_err(|e| io::Error::other(e.to_string()))?; + self.pos += n; + Ok(n) + } +} + +impl Connection { + pub fn open_blob<'a>( + &'a self, + schema: &str, + table: &str, + column: &str, + rowid: i64, + write: bool, + ) -> Result, SqliteError> { + let schema = CString::new(schema)?; + let table = CString::new(table)?; + let column = CString::new(column)?; + let mut blob = std::ptr::null_mut(); + + let rc = unsafe { + blob_open_raw( + self.as_ptr(), + schema.as_ptr(), + table.as_ptr(), + column.as_ptr(), + rowid, + if write { 1 } else { 0 }, + &mut blob, + ) + }; + + if rc != SQLITE_OK { + return Err(unsafe { SqliteError::from_db(self.as_ptr(), "sqlite3_blob_open", rc) }); + } + + Ok(SqliteBlob { + db: &self.conn, + handle: NonNull::new(blob).expect("sqlite3_blob_open returned a null handle"), + }) + } +} + +impl Drop for SqliteBlob<'_> { + fn drop(&mut self) { + unsafe { sqlite3_blob_close(self.handle.as_ptr()) }; + } +} + +unsafe fn blob_open_raw( + db: *mut libsqlite3_sys::sqlite3, + schema: *const c_char, + table: *const c_char, + column: *const c_char, + rowid: i64, + flags: c_int, + blob: *mut *mut sqlite3_blob, +) -> c_int { + unsafe { sqlite3_blob_open(db, schema, table, column, rowid, flags, blob) } +} + +unsafe fn blob_read_raw( + blob: *mut sqlite3_blob, + buf: *mut c_void, + n: c_int, + offset: c_int, +) -> c_int { + unsafe { sqlite3_blob_read(blob, buf, n, offset) } +} + +unsafe fn blob_bytes_raw(blob: *mut sqlite3_blob) -> c_int { + unsafe { sqlite3_blob_bytes(blob) } +} diff --git a/crates/sqlite-ffi/src/context/mod.rs b/crates/sqlite-ffi/src/context/mod.rs index e151444..7730d37 100644 --- a/crates/sqlite-ffi/src/context/mod.rs +++ b/crates/sqlite-ffi/src/context/mod.rs @@ -48,6 +48,10 @@ impl Context { sqlite3_result_error(self.ctx.as_ptr(), error.as_ptr().cast(), error.len() as i32); } } + + pub fn result_int64(&self, value: i64) { + unsafe { result_int_raw(self.ctx.as_ptr(), value) }; + } } unsafe fn result_blob_raw(context: *mut sqlite3_context, blob: *const c_void, size: i32) { diff --git a/crates/sqlite-ffi/src/error.rs b/crates/sqlite-ffi/src/error.rs index 62d8010..c6f63c0 100644 --- a/crates/sqlite-ffi/src/error.rs +++ b/crates/sqlite-ffi/src/error.rs @@ -1,9 +1,9 @@ use std::{ - ffi::{CString, c_char}, + ffi::{CStr, CString, c_char}, ptr, }; -use libsqlite3_sys::{SQLITE_ERROR, sqlite3_malloc}; +use libsqlite3_sys::{SQLITE_ERROR, sqlite3, sqlite3_errmsg, sqlite3_malloc}; use thiserror::Error; #[derive(Debug, Error)] @@ -36,6 +36,22 @@ impl SqliteError { } } + /// # Safety + /// `db` is the database connection pointer passed by SQLite into the C entrypoint + #[cold] + pub(crate) unsafe fn from_db(db: *mut sqlite3, operation: &'static str, code: i32) -> Self { + let message = unsafe { + CStr::from_ptr(sqlite3_errmsg(db)) + .to_string_lossy() + .into_owned() + }; + Self::Sqlite { + operation, + code, + message, + } + } + /// # Safety /// `pz_err_msg` is the out-pointer passed by SQLite into the C entrypoint pub unsafe fn report(&self, pz_err_msg: *mut *mut c_char) -> i32 { diff --git a/crates/sqlite-ffi/src/function.rs b/crates/sqlite-ffi/src/function.rs index 479d125..5cc6ffd 100644 --- a/crates/sqlite-ffi/src/function.rs +++ b/crates/sqlite-ffi/src/function.rs @@ -1,11 +1,11 @@ use std::{ - ffi::{CStr, CString, c_char, c_int, c_void}, + ffi::{CString, c_char, c_int, c_void}, slice, }; use libsqlite3_sys::{ - SQLITE_OK, SQLITE_UTF8, sqlite3_context, sqlite3_create_function_v2, sqlite3_errmsg, - sqlite3_user_data, sqlite3_value, + SQLITE_OK, SQLITE_UTF8, sqlite3_context, sqlite3_create_function_v2, sqlite3_user_data, + sqlite3_value, }; use crate::{Connection, Context, Value, error::SqliteError}; @@ -48,15 +48,8 @@ impl Connection { }; if rc != SQLITE_OK { - let message = unsafe { - CStr::from_ptr(sqlite3_errmsg(self.conn.as_ptr())) - .to_string_lossy() - .into_owned() - }; - return Err(SqliteError::Sqlite { - operation: "create function v2", - code: rc, - message, + return Err(unsafe { + SqliteError::from_db(self.conn.as_ptr(), "sqlite3_create_function", rc) }); } diff --git a/crates/sqlite-ffi/src/lib.rs b/crates/sqlite-ffi/src/lib.rs index f23b155..c242de7 100644 --- a/crates/sqlite-ffi/src/lib.rs +++ b/crates/sqlite-ffi/src/lib.rs @@ -1,3 +1,4 @@ +mod blob; mod client_data; mod connection; mod context; @@ -10,6 +11,7 @@ mod register; mod utils; mod values; +pub use blob::SqliteBlob; pub use connection::Connection; pub use context::{Aux, Context}; pub use error::SqliteError; diff --git a/sqlite-compress/Cargo.toml b/sqlite-compress/Cargo.toml index 44f79c0..97cb28c 100644 --- a/sqlite-compress/Cargo.toml +++ b/sqlite-compress/Cargo.toml @@ -14,12 +14,14 @@ thiserror = { workspace = true } libsqlite3-sys = { workspace = true } sqlite-ffi = { workspace = true } sqlite-ffi-macros = { workspace = true } +matcher = { workspace = true, optional = true } [lib] crate-type = ["cdylib", "rlib"] [features] default = ["bundled"] +like = ["dep:matcher"] bundled = [ "sqlite-ffi/bundled", "libsqlite3-sys/bundled", @@ -27,4 +29,4 @@ bundled = [ loadable_extension = [ "sqlite-ffi/loadable_extension", "libsqlite3-sys/loadable_extension", -] \ No newline at end of file +] diff --git a/sqlite-compress/src/functions/like/mod.rs b/sqlite-compress/src/functions/like/mod.rs new file mode 100644 index 0000000..85a2110 --- /dev/null +++ b/sqlite-compress/src/functions/like/mod.rs @@ -0,0 +1,47 @@ +use matcher::Matcher; +use sqlite_ffi::{Context, SqliteError, Value}; + +use crate::{functions::like::search::search_blob, state::ExtensionState}; + +const PATTERN_ARG: i32 = 4; + +mod search; + +/// `decompress_like(schema, table, column, rowid, pattern, offset) -> 0|1` +/// +/// `pattern` uses `LIKE` wildcards (`%`, `_`) and matches anywhere in the +/// value, ASCII case-insensitively. +pub fn sqlite_decompress_like(mut context: Context, values: &[Value]) -> Result<(), SqliteError> { + if values.len() < 6 { + return Err(SqliteError::Message( + "decompress_like expects 6 arguments: schema, table, column, rowid, pattern, offset" + .into(), + )); + } + + let schema = values[0].to_text()?; + let table = values[1].to_text()?; + let column = values[2].to_text()?; + let rowid = values[3].to_i64(); + let pattern = values[PATTERN_ARG as usize].to_blob(); + let offset = values[5].to_i64().max(0) as usize; + + let state = ExtensionState::from_context(&context)?; + let blob = state + .connection + .open_blob(schema, table, column, rowid, false)?; + + let aux = context.aux::(PATTERN_ARG); + let found = match aux.get() { + Ok(matcher) => search_blob(&state, schema, &blob, matcher, offset)?, + Err(_) => { + let matcher = Matcher::compile(pattern); + let found = search_blob(&state, schema, &blob, &matcher, offset)?; + aux.set(matcher); + found + } + }; + + context.result_bool(found); + Ok(()) +} diff --git a/sqlite-compress/src/functions/like/search.rs b/sqlite-compress/src/functions/like/search.rs new file mode 100644 index 0000000..494bb31 --- /dev/null +++ b/sqlite-compress/src/functions/like/search.rs @@ -0,0 +1,66 @@ +use std::io::{self, BufReader}; + +use matcher::Matcher; +use sqlite_ffi::{SqliteBlob, SqliteError}; +use zstd::{stream::read::Decoder, zstd_safe}; + +use crate::{ + DictError, DictId, + cache::get_decoder_in_cache, + functions::{ParsedBlob, header::Header}, + state::ExtensionState, +}; + +pub(crate) fn search_blob( + state: &ExtensionState, + schema: &str, + blob: &SqliteBlob<'_>, + matcher: &Matcher, + offset: usize, +) -> Result { + let total = blob.len(); + if total == 0 { + return matcher.find_in(io::empty(), offset).map_err(to_sqlite); + } + + // Peek just enough to parse the header (no full materialization). + let peek_len = Header::PREFIX_SIZE.min(total); + let mut peek = vec![0u8; peek_len]; + blob.read_at(0, &mut peek)?; + + match Header::parse_blob(&peek) { + Ok(ParsedBlob::Plain) | Err(_) if peek_len < Header::PREFIX_SIZE => { + matcher.find_in(blob.reader(offset), 0).map_err(to_sqlite) + } + Ok(ParsedBlob::Plain) => matcher.find_in(blob.reader(offset), 0).map_err(to_sqlite), + Ok(ParsedBlob::Framed(header, _)) => { + let header_len = header.header_len as usize; + + let dict_id = DictId::from(header.dict_id.get()); + let prepared = if dict_id.get() != 0 { + match get_decoder_in_cache(state, schema, dict_id) { + Ok(d) => Some(d), + Err(DictError::NotReady) => None, + Err(e) => return Err(SqliteError::Message(e.to_string())), + } + } else { + None + }; + + let reader = + BufReader::with_capacity(zstd_safe::DCtx::in_size(), blob.reader(header_len)); + let decoder = match prepared.as_deref() { + Some(dict) => Decoder::with_prepared_dictionary(reader, dict), + None => Decoder::with_dictionary(reader, &[]), + } + .map_err(to_sqlite)?; + + matcher.find_in(decoder, offset).map_err(to_sqlite) + } + Err(e) => Err(SqliteError::Message(e.to_string())), + } +} + +pub(crate) fn to_sqlite(e: impl std::fmt::Display) -> SqliteError { + SqliteError::Message(e.to_string()) +} diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index b7954bc..68689da 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -2,6 +2,8 @@ mod compress; mod decompress; mod errors; mod header; +#[cfg(feature = "like")] +mod like; mod types; /// Compression level used when configuration does not select another level. @@ -13,4 +15,6 @@ pub(crate) use decompress::{decompress_raw, decompress_with_decoder}; pub use errors::CodecError; pub use header::Header; pub(crate) use header::ParsedBlob; +#[cfg(feature = "like")] +pub use like::sqlite_decompress_like; pub use types::Level; diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index 7f0383c..056e337 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -34,5 +34,8 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> connection.create_function("decompress", 2, functions::sqlite_decompress)?; + #[cfg(feature = "like")] + connection.create_function("decompress_like", 6, functions::sqlite_decompress_like)?; + Ok(()) }