From 836789da8ee6a6f888aa25f636d180991e3e1110 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Wed, 23 Sep 2026 17:01:37 -0400 Subject: [PATCH 01/17] =?UTF-8?q?=F0=9F=9A=A7=20Test=20scan=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/sqlite-ffi/src/blob.rs | 120 ++++++++++++++++++++++++++++++++ crates/sqlite-ffi/src/lib.rs | 2 + crates/sqlite-ffi/src/status.rs | 44 ++++++++++++ sqlite-compress/tests/scan.rs | 41 +++++++++++ 4 files changed, 207 insertions(+) create mode 100644 crates/sqlite-ffi/src/blob.rs create mode 100644 crates/sqlite-ffi/src/status.rs create mode 100644 sqlite-compress/tests/scan.rs diff --git a/crates/sqlite-ffi/src/blob.rs b/crates/sqlite-ffi/src/blob.rs new file mode 100644 index 0000000..2315135 --- /dev/null +++ b/crates/sqlite-ffi/src/blob.rs @@ -0,0 +1,120 @@ +use std::{ + ffi::{CStr, CString, c_int, c_void}, + ptr::NonNull, +}; + +use libsqlite3_sys::{ + SQLITE_OK, sqlite3_blob, sqlite3_blob_bytes, sqlite3_blob_close, sqlite3_blob_open, + sqlite3_blob_read, sqlite3_errmsg, +}; + +use crate::{Connection, error::SqliteError}; + +pub struct SqliteBlob { + handle: NonNull, +} + +impl SqliteBlob { + pub fn len(&self) -> usize { + unsafe { blob_bytes_raw(self.handle.as_ptr()) as usize } + } + + pub fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<(), SqliteError> { + let n = i32::try_from(buf.len()) + .map_err(|_| SqliteError::Message("read length exceeds i32".into()))?; + let offset = i32::try_from(offset) + .map_err(|_| SqliteError::Message("read offset exceeds i32".into()))?; + + let rc = unsafe { blob_read_raw(self.handle.as_ptr(), buf.as_mut_ptr().cast(), n, offset) }; + + if rc != SQLITE_OK { + let message = unsafe { + CStr::from_ptr(sqlite3_errmsg(self.db.as_ptr())) + .to_string_lossy() + .into_owned() + }; + return Err(SqliteError::Sqlite { + operation: "sqlite3_blob_read", + code: rc, + message, + }); + } + Ok(()) + } +} + +impl Connection { + pub fn open_blob( + &self, + schema: &str, + table: &str, + column: &str, + rowid: i64, + write: bool, + ) -> Result { + let schema = CString::new(schema)?; + let table = CString::new(table)?; + let column = CString::new(column)?; + let mut blob = std::ptr::null_mut(); + + let rc = unsafe { + blob_open_raw( + self.as_ptr(), + schema.as_ptr(), + table.as_ptr(), + column.as_ptr(), + rowid, + if write { 1 } else { 0 }, + &mut blob, + ) + }; + + if rc != SQLITE_OK { + let message = unsafe { + CStr::from_ptr(sqlite3_errmsg(self.as_ptr())) + .to_string_lossy() + .into_owned() + }; + return Err(SqliteError::Sqlite { + operation: "sqlite3_blob_open", + code: rc, + message, + }); + } + + Ok(SqliteBlob { + handle: NonNull::new(blob).expect("sqlite3_blob_open returned a null handle"), + }) + } +} + +impl Drop for SqliteBlob { + fn drop(&mut self) { + unsafe { sqlite3_blob_close(self.handle.as_ptr()) }; + } +} + +unsafe fn blob_open_raw( + db: *mut libsqlite3_sys::sqlite3, + schema: *const std::ffi::c_char, + table: *const std::ffi::c_char, + column: *const std::ffi::c_char, + rowid: i64, + flags: std::ffi::c_int, + blob: *mut *mut sqlite3_blob, +) -> std::ffi::c_int { + unsafe { sqlite3_blob_open(db, schema, table, column, rowid, flags, blob) } +} + +unsafe fn blob_read_raw( + blob: *mut sqlite3_blob, + buf: *mut c_void, + n: c_int, + offset: std::ffi::c_int, +) -> std::ffi::c_int { + unsafe { sqlite3_blob_read(blob, buf, n, offset) } +} + +unsafe fn blob_bytes_raw(blob: *mut sqlite3_blob) -> std::ffi::c_int { + unsafe { sqlite3_blob_bytes(blob) } +} diff --git a/crates/sqlite-ffi/src/lib.rs b/crates/sqlite-ffi/src/lib.rs index 232562a..3a649c2 100644 --- a/crates/sqlite-ffi/src/lib.rs +++ b/crates/sqlite-ffi/src/lib.rs @@ -1,3 +1,4 @@ +mod blob; mod client_data; mod connection; mod context; @@ -6,6 +7,7 @@ mod error; mod extension; mod function; mod queries; +mod status; mod values; pub use connection::Connection; diff --git a/crates/sqlite-ffi/src/status.rs b/crates/sqlite-ffi/src/status.rs new file mode 100644 index 0000000..d58d9e3 --- /dev/null +++ b/crates/sqlite-ffi/src/status.rs @@ -0,0 +1,44 @@ +use std::ffi::c_int; + +use libsqlite3_sys::{SQLITE_OK, sqlite3_int64, sqlite3_status64}; + +use crate::{Connection, error::SqliteError}; + +#[repr(i32)] +enum SqliteStatus { + MallocSize = 5, +} + +impl Connection { + pub fn malloc_highwater(&self) -> Result { + let mut current = 0; + let mut highwater = 0; + let rc = unsafe { + status_raw( + SqliteStatus::MallocSize as c_int, + &mut current, + &mut highwater, + 1, + ) + }; + + if rc != SQLITE_OK { + return Err(SqliteError::Sqlite { + operation: "sqlite3_status64", + code: rc, + message: "malloc highwater".into(), + }); + } + + Ok(highwater) + } +} + +unsafe fn status_raw( + op: c_int, + current: *mut sqlite3_int64, + highwater: *mut sqlite3_int64, + reset_flag: c_int, +) -> c_int { + unsafe { sqlite3_status64(op, current, highwater, reset_flag) } +} diff --git a/sqlite-compress/tests/scan.rs b/sqlite-compress/tests/scan.rs new file mode 100644 index 0000000..20f2fe1 --- /dev/null +++ b/sqlite-compress/tests/scan.rs @@ -0,0 +1,41 @@ +use sqlite_ffi::{first_value, SqlValue}; +mod common; + +use common::TestDb; + +#[test] +fn like_loads_full_blob_when_match_is_at_start() { + const BODY_LEN: usize = 1024 * 1024; + let needle = b"NEEDLE"; + + let mut body = vec![b'a'; BODY_LEN]; + body[..needle.len()].copy_from_slice(needle); + + let db = TestDb::open(); + let conn = db.state(); + conn.as_ref() + .batch_execute("CREATE TABLE t(id INTEGER PRIMARY KEY, data BLOB)") + .unwrap(); + + conn.as_ref() + .execute("INSERT INTO t(data) VALUES (?1)", &[SqlValue::Blob(body)]) + .unwrap(); + + let id_hw = conn.as_ref().malloc_highwater().unwrap(); + let id_rows = conn + .as_ref() + .query("SELECT id FROM t WHERE id = 1", &[]) + .unwrap(); + assert_eq!(first_value(&id_rows).and_then(SqlValue::as_i64), Some(1)); + println!("id_hw: {id_hw}"); + + let _ = conn.as_ref().malloc_highwater().unwrap(); + let like_rows = conn + .as_ref() + .query("SELECT data LIKE 'NEEDLE%' FROM t WHERE id = 1", &[]) + .unwrap(); + assert_eq!(first_value(&like_rows).and_then(SqlValue::as_i64), Some(1)); + let like_hw = conn.as_ref().malloc_highwater().unwrap(); + + println!("like_hw: {like_hw}"); +} From 0e417a2667e497b0fc02907726ecdb0dd49ea2cb Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Wed, 23 Sep 2026 17:58:12 -0400 Subject: [PATCH 02/17] =?UTF-8?q?=F0=9F=9A=A7=20wip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/sqlite-ffi/src/blob.rs | 24 ++- crates/sqlite-ffi/src/context.rs | 4 + crates/sqlite-ffi/src/lib.rs | 1 + sqlite-compress/src/functions/clike.rs | 194 ++++++++++++++++++++ sqlite-compress/src/functions/decompress.rs | 2 + sqlite-compress/src/functions/mod.rs | 2 + sqlite-compress/src/lib.rs | 2 + sqlite-compress/tests/scan.rs | 53 ++++++ 8 files changed, 272 insertions(+), 10 deletions(-) create mode 100644 sqlite-compress/src/functions/clike.rs diff --git a/crates/sqlite-ffi/src/blob.rs b/crates/sqlite-ffi/src/blob.rs index 2315135..462af9a 100644 --- a/crates/sqlite-ffi/src/blob.rs +++ b/crates/sqlite-ffi/src/blob.rs @@ -1,16 +1,17 @@ use std::{ - ffi::{CStr, CString, c_int, c_void}, + ffi::{CStr, CString, c_char, c_int, c_void}, ptr::NonNull, }; use libsqlite3_sys::{ - SQLITE_OK, sqlite3_blob, sqlite3_blob_bytes, sqlite3_blob_close, sqlite3_blob_open, + SQLITE_OK, sqlite3, sqlite3_blob, sqlite3_blob_bytes, sqlite3_blob_close, sqlite3_blob_open, sqlite3_blob_read, sqlite3_errmsg, }; use crate::{Connection, error::SqliteError}; pub struct SqliteBlob { + db: NonNull, handle: NonNull, } @@ -22,6 +23,7 @@ impl SqliteBlob { pub fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<(), SqliteError> { let n = i32::try_from(buf.len()) .map_err(|_| SqliteError::Message("read length exceeds i32".into()))?; + let offset = i32::try_from(offset) .map_err(|_| SqliteError::Message("read offset exceeds i32".into()))?; @@ -39,6 +41,7 @@ impl SqliteBlob { message, }); } + Ok(()) } } @@ -83,6 +86,7 @@ impl Connection { } Ok(SqliteBlob { + db: NonNull::new(self.as_ptr()).expect("database pointer is not valid"), handle: NonNull::new(blob).expect("sqlite3_blob_open returned a null handle"), }) } @@ -96,13 +100,13 @@ impl Drop for SqliteBlob { unsafe fn blob_open_raw( db: *mut libsqlite3_sys::sqlite3, - schema: *const std::ffi::c_char, - table: *const std::ffi::c_char, - column: *const std::ffi::c_char, + schema: *const c_char, + table: *const c_char, + column: *const c_char, rowid: i64, - flags: std::ffi::c_int, + flags: c_int, blob: *mut *mut sqlite3_blob, -) -> std::ffi::c_int { +) -> c_int { unsafe { sqlite3_blob_open(db, schema, table, column, rowid, flags, blob) } } @@ -110,11 +114,11 @@ unsafe fn blob_read_raw( blob: *mut sqlite3_blob, buf: *mut c_void, n: c_int, - offset: std::ffi::c_int, -) -> std::ffi::c_int { + offset: c_int, +) -> c_int { unsafe { sqlite3_blob_read(blob, buf, n, offset) } } -unsafe fn blob_bytes_raw(blob: *mut sqlite3_blob) -> std::ffi::c_int { +unsafe fn blob_bytes_raw(blob: *mut sqlite3_blob) -> c_int { unsafe { sqlite3_blob_bytes(blob) } } diff --git a/crates/sqlite-ffi/src/context.rs b/crates/sqlite-ffi/src/context.rs index ef64860..ffb0ecf 100644 --- a/crates/sqlite-ffi/src/context.rs +++ b/crates/sqlite-ffi/src/context.rs @@ -28,6 +28,10 @@ impl Context { sqlite3_result_error(self.ctx.as_ptr(), error.as_ptr().cast(), error.len() as i32); } } + + pub fn result_int64(&self, value: i64) { + unsafe { libsqlite3_sys::sqlite3_result_int64(self.ctx.as_ptr(), value) }; + } } unsafe fn result_blob_raw(context: *mut sqlite3_context, blob: *const c_void, size: i32) { diff --git a/crates/sqlite-ffi/src/lib.rs b/crates/sqlite-ffi/src/lib.rs index 3a649c2..e1ba967 100644 --- a/crates/sqlite-ffi/src/lib.rs +++ b/crates/sqlite-ffi/src/lib.rs @@ -10,6 +10,7 @@ mod queries; mod status; mod values; +pub use blob::SqliteBlob; pub use connection::Connection; pub use context::Context; pub use error::SqliteError; diff --git a/sqlite-compress/src/functions/clike.rs b/sqlite-compress/src/functions/clike.rs new file mode 100644 index 0000000..bb45571 --- /dev/null +++ b/sqlite-compress/src/functions/clike.rs @@ -0,0 +1,194 @@ +use sqlite_ffi::{Context, SqliteError, Value}; +use zstd::stream::raw::{Decoder, Operation}; + +use crate::{ + cache::get_decoder_in_cache, functions::header::Header, DictError, DictId, ExtensionState, +}; + +const CHUNK: usize = 64 * 1024; + +struct Progress { + matched: bool, + frame_done: bool, +} + +pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteError> { + let schema = values[0].to_text()?; + let table = values[1].to_text()?; + let column = values[2].to_text()?; + let rowid = values[3].to_i64(); + let needle = values[4].to_text()?.as_bytes(); + + let state = ExtensionState::from_context(&context)?; + let blob = state + .connection + .open_blob(schema, table, column, rowid, false)?; + + let total = blob.len(); + if total < Header::PREFIX_SIZE { + return Err(SqliteError::Message( + "compressed blob is shorter than the header".into(), + )); + } + + let mut compressed = vec![0u8; CHUNK.min(total)]; + blob.read_at(0, &mut compressed)?; + + let (header, _) = + Header::parse(&compressed).map_err(|e| SqliteError::Message(e.to_string()))?; + let dict_id = DictId::from(header.dict_id.get()); + let header_len = header.header_len as usize; + let first_len = compressed.len(); + + let prepared = if dict_id.get() != 0 { + match get_decoder_in_cache(&state, schema, dict_id) { + Ok(dict) => Some(dict), + Err(DictError::NotReady) => None, + Err(e) => return Err(SqliteError::Message(e.to_string())), + } + } else { + None + }; + + let matched = if let Some(dict) = &prepared { + let mut decoder = Decoder::with_prepared_dictionary(dict.as_ref()) + .map_err(|e| SqliteError::Message(e.to_string()))?; + scan( + &mut decoder, + &blob, + &mut compressed, + header_len, + first_len, + needle, + )? + } else { + let mut decoder = Decoder::new().map_err(|e| SqliteError::Message(e.to_string()))?; + scan( + &mut decoder, + &blob, + &mut compressed, + header_len, + first_len, + needle, + )? + }; + + context.result_int64(if matched { 1 } else { 0 }); + Ok(()) +} + +fn scan( + decoder: &mut Decoder<'_>, + blob: &sqlite_ffi::SqliteBlob, + compressed: &mut [u8], + header_len: usize, + first_len: usize, + needle: &[u8], +) -> Result { + let mut plain = vec![0u8; CHUNK]; + let mut overlap = Vec::new(); + + let first = feed( + decoder, + &compressed[header_len..first_len], + &mut plain, + &mut overlap, + needle, + )?; + + if first.matched || first.frame_done { + eprintln!( + "clike rust vecs: compressed={} plain={} overlap={}", + compressed.len(), + plain.capacity(), + overlap.capacity(), + ); + + return Ok(first.matched); + } + + let total = blob.len(); + let mut pos = first_len; + while pos < total { + let n = CHUNK.min(total - pos); + blob.read_at(pos, &mut compressed[..n])?; + pos += n; + + let step = feed(decoder, &compressed[..n], &mut plain, &mut overlap, needle)?; + if step.matched || step.frame_done { + eprintln!( + "clike rust vecs: compressed={} plain={} overlap={}", + compressed.len(), + plain.capacity(), + overlap.capacity(), + ); + + return Ok(step.matched); + } + } + + let tail = feed(decoder, &[], &mut plain, &mut overlap, needle)?; + + eprintln!( + "clike rust vecs: compressed={} plain={} overlap={}", + compressed.len(), + plain.capacity(), + overlap.capacity(), + ); + Ok(tail.matched) +} + +fn feed( + decoder: &mut Decoder<'_>, + mut input: &[u8], + plain: &mut [u8], + overlap: &mut Vec, + needle: &[u8], +) -> Result { + loop { + let status = decoder + .run_on_buffers(input, plain) + .map_err(|e| SqliteError::Message(e.to_string()))?; + + if status.bytes_written > 0 && take_match(overlap, &plain[..status.bytes_written], needle) { + return Ok(Progress { + matched: true, + frame_done: status.remaining == 0, + }); + } + + input = &input[status.bytes_read..]; + + if status.remaining == 0 && status.bytes_written < plain.len() { + return Ok(Progress { + matched: false, + frame_done: true, + }); + } + + if input.is_empty() && status.bytes_written < plain.len() { + return Ok(Progress { + matched: false, + frame_done: false, + }); + } + + if status.bytes_read == 0 && status.bytes_written == 0 { + return Err(SqliteError::Message("zstd decoder made no progress".into())); + } + } +} + +fn take_match(overlap: &mut Vec, chunk: &[u8], needle: &[u8]) -> bool { + if needle.is_empty() { + return false; + } + + overlap.extend_from_slice(chunk); + let matched = overlap.windows(needle.len()).any(|window| window == needle); + let keep = needle.len() - 1; + if overlap.len() > keep { + overlap.drain(..overlap.len() - keep); + } + matched +} diff --git a/sqlite-compress/src/functions/decompress.rs b/sqlite-compress/src/functions/decompress.rs index 436d273..57cd775 100644 --- a/sqlite-compress/src/functions/decompress.rs +++ b/sqlite-compress/src/functions/decompress.rs @@ -58,6 +58,8 @@ pub fn sqlite_decompress(context: Context, values: &[Value]) -> Result<(), Sqlit decompress_raw(payload, len).map_err(|e| SqliteError::Message(e.to_string()))? }; + eprintln!("decompress rust vec: {}", decompressed.capacity()); + context.result_blob(&decompressed); Ok(()) diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index cc193d2..2f5fbd0 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -1,3 +1,4 @@ +mod clike; mod compress; mod decompress; mod errors; @@ -6,6 +7,7 @@ mod types; pub const DEFAULT_LEVEL: types::Level = types::Level::new(3); +pub use clike::sqlite_clike; pub use compress::sqlite_compress; pub use decompress::sqlite_decompress; pub(crate) use decompress::{decompress_raw, decompress_with_decoder}; diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index 0e1a206..9830a67 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -28,5 +28,7 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> connection.create_function("decompress", 2, functions::sqlite_decompress)?; + connection.create_function("clike", 5, functions::sqlite_clike)?; + Ok(()) } diff --git a/sqlite-compress/tests/scan.rs b/sqlite-compress/tests/scan.rs index 20f2fe1..b1a4153 100644 --- a/sqlite-compress/tests/scan.rs +++ b/sqlite-compress/tests/scan.rs @@ -1,10 +1,16 @@ +use std::sync::Mutex; + use sqlite_ffi::{first_value, SqlValue}; + mod common; use common::TestDb; +static STATUS: Mutex<()> = Mutex::new(()); + #[test] fn like_loads_full_blob_when_match_is_at_start() { + let _guard = STATUS.lock().unwrap(); const BODY_LEN: usize = 1024 * 1024; let needle = b"NEEDLE"; @@ -39,3 +45,50 @@ fn like_loads_full_blob_when_match_is_at_start() { println!("like_hw: {like_hw}"); } + +#[test] +fn clike_allocates_less_than_decompress_like_when_match_is_at_start() { + let _guard = STATUS.lock().unwrap(); + const BODY_LEN: usize = 1024 * 1024; + let needle = b"NEEDLE"; + + let mut body = vec![b'a'; BODY_LEN]; + body[..needle.len()].copy_from_slice(needle); + + let db = TestDb::open(); + let conn = db.state(); + conn.as_ref() + .batch_execute("CREATE TABLE t(id INTEGER PRIMARY KEY, data BLOB)") + .unwrap(); + + conn.as_ref() + .execute( + "INSERT INTO t(data) VALUES (compress(?1, 'main', 't', 'data'))", + &[SqlValue::Blob(body)], + ) + .unwrap(); + + let _ = conn.as_ref().malloc_highwater().unwrap(); + let like_rows = conn + .as_ref() + .query( + "SELECT decompress(data, 'main') LIKE 'NEEDLE%' FROM t WHERE id = 1", + &[], + ) + .unwrap(); + assert_eq!(first_value(&like_rows).and_then(SqlValue::as_i64), Some(1)); + let like_hw = conn.as_ref().malloc_highwater().unwrap(); + + let clike_rows = conn + .as_ref() + .query( + "SELECT clike('main', 't', 'data', id, 'NEEDLE') FROM t WHERE id = 1", + &[], + ) + .unwrap(); + assert_eq!(first_value(&clike_rows).and_then(SqlValue::as_i64), Some(1)); + let clike_hw = conn.as_ref().malloc_highwater().unwrap(); + + println!("decompress_like_hw: {like_hw}"); + println!("clike_hw: {clike_hw}"); +} From e0ad7fe5dff3396baa8620fe5b6a85c5def082cc Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Wed, 23 Sep 2026 18:32:59 -0400 Subject: [PATCH 03/17] =?UTF-8?q?=F0=9F=9A=A7=20benches=20+=20memchr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 517 ++++++++++++++++++++ sqlite-compress/Cargo.toml | 10 +- sqlite-compress/benches/scan.rs | 97 ++++ sqlite-compress/src/functions/clike.rs | 74 ++- sqlite-compress/src/functions/decompress.rs | 2 - 5 files changed, 658 insertions(+), 42 deletions(-) create mode 100644 sqlite-compress/benches/scan.rs diff --git a/Cargo.lock b/Cargo.lock index b183334..10b94f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,51 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.4.6" @@ -26,12 +65,161 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + [[package]] name = "find-msvc-tools" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -43,6 +231,49 @@ dependencies = [ "r-efi", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "jobserver" version = "0.1.35" @@ -53,6 +284,17 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.189" @@ -81,6 +323,33 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "parking_lot" version = "0.12.5" @@ -104,12 +373,46 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "prettyplease" version = "0.3.0" @@ -144,6 +447,26 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -153,18 +476,111 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.16.1" @@ -175,7 +591,9 @@ checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" name = "sqlite-compress" version = "0.1.0" dependencies = [ + "criterion", "libsqlite3-sys", + "memchr", "parking_lot", "sqlite-ffi", "sqlite-ffi-macros", @@ -244,6 +662,16 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "unicode-ident" version = "1.0.24" @@ -256,12 +684,95 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.5", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "zerocopy" version = "0.8.57" @@ -282,6 +793,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + [[package]] name = "zstd" version = "0.13.3" diff --git a/sqlite-compress/Cargo.toml b/sqlite-compress/Cargo.toml index 31f5b51..8534ef1 100644 --- a/sqlite-compress/Cargo.toml +++ b/sqlite-compress/Cargo.toml @@ -11,6 +11,7 @@ thiserror = { workspace = true } sqlite-ffi = { version = "0.1.0", path = "../crates/sqlite-ffi", default-features = false } libsqlite3-sys = { workspace = true, default-features = false } sqlite-ffi-macros = { version = "0.1.0", path = "../crates/sqlite-ffi-macros" } +memchr = "2.8.3" [lib] crate-type = ["cdylib", "rlib"] @@ -24,4 +25,11 @@ bundled = [ loadable_extension = [ "sqlite-ffi/loadable_extension", "libsqlite3-sys/loadable_extension", -] \ No newline at end of file +] + +[dev-dependencies] +criterion = "0.5" + +[[bench]] +name = "scan" +harness = false diff --git a/sqlite-compress/benches/scan.rs b/sqlite-compress/benches/scan.rs new file mode 100644 index 0000000..b563694 --- /dev/null +++ b/sqlite-compress/benches/scan.rs @@ -0,0 +1,97 @@ +use std::{hint::black_box, ptr, sync::Once}; + +use criterion::{criterion_group, criterion_main, Criterion}; +use libsqlite3_sys::{sqlite3, sqlite3_auto_extension, sqlite3_close, sqlite3_open, SQLITE_OK}; +use sqlite_compress::{sqlite3_compress_init, ExtensionState}; +use sqlite_ffi::{first_value, SqlValue}; + +struct BenchDb(*mut sqlite3); + +impl BenchDb { + fn open() -> Self { + static REGISTER: Once = Once::new(); + REGISTER.call_once(|| unsafe { + sqlite3_auto_extension(Some(std::mem::transmute( + sqlite3_compress_init as *const (), + ))); + }); + let mut db = ptr::null_mut(); + assert_eq!( + unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }, + SQLITE_OK + ); + Self(db) + } + + fn state(&self) -> ExtensionState { + ExtensionState::from_db(self.0).unwrap() + } +} + +impl Drop for BenchDb { + fn drop(&mut self) { + unsafe { sqlite3_close(self.0) }; + } +} + +fn insert(conn: &ExtensionState, kind: &str, at: Option) { + const BODY_LEN: usize = 8 * 1024 * 1024; + let mut body = vec![b'a'; BODY_LEN]; + if let Some(at) = at { + body[at..at + 6].copy_from_slice(b"NEEDLE"); + } + conn.as_ref() + .execute( + "INSERT INTO t(data, kind) VALUES (compress(?1, 'main', 't', 'data'), ?2)", + &[SqlValue::Blob(body), SqlValue::Text(kind.to_string())], + ) + .unwrap(); +} + +fn hit(conn: &ExtensionState, sql: &str) -> i64 { + let rows = conn.as_ref().query(sql, &[]).unwrap(); + first_value(&rows).and_then(SqlValue::as_i64).unwrap() +} + +fn bench_scan(c: &mut Criterion) { + let db = BenchDb::open(); + let conn = db.state(); + conn.as_ref() + .batch_execute("CREATE TABLE t(id INTEGER PRIMARY KEY, data BLOB, kind TEXT)") + .unwrap(); + insert(&conn, "start", Some(0)); + insert(&conn, "middle", Some(4 * 1024 * 1024)); + insert(&conn, "end", Some(8 * 1024 * 1024 - 6)); + insert(&conn, "absent", None); + + let mut group = c.benchmark_group("clike_vs_like"); + group.sample_size(10); + + for (kind, like_pat, clike_pat, expect) in [ + ("start", "NEEDLE%", "NEEDLE", 1), + ("middle", "%NEEDLE%", "NEEDLE", 1), + ("end", "%NEEDLE", "NEEDLE", 1), + ("absent", "%ABSENT%", "ABSENT", 0), + ] { + let like_sql = format!( + "SELECT decompress(data, 'main') LIKE '{like_pat}' FROM t WHERE kind = '{kind}'" + ); + let clike_sql = format!( + "SELECT clike('main', 't', 'data', id, '{clike_pat}') FROM t WHERE kind = '{kind}'" + ); + assert_eq!(hit(&conn, &like_sql), expect); + assert_eq!(hit(&conn, &clike_sql), expect); + + group.bench_function(format!("decompress-like-{kind}"), |b| { + b.iter(|| black_box(hit(&conn, &like_sql))) + }); + group.bench_function(format!("clike-{kind}"), |b| { + b.iter(|| black_box(hit(&conn, &clike_sql))) + }); + } + + group.finish(); +} + +criterion_group!(benches, bench_scan); +criterion_main!(benches); diff --git a/sqlite-compress/src/functions/clike.rs b/sqlite-compress/src/functions/clike.rs index bb45571..d86287a 100644 --- a/sqlite-compress/src/functions/clike.rs +++ b/sqlite-compress/src/functions/clike.rs @@ -1,3 +1,4 @@ +use memchr::memmem::Finder; use sqlite_ffi::{Context, SqliteError, Value}; use zstd::stream::raw::{Decoder, Operation}; @@ -85,25 +86,20 @@ fn scan( first_len: usize, needle: &[u8], ) -> Result { + let finder = Finder::new(needle); + let mut plain = vec![0u8; CHUNK]; - let mut overlap = Vec::new(); + let mut tail = Vec::with_capacity(needle.len() - 1); let first = feed( decoder, &compressed[header_len..first_len], &mut plain, - &mut overlap, - needle, + &mut tail, + &finder, )?; if first.matched || first.frame_done { - eprintln!( - "clike rust vecs: compressed={} plain={} overlap={}", - compressed.len(), - plain.capacity(), - overlap.capacity(), - ); - return Ok(first.matched); } @@ -114,43 +110,29 @@ fn scan( blob.read_at(pos, &mut compressed[..n])?; pos += n; - let step = feed(decoder, &compressed[..n], &mut plain, &mut overlap, needle)?; + let step = feed(decoder, &compressed[..n], &mut plain, &mut tail, &finder)?; if step.matched || step.frame_done { - eprintln!( - "clike rust vecs: compressed={} plain={} overlap={}", - compressed.len(), - plain.capacity(), - overlap.capacity(), - ); - return Ok(step.matched); } } - let tail = feed(decoder, &[], &mut plain, &mut overlap, needle)?; - - eprintln!( - "clike rust vecs: compressed={} plain={} overlap={}", - compressed.len(), - plain.capacity(), - overlap.capacity(), - ); - Ok(tail.matched) + let last = feed(decoder, &[], &mut plain, &mut tail, &finder)?; + Ok(last.matched) } fn feed( decoder: &mut Decoder<'_>, mut input: &[u8], plain: &mut [u8], - overlap: &mut Vec, - needle: &[u8], + tail: &mut Vec, + finder: &Finder<'_>, ) -> Result { loop { let status = decoder .run_on_buffers(input, plain) .map_err(|e| SqliteError::Message(e.to_string()))?; - if status.bytes_written > 0 && take_match(overlap, &plain[..status.bytes_written], needle) { + if status.bytes_written > 0 && take_match(finder, tail, &plain[..status.bytes_written]) { return Ok(Progress { matched: true, frame_done: status.remaining == 0, @@ -179,16 +161,30 @@ fn feed( } } -fn take_match(overlap: &mut Vec, chunk: &[u8], needle: &[u8]) -> bool { - if needle.is_empty() { - return false; +fn take_match(finder: &Finder<'_>, tail: &mut Vec, chunk: &[u8]) -> bool { + let k = finder.needle().len() - 1; + + if !tail.is_empty() { + let head = &chunk[..k.min(chunk.len())]; + let base = tail.len(); + tail.extend_from_slice(head); + if finder.find(tail).is_some() { + return true; + } + tail.truncate(base); } - overlap.extend_from_slice(chunk); - let matched = overlap.windows(needle.len()).any(|window| window == needle); - let keep = needle.len() - 1; - if overlap.len() > keep { - overlap.drain(..overlap.len() - keep); + if finder.find(chunk).is_some() { + return true; + } + + if chunk.len() >= k { + tail.clear(); + tail.extend_from_slice(&chunk[chunk.len() - k..]); + } else { + tail.extend_from_slice(chunk); + let excess = tail.len().saturating_sub(k); + tail.drain(..excess); } - matched + false } diff --git a/sqlite-compress/src/functions/decompress.rs b/sqlite-compress/src/functions/decompress.rs index 57cd775..436d273 100644 --- a/sqlite-compress/src/functions/decompress.rs +++ b/sqlite-compress/src/functions/decompress.rs @@ -58,8 +58,6 @@ pub fn sqlite_decompress(context: Context, values: &[Value]) -> Result<(), Sqlit decompress_raw(payload, len).map_err(|e| SqliteError::Message(e.to_string()))? }; - eprintln!("decompress rust vec: {}", decompressed.capacity()); - context.result_blob(&decompressed); Ok(()) From bd8b2a9e938ffe94256dcc14a672636985979993 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Thu, 24 Sep 2026 10:42:59 -0400 Subject: [PATCH 04/17] =?UTF-8?q?=E2=9C=85=20Add=20test=20over=20scan=20an?= =?UTF-8?q?d=20test=5Fmatch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/sqlite-ffi/src/blob.rs | 4 + crates/sqlite-ffi/src/lib.rs | 1 - crates/sqlite-ffi/src/status.rs | 44 ------- sqlite-compress/benches/scan.rs | 1 + sqlite-compress/src/functions/clike.rs | 156 +++++++++++++++++++------ sqlite-compress/tests/scan.rs | 94 --------------- 6 files changed, 124 insertions(+), 176 deletions(-) delete mode 100644 crates/sqlite-ffi/src/status.rs delete mode 100644 sqlite-compress/tests/scan.rs diff --git a/crates/sqlite-ffi/src/blob.rs b/crates/sqlite-ffi/src/blob.rs index 462af9a..ce60037 100644 --- a/crates/sqlite-ffi/src/blob.rs +++ b/crates/sqlite-ffi/src/blob.rs @@ -20,6 +20,10 @@ impl SqliteBlob { unsafe { blob_bytes_raw(self.handle.as_ptr()) as usize } } + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + pub fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<(), SqliteError> { let n = i32::try_from(buf.len()) .map_err(|_| SqliteError::Message("read length exceeds i32".into()))?; diff --git a/crates/sqlite-ffi/src/lib.rs b/crates/sqlite-ffi/src/lib.rs index e1ba967..877d59d 100644 --- a/crates/sqlite-ffi/src/lib.rs +++ b/crates/sqlite-ffi/src/lib.rs @@ -7,7 +7,6 @@ mod error; mod extension; mod function; mod queries; -mod status; mod values; pub use blob::SqliteBlob; diff --git a/crates/sqlite-ffi/src/status.rs b/crates/sqlite-ffi/src/status.rs deleted file mode 100644 index d58d9e3..0000000 --- a/crates/sqlite-ffi/src/status.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::ffi::c_int; - -use libsqlite3_sys::{SQLITE_OK, sqlite3_int64, sqlite3_status64}; - -use crate::{Connection, error::SqliteError}; - -#[repr(i32)] -enum SqliteStatus { - MallocSize = 5, -} - -impl Connection { - pub fn malloc_highwater(&self) -> Result { - let mut current = 0; - let mut highwater = 0; - let rc = unsafe { - status_raw( - SqliteStatus::MallocSize as c_int, - &mut current, - &mut highwater, - 1, - ) - }; - - if rc != SQLITE_OK { - return Err(SqliteError::Sqlite { - operation: "sqlite3_status64", - code: rc, - message: "malloc highwater".into(), - }); - } - - Ok(highwater) - } -} - -unsafe fn status_raw( - op: c_int, - current: *mut sqlite3_int64, - highwater: *mut sqlite3_int64, - reset_flag: c_int, -) -> c_int { - unsafe { sqlite3_status64(op, current, highwater, reset_flag) } -} diff --git a/sqlite-compress/benches/scan.rs b/sqlite-compress/benches/scan.rs index b563694..b951caa 100644 --- a/sqlite-compress/benches/scan.rs +++ b/sqlite-compress/benches/scan.rs @@ -11,6 +11,7 @@ impl BenchDb { fn open() -> Self { static REGISTER: Once = Once::new(); REGISTER.call_once(|| unsafe { + #[allow(clippy::missing_transmute_annotations)] sqlite3_auto_extension(Some(std::mem::transmute( sqlite3_compress_init as *const (), ))); diff --git a/sqlite-compress/src/functions/clike.rs b/sqlite-compress/src/functions/clike.rs index d86287a..f139bae 100644 --- a/sqlite-compress/src/functions/clike.rs +++ b/sqlite-compress/src/functions/clike.rs @@ -1,5 +1,5 @@ use memchr::memmem::Finder; -use sqlite_ffi::{Context, SqliteError, Value}; +use sqlite_ffi::{Context, SqliteBlob, SqliteError, Value}; use zstd::stream::raw::{Decoder, Operation}; use crate::{ @@ -39,7 +39,6 @@ pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteErro Header::parse(&compressed).map_err(|e| SqliteError::Message(e.to_string()))?; let dict_id = DictId::from(header.dict_id.get()); let header_len = header.header_len as usize; - let first_len = compressed.len(); let prepared = if dict_id.get() != 0 { match get_decoder_in_cache(&state, schema, dict_id) { @@ -54,36 +53,22 @@ pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteErro let matched = if let Some(dict) = &prepared { let mut decoder = Decoder::with_prepared_dictionary(dict.as_ref()) .map_err(|e| SqliteError::Message(e.to_string()))?; - scan( - &mut decoder, - &blob, - &mut compressed, - header_len, - first_len, - needle, - )? + scan(&mut decoder, &blob, &mut compressed, header_len, needle)? } else { let mut decoder = Decoder::new().map_err(|e| SqliteError::Message(e.to_string()))?; - scan( - &mut decoder, - &blob, - &mut compressed, - header_len, - first_len, - needle, - )? + scan(&mut decoder, &blob, &mut compressed, header_len, needle)? }; context.result_int64(if matched { 1 } else { 0 }); + Ok(()) } fn scan( decoder: &mut Decoder<'_>, - blob: &sqlite_ffi::SqliteBlob, + blob: &SqliteBlob, compressed: &mut [u8], header_len: usize, - first_len: usize, needle: &[u8], ) -> Result { let finder = Finder::new(needle); @@ -91,33 +76,34 @@ fn scan( let mut plain = vec![0u8; CHUNK]; let mut tail = Vec::with_capacity(needle.len() - 1); - let first = feed( - decoder, - &compressed[header_len..first_len], - &mut plain, - &mut tail, - &finder, - )?; - - if first.matched || first.frame_done { - return Ok(first.matched); - } - let total = blob.len(); - let mut pos = first_len; + let mut pos = 0; + while pos < total { let n = CHUNK.min(total - pos); - blob.read_at(pos, &mut compressed[..n])?; + let start = if pos == 0 { + header_len + } else { + blob.read_at(pos, &mut compressed[..n])?; + 0 + }; + pos += n; - let step = feed(decoder, &compressed[..n], &mut plain, &mut tail, &finder)?; + let step = feed( + decoder, + &compressed[start..n], + &mut plain, + &mut tail, + &finder, + )?; + if step.matched || step.frame_done { return Ok(step.matched); } } - let last = feed(decoder, &[], &mut plain, &mut tail, &finder)?; - Ok(last.matched) + Ok(false) } fn feed( @@ -188,3 +174,99 @@ fn take_match(finder: &Finder<'_>, tail: &mut Vec, chunk: &[u8]) -> bool { } false } + +#[cfg(test)] +mod tests { + use std::ptr; + + use libsqlite3_sys::{sqlite3_close, sqlite3_open, SQLITE_OK}; + use sqlite_ffi::{Connection, SqlValue}; + use zstd::stream::raw::Decoder; + + use super::*; + use crate::{functions::header::wrap, DictId}; + + fn search(plain: &[u8], needle: &[u8]) -> bool { + let payload = zstd::stream::encode_all(plain, 1).unwrap(); + let stored = wrap(DictId::from(0), plain.len(), payload).unwrap(); + + let mut db = ptr::null_mut(); + let rc = unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }; + assert_eq!(rc, SQLITE_OK); + let conn = Connection::from_raw(db); + conn.batch_execute("CREATE TABLE t (id INTEGER PRIMARY KEY, data BLOB)") + .unwrap(); + + let rowid = conn + .execute( + "INSERT INTO t (data) VALUES (?) RETURNING id", + &[SqlValue::Blob(stored)], + ) + .unwrap(); + + let found = { + let blob = conn.open_blob("main", "t", "data", rowid, false).unwrap(); + let mut compressed = vec![0u8; CHUNK.min(blob.len())]; + blob.read_at(0, &mut compressed).unwrap(); + let mut decoder = Decoder::new().unwrap(); + scan( + &mut decoder, + &blob, + &mut compressed, + Header::PREFIX_SIZE, + needle, + ) + .unwrap() + }; + unsafe { sqlite3_close(db) }; + found + } + + #[test] + fn finds_needle_in_first_block() { + let plain = b"alpha bravo needle charlie"; + assert!(search(plain, b"needle")); + } + + #[test] + fn misses_absent_needle() { + assert!(!search(b"alpha bravo charlie", b"needle")); + } + + #[test] + fn finds_needle_across_plain_chunks() { + let mut plain = vec![b'a'; CHUNK + 8]; + let needle = b"xyz"; + let at = CHUNK - 1; + plain[at..at + needle.len()].copy_from_slice(needle); + assert!(search(&plain, needle)); + } + + #[test] + fn finds_needle_after_first_compressed_read() { + let mut plain = vec![0u8; CHUNK + 4096]; + for (i, byte) in plain.iter_mut().enumerate() { + *byte = (i % 251) as u8; + } + let needle = b"NEEDLE!!"; + let at = plain.len() - needle.len(); + plain[at..].copy_from_slice(needle); + assert!(search(&plain, needle)); + } + + #[test] + fn finds_needle_between_two_windows() { + let needle = b"abcdef"; + let finder = Finder::new(needle); + let mut tail = Vec::new(); + + let mut first = vec![b'q'; 32]; + first[29..].copy_from_slice(b"abc"); + + let mut second = vec![b'z'; 16]; + second[..3].copy_from_slice(b"def"); + + assert!(!take_match(&finder, &mut tail, &first)); + assert!(take_match(&finder, &mut tail, &second)); + } +} diff --git a/sqlite-compress/tests/scan.rs b/sqlite-compress/tests/scan.rs deleted file mode 100644 index b1a4153..0000000 --- a/sqlite-compress/tests/scan.rs +++ /dev/null @@ -1,94 +0,0 @@ -use std::sync::Mutex; - -use sqlite_ffi::{first_value, SqlValue}; - -mod common; - -use common::TestDb; - -static STATUS: Mutex<()> = Mutex::new(()); - -#[test] -fn like_loads_full_blob_when_match_is_at_start() { - let _guard = STATUS.lock().unwrap(); - const BODY_LEN: usize = 1024 * 1024; - let needle = b"NEEDLE"; - - let mut body = vec![b'a'; BODY_LEN]; - body[..needle.len()].copy_from_slice(needle); - - let db = TestDb::open(); - let conn = db.state(); - conn.as_ref() - .batch_execute("CREATE TABLE t(id INTEGER PRIMARY KEY, data BLOB)") - .unwrap(); - - conn.as_ref() - .execute("INSERT INTO t(data) VALUES (?1)", &[SqlValue::Blob(body)]) - .unwrap(); - - let id_hw = conn.as_ref().malloc_highwater().unwrap(); - let id_rows = conn - .as_ref() - .query("SELECT id FROM t WHERE id = 1", &[]) - .unwrap(); - assert_eq!(first_value(&id_rows).and_then(SqlValue::as_i64), Some(1)); - println!("id_hw: {id_hw}"); - - let _ = conn.as_ref().malloc_highwater().unwrap(); - let like_rows = conn - .as_ref() - .query("SELECT data LIKE 'NEEDLE%' FROM t WHERE id = 1", &[]) - .unwrap(); - assert_eq!(first_value(&like_rows).and_then(SqlValue::as_i64), Some(1)); - let like_hw = conn.as_ref().malloc_highwater().unwrap(); - - println!("like_hw: {like_hw}"); -} - -#[test] -fn clike_allocates_less_than_decompress_like_when_match_is_at_start() { - let _guard = STATUS.lock().unwrap(); - const BODY_LEN: usize = 1024 * 1024; - let needle = b"NEEDLE"; - - let mut body = vec![b'a'; BODY_LEN]; - body[..needle.len()].copy_from_slice(needle); - - let db = TestDb::open(); - let conn = db.state(); - conn.as_ref() - .batch_execute("CREATE TABLE t(id INTEGER PRIMARY KEY, data BLOB)") - .unwrap(); - - conn.as_ref() - .execute( - "INSERT INTO t(data) VALUES (compress(?1, 'main', 't', 'data'))", - &[SqlValue::Blob(body)], - ) - .unwrap(); - - let _ = conn.as_ref().malloc_highwater().unwrap(); - let like_rows = conn - .as_ref() - .query( - "SELECT decompress(data, 'main') LIKE 'NEEDLE%' FROM t WHERE id = 1", - &[], - ) - .unwrap(); - assert_eq!(first_value(&like_rows).and_then(SqlValue::as_i64), Some(1)); - let like_hw = conn.as_ref().malloc_highwater().unwrap(); - - let clike_rows = conn - .as_ref() - .query( - "SELECT clike('main', 't', 'data', id, 'NEEDLE') FROM t WHERE id = 1", - &[], - ) - .unwrap(); - assert_eq!(first_value(&clike_rows).and_then(SqlValue::as_i64), Some(1)); - let clike_hw = conn.as_ref().malloc_highwater().unwrap(); - - println!("decompress_like_hw: {like_hw}"); - println!("clike_hw: {clike_hw}"); -} From 4235bf73171eaf7eb62b56a813ada20c466bc3f7 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Thu, 24 Sep 2026 10:48:05 -0400 Subject: [PATCH 05/17] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Wrap=20unsafe=20fct?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/sqlite-ffi/src/context.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/sqlite-ffi/src/context.rs b/crates/sqlite-ffi/src/context.rs index ffb0ecf..d1031e5 100644 --- a/crates/sqlite-ffi/src/context.rs +++ b/crates/sqlite-ffi/src/context.rs @@ -1,6 +1,6 @@ use std::{ffi::c_void, ptr::NonNull}; -use libsqlite3_sys::{sqlite3_context, sqlite3_result_blob, sqlite3_result_error}; +use libsqlite3_sys::{sqlite3_context, sqlite3_result_blob, sqlite3_result_error, sqlite3_result_int64}; pub struct Context { pub(crate) ctx: NonNull, @@ -30,7 +30,7 @@ impl Context { } pub fn result_int64(&self, value: i64) { - unsafe { libsqlite3_sys::sqlite3_result_int64(self.ctx.as_ptr(), value) }; + unsafe { result_int64_raw(self.ctx.as_ptr(), value) }; } } @@ -39,3 +39,7 @@ unsafe fn result_blob_raw(context: *mut sqlite3_context, blob: *const c_void, si sqlite3_result_blob(context, blob, size, libsqlite3_sys::SQLITE_TRANSIENT()); } } + +unsafe fn result_int64_raw(context: *mut sqlite3_context, value: i64) { + unsafe { sqlite3_result_int64(context, value) }; +} From 5ef7e09a814f65ed1a213d108576244700f4e16f Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Thu, 24 Sep 2026 18:07:35 -0400 Subject: [PATCH 06/17] =?UTF-8?q?=F0=9F=9A=A7=20Handle=20special=20char=20?= =?UTF-8?q?like=20%=20and=20=5F=20for=20search?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/sqlite-ffi/src/blob.rs | 42 ++--- crates/sqlite-ffi/src/context.rs | 4 +- crates/sqlite-ffi/src/error.rs | 19 ++- crates/sqlite-ffi/src/function.rs | 17 +- sqlite-compress/benches/scan.rs | 18 +- sqlite-compress/src/functions/clike.rs | 228 +++++++++++++------------ sqlite-compress/src/functions/like.rs | 202 ++++++++++++++++++++++ sqlite-compress/src/functions/mod.rs | 1 + 8 files changed, 368 insertions(+), 163 deletions(-) create mode 100644 sqlite-compress/src/functions/like.rs diff --git a/crates/sqlite-ffi/src/blob.rs b/crates/sqlite-ffi/src/blob.rs index ce60037..6d846b8 100644 --- a/crates/sqlite-ffi/src/blob.rs +++ b/crates/sqlite-ffi/src/blob.rs @@ -1,21 +1,21 @@ use std::{ - ffi::{CStr, CString, c_char, c_int, c_void}, + ffi::{CString, c_char, c_int, c_void}, ptr::NonNull, }; use libsqlite3_sys::{ SQLITE_OK, sqlite3, sqlite3_blob, sqlite3_blob_bytes, sqlite3_blob_close, sqlite3_blob_open, - sqlite3_blob_read, sqlite3_errmsg, + sqlite3_blob_read, }; use crate::{Connection, error::SqliteError}; -pub struct SqliteBlob { - db: NonNull, +pub struct SqliteBlob<'a> { + db: &'a NonNull, handle: NonNull, } -impl SqliteBlob { +impl SqliteBlob<'_> { pub fn len(&self) -> usize { unsafe { blob_bytes_raw(self.handle.as_ptr()) as usize } } @@ -34,16 +34,7 @@ impl SqliteBlob { let rc = unsafe { blob_read_raw(self.handle.as_ptr(), buf.as_mut_ptr().cast(), n, offset) }; if rc != SQLITE_OK { - let message = unsafe { - CStr::from_ptr(sqlite3_errmsg(self.db.as_ptr())) - .to_string_lossy() - .into_owned() - }; - return Err(SqliteError::Sqlite { - operation: "sqlite3_blob_read", - code: rc, - message, - }); + return Err(unsafe { SqliteError::from_db(self.db.as_ptr(), "sqlite3_blob_read", rc) }); } Ok(()) @@ -51,14 +42,14 @@ impl SqliteBlob { } impl Connection { - pub fn open_blob( - &self, + pub fn open_blob<'a>( + &'a self, schema: &str, table: &str, column: &str, rowid: i64, write: bool, - ) -> Result { + ) -> Result, SqliteError> { let schema = CString::new(schema)?; let table = CString::new(table)?; let column = CString::new(column)?; @@ -77,26 +68,17 @@ impl Connection { }; if rc != SQLITE_OK { - let message = unsafe { - CStr::from_ptr(sqlite3_errmsg(self.as_ptr())) - .to_string_lossy() - .into_owned() - }; - return Err(SqliteError::Sqlite { - operation: "sqlite3_blob_open", - code: rc, - message, - }); + return Err(unsafe { SqliteError::from_db(self.as_ptr(), "sqlite3_blob_open", rc) }); } Ok(SqliteBlob { - db: NonNull::new(self.as_ptr()).expect("database pointer is not valid"), + db: &self.conn, handle: NonNull::new(blob).expect("sqlite3_blob_open returned a null handle"), }) } } -impl Drop for SqliteBlob { +impl Drop for SqliteBlob<'_> { fn drop(&mut self) { unsafe { sqlite3_blob_close(self.handle.as_ptr()) }; } diff --git a/crates/sqlite-ffi/src/context.rs b/crates/sqlite-ffi/src/context.rs index d1031e5..0681809 100644 --- a/crates/sqlite-ffi/src/context.rs +++ b/crates/sqlite-ffi/src/context.rs @@ -1,6 +1,8 @@ use std::{ffi::c_void, ptr::NonNull}; -use libsqlite3_sys::{sqlite3_context, sqlite3_result_blob, sqlite3_result_error, sqlite3_result_int64}; +use libsqlite3_sys::{ + sqlite3_context, sqlite3_result_blob, sqlite3_result_error, sqlite3_result_int64, +}; pub struct Context { pub(crate) ctx: NonNull, diff --git a/crates/sqlite-ffi/src/error.rs b/crates/sqlite-ffi/src/error.rs index 2ce12c8..85c2ae1 100644 --- a/crates/sqlite-ffi/src/error.rs +++ b/crates/sqlite-ffi/src/error.rs @@ -1,9 +1,9 @@ use std::{ - ffi::{CString, c_char}, + ffi::{CStr, CString, c_char}, ptr, }; -use libsqlite3_sys::{SQLITE_ERROR, sqlite3_malloc}; +use libsqlite3_sys::{SQLITE_ERROR, sqlite3, sqlite3_errmsg, sqlite3_malloc}; use thiserror::Error; #[derive(Debug, Error)] @@ -36,6 +36,21 @@ impl SqliteError { } } + /// # Safety + #[cold] + pub(crate) unsafe fn from_db(db: *mut sqlite3, operation: &'static str, code: i32) -> Self { + let message = unsafe { + CStr::from_ptr(sqlite3_errmsg(db)) + .to_string_lossy() + .into_owned() + }; + Self::Sqlite { + operation, + code, + message, + } + } + /// # Safety /// `pz_err_msg` is the out-pointer passed by SQLite into the C entrypoint pub unsafe fn report(&self, pz_err_msg: *mut *mut c_char) -> i32 { diff --git a/crates/sqlite-ffi/src/function.rs b/crates/sqlite-ffi/src/function.rs index 99783b1..d349037 100644 --- a/crates/sqlite-ffi/src/function.rs +++ b/crates/sqlite-ffi/src/function.rs @@ -1,11 +1,11 @@ use std::{ - ffi::{CStr, CString, c_char, c_int, c_void}, + ffi::{CString, c_char, c_int, c_void}, slice, }; use libsqlite3_sys::{ - SQLITE_OK, SQLITE_UTF8, sqlite3_context, sqlite3_create_function_v2, sqlite3_errmsg, - sqlite3_user_data, sqlite3_value, + SQLITE_OK, SQLITE_UTF8, sqlite3_context, sqlite3_create_function_v2, sqlite3_user_data, + sqlite3_value, }; use crate::{Connection, Context, Value, error::SqliteError}; @@ -48,15 +48,8 @@ impl Connection { }; if rc != SQLITE_OK { - let message = unsafe { - CStr::from_ptr(sqlite3_errmsg(self.conn.as_ptr())) - .to_string_lossy() - .into_owned() - }; - return Err(SqliteError::Sqlite { - operation: "create function v2", - code: rc, - message, + return Err(unsafe { + SqliteError::from_db(self.conn.as_ptr(), "sqlite3_create_function", rc) }); } diff --git a/sqlite-compress/benches/scan.rs b/sqlite-compress/benches/scan.rs index b951caa..45eb502 100644 --- a/sqlite-compress/benches/scan.rs +++ b/sqlite-compress/benches/scan.rs @@ -68,17 +68,21 @@ fn bench_scan(c: &mut Criterion) { let mut group = c.benchmark_group("clike_vs_like"); group.sample_size(10); - for (kind, like_pat, clike_pat, expect) in [ - ("start", "NEEDLE%", "NEEDLE", 1), - ("middle", "%NEEDLE%", "NEEDLE", 1), - ("end", "%NEEDLE", "NEEDLE", 1), - ("absent", "%ABSENT%", "ABSENT", 0), + conn.as_ref() + .batch_execute("PRAGMA case_sensitive_like = ON") + .unwrap(); + + for (kind, pattern, expect) in [ + ("start", "NEEDLE%", 1), + ("middle", "%NEEDLE%", 1), + ("end", "%NEEDLE", 1), + ("absent", "%ABSENT%", 0), ] { let like_sql = format!( - "SELECT decompress(data, 'main') LIKE '{like_pat}' FROM t WHERE kind = '{kind}'" + "SELECT decompress(data, 'main') LIKE '{pattern}' FROM t WHERE kind = '{kind}'" ); let clike_sql = format!( - "SELECT clike('main', 't', 'data', id, '{clike_pat}') FROM t WHERE kind = '{kind}'" + "SELECT clike('main', 't', 'data', id, '{pattern}') FROM t WHERE kind = '{kind}'" ); assert_eq!(hit(&conn, &like_sql), expect); assert_eq!(hit(&conn, &clike_sql), expect); diff --git a/sqlite-compress/src/functions/clike.rs b/sqlite-compress/src/functions/clike.rs index f139bae..0ec5957 100644 --- a/sqlite-compress/src/functions/clike.rs +++ b/sqlite-compress/src/functions/clike.rs @@ -1,24 +1,28 @@ -use memchr::memmem::Finder; use sqlite_ffi::{Context, SqliteBlob, SqliteError, Value}; use zstd::stream::raw::{Decoder, Operation}; use crate::{ - cache::get_decoder_in_cache, functions::header::Header, DictError, DictId, ExtensionState, + cache::get_decoder_in_cache, + functions::{ + header::Header, + like::{LikeMatcher, Verdict}, + }, + DictError, DictId, ExtensionState, }; const CHUNK: usize = 64 * 1024; -struct Progress { - matched: bool, - frame_done: bool, -} - pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteError> { let schema = values[0].to_text()?; let table = values[1].to_text()?; let column = values[2].to_text()?; let rowid = values[3].to_i64(); - let needle = values[4].to_text()?.as_bytes(); + let mut matcher = LikeMatcher::new(values[4].to_text()?.as_bytes(), None)?; + + if matcher.verdict() == Verdict::Match { + context.result_int64(1); + return Ok(()); + } let state = ExtensionState::from_context(&context)?; let blob = state @@ -50,35 +54,39 @@ pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteErro None }; - let matched = if let Some(dict) = &prepared { - let mut decoder = Decoder::with_prepared_dictionary(dict.as_ref()) - .map_err(|e| SqliteError::Message(e.to_string()))?; - scan(&mut decoder, &blob, &mut compressed, header_len, needle)? + let mut decoder = if let Some(dict) = &prepared { + Decoder::with_prepared_dictionary(dict.as_ref()) } else { - let mut decoder = Decoder::new().map_err(|e| SqliteError::Message(e.to_string()))?; - scan(&mut decoder, &blob, &mut compressed, header_len, needle)? - }; - - context.result_int64(if matched { 1 } else { 0 }); - + Decoder::new() + } + .map_err(|e| SqliteError::Message(e.to_string()))?; + + let matched = scan( + &mut decoder, + &blob, + &mut compressed, + header_len, + &mut matcher, + )?; + context.result_int64(i64::from(matched)); Ok(()) } +enum Step { + Done(bool), + NeedInput, +} + fn scan( decoder: &mut Decoder<'_>, blob: &SqliteBlob, compressed: &mut [u8], header_len: usize, - needle: &[u8], + matcher: &mut LikeMatcher, ) -> Result { - let finder = Finder::new(needle); - let mut plain = vec![0u8; CHUNK]; - let mut tail = Vec::with_capacity(needle.len() - 1); - let total = blob.len(); let mut pos = 0; - while pos < total { let n = CHUNK.min(total - pos); let start = if pos == 0 { @@ -87,22 +95,11 @@ fn scan( blob.read_at(pos, &mut compressed[..n])?; 0 }; - pos += n; - - let step = feed( - decoder, - &compressed[start..n], - &mut plain, - &mut tail, - &finder, - )?; - - if step.matched || step.frame_done { - return Ok(step.matched); + if let Step::Done(matched) = feed(decoder, &compressed[start..n], &mut plain, matcher)? { + return Ok(matched); } } - Ok(false) } @@ -110,69 +107,28 @@ fn feed( decoder: &mut Decoder<'_>, mut input: &[u8], plain: &mut [u8], - tail: &mut Vec, - finder: &Finder<'_>, -) -> Result { + matcher: &mut LikeMatcher, +) -> Result { loop { let status = decoder .run_on_buffers(input, plain) .map_err(|e| SqliteError::Message(e.to_string()))?; - - if status.bytes_written > 0 && take_match(finder, tail, &plain[..status.bytes_written]) { - return Ok(Progress { - matched: true, - frame_done: status.remaining == 0, - }); + match matcher.push(&plain[..status.bytes_written]) { + Verdict::Match => return Ok(Step::Done(true)), + Verdict::NoMatch => return Ok(Step::Done(false)), + Verdict::NeedMore => {} } - input = &input[status.bytes_read..]; - - if status.remaining == 0 && status.bytes_written < plain.len() { - return Ok(Progress { - matched: false, - frame_done: true, - }); + if status.remaining == 0 { + return Ok(Step::Done(matcher.finish())); } - if input.is_empty() && status.bytes_written < plain.len() { - return Ok(Progress { - matched: false, - frame_done: false, - }); + return Ok(Step::NeedInput); } - if status.bytes_read == 0 && status.bytes_written == 0 { - return Err(SqliteError::Message("zstd decoder made no progress".into())); - } - } -} - -fn take_match(finder: &Finder<'_>, tail: &mut Vec, chunk: &[u8]) -> bool { - let k = finder.needle().len() - 1; - - if !tail.is_empty() { - let head = &chunk[..k.min(chunk.len())]; - let base = tail.len(); - tail.extend_from_slice(head); - if finder.find(tail).is_some() { - return true; + return Err(SqliteError::Message("compressed frame is truncated".into())); } - tail.truncate(base); } - - if finder.find(chunk).is_some() { - return true; - } - - if chunk.len() >= k { - tail.clear(); - tail.extend_from_slice(&chunk[chunk.len() - k..]); - } else { - tail.extend_from_slice(chunk); - let excess = tail.len().saturating_sub(k); - tail.drain(..excess); - } - false } #[cfg(test)] @@ -186,7 +142,7 @@ mod tests { use super::*; use crate::{functions::header::wrap, DictId}; - fn search(plain: &[u8], needle: &[u8]) -> bool { + fn search(plain: &[u8], pattern: &str) -> bool { let payload = zstd::stream::encode_all(plain, 1).unwrap(); let stored = wrap(DictId::from(0), plain.len(), payload).unwrap(); @@ -209,12 +165,13 @@ mod tests { let mut compressed = vec![0u8; CHUNK.min(blob.len())]; blob.read_at(0, &mut compressed).unwrap(); let mut decoder = Decoder::new().unwrap(); + let mut matcher = LikeMatcher::new(pattern.as_bytes(), None).unwrap(); scan( &mut decoder, &blob, &mut compressed, Header::PREFIX_SIZE, - needle, + &mut matcher, ) .unwrap() }; @@ -222,28 +179,66 @@ mod tests { found } + const TEXT: &[u8] = b"alpha bravo needle charlie"; + + #[test] + fn literal_pattern_is_whole_value_equality() { + assert!(search(TEXT, "alpha bravo needle charlie")); + assert!(!search(TEXT, "needle")); + assert!(!search(TEXT, "alpha bravo needle charli")); + } + + #[test] + fn percent_anchors_prefix_suffix_and_contains() { + assert!(search(TEXT, "alpha%")); + assert!(search(TEXT, "%charlie")); + assert!(search(TEXT, "%needle%")); + assert!(!search(TEXT, "%needle")); + assert!(!search(TEXT, "needle%")); + assert!(!search(TEXT, "%absent%")); + } + + #[test] + fn underscore_matches_exactly_one_character() { + assert!(search(TEXT, "alph_ bravo%")); + assert!(search(TEXT, "%n__dle%")); + assert!(!search(TEXT, "%n_dle%")); + assert!(search("aéc".as_bytes(), "a_c")); + assert!(!search("aéc".as_bytes(), "a__c")); + } + #[test] - fn finds_needle_in_first_block() { - let plain = b"alpha bravo needle charlie"; - assert!(search(plain, b"needle")); + fn percent_backtracks_to_a_later_occurrence() { + assert!(search(b"aaab", "%a%ab")); + assert!(search(b"axxbyyc", "a%b%c")); + assert!(!search(b"axxbyy", "a%b%c")); } #[test] - fn misses_absent_needle() { - assert!(!search(b"alpha bravo charlie", b"needle")); + fn empty_pattern_only_matches_empty_value() { + assert!(!search(TEXT, "")); } #[test] - fn finds_needle_across_plain_chunks() { + fn finds_literal_across_plain_chunks() { let mut plain = vec![b'a'; CHUNK + 8]; - let needle = b"xyz"; let at = CHUNK - 1; - plain[at..at + needle.len()].copy_from_slice(needle); - assert!(search(&plain, needle)); + plain[at..at + 3].copy_from_slice(b"xyz"); + assert!(search(&plain, "%xyz%")); + } + + #[test] + fn underscore_matches_char_split_across_plain_chunks() { + let mut plain = vec![b'z'; CHUNK + 8]; + plain[CHUNK - 2] = b'a'; + plain[CHUNK - 1..CHUNK + 1].copy_from_slice("é".as_bytes()); + plain[CHUNK + 1] = b'b'; + assert!(search(&plain, "%a_b%")); + assert!(!search(&plain, "%a__b%")); } #[test] - fn finds_needle_after_first_compressed_read() { + fn finds_suffix_after_first_compressed_read() { let mut plain = vec![0u8; CHUNK + 4096]; for (i, byte) in plain.iter_mut().enumerate() { *byte = (i % 251) as u8; @@ -251,22 +246,33 @@ mod tests { let needle = b"NEEDLE!!"; let at = plain.len() - needle.len(); plain[at..].copy_from_slice(needle); - assert!(search(&plain, needle)); + assert!(search(&plain, "%NEEDLE!!")); + assert!(!search(&plain, "%NEEDLE!")); } #[test] - fn finds_needle_between_two_windows() { - let needle = b"abcdef"; - let finder = Finder::new(needle); - let mut tail = Vec::new(); + fn lone_percent_matches_without_opening_the_blob() { + let mut db = ptr::null_mut(); + let rc = unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }; + assert_eq!(rc, SQLITE_OK); - let mut first = vec![b'q'; 32]; - first[29..].copy_from_slice(b"abc"); + let rc = unsafe { crate::sqlite3_compress_init(db, ptr::null_mut(), ptr::null_mut()) }; + assert_eq!(rc, SQLITE_OK); - let mut second = vec![b'z'; 16]; - second[..3].copy_from_slice(b"def"); + let conn = Connection::from_raw(db); + let rows = conn + .query("SELECT clike('main', 't', 'data', 1, '%')", &[]) + .unwrap(); + assert_eq!(rows[0][0].as_i64(), Some(1)); - assert!(!take_match(&finder, &mut tail, &first)); - assert!(take_match(&finder, &mut tail, &second)); + unsafe { sqlite3_close(db) }; + } + + #[test] + fn finds_suffix_when_frame_ends_on_chunk_boundary() { + let mut plain = vec![b'a'; 2 * CHUNK]; + let at = plain.len() - 3; + plain[at..].copy_from_slice(b"xyz"); + assert!(search(&plain, "%xyz")); } } diff --git a/sqlite-compress/src/functions/like.rs b/sqlite-compress/src/functions/like.rs new file mode 100644 index 0000000..e0a842d --- /dev/null +++ b/sqlite-compress/src/functions/like.rs @@ -0,0 +1,202 @@ +use memchr::memchr; +use sqlite_ffi::SqliteError; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Token { + Byte(u8), + Any, + Star, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Verdict { + Match, + NoMatch, + NeedMore, +} + +pub struct LikeMatcher { + tokens: Vec, + trailing_star: bool, + active: Vec, + next: Vec, + skip: Option, + pending: [u8; 4], + pending_len: usize, + pending_width: usize, +} + +impl LikeMatcher { + pub fn new(pattern: &[u8], escape: Option) -> Result { + let mut tokens = Vec::with_capacity(pattern.len()); + let mut bytes = pattern.iter().copied(); + while let Some(b) = bytes.next() { + let token = if Some(b) == escape { + Token::Byte(bytes.next().ok_or_else(|| { + SqliteError::Message("escape character must be followed by a character".into()) + })?) + } else { + match b { + b'%' => Token::Star, + b'_' => Token::Any, + b => Token::Byte(b), + } + }; + if token == Token::Star && tokens.last() == Some(&Token::Star) { + continue; + } + tokens.push(token); + } + + let n = tokens.len(); + let mut matcher = Self { + trailing_star: tokens.last() == Some(&Token::Star), + tokens, + active: vec![false; n + 1], + next: vec![false; n + 1], + skip: None, + pending: [0; 4], + pending_len: 0, + pending_width: 0, + }; + matcher.active[0] = true; + close(&mut matcher.active, &matcher.tokens); + matcher.skip = matcher.compute_skip(); + Ok(matcher) + } + + pub fn verdict(&self) -> Verdict { + if self.trailing_star && self.active[self.tokens.len()] { + Verdict::Match + } else if self.active.iter().any(|&a| a) { + Verdict::NeedMore + } else { + Verdict::NoMatch + } + } + + pub fn push(&mut self, mut chunk: &[u8]) -> Verdict { + while let Some((&b, rest)) = chunk.split_first() { + if self.pending_len == 0 { + if let Some(target) = self.skip { + if b != target { + match memchr(target, chunk) { + Some(i) => chunk = &chunk[i..], + None => return Verdict::NeedMore, + } + continue; + } + } + } + + chunk = rest; + self.push_byte(b); + match self.verdict() { + Verdict::NeedMore => {} + verdict => return verdict, + } + } + Verdict::NeedMore + } + + pub fn finish(&mut self) -> bool { + self.flush_pending(); + self.active[self.tokens.len()] + } + + fn push_byte(&mut self, b: u8) { + if self.pending_len > 0 { + if b & 0xC0 == 0x80 { + self.pending[self.pending_len] = b; + self.pending_len += 1; + if self.pending_len == self.pending_width { + let unit = self.pending; + let len = self.pending_len; + self.pending_len = 0; + self.step(&unit[..len]); + } + return; + } + self.flush_pending(); + } + + let width = match b { + 0xC0..=0xDF => 2, + 0xE0..=0xEF => 3, + 0xF0..=0xF7 => 4, + _ => 1, + }; + if width == 1 { + self.step(&[b]); + } else { + self.pending[0] = b; + self.pending_len = 1; + self.pending_width = width; + } + } + + fn flush_pending(&mut self) { + let unit = self.pending; + let len = self.pending_len; + self.pending_len = 0; + for &b in &unit[..len] { + self.step(&[b]); + } + } + + fn step(&mut self, unit: &[u8]) { + let n = self.tokens.len(); + self.next.fill(false); + + for s in 0..n { + if !self.active[s] { + continue; + } + match self.tokens[s] { + Token::Star => self.next[s] = true, + Token::Any => self.next[s + 1] = true, + Token::Byte(_) => { + let end = s + unit.len(); + if end <= n + && self.tokens[s..end] + .iter() + .zip(unit) + .all(|(t, &b)| *t == Token::Byte(b)) + { + self.next[end] = true; + } + } + } + } + + close(&mut self.next, &self.tokens); + std::mem::swap(&mut self.active, &mut self.next); + self.skip = self.compute_skip(); + } + + fn compute_skip(&self) -> Option { + let mut on = self + .active + .iter() + .enumerate() + .filter(|(_, &a)| a) + .map(|(i, _)| i); + let s = on.next()?; + if on.next() != Some(s + 1) || on.next().is_some() { + return None; + } + match (self.tokens[s], self.tokens.get(s + 1)) { + // A continuation byte can't be a jump target: we'd land mid-character. + (Token::Star, Some(&Token::Byte(b))) if b & 0xC0 != 0x80 => Some(b), + _ => None, + } + } +} + +fn close(set: &mut [bool], tokens: &[Token]) { + for (i, token) in tokens.iter().enumerate() { + if set[i] && *token == Token::Star { + set[i + 1] = true; + } + } +} diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index 2f5fbd0..ac7f5f5 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -3,6 +3,7 @@ mod compress; mod decompress; mod errors; mod header; +mod like; mod types; pub const DEFAULT_LEVEL: types::Level = types::Level::new(3); From 9d482fc793c75528f271ec04ad6b0e8edc0b3ff2 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Fri, 25 Sep 2026 11:30:56 -0400 Subject: [PATCH 07/17] =?UTF-8?q?=E2=9C=A8=20Split=20clike=20files=20to=20?= =?UTF-8?q?better=20maintenance=20and=20understanding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .mise/tasks/sqlite-compress/benches | 3 + sqlite-compress/benches/scan.rs | 75 ++++--- .../src/functions/clike/matcher.rs | 77 +++++++ sqlite-compress/src/functions/clike/mod.rs | 113 ++++++++++ sqlite-compress/src/functions/clike/nfa.rs | 126 +++++++++++ .../src/functions/clike/pattern.rs | 28 +++ .../src/functions/{clike.rs => clike/scan.rs} | 106 ++------- sqlite-compress/src/functions/clike/utf8.rs | 63 ++++++ sqlite-compress/src/functions/like.rs | 202 ------------------ sqlite-compress/src/functions/mod.rs | 1 - 10 files changed, 469 insertions(+), 325 deletions(-) create mode 100755 .mise/tasks/sqlite-compress/benches create mode 100644 sqlite-compress/src/functions/clike/matcher.rs create mode 100644 sqlite-compress/src/functions/clike/mod.rs create mode 100644 sqlite-compress/src/functions/clike/nfa.rs create mode 100644 sqlite-compress/src/functions/clike/pattern.rs rename sqlite-compress/src/functions/{clike.rs => clike/scan.rs} (67%) create mode 100644 sqlite-compress/src/functions/clike/utf8.rs delete mode 100644 sqlite-compress/src/functions/like.rs diff --git a/.mise/tasks/sqlite-compress/benches b/.mise/tasks/sqlite-compress/benches new file mode 100755 index 0000000..0b4eafb --- /dev/null +++ b/.mise/tasks/sqlite-compress/benches @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +#MISE description="Benchmarks" +cargo bench -p sqlite-compress --bench scan \ No newline at end of file diff --git a/sqlite-compress/benches/scan.rs b/sqlite-compress/benches/scan.rs index 45eb502..853e090 100644 --- a/sqlite-compress/benches/scan.rs +++ b/sqlite-compress/benches/scan.rs @@ -1,4 +1,4 @@ -use std::{hint::black_box, ptr, sync::Once}; +use std::{hint::black_box, iter::repeat_n, ptr, sync::Once}; use criterion::{criterion_group, criterion_main, Criterion}; use libsqlite3_sys::{sqlite3, sqlite3_auto_extension, sqlite3_close, sqlite3_open, SQLITE_OK}; @@ -35,11 +35,18 @@ impl Drop for BenchDb { } } -fn insert(conn: &ExtensionState, kind: &str, at: Option) { - const BODY_LEN: usize = 8 * 1024 * 1024; +const BODY_LEN: usize = 8 * 1024 * 1024; + +fn needle(len: usize) -> String { + let mut s = String::from("NEEDLE"); + s.extend(repeat_n('X', len - s.len())); + s +} + +fn insert(conn: &ExtensionState, kind: &str, needle: &str, at: Option) { let mut body = vec![b'a'; BODY_LEN]; if let Some(at) = at { - body[at..at + 6].copy_from_slice(b"NEEDLE"); + body[at..at + needle.len()].copy_from_slice(needle.as_bytes()); } conn.as_ref() .execute( @@ -57,42 +64,48 @@ fn hit(conn: &ExtensionState, sql: &str) -> i64 { fn bench_scan(c: &mut Criterion) { let db = BenchDb::open(); let conn = db.state(); + conn.as_ref() .batch_execute("CREATE TABLE t(id INTEGER PRIMARY KEY, data BLOB, kind TEXT)") .unwrap(); - insert(&conn, "start", Some(0)); - insert(&conn, "middle", Some(4 * 1024 * 1024)); - insert(&conn, "end", Some(8 * 1024 * 1024 - 6)); - insert(&conn, "absent", None); let mut group = c.benchmark_group("clike_vs_like"); group.sample_size(10); - conn.as_ref() - .batch_execute("PRAGMA case_sensitive_like = ON") - .unwrap(); + for len in [6, 64, 256, 1024] { + let needle = needle(len); + for (place, at) in [ + ("start", Some(0)), + ("middle", Some(BODY_LEN / 2)), + ("end", Some(BODY_LEN - needle.len())), + ("absent", None), + ] { + insert(&conn, &format!("{place}-{len}"), &needle, at); + } - for (kind, pattern, expect) in [ - ("start", "NEEDLE%", 1), - ("middle", "%NEEDLE%", 1), - ("end", "%NEEDLE", 1), - ("absent", "%ABSENT%", 0), - ] { - let like_sql = format!( - "SELECT decompress(data, 'main') LIKE '{pattern}' FROM t WHERE kind = '{kind}'" - ); - let clike_sql = format!( - "SELECT clike('main', 't', 'data', id, '{pattern}') FROM t WHERE kind = '{kind}'" - ); - assert_eq!(hit(&conn, &like_sql), expect); - assert_eq!(hit(&conn, &clike_sql), expect); + for (place, pattern, expect) in [ + ("start", format!("{needle}%"), 1), + ("middle", format!("%{needle}%"), 1), + ("end", format!("%{needle}"), 1), + ("absent", format!("%{needle}%"), 0), + ] { + let kind = format!("{place}-{len}"); + let like_sql = format!( + "SELECT decompress(data, 'main') LIKE '{pattern}' FROM t WHERE kind = '{kind}'" + ); + let clike_sql = format!( + "SELECT clike('main', 't', 'data', id, '{pattern}') FROM t WHERE kind = '{kind}'" + ); + assert_eq!(hit(&conn, &like_sql), expect, "{kind}"); + assert_eq!(hit(&conn, &clike_sql), expect, "{kind}"); - group.bench_function(format!("decompress-like-{kind}"), |b| { - b.iter(|| black_box(hit(&conn, &like_sql))) - }); - group.bench_function(format!("clike-{kind}"), |b| { - b.iter(|| black_box(hit(&conn, &clike_sql))) - }); + group.bench_function(format!("decompress-like-{place}-{len}"), |b| { + b.iter(|| black_box(hit(&conn, &like_sql))) + }); + group.bench_function(format!("clike-{place}-{len}"), |b| { + b.iter(|| black_box(hit(&conn, &clike_sql))) + }); + } } group.finish(); diff --git a/sqlite-compress/src/functions/clike/matcher.rs b/sqlite-compress/src/functions/clike/matcher.rs new file mode 100644 index 0000000..e51d48f --- /dev/null +++ b/sqlite-compress/src/functions/clike/matcher.rs @@ -0,0 +1,77 @@ +//! Streaming entry point: plaintext bytes in, verdict out. +//! +//! Bytes are grouped into whole characters, each character is one NFA step, +//! and the verdict is checked after every step so the scan can stop early. +//! While the NFA is only waiting inside a star for one literal byte, input is +//! skipped straight to that byte with `memchr`. + +use memchr::memchr; + +use super::{nfa::Nfa, pattern, utf8::CharBuf}; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum Verdict { + Match, + NoMatch, + NeedMore, +} + +pub(super) struct LikeMatcher { + nfa: Nfa, + chars: CharBuf, +} + +impl LikeMatcher { + pub(super) fn new(pattern: &[u8]) -> Self { + Self { + nfa: Nfa::new(pattern::compile(pattern)), + chars: CharBuf::default(), + } + } + + /// Answer so far, from what has been pushed. + pub(super) fn verdict(&self) -> Verdict { + if self.nfa.is_settled() { + Verdict::Match + } else if self.nfa.is_dead() { + Verdict::NoMatch + } else { + Verdict::NeedMore + } + } + + /// Feeds the next plaintext chunk. Returns as soon as the answer is known. + pub(super) fn push(&mut self, mut chunk: &[u8]) -> Verdict { + while !chunk.is_empty() { + if let Some(target) = self.skip_target() { + match memchr(target, chunk) { + Some(i) => chunk = &chunk[i..], + None => return Verdict::NeedMore, + } + } + + self.chars.push(chunk[0], |unit| self.nfa.step(unit)); + chunk = &chunk[1..]; + + match self.verdict() { + Verdict::NeedMore => {} + verdict => return verdict, + } + } + Verdict::NeedMore + } + + /// End of the value: whether the whole pattern was matched. + pub(super) fn finish(&mut self) -> bool { + self.chars.flush(|unit| self.nfa.step(unit)); + self.nfa.is_complete() + } + + fn skip_target(&self) -> Option { + if self.chars.is_empty() { + self.nfa.skip_byte() + } else { + None + } + } +} diff --git a/sqlite-compress/src/functions/clike/mod.rs b/sqlite-compress/src/functions/clike/mod.rs new file mode 100644 index 0000000..41f0a02 --- /dev/null +++ b/sqlite-compress/src/functions/clike/mod.rs @@ -0,0 +1,113 @@ +//! `clike(schema, table, column, rowid, pattern)`: `LIKE` on a compressed +//! value, without materializing the plaintext. +//! +//! The blob is read through SQLite incremental I/O and decompressed in fixed +//! chunks (`scan`). Each plaintext chunk goes to a `LikeMatcher` +//! (`matcher`), which reassembles UTF-8 characters (`utf8`) and runs the +//! compiled pattern (`pattern`) as an NFA (`nfa`). Decompression stops as +//! soon as the matcher knows the answer. + +mod matcher; +mod nfa; +mod pattern; +mod scan; +mod utf8; + +use matcher::{LikeMatcher, Verdict}; +use scan::{scan, CHUNK}; +use sqlite_ffi::{Context, SqliteError, Value}; +use zstd::stream::raw::Decoder; + +use crate::{ + cache::get_decoder_in_cache, functions::header::Header, DictError, DictId, ExtensionState, +}; + +pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteError> { + let schema = values[0].to_text()?; + let table = values[1].to_text()?; + let column = values[2].to_text()?; + let rowid = values[3].to_i64(); + let mut matcher = LikeMatcher::new(values[4].to_text()?.as_bytes()); + + if matcher.verdict() == Verdict::Match { + context.result_int64(1); + return Ok(()); + } + + let state = ExtensionState::from_context(&context)?; + let blob = state + .connection + .open_blob(schema, table, column, rowid, false)?; + + let total = blob.len(); + if total < Header::PREFIX_SIZE { + return Err(SqliteError::Message( + "compressed blob is shorter than the header".into(), + )); + } + + let mut compressed = vec![0u8; CHUNK.min(total)]; + blob.read_at(0, &mut compressed)?; + + let (header, _) = + Header::parse(&compressed).map_err(|e| SqliteError::Message(e.to_string()))?; + let dict_id = DictId::from(header.dict_id.get()); + let header_len = header.header_len as usize; + + let prepared = if dict_id.get() != 0 { + match get_decoder_in_cache(&state, schema, dict_id) { + Ok(dict) => Some(dict), + Err(DictError::NotReady) => None, + Err(e) => return Err(SqliteError::Message(e.to_string())), + } + } else { + None + }; + + let mut decoder = if let Some(dict) = &prepared { + Decoder::with_prepared_dictionary(dict.as_ref()) + } else { + Decoder::new() + } + .map_err(|e| SqliteError::Message(e.to_string()))?; + + let matched = scan( + &mut decoder, + &blob, + &mut compressed, + header_len, + &mut matcher, + )?; + + context.result_int64(i64::from(matched)); + + Ok(()) +} + +#[cfg(test)] +mod tests { + + use std::ptr; + + use libsqlite3_sys::{sqlite3_close, sqlite3_open, SQLITE_OK}; + + use crate::Connection; + + #[test] + fn lone_percent_matches_without_opening_the_blob() { + let mut db = ptr::null_mut(); + let rc = unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }; + assert_eq!(rc, SQLITE_OK); + + let rc = unsafe { crate::sqlite3_compress_init(db, ptr::null_mut(), ptr::null_mut()) }; + assert_eq!(rc, SQLITE_OK); + + let conn = Connection::from_raw(db); + let rows = conn + .query("SELECT clike('main', 't', 'data', 1, '%')", &[]) + .unwrap(); + assert_eq!(rows[0][0].as_i64(), Some(1)); + + unsafe { sqlite3_close(db) }; + } +} diff --git a/sqlite-compress/src/functions/clike/nfa.rs b/sqlite-compress/src/functions/clike/nfa.rs new file mode 100644 index 0000000..474bc86 --- /dev/null +++ b/sqlite-compress/src/functions/clike/nfa.rs @@ -0,0 +1,126 @@ +//! The pattern as an NFA over pattern positions. +//! +//! Position `i` means "the first `i` tokens are matched". The last position, +//! `tokens.len()`, means the whole pattern is matched. The state is the set of +//! positions the input seen so far can have reached. All of them are tracked +//! at once, so the matcher never goes back over input. +//! +//! One step takes one character and builds the next set: +//! - a star keeps its position, since it can absorb the character; +//! - `Any` moves forward one position; +//! - `Byte` moves forward only if the character's bytes are the next literals. +//! +//! Positions that cannot move are dropped. A star can also match nothing, so +//! after each step the position just past a live star is made live too. +//! +//! The outcome is known early in two cases: no position is left, or a trailing +//! star is reached and whatever follows cannot reject it. + +use super::{pattern::Token, utf8::is_continuation}; + +pub(super) struct Nfa { + tokens: Vec, + live: Vec, + next: Vec, + skip: Option, +} + +impl Nfa { + pub(super) fn new(tokens: Vec) -> Self { + let n = tokens.len(); + let mut live = vec![false; n + 1]; + live[0] = true; + close(&tokens, &mut live); + let mut nfa = Self { + tokens, + live, + next: vec![false; n + 1], + skip: None, + }; + nfa.skip = nfa.find_skip(); + nfa + } + + /// Advances every live position over one character. + pub(super) fn step(&mut self, unit: &[u8]) { + self.next.fill(false); + for pos in 0..self.tokens.len() { + if !self.live[pos] { + continue; + } + match self.tokens[pos] { + Token::Star => self.next[pos] = true, + Token::Any => self.next[pos + 1] = true, + Token::Byte(_) => { + // A multi-byte character must match as many consecutive literals. + if matches_literal(&self.tokens[pos..], unit) { + self.next[pos + unit.len()] = true; + } + } + } + } + close(&self.tokens, &mut self.next); + std::mem::swap(&mut self.live, &mut self.next); + self.skip = self.find_skip(); + } + + /// No position is left: no continuation of the input can match. + pub(super) fn is_dead(&self) -> bool { + !self.live.iter().any(|&live| live) + } + + /// The whole pattern is matched by the input so far. + pub(super) fn is_complete(&self) -> bool { + self.live[self.tokens.len()] + } + + /// Matched through a trailing star: any further input still matches. + pub(super) fn is_settled(&self) -> bool { + self.tokens.last() == Some(&Token::Star) && self.is_complete() + } + + /// A byte the input can be skipped to without changing the state. + pub(super) fn skip_byte(&self) -> Option { + self.skip + } + + /// When the only live positions are a star and the literal right after it, + /// every other byte leads back to that same pair. + fn find_skip(&self) -> Option { + let mut live = self + .live + .iter() + .enumerate() + .filter(|(_, live)| **live) + .map(|(pos, _)| pos); + + let star = live.next()?; + let literal = live.next()?; + if literal != star + 1 || live.next().is_some() { + return None; + } + match (self.tokens[star], self.tokens.get(literal)) { + // Jumping to a continuation byte would land mid-character. + (Token::Star, Some(&Token::Byte(b))) if !is_continuation(b) => Some(b), + _ => None, + } + } +} + +/// A star matches the empty string, so a live position on a star makes the +/// next one live. Left to right, so it chains within one pass. +fn close(tokens: &[Token], set: &mut [bool]) { + for (pos, token) in tokens.iter().enumerate() { + if set[pos] && *token == Token::Star { + set[pos + 1] = true; + } + } +} + +fn matches_literal(tokens: &[Token], unit: &[u8]) -> bool { + tokens.len() >= unit.len() + && tokens + .iter() + .zip(unit) + .all(|(token, &byte)| *token == Token::Byte(byte)) +} diff --git a/sqlite-compress/src/functions/clike/pattern.rs b/sqlite-compress/src/functions/clike/pattern.rs new file mode 100644 index 0000000..089dacb --- /dev/null +++ b/sqlite-compress/src/functions/clike/pattern.rs @@ -0,0 +1,28 @@ +//! Turns a `LIKE` pattern into the tokens the NFA walks. + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum Token { + /// A byte that must appear as-is. + Byte(u8), + /// `_`: exactly one character. + Any, + /// `%`: any run of characters, including none. + Star, +} + +pub(super) fn compile(pattern: &[u8]) -> Vec { + let mut tokens = Vec::with_capacity(pattern.len()); + for &byte in pattern { + let token = match byte { + b'%' => Token::Star, + b'_' => Token::Any, + byte => Token::Byte(byte), + }; + // `%%` matches what `%` matches. One star keeps the skip fast path reachable. + if token == Token::Star && tokens.last() == Some(&Token::Star) { + continue; + } + tokens.push(token); + } + tokens +} diff --git a/sqlite-compress/src/functions/clike.rs b/sqlite-compress/src/functions/clike/scan.rs similarity index 67% rename from sqlite-compress/src/functions/clike.rs rename to sqlite-compress/src/functions/clike/scan.rs index 0ec5957..521d6a0 100644 --- a/sqlite-compress/src/functions/clike.rs +++ b/sqlite-compress/src/functions/clike/scan.rs @@ -1,83 +1,19 @@ -use sqlite_ffi::{Context, SqliteBlob, SqliteError, Value}; -use zstd::stream::raw::{Decoder, Operation}; - -use crate::{ - cache::get_decoder_in_cache, - functions::{ - header::Header, - like::{LikeMatcher, Verdict}, - }, - DictError, DictId, ExtensionState, -}; - -const CHUNK: usize = 64 * 1024; - -pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteError> { - let schema = values[0].to_text()?; - let table = values[1].to_text()?; - let column = values[2].to_text()?; - let rowid = values[3].to_i64(); - let mut matcher = LikeMatcher::new(values[4].to_text()?.as_bytes(), None)?; - - if matcher.verdict() == Verdict::Match { - context.result_int64(1); - return Ok(()); - } - - let state = ExtensionState::from_context(&context)?; - let blob = state - .connection - .open_blob(schema, table, column, rowid, false)?; - - let total = blob.len(); - if total < Header::PREFIX_SIZE { - return Err(SqliteError::Message( - "compressed blob is shorter than the header".into(), - )); - } +//! Drives zstd over the blob and feeds each plaintext chunk to the matcher. +//! Neither the compressed value nor the plaintext is held in full. - let mut compressed = vec![0u8; CHUNK.min(total)]; - blob.read_at(0, &mut compressed)?; +use sqlite_ffi::{SqliteBlob, SqliteError}; +use zstd::stream::raw::{Decoder, Operation}; - let (header, _) = - Header::parse(&compressed).map_err(|e| SqliteError::Message(e.to_string()))?; - let dict_id = DictId::from(header.dict_id.get()); - let header_len = header.header_len as usize; +use super::matcher::{LikeMatcher, Verdict}; - let prepared = if dict_id.get() != 0 { - match get_decoder_in_cache(&state, schema, dict_id) { - Ok(dict) => Some(dict), - Err(DictError::NotReady) => None, - Err(e) => return Err(SqliteError::Message(e.to_string())), - } - } else { - None - }; - - let mut decoder = if let Some(dict) = &prepared { - Decoder::with_prepared_dictionary(dict.as_ref()) - } else { - Decoder::new() - } - .map_err(|e| SqliteError::Message(e.to_string()))?; - - let matched = scan( - &mut decoder, - &blob, - &mut compressed, - header_len, - &mut matcher, - )?; - context.result_int64(i64::from(matched)); - Ok(()) -} +pub(super) const CHUNK: usize = 64 * 1024; enum Step { Done(bool), NeedInput, } -fn scan( +pub(super) fn scan( decoder: &mut Decoder<'_>, blob: &SqliteBlob, compressed: &mut [u8], @@ -113,18 +49,23 @@ fn feed( let status = decoder .run_on_buffers(input, plain) .map_err(|e| SqliteError::Message(e.to_string()))?; + match matcher.push(&plain[..status.bytes_written]) { Verdict::Match => return Ok(Step::Done(true)), Verdict::NoMatch => return Ok(Step::Done(false)), Verdict::NeedMore => {} } + input = &input[status.bytes_read..]; + if status.remaining == 0 { return Ok(Step::Done(matcher.finish())); } + if input.is_empty() && status.bytes_written < plain.len() { return Ok(Step::NeedInput); } + if status.bytes_read == 0 && status.bytes_written == 0 { return Err(SqliteError::Message("compressed frame is truncated".into())); } @@ -140,7 +81,7 @@ mod tests { use zstd::stream::raw::Decoder; use super::*; - use crate::{functions::header::wrap, DictId}; + use crate::{functions::header::wrap, DictId, Header}; fn search(plain: &[u8], pattern: &str) -> bool { let payload = zstd::stream::encode_all(plain, 1).unwrap(); @@ -165,7 +106,8 @@ mod tests { let mut compressed = vec![0u8; CHUNK.min(blob.len())]; blob.read_at(0, &mut compressed).unwrap(); let mut decoder = Decoder::new().unwrap(); - let mut matcher = LikeMatcher::new(pattern.as_bytes(), None).unwrap(); + let mut matcher = LikeMatcher::new(pattern.as_bytes()); + scan( &mut decoder, &blob, @@ -250,24 +192,6 @@ mod tests { assert!(!search(&plain, "%NEEDLE!")); } - #[test] - fn lone_percent_matches_without_opening_the_blob() { - let mut db = ptr::null_mut(); - let rc = unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }; - assert_eq!(rc, SQLITE_OK); - - let rc = unsafe { crate::sqlite3_compress_init(db, ptr::null_mut(), ptr::null_mut()) }; - assert_eq!(rc, SQLITE_OK); - - let conn = Connection::from_raw(db); - let rows = conn - .query("SELECT clike('main', 't', 'data', 1, '%')", &[]) - .unwrap(); - assert_eq!(rows[0][0].as_i64(), Some(1)); - - unsafe { sqlite3_close(db) }; - } - #[test] fn finds_suffix_when_frame_ends_on_chunk_boundary() { let mut plain = vec![b'a'; 2 * CHUNK]; diff --git a/sqlite-compress/src/functions/clike/utf8.rs b/sqlite-compress/src/functions/clike/utf8.rs new file mode 100644 index 0000000..bb0e706 --- /dev/null +++ b/sqlite-compress/src/functions/clike/utf8.rs @@ -0,0 +1,63 @@ +//! Groups plaintext bytes into characters for the NFA. A character can be +//! split across two decompressed chunks, so its leading bytes are held here +//! until it is complete. + +#[derive(Default)] +pub(super) struct CharBuf { + bytes: [u8; 4], + len: usize, + width: usize, +} + +impl CharBuf { + pub(super) fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Takes one byte and calls `emit` for every unit that is now ready: a + /// whole character, or single bytes when the input is not valid UTF-8. + pub(super) fn push(&mut self, byte: u8, mut emit: impl FnMut(&[u8])) { + if self.len > 0 { + if is_continuation(byte) { + self.bytes[self.len] = byte; + self.len += 1; + if self.len == self.width { + emit(&self.bytes[..self.len]); + self.len = 0; + } + return; + } + self.flush(&mut emit); + } + + match width(byte) { + 1 => emit(&[byte]), + width => { + self.bytes[0] = byte; + self.len = 1; + self.width = width; + } + } + } + + /// Emits the bytes of an unfinished character one at a time. + pub(super) fn flush(&mut self, mut emit: impl FnMut(&[u8])) { + for &byte in &self.bytes[..self.len] { + emit(&[byte]); + } + self.len = 0; + } +} + +pub(super) fn is_continuation(byte: u8) -> bool { + byte & 0xC0 == 0x80 +} + +fn width(lead: u8) -> usize { + match lead { + 0xC0..=0xDF => 2, + 0xE0..=0xEF => 3, + 0xF0..=0xF7 => 4, + _ => 1, + } +} diff --git a/sqlite-compress/src/functions/like.rs b/sqlite-compress/src/functions/like.rs deleted file mode 100644 index e0a842d..0000000 --- a/sqlite-compress/src/functions/like.rs +++ /dev/null @@ -1,202 +0,0 @@ -use memchr::memchr; -use sqlite_ffi::SqliteError; - -#[derive(Clone, Copy, PartialEq, Eq)] -enum Token { - Byte(u8), - Any, - Star, -} - -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum Verdict { - Match, - NoMatch, - NeedMore, -} - -pub struct LikeMatcher { - tokens: Vec, - trailing_star: bool, - active: Vec, - next: Vec, - skip: Option, - pending: [u8; 4], - pending_len: usize, - pending_width: usize, -} - -impl LikeMatcher { - pub fn new(pattern: &[u8], escape: Option) -> Result { - let mut tokens = Vec::with_capacity(pattern.len()); - let mut bytes = pattern.iter().copied(); - while let Some(b) = bytes.next() { - let token = if Some(b) == escape { - Token::Byte(bytes.next().ok_or_else(|| { - SqliteError::Message("escape character must be followed by a character".into()) - })?) - } else { - match b { - b'%' => Token::Star, - b'_' => Token::Any, - b => Token::Byte(b), - } - }; - if token == Token::Star && tokens.last() == Some(&Token::Star) { - continue; - } - tokens.push(token); - } - - let n = tokens.len(); - let mut matcher = Self { - trailing_star: tokens.last() == Some(&Token::Star), - tokens, - active: vec![false; n + 1], - next: vec![false; n + 1], - skip: None, - pending: [0; 4], - pending_len: 0, - pending_width: 0, - }; - matcher.active[0] = true; - close(&mut matcher.active, &matcher.tokens); - matcher.skip = matcher.compute_skip(); - Ok(matcher) - } - - pub fn verdict(&self) -> Verdict { - if self.trailing_star && self.active[self.tokens.len()] { - Verdict::Match - } else if self.active.iter().any(|&a| a) { - Verdict::NeedMore - } else { - Verdict::NoMatch - } - } - - pub fn push(&mut self, mut chunk: &[u8]) -> Verdict { - while let Some((&b, rest)) = chunk.split_first() { - if self.pending_len == 0 { - if let Some(target) = self.skip { - if b != target { - match memchr(target, chunk) { - Some(i) => chunk = &chunk[i..], - None => return Verdict::NeedMore, - } - continue; - } - } - } - - chunk = rest; - self.push_byte(b); - match self.verdict() { - Verdict::NeedMore => {} - verdict => return verdict, - } - } - Verdict::NeedMore - } - - pub fn finish(&mut self) -> bool { - self.flush_pending(); - self.active[self.tokens.len()] - } - - fn push_byte(&mut self, b: u8) { - if self.pending_len > 0 { - if b & 0xC0 == 0x80 { - self.pending[self.pending_len] = b; - self.pending_len += 1; - if self.pending_len == self.pending_width { - let unit = self.pending; - let len = self.pending_len; - self.pending_len = 0; - self.step(&unit[..len]); - } - return; - } - self.flush_pending(); - } - - let width = match b { - 0xC0..=0xDF => 2, - 0xE0..=0xEF => 3, - 0xF0..=0xF7 => 4, - _ => 1, - }; - if width == 1 { - self.step(&[b]); - } else { - self.pending[0] = b; - self.pending_len = 1; - self.pending_width = width; - } - } - - fn flush_pending(&mut self) { - let unit = self.pending; - let len = self.pending_len; - self.pending_len = 0; - for &b in &unit[..len] { - self.step(&[b]); - } - } - - fn step(&mut self, unit: &[u8]) { - let n = self.tokens.len(); - self.next.fill(false); - - for s in 0..n { - if !self.active[s] { - continue; - } - match self.tokens[s] { - Token::Star => self.next[s] = true, - Token::Any => self.next[s + 1] = true, - Token::Byte(_) => { - let end = s + unit.len(); - if end <= n - && self.tokens[s..end] - .iter() - .zip(unit) - .all(|(t, &b)| *t == Token::Byte(b)) - { - self.next[end] = true; - } - } - } - } - - close(&mut self.next, &self.tokens); - std::mem::swap(&mut self.active, &mut self.next); - self.skip = self.compute_skip(); - } - - fn compute_skip(&self) -> Option { - let mut on = self - .active - .iter() - .enumerate() - .filter(|(_, &a)| a) - .map(|(i, _)| i); - let s = on.next()?; - if on.next() != Some(s + 1) || on.next().is_some() { - return None; - } - match (self.tokens[s], self.tokens.get(s + 1)) { - // A continuation byte can't be a jump target: we'd land mid-character. - (Token::Star, Some(&Token::Byte(b))) if b & 0xC0 != 0x80 => Some(b), - _ => None, - } - } -} - -fn close(set: &mut [bool], tokens: &[Token]) { - for (i, token) in tokens.iter().enumerate() { - if set[i] && *token == Token::Star { - set[i + 1] = true; - } - } -} diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index ac7f5f5..2f5fbd0 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -3,7 +3,6 @@ mod compress; mod decompress; mod errors; mod header; -mod like; mod types; pub const DEFAULT_LEVEL: types::Level = types::Level::new(3); From d673ca7ef4a9cac9d3c58f06d19842736ed1bc90 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Wed, 21 Oct 2026 21:56:57 +0200 Subject: [PATCH 08/17] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Update=20code=20to?= =?UTF-8?q?=20match=20new=20header=20parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 11 +++++++++++ crates/sqlite-ffi/src/context/mod.rs | 2 +- sqlite-compress/src/functions/clike/mod.rs | 17 ++++++++++++----- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45d1722..55c8ec4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,6 +506,17 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reglite" +version = "0.1.0" +dependencies = [ + "libsqlite3-sys", + "regex", + "sqlite-ffi", + "sqlite-ffi-macros", + "thiserror", +] + [[package]] name = "rustversion" version = "1.0.23" diff --git a/crates/sqlite-ffi/src/context/mod.rs b/crates/sqlite-ffi/src/context/mod.rs index c1108c5..7730d37 100644 --- a/crates/sqlite-ffi/src/context/mod.rs +++ b/crates/sqlite-ffi/src/context/mod.rs @@ -50,7 +50,7 @@ impl Context { } pub fn result_int64(&self, value: i64) { - unsafe { result_int64_raw(self.ctx.as_ptr(), value) }; + unsafe { result_int_raw(self.ctx.as_ptr(), value) }; } } diff --git a/sqlite-compress/src/functions/clike/mod.rs b/sqlite-compress/src/functions/clike/mod.rs index 41f0a02..0899d0c 100644 --- a/sqlite-compress/src/functions/clike/mod.rs +++ b/sqlite-compress/src/functions/clike/mod.rs @@ -14,12 +14,14 @@ mod scan; mod utf8; use matcher::{LikeMatcher, Verdict}; -use scan::{scan, CHUNK}; +use scan::{CHUNK, scan}; use sqlite_ffi::{Context, SqliteError, Value}; use zstd::stream::raw::Decoder; use crate::{ - cache::get_decoder_in_cache, functions::header::Header, DictError, DictId, ExtensionState, + DictError, DictId, ExtensionState, + cache::get_decoder_in_cache, + functions::{ParsedBlob, header::Header}, }; pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteError> { @@ -49,8 +51,13 @@ pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteErro let mut compressed = vec![0u8; CHUNK.min(total)]; blob.read_at(0, &mut compressed)?; - let (header, _) = - Header::parse(&compressed).map_err(|e| SqliteError::Message(e.to_string()))?; + let ParsedBlob::Framed(header, _) = + Header::parse_blob(&compressed).map_err(|e| SqliteError::Message(e.to_string()))? + else { + context.result_blob(&compressed); + return Ok(()); + }; + let dict_id = DictId::from(header.dict_id.get()); let header_len = header.header_len as usize; @@ -89,7 +96,7 @@ mod tests { use std::ptr; - use libsqlite3_sys::{sqlite3_close, sqlite3_open, SQLITE_OK}; + use libsqlite3_sys::{SQLITE_OK, sqlite3_close, sqlite3_open}; use crate::Connection; From 675ed58ae1f90105b20a7b530912939214788820 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Wed, 21 Oct 2026 22:58:08 +0200 Subject: [PATCH 09/17] =?UTF-8?q?=F0=9F=9A=A7=20Test=20light=20like?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sqlite-compress/src/functions/light.rs | 128 +++++++++++++++++++++++++ sqlite-compress/src/functions/mod.rs | 2 + sqlite-compress/src/lib.rs | 2 + 3 files changed, 132 insertions(+) create mode 100644 sqlite-compress/src/functions/light.rs diff --git a/sqlite-compress/src/functions/light.rs b/sqlite-compress/src/functions/light.rs new file mode 100644 index 0000000..7b6818e --- /dev/null +++ b/sqlite-compress/src/functions/light.rs @@ -0,0 +1,128 @@ +use std::io::{self, Read}; + +use memchr::memmem::Finder; +use sqlite_ffi::{Context, SqliteError, Value}; +use zstd::dict::DecoderDictionary; +use zstd::stream::read::Decoder; + +use crate::{ + DictError, DictId, + cache::get_decoder_in_cache, + functions::{ParsedBlob, errors::CodecError, header::Header}, + state::ExtensionState, +}; + +const STREAM_CHUNK: usize = 16 * 1024; + +/// `decompress_cont(blob, schema, needle, offset) -> 0|1` +/// +/// Literal byte substring search (no LIKE wildcards / escape). +/// `offset` skips that many decompressed bytes before searching (e.g. body_position). +pub fn sqlite_decompress_cont(context: Context, values: &[Value]) -> Result<(), SqliteError> { + if values.len() < 4 { + return Err(SqliteError::Message( + "decompress_cont expects 4 arguments: blob, schema, needle, offset".into(), + )); + } + + let blob = values[0].to_blob(); + let schema = values[1].to_text().unwrap_or("raw"); + let needle = values[2].to_blob(); + let offset = values[3].to_i64().max(0) as usize; + + let found = match Header::parse_blob(blob).map_err(|e| SqliteError::Message(e.to_string()))? { + ParsedBlob::Plain => contains_bytes(blob, needle, offset), + ParsedBlob::Framed(header, payload) => { + let dict_id = DictId::from(header.dict_id.get()); + + let prepared = if dict_id.get() != 0 { + let state = ExtensionState::from_context(&context)?; + match get_decoder_in_cache(&state, schema, dict_id) { + Ok(decoder) => Some(decoder), + Err(DictError::NotReady) => None, + Err(e) => return Err(SqliteError::Message(e.to_string())), + } + } else { + None + }; + + contains_stream_zstd(payload, prepared.as_deref(), needle, offset) + .map_err(|e| SqliteError::Message(e.to_string()))? + } + }; + + context.result_bool(found); + Ok(()) +} + +/// Literal search with optional skip. Empty needle → true (like LIKE '%%'). +fn contains_bytes(haystack: &[u8], needle: &[u8], offset: usize) -> bool { + if needle.is_empty() { + return true; + } + let data = match haystack.get(offset..) { + Some(d) => d, + None => return false, + }; + Finder::new(needle).find(data).is_some() +} + +/// Sliding-window search over a byte stream. +/// +/// Keeps `needle.len() - 1` bytes of overlap so matches spanning chunks are found. +fn contains_streaming(mut reader: R, needle: &[u8], mut skip: usize) -> io::Result { + if needle.is_empty() { + return Ok(true); + } + + let finder = Finder::new(needle); + let overlap_len = needle.len() - 1; + let mut overlap = Vec::with_capacity(overlap_len); + let mut chunk = vec![0u8; STREAM_CHUNK]; + + loop { + let n = reader.read(&mut chunk)?; + if n == 0 { + break; + } + let mut data = &chunk[..n]; + + if skip > 0 { + if skip >= data.len() { + skip -= data.len(); + continue; + } + data = &data[skip..]; + skip = 0; + } + + let mut window = std::mem::take(&mut overlap); + window.extend_from_slice(data); + + if finder.find(&window).is_some() { + return Ok(true); + } + + let keep = overlap_len.min(window.len()); + overlap.clear(); + overlap.extend_from_slice(&window[window.len() - keep..]); + } + + Ok(false) +} + +/// Stream-decompress then sliding-window search. +fn contains_stream_zstd( + payload: &[u8], + dict: Option<&DecoderDictionary<'static>>, + needle: &[u8], + offset: usize, +) -> Result { + let decoder = match dict { + Some(dict) => Decoder::with_prepared_dictionary(payload, dict), + None => Decoder::with_dictionary(payload, &[]), + } + .map_err(CodecError::DecompressionFailed)?; + + contains_streaming(decoder, needle, offset).map_err(CodecError::DecompressionFailed) +} diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index e6664f3..e2104d6 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -4,6 +4,7 @@ mod decompress; mod errors; mod header; mod types; +mod light; /// Compression level used when configuration does not select another level. pub const DEFAULT_LEVEL: types::Level = types::Level::new(3); @@ -16,3 +17,4 @@ pub use errors::CodecError; pub use header::Header; pub(crate) use header::ParsedBlob; pub use types::Level; +pub use light::sqlite_decompress_cont; \ No newline at end of file diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index a7a6957..cc11483 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -36,5 +36,7 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> connection.create_function("clike", 5, functions::sqlite_clike)?; + connection.create_function("decompress_cont", 4, functions::sqlite_decompress_cont)?; + Ok(()) } From 9f7c3b6a96b452f0f9b16ab10ace79730ba82974 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Thu, 22 Oct 2026 09:03:43 +0200 Subject: [PATCH 10/17] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Stream=20SQL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sqlite-compress/src/functions/light.rs | 118 +++++++++++++++++++++---- sqlite-compress/src/lib.rs | 2 +- 2 files changed, 100 insertions(+), 20 deletions(-) diff --git a/sqlite-compress/src/functions/light.rs b/sqlite-compress/src/functions/light.rs index 7b6818e..1d5380a 100644 --- a/sqlite-compress/src/functions/light.rs +++ b/sqlite-compress/src/functions/light.rs @@ -1,7 +1,7 @@ -use std::io::{self, Read}; - use memchr::memmem::Finder; +use sqlite_ffi::SqliteBlob; use sqlite_ffi::{Context, SqliteError, Value}; +use std::io::{self, BufReader, Read}; use zstd::dict::DecoderDictionary; use zstd::stream::read::Decoder; @@ -14,31 +14,62 @@ use crate::{ const STREAM_CHUNK: usize = 16 * 1024; -/// `decompress_cont(blob, schema, needle, offset) -> 0|1` -/// -/// Literal byte substring search (no LIKE wildcards / escape). -/// `offset` skips that many decompressed bytes before searching (e.g. body_position). +/// `decompress_cont(schema, table, column, rowid, needle, offset) -> 0|1` pub fn sqlite_decompress_cont(context: Context, values: &[Value]) -> Result<(), SqliteError> { - if values.len() < 4 { + if values.len() < 6 { return Err(SqliteError::Message( - "decompress_cont expects 4 arguments: blob, schema, needle, offset".into(), + "decompress_cont expects 6 arguments: schema, table, column, rowid, needle, offset" + .into(), )); } - let blob = values[0].to_blob(); - let schema = values[1].to_text().unwrap_or("raw"); - let needle = values[2].to_blob(); - let offset = values[3].to_i64().max(0) as usize; + let schema = values[0].to_text()?; + let table = values[1].to_text()?; + let column = values[2].to_text()?; + let rowid = values[3].to_i64(); + let needle = values[4].to_blob(); + let offset = values[5].to_i64().max(0) as usize; + + let state = ExtensionState::from_context(&context)?; + let blob = state + .connection + .open_blob(schema, table, column, rowid, false)?; + + let total = blob.len(); + if total == 0 { + context.result_bool(contains_bytes(&[], needle, offset)); + return Ok(()); + } - let found = match Header::parse_blob(blob).map_err(|e| SqliteError::Message(e.to_string()))? { - ParsedBlob::Plain => contains_bytes(blob, needle, offset), - ParsedBlob::Framed(header, payload) => { - let dict_id = DictId::from(header.dict_id.get()); + // Peek just enough to parse the header (no full materialization). + let peek_len = Header::PREFIX_SIZE.min(total); + let mut peek = vec![0u8; peek_len]; + blob.read_at(0, &mut peek)?; + let found = match Header::parse_blob(&peek) { + Ok(ParsedBlob::Plain) | Err(_) if peek_len < Header::PREFIX_SIZE => { + contains_streaming(BlobReader::new(&blob, 0), needle, offset) + .map_err(|e| SqliteError::Message(e.to_string()))? + } + Ok(ParsedBlob::Plain) => contains_streaming(BlobReader::new(&blob, 0), needle, offset) + .map_err(|e| SqliteError::Message(e.to_string()))?, + Ok(ParsedBlob::Framed(header, _)) => { + let header_len = header.header_len as usize; + if total < header_len { + return Err(SqliteError::Message( + "compressed blob is shorter than the header".into(), + )); + } + + if header_len > peek.len() { + peek.resize(header_len, 0); + blob.read_at(0, &mut peek)?; + } + + let dict_id = DictId::from(header.dict_id.get()); let prepared = if dict_id.get() != 0 { - let state = ExtensionState::from_context(&context)?; match get_decoder_in_cache(&state, schema, dict_id) { - Ok(decoder) => Some(decoder), + Ok(d) => Some(d), Err(DictError::NotReady) => None, Err(e) => return Err(SqliteError::Message(e.to_string())), } @@ -46,9 +77,17 @@ pub fn sqlite_decompress_cont(context: Context, values: &[Value]) -> Result<(), None }; - contains_stream_zstd(payload, prepared.as_deref(), needle, offset) + let reader = BufReader::new(BlobReader::new(&blob, header_len)); + let decoder = match prepared.as_deref() { + Some(dict) => Decoder::with_prepared_dictionary(reader, dict), + None => Decoder::with_dictionary(reader, &[]), + } + .map_err(|e| SqliteError::Message(e.to_string()))?; + + contains_streaming(decoder, needle, offset) .map_err(|e| SqliteError::Message(e.to_string()))? } + Err(e) => return Err(SqliteError::Message(e.to_string())), }; context.result_bool(found); @@ -126,3 +165,44 @@ fn contains_stream_zstd( contains_streaming(decoder, needle, offset).map_err(CodecError::DecompressionFailed) } + +struct BlobReader<'a> { + blob: &'a SqliteBlob<'a>, + pos: usize, + end: usize, +} + +impl<'a> BlobReader<'a> { + fn new(blob: &'a SqliteBlob<'a>, start: usize) -> Self { + let end = blob.len(); + Self { + blob, + pos: start.min(end), + end, + } + } + + fn from_range(blob: &'a SqliteBlob<'a>, start: usize, end: usize) -> Self { + let end = end.min(blob.len()); + Self { + blob, + pos: start.min(end), + end, + } + } +} + +impl Read for BlobReader<'_> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let remaining = self.end.saturating_sub(self.pos); + if remaining == 0 || buf.is_empty() { + return Ok(0); + } + let n = buf.len().min(remaining); + self.blob + .read_at(self.pos, &mut buf[..n]) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + self.pos += n; + Ok(n) + } +} diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index cc11483..a5fc07f 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -36,7 +36,7 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> connection.create_function("clike", 5, functions::sqlite_clike)?; - connection.create_function("decompress_cont", 4, functions::sqlite_decompress_cont)?; + connection.create_function("decompress_cont", 6, functions::sqlite_decompress_cont)?; Ok(()) } From 8ea169449a89a989d85e0a7aa8780fc59417c35d Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Thu, 22 Oct 2026 10:39:13 +0200 Subject: [PATCH 11/17] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Revert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/functions/clike/matcher.rs | 77 ------- sqlite-compress/src/functions/clike/mod.rs | 120 ----------- sqlite-compress/src/functions/clike/nfa.rs | 126 ----------- .../src/functions/clike/pattern.rs | 28 --- sqlite-compress/src/functions/clike/scan.rs | 202 ------------------ sqlite-compress/src/functions/clike/utf8.rs | 63 ------ .../src/functions/{light.rs => contains.rs} | 21 +- sqlite-compress/src/functions/mod.rs | 6 +- sqlite-compress/src/lib.rs | 2 - 9 files changed, 4 insertions(+), 641 deletions(-) delete mode 100644 sqlite-compress/src/functions/clike/matcher.rs delete mode 100644 sqlite-compress/src/functions/clike/mod.rs delete mode 100644 sqlite-compress/src/functions/clike/nfa.rs delete mode 100644 sqlite-compress/src/functions/clike/pattern.rs delete mode 100644 sqlite-compress/src/functions/clike/scan.rs delete mode 100644 sqlite-compress/src/functions/clike/utf8.rs rename sqlite-compress/src/functions/{light.rs => contains.rs} (89%) diff --git a/sqlite-compress/src/functions/clike/matcher.rs b/sqlite-compress/src/functions/clike/matcher.rs deleted file mode 100644 index e51d48f..0000000 --- a/sqlite-compress/src/functions/clike/matcher.rs +++ /dev/null @@ -1,77 +0,0 @@ -//! Streaming entry point: plaintext bytes in, verdict out. -//! -//! Bytes are grouped into whole characters, each character is one NFA step, -//! and the verdict is checked after every step so the scan can stop early. -//! While the NFA is only waiting inside a star for one literal byte, input is -//! skipped straight to that byte with `memchr`. - -use memchr::memchr; - -use super::{nfa::Nfa, pattern, utf8::CharBuf}; - -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(super) enum Verdict { - Match, - NoMatch, - NeedMore, -} - -pub(super) struct LikeMatcher { - nfa: Nfa, - chars: CharBuf, -} - -impl LikeMatcher { - pub(super) fn new(pattern: &[u8]) -> Self { - Self { - nfa: Nfa::new(pattern::compile(pattern)), - chars: CharBuf::default(), - } - } - - /// Answer so far, from what has been pushed. - pub(super) fn verdict(&self) -> Verdict { - if self.nfa.is_settled() { - Verdict::Match - } else if self.nfa.is_dead() { - Verdict::NoMatch - } else { - Verdict::NeedMore - } - } - - /// Feeds the next plaintext chunk. Returns as soon as the answer is known. - pub(super) fn push(&mut self, mut chunk: &[u8]) -> Verdict { - while !chunk.is_empty() { - if let Some(target) = self.skip_target() { - match memchr(target, chunk) { - Some(i) => chunk = &chunk[i..], - None => return Verdict::NeedMore, - } - } - - self.chars.push(chunk[0], |unit| self.nfa.step(unit)); - chunk = &chunk[1..]; - - match self.verdict() { - Verdict::NeedMore => {} - verdict => return verdict, - } - } - Verdict::NeedMore - } - - /// End of the value: whether the whole pattern was matched. - pub(super) fn finish(&mut self) -> bool { - self.chars.flush(|unit| self.nfa.step(unit)); - self.nfa.is_complete() - } - - fn skip_target(&self) -> Option { - if self.chars.is_empty() { - self.nfa.skip_byte() - } else { - None - } - } -} diff --git a/sqlite-compress/src/functions/clike/mod.rs b/sqlite-compress/src/functions/clike/mod.rs deleted file mode 100644 index 0899d0c..0000000 --- a/sqlite-compress/src/functions/clike/mod.rs +++ /dev/null @@ -1,120 +0,0 @@ -//! `clike(schema, table, column, rowid, pattern)`: `LIKE` on a compressed -//! value, without materializing the plaintext. -//! -//! The blob is read through SQLite incremental I/O and decompressed in fixed -//! chunks (`scan`). Each plaintext chunk goes to a `LikeMatcher` -//! (`matcher`), which reassembles UTF-8 characters (`utf8`) and runs the -//! compiled pattern (`pattern`) as an NFA (`nfa`). Decompression stops as -//! soon as the matcher knows the answer. - -mod matcher; -mod nfa; -mod pattern; -mod scan; -mod utf8; - -use matcher::{LikeMatcher, Verdict}; -use scan::{CHUNK, scan}; -use sqlite_ffi::{Context, SqliteError, Value}; -use zstd::stream::raw::Decoder; - -use crate::{ - DictError, DictId, ExtensionState, - cache::get_decoder_in_cache, - functions::{ParsedBlob, header::Header}, -}; - -pub fn sqlite_clike(context: Context, values: &[Value]) -> Result<(), SqliteError> { - let schema = values[0].to_text()?; - let table = values[1].to_text()?; - let column = values[2].to_text()?; - let rowid = values[3].to_i64(); - let mut matcher = LikeMatcher::new(values[4].to_text()?.as_bytes()); - - if matcher.verdict() == Verdict::Match { - context.result_int64(1); - return Ok(()); - } - - let state = ExtensionState::from_context(&context)?; - let blob = state - .connection - .open_blob(schema, table, column, rowid, false)?; - - let total = blob.len(); - if total < Header::PREFIX_SIZE { - return Err(SqliteError::Message( - "compressed blob is shorter than the header".into(), - )); - } - - let mut compressed = vec![0u8; CHUNK.min(total)]; - blob.read_at(0, &mut compressed)?; - - let ParsedBlob::Framed(header, _) = - Header::parse_blob(&compressed).map_err(|e| SqliteError::Message(e.to_string()))? - else { - context.result_blob(&compressed); - return Ok(()); - }; - - let dict_id = DictId::from(header.dict_id.get()); - let header_len = header.header_len as usize; - - let prepared = if dict_id.get() != 0 { - match get_decoder_in_cache(&state, schema, dict_id) { - Ok(dict) => Some(dict), - Err(DictError::NotReady) => None, - Err(e) => return Err(SqliteError::Message(e.to_string())), - } - } else { - None - }; - - let mut decoder = if let Some(dict) = &prepared { - Decoder::with_prepared_dictionary(dict.as_ref()) - } else { - Decoder::new() - } - .map_err(|e| SqliteError::Message(e.to_string()))?; - - let matched = scan( - &mut decoder, - &blob, - &mut compressed, - header_len, - &mut matcher, - )?; - - context.result_int64(i64::from(matched)); - - Ok(()) -} - -#[cfg(test)] -mod tests { - - use std::ptr; - - use libsqlite3_sys::{SQLITE_OK, sqlite3_close, sqlite3_open}; - - use crate::Connection; - - #[test] - fn lone_percent_matches_without_opening_the_blob() { - let mut db = ptr::null_mut(); - let rc = unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }; - assert_eq!(rc, SQLITE_OK); - - let rc = unsafe { crate::sqlite3_compress_init(db, ptr::null_mut(), ptr::null_mut()) }; - assert_eq!(rc, SQLITE_OK); - - let conn = Connection::from_raw(db); - let rows = conn - .query("SELECT clike('main', 't', 'data', 1, '%')", &[]) - .unwrap(); - assert_eq!(rows[0][0].as_i64(), Some(1)); - - unsafe { sqlite3_close(db) }; - } -} diff --git a/sqlite-compress/src/functions/clike/nfa.rs b/sqlite-compress/src/functions/clike/nfa.rs deleted file mode 100644 index 474bc86..0000000 --- a/sqlite-compress/src/functions/clike/nfa.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! The pattern as an NFA over pattern positions. -//! -//! Position `i` means "the first `i` tokens are matched". The last position, -//! `tokens.len()`, means the whole pattern is matched. The state is the set of -//! positions the input seen so far can have reached. All of them are tracked -//! at once, so the matcher never goes back over input. -//! -//! One step takes one character and builds the next set: -//! - a star keeps its position, since it can absorb the character; -//! - `Any` moves forward one position; -//! - `Byte` moves forward only if the character's bytes are the next literals. -//! -//! Positions that cannot move are dropped. A star can also match nothing, so -//! after each step the position just past a live star is made live too. -//! -//! The outcome is known early in two cases: no position is left, or a trailing -//! star is reached and whatever follows cannot reject it. - -use super::{pattern::Token, utf8::is_continuation}; - -pub(super) struct Nfa { - tokens: Vec, - live: Vec, - next: Vec, - skip: Option, -} - -impl Nfa { - pub(super) fn new(tokens: Vec) -> Self { - let n = tokens.len(); - let mut live = vec![false; n + 1]; - live[0] = true; - close(&tokens, &mut live); - let mut nfa = Self { - tokens, - live, - next: vec![false; n + 1], - skip: None, - }; - nfa.skip = nfa.find_skip(); - nfa - } - - /// Advances every live position over one character. - pub(super) fn step(&mut self, unit: &[u8]) { - self.next.fill(false); - for pos in 0..self.tokens.len() { - if !self.live[pos] { - continue; - } - match self.tokens[pos] { - Token::Star => self.next[pos] = true, - Token::Any => self.next[pos + 1] = true, - Token::Byte(_) => { - // A multi-byte character must match as many consecutive literals. - if matches_literal(&self.tokens[pos..], unit) { - self.next[pos + unit.len()] = true; - } - } - } - } - close(&self.tokens, &mut self.next); - std::mem::swap(&mut self.live, &mut self.next); - self.skip = self.find_skip(); - } - - /// No position is left: no continuation of the input can match. - pub(super) fn is_dead(&self) -> bool { - !self.live.iter().any(|&live| live) - } - - /// The whole pattern is matched by the input so far. - pub(super) fn is_complete(&self) -> bool { - self.live[self.tokens.len()] - } - - /// Matched through a trailing star: any further input still matches. - pub(super) fn is_settled(&self) -> bool { - self.tokens.last() == Some(&Token::Star) && self.is_complete() - } - - /// A byte the input can be skipped to without changing the state. - pub(super) fn skip_byte(&self) -> Option { - self.skip - } - - /// When the only live positions are a star and the literal right after it, - /// every other byte leads back to that same pair. - fn find_skip(&self) -> Option { - let mut live = self - .live - .iter() - .enumerate() - .filter(|(_, live)| **live) - .map(|(pos, _)| pos); - - let star = live.next()?; - let literal = live.next()?; - if literal != star + 1 || live.next().is_some() { - return None; - } - match (self.tokens[star], self.tokens.get(literal)) { - // Jumping to a continuation byte would land mid-character. - (Token::Star, Some(&Token::Byte(b))) if !is_continuation(b) => Some(b), - _ => None, - } - } -} - -/// A star matches the empty string, so a live position on a star makes the -/// next one live. Left to right, so it chains within one pass. -fn close(tokens: &[Token], set: &mut [bool]) { - for (pos, token) in tokens.iter().enumerate() { - if set[pos] && *token == Token::Star { - set[pos + 1] = true; - } - } -} - -fn matches_literal(tokens: &[Token], unit: &[u8]) -> bool { - tokens.len() >= unit.len() - && tokens - .iter() - .zip(unit) - .all(|(token, &byte)| *token == Token::Byte(byte)) -} diff --git a/sqlite-compress/src/functions/clike/pattern.rs b/sqlite-compress/src/functions/clike/pattern.rs deleted file mode 100644 index 089dacb..0000000 --- a/sqlite-compress/src/functions/clike/pattern.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! Turns a `LIKE` pattern into the tokens the NFA walks. - -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub(super) enum Token { - /// A byte that must appear as-is. - Byte(u8), - /// `_`: exactly one character. - Any, - /// `%`: any run of characters, including none. - Star, -} - -pub(super) fn compile(pattern: &[u8]) -> Vec { - let mut tokens = Vec::with_capacity(pattern.len()); - for &byte in pattern { - let token = match byte { - b'%' => Token::Star, - b'_' => Token::Any, - byte => Token::Byte(byte), - }; - // `%%` matches what `%` matches. One star keeps the skip fast path reachable. - if token == Token::Star && tokens.last() == Some(&Token::Star) { - continue; - } - tokens.push(token); - } - tokens -} diff --git a/sqlite-compress/src/functions/clike/scan.rs b/sqlite-compress/src/functions/clike/scan.rs deleted file mode 100644 index 521d6a0..0000000 --- a/sqlite-compress/src/functions/clike/scan.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! Drives zstd over the blob and feeds each plaintext chunk to the matcher. -//! Neither the compressed value nor the plaintext is held in full. - -use sqlite_ffi::{SqliteBlob, SqliteError}; -use zstd::stream::raw::{Decoder, Operation}; - -use super::matcher::{LikeMatcher, Verdict}; - -pub(super) const CHUNK: usize = 64 * 1024; - -enum Step { - Done(bool), - NeedInput, -} - -pub(super) fn scan( - decoder: &mut Decoder<'_>, - blob: &SqliteBlob, - compressed: &mut [u8], - header_len: usize, - matcher: &mut LikeMatcher, -) -> Result { - let mut plain = vec![0u8; CHUNK]; - let total = blob.len(); - let mut pos = 0; - while pos < total { - let n = CHUNK.min(total - pos); - let start = if pos == 0 { - header_len - } else { - blob.read_at(pos, &mut compressed[..n])?; - 0 - }; - pos += n; - if let Step::Done(matched) = feed(decoder, &compressed[start..n], &mut plain, matcher)? { - return Ok(matched); - } - } - Ok(false) -} - -fn feed( - decoder: &mut Decoder<'_>, - mut input: &[u8], - plain: &mut [u8], - matcher: &mut LikeMatcher, -) -> Result { - loop { - let status = decoder - .run_on_buffers(input, plain) - .map_err(|e| SqliteError::Message(e.to_string()))?; - - match matcher.push(&plain[..status.bytes_written]) { - Verdict::Match => return Ok(Step::Done(true)), - Verdict::NoMatch => return Ok(Step::Done(false)), - Verdict::NeedMore => {} - } - - input = &input[status.bytes_read..]; - - if status.remaining == 0 { - return Ok(Step::Done(matcher.finish())); - } - - if input.is_empty() && status.bytes_written < plain.len() { - return Ok(Step::NeedInput); - } - - if status.bytes_read == 0 && status.bytes_written == 0 { - return Err(SqliteError::Message("compressed frame is truncated".into())); - } - } -} - -#[cfg(test)] -mod tests { - use std::ptr; - - use libsqlite3_sys::{sqlite3_close, sqlite3_open, SQLITE_OK}; - use sqlite_ffi::{Connection, SqlValue}; - use zstd::stream::raw::Decoder; - - use super::*; - use crate::{functions::header::wrap, DictId, Header}; - - fn search(plain: &[u8], pattern: &str) -> bool { - let payload = zstd::stream::encode_all(plain, 1).unwrap(); - let stored = wrap(DictId::from(0), plain.len(), payload).unwrap(); - - let mut db = ptr::null_mut(); - let rc = unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }; - assert_eq!(rc, SQLITE_OK); - let conn = Connection::from_raw(db); - conn.batch_execute("CREATE TABLE t (id INTEGER PRIMARY KEY, data BLOB)") - .unwrap(); - - let rowid = conn - .execute( - "INSERT INTO t (data) VALUES (?) RETURNING id", - &[SqlValue::Blob(stored)], - ) - .unwrap(); - - let found = { - let blob = conn.open_blob("main", "t", "data", rowid, false).unwrap(); - let mut compressed = vec![0u8; CHUNK.min(blob.len())]; - blob.read_at(0, &mut compressed).unwrap(); - let mut decoder = Decoder::new().unwrap(); - let mut matcher = LikeMatcher::new(pattern.as_bytes()); - - scan( - &mut decoder, - &blob, - &mut compressed, - Header::PREFIX_SIZE, - &mut matcher, - ) - .unwrap() - }; - unsafe { sqlite3_close(db) }; - found - } - - const TEXT: &[u8] = b"alpha bravo needle charlie"; - - #[test] - fn literal_pattern_is_whole_value_equality() { - assert!(search(TEXT, "alpha bravo needle charlie")); - assert!(!search(TEXT, "needle")); - assert!(!search(TEXT, "alpha bravo needle charli")); - } - - #[test] - fn percent_anchors_prefix_suffix_and_contains() { - assert!(search(TEXT, "alpha%")); - assert!(search(TEXT, "%charlie")); - assert!(search(TEXT, "%needle%")); - assert!(!search(TEXT, "%needle")); - assert!(!search(TEXT, "needle%")); - assert!(!search(TEXT, "%absent%")); - } - - #[test] - fn underscore_matches_exactly_one_character() { - assert!(search(TEXT, "alph_ bravo%")); - assert!(search(TEXT, "%n__dle%")); - assert!(!search(TEXT, "%n_dle%")); - assert!(search("aéc".as_bytes(), "a_c")); - assert!(!search("aéc".as_bytes(), "a__c")); - } - - #[test] - fn percent_backtracks_to_a_later_occurrence() { - assert!(search(b"aaab", "%a%ab")); - assert!(search(b"axxbyyc", "a%b%c")); - assert!(!search(b"axxbyy", "a%b%c")); - } - - #[test] - fn empty_pattern_only_matches_empty_value() { - assert!(!search(TEXT, "")); - } - - #[test] - fn finds_literal_across_plain_chunks() { - let mut plain = vec![b'a'; CHUNK + 8]; - let at = CHUNK - 1; - plain[at..at + 3].copy_from_slice(b"xyz"); - assert!(search(&plain, "%xyz%")); - } - - #[test] - fn underscore_matches_char_split_across_plain_chunks() { - let mut plain = vec![b'z'; CHUNK + 8]; - plain[CHUNK - 2] = b'a'; - plain[CHUNK - 1..CHUNK + 1].copy_from_slice("é".as_bytes()); - plain[CHUNK + 1] = b'b'; - assert!(search(&plain, "%a_b%")); - assert!(!search(&plain, "%a__b%")); - } - - #[test] - fn finds_suffix_after_first_compressed_read() { - let mut plain = vec![0u8; CHUNK + 4096]; - for (i, byte) in plain.iter_mut().enumerate() { - *byte = (i % 251) as u8; - } - let needle = b"NEEDLE!!"; - let at = plain.len() - needle.len(); - plain[at..].copy_from_slice(needle); - assert!(search(&plain, "%NEEDLE!!")); - assert!(!search(&plain, "%NEEDLE!")); - } - - #[test] - fn finds_suffix_when_frame_ends_on_chunk_boundary() { - let mut plain = vec![b'a'; 2 * CHUNK]; - let at = plain.len() - 3; - plain[at..].copy_from_slice(b"xyz"); - assert!(search(&plain, "%xyz")); - } -} diff --git a/sqlite-compress/src/functions/clike/utf8.rs b/sqlite-compress/src/functions/clike/utf8.rs deleted file mode 100644 index bb0e706..0000000 --- a/sqlite-compress/src/functions/clike/utf8.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! Groups plaintext bytes into characters for the NFA. A character can be -//! split across two decompressed chunks, so its leading bytes are held here -//! until it is complete. - -#[derive(Default)] -pub(super) struct CharBuf { - bytes: [u8; 4], - len: usize, - width: usize, -} - -impl CharBuf { - pub(super) fn is_empty(&self) -> bool { - self.len == 0 - } - - /// Takes one byte and calls `emit` for every unit that is now ready: a - /// whole character, or single bytes when the input is not valid UTF-8. - pub(super) fn push(&mut self, byte: u8, mut emit: impl FnMut(&[u8])) { - if self.len > 0 { - if is_continuation(byte) { - self.bytes[self.len] = byte; - self.len += 1; - if self.len == self.width { - emit(&self.bytes[..self.len]); - self.len = 0; - } - return; - } - self.flush(&mut emit); - } - - match width(byte) { - 1 => emit(&[byte]), - width => { - self.bytes[0] = byte; - self.len = 1; - self.width = width; - } - } - } - - /// Emits the bytes of an unfinished character one at a time. - pub(super) fn flush(&mut self, mut emit: impl FnMut(&[u8])) { - for &byte in &self.bytes[..self.len] { - emit(&[byte]); - } - self.len = 0; - } -} - -pub(super) fn is_continuation(byte: u8) -> bool { - byte & 0xC0 == 0x80 -} - -fn width(lead: u8) -> usize { - match lead { - 0xC0..=0xDF => 2, - 0xE0..=0xEF => 3, - 0xF0..=0xF7 => 4, - _ => 1, - } -} diff --git a/sqlite-compress/src/functions/light.rs b/sqlite-compress/src/functions/contains.rs similarity index 89% rename from sqlite-compress/src/functions/light.rs rename to sqlite-compress/src/functions/contains.rs index 1d5380a..87e6df4 100644 --- a/sqlite-compress/src/functions/light.rs +++ b/sqlite-compress/src/functions/contains.rs @@ -2,13 +2,12 @@ use memchr::memmem::Finder; use sqlite_ffi::SqliteBlob; use sqlite_ffi::{Context, SqliteError, Value}; use std::io::{self, BufReader, Read}; -use zstd::dict::DecoderDictionary; use zstd::stream::read::Decoder; use crate::{ DictError, DictId, cache::get_decoder_in_cache, - functions::{ParsedBlob, errors::CodecError, header::Header}, + functions::{ParsedBlob, header::Header}, state::ExtensionState, }; @@ -77,7 +76,7 @@ pub fn sqlite_decompress_cont(context: Context, values: &[Value]) -> Result<(), None }; - let reader = BufReader::new(BlobReader::new(&blob, header_len)); + let reader = BufReader::with_capacity(64 * 1024, BlobReader::new(&blob, header_len)); let decoder = match prepared.as_deref() { Some(dict) => Decoder::with_prepared_dictionary(reader, dict), None => Decoder::with_dictionary(reader, &[]), @@ -150,22 +149,6 @@ fn contains_streaming(mut reader: R, needle: &[u8], mut skip: usize) -> Ok(false) } -/// Stream-decompress then sliding-window search. -fn contains_stream_zstd( - payload: &[u8], - dict: Option<&DecoderDictionary<'static>>, - needle: &[u8], - offset: usize, -) -> Result { - let decoder = match dict { - Some(dict) => Decoder::with_prepared_dictionary(payload, dict), - None => Decoder::with_dictionary(payload, &[]), - } - .map_err(CodecError::DecompressionFailed)?; - - contains_streaming(decoder, needle, offset).map_err(CodecError::DecompressionFailed) -} - struct BlobReader<'a> { blob: &'a SqliteBlob<'a>, pos: usize, diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index e2104d6..86c3171 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -1,15 +1,13 @@ -mod clike; mod compress; mod decompress; mod errors; mod header; mod types; -mod light; +mod contains; /// Compression level used when configuration does not select another level. pub const DEFAULT_LEVEL: types::Level = types::Level::new(3); -pub use clike::sqlite_clike; pub use compress::sqlite_compress; pub use decompress::sqlite_decompress; pub(crate) use decompress::{decompress_raw, decompress_with_decoder}; @@ -17,4 +15,4 @@ pub use errors::CodecError; pub use header::Header; pub(crate) use header::ParsedBlob; pub use types::Level; -pub use light::sqlite_decompress_cont; \ No newline at end of file +pub use contains::sqlite_decompress_cont; \ No newline at end of file diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index a5fc07f..1d0f00e 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -34,8 +34,6 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> connection.create_function("decompress", 2, functions::sqlite_decompress)?; - connection.create_function("clike", 5, functions::sqlite_clike)?; - connection.create_function("decompress_cont", 6, functions::sqlite_decompress_cont)?; Ok(()) From a54dbee013d3d75d93f518551bcab3a1bb1313e8 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Thu, 22 Oct 2026 11:18:21 +0200 Subject: [PATCH 12/17] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Use=20regex=20approa?= =?UTF-8?q?ch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 1 + crates/sqlite-ffi/src/blob.rs | 37 +++ sqlite-compress/Cargo.toml | 7 + sqlite-compress/src/functions/contains.rs | 191 --------------- .../src/functions/contains/contains.rs | 26 +++ .../src/functions/contains/matcher.rs | 219 ++++++++++++++++++ sqlite-compress/src/functions/contains/mod.rs | 48 ++++ .../src/functions/contains/search.rs | 61 +++++ .../src/functions/contains/utils.rs | 26 +++ 9 files changed, 425 insertions(+), 191 deletions(-) delete mode 100644 sqlite-compress/src/functions/contains.rs create mode 100644 sqlite-compress/src/functions/contains/contains.rs create mode 100644 sqlite-compress/src/functions/contains/matcher.rs create mode 100644 sqlite-compress/src/functions/contains/mod.rs create mode 100644 sqlite-compress/src/functions/contains/search.rs create mode 100644 sqlite-compress/src/functions/contains/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 55c8ec4..790a661 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -607,6 +607,7 @@ dependencies = [ "libsqlite3-sys", "memchr", "parking_lot", + "regex-automata", "sqlite-ffi", "sqlite-ffi-macros", "thiserror", diff --git a/crates/sqlite-ffi/src/blob.rs b/crates/sqlite-ffi/src/blob.rs index 6d846b8..4ff2347 100644 --- a/crates/sqlite-ffi/src/blob.rs +++ b/crates/sqlite-ffi/src/blob.rs @@ -1,5 +1,6 @@ use std::{ ffi::{CString, c_char, c_int, c_void}, + io::{self, Read}, ptr::NonNull, }; @@ -39,6 +40,42 @@ impl SqliteBlob<'_> { Ok(()) } + + pub fn reader(&self, start: usize) -> BlobReader<'_> { + BlobReader::new(self, start) + } +} + +pub struct BlobReader<'a> { + blob: &'a SqliteBlob<'a>, + pos: usize, + end: usize, +} + +impl<'a> BlobReader<'a> { + fn new(blob: &'a SqliteBlob<'a>, start: usize) -> Self { + let end = blob.len(); + Self { + blob, + pos: start.min(end), + end, + } + } +} + +impl Read for BlobReader<'_> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let remaining = self.end.saturating_sub(self.pos); + if remaining == 0 || buf.is_empty() { + return Ok(0); + } + let n = buf.len().min(remaining); + self.blob + .read_at(self.pos, &mut buf[..n]) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + self.pos += n; + Ok(n) + } } impl Connection { diff --git a/sqlite-compress/Cargo.toml b/sqlite-compress/Cargo.toml index c28d252..9a7854d 100644 --- a/sqlite-compress/Cargo.toml +++ b/sqlite-compress/Cargo.toml @@ -15,6 +15,13 @@ libsqlite3-sys = { workspace = true } sqlite-ffi = { workspace = true } sqlite-ffi-macros = { workspace = true } memchr = "2.8.3" +regex-automata = { version = "0.4", default-features = false, features = [ + "std", + "syntax", + "unicode", + "hybrid", + "perf-inline", +] } [lib] crate-type = ["cdylib", "rlib"] diff --git a/sqlite-compress/src/functions/contains.rs b/sqlite-compress/src/functions/contains.rs deleted file mode 100644 index 87e6df4..0000000 --- a/sqlite-compress/src/functions/contains.rs +++ /dev/null @@ -1,191 +0,0 @@ -use memchr::memmem::Finder; -use sqlite_ffi::SqliteBlob; -use sqlite_ffi::{Context, SqliteError, Value}; -use std::io::{self, BufReader, Read}; -use zstd::stream::read::Decoder; - -use crate::{ - DictError, DictId, - cache::get_decoder_in_cache, - functions::{ParsedBlob, header::Header}, - state::ExtensionState, -}; - -const STREAM_CHUNK: usize = 16 * 1024; - -/// `decompress_cont(schema, table, column, rowid, needle, offset) -> 0|1` -pub fn sqlite_decompress_cont(context: Context, values: &[Value]) -> Result<(), SqliteError> { - if values.len() < 6 { - return Err(SqliteError::Message( - "decompress_cont expects 6 arguments: schema, table, column, rowid, needle, offset" - .into(), - )); - } - - let schema = values[0].to_text()?; - let table = values[1].to_text()?; - let column = values[2].to_text()?; - let rowid = values[3].to_i64(); - let needle = values[4].to_blob(); - let offset = values[5].to_i64().max(0) as usize; - - let state = ExtensionState::from_context(&context)?; - let blob = state - .connection - .open_blob(schema, table, column, rowid, false)?; - - let total = blob.len(); - if total == 0 { - context.result_bool(contains_bytes(&[], needle, offset)); - return Ok(()); - } - - // Peek just enough to parse the header (no full materialization). - let peek_len = Header::PREFIX_SIZE.min(total); - let mut peek = vec![0u8; peek_len]; - blob.read_at(0, &mut peek)?; - - let found = match Header::parse_blob(&peek) { - Ok(ParsedBlob::Plain) | Err(_) if peek_len < Header::PREFIX_SIZE => { - contains_streaming(BlobReader::new(&blob, 0), needle, offset) - .map_err(|e| SqliteError::Message(e.to_string()))? - } - Ok(ParsedBlob::Plain) => contains_streaming(BlobReader::new(&blob, 0), needle, offset) - .map_err(|e| SqliteError::Message(e.to_string()))?, - Ok(ParsedBlob::Framed(header, _)) => { - let header_len = header.header_len as usize; - if total < header_len { - return Err(SqliteError::Message( - "compressed blob is shorter than the header".into(), - )); - } - - if header_len > peek.len() { - peek.resize(header_len, 0); - blob.read_at(0, &mut peek)?; - } - - let dict_id = DictId::from(header.dict_id.get()); - let prepared = if dict_id.get() != 0 { - match get_decoder_in_cache(&state, schema, dict_id) { - Ok(d) => Some(d), - Err(DictError::NotReady) => None, - Err(e) => return Err(SqliteError::Message(e.to_string())), - } - } else { - None - }; - - let reader = BufReader::with_capacity(64 * 1024, BlobReader::new(&blob, header_len)); - let decoder = match prepared.as_deref() { - Some(dict) => Decoder::with_prepared_dictionary(reader, dict), - None => Decoder::with_dictionary(reader, &[]), - } - .map_err(|e| SqliteError::Message(e.to_string()))?; - - contains_streaming(decoder, needle, offset) - .map_err(|e| SqliteError::Message(e.to_string()))? - } - Err(e) => return Err(SqliteError::Message(e.to_string())), - }; - - context.result_bool(found); - Ok(()) -} - -/// Literal search with optional skip. Empty needle → true (like LIKE '%%'). -fn contains_bytes(haystack: &[u8], needle: &[u8], offset: usize) -> bool { - if needle.is_empty() { - return true; - } - let data = match haystack.get(offset..) { - Some(d) => d, - None => return false, - }; - Finder::new(needle).find(data).is_some() -} - -/// Sliding-window search over a byte stream. -/// -/// Keeps `needle.len() - 1` bytes of overlap so matches spanning chunks are found. -fn contains_streaming(mut reader: R, needle: &[u8], mut skip: usize) -> io::Result { - if needle.is_empty() { - return Ok(true); - } - - let finder = Finder::new(needle); - let overlap_len = needle.len() - 1; - let mut overlap = Vec::with_capacity(overlap_len); - let mut chunk = vec![0u8; STREAM_CHUNK]; - - loop { - let n = reader.read(&mut chunk)?; - if n == 0 { - break; - } - let mut data = &chunk[..n]; - - if skip > 0 { - if skip >= data.len() { - skip -= data.len(); - continue; - } - data = &data[skip..]; - skip = 0; - } - - let mut window = std::mem::take(&mut overlap); - window.extend_from_slice(data); - - if finder.find(&window).is_some() { - return Ok(true); - } - - let keep = overlap_len.min(window.len()); - overlap.clear(); - overlap.extend_from_slice(&window[window.len() - keep..]); - } - - Ok(false) -} - -struct BlobReader<'a> { - blob: &'a SqliteBlob<'a>, - pos: usize, - end: usize, -} - -impl<'a> BlobReader<'a> { - fn new(blob: &'a SqliteBlob<'a>, start: usize) -> Self { - let end = blob.len(); - Self { - blob, - pos: start.min(end), - end, - } - } - - fn from_range(blob: &'a SqliteBlob<'a>, start: usize, end: usize) -> Self { - let end = end.min(blob.len()); - Self { - blob, - pos: start.min(end), - end, - } - } -} - -impl Read for BlobReader<'_> { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let remaining = self.end.saturating_sub(self.pos); - if remaining == 0 || buf.is_empty() { - return Ok(0); - } - let n = buf.len().min(remaining); - self.blob - .read_at(self.pos, &mut buf[..n]) - .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; - self.pos += n; - Ok(n) - } -} diff --git a/sqlite-compress/src/functions/contains/contains.rs b/sqlite-compress/src/functions/contains/contains.rs new file mode 100644 index 0000000..2aac78d --- /dev/null +++ b/sqlite-compress/src/functions/contains/contains.rs @@ -0,0 +1,26 @@ +use memchr::memmem::Finder; +use regex_automata::{ + Anchored, + hybrid::dfa::{Cache, DFA}, + nfa::thompson, + util::{start, syntax}, +}; +use sqlite_ffi::SqliteBlob; +use sqlite_ffi::{Context, SqliteError, Value}; +use std::cell::RefCell; +use std::fmt::Write as _; +use std::io::{self, BufReader, Read}; +use zstd::stream::read::Decoder; + +use crate::{ + DictError, DictId, + cache::get_decoder_in_cache, + functions::{ParsedBlob, header::Header}, + state::ExtensionState, +}; + +const STREAM_CHUNK: usize = 16 * 1024; + + + + diff --git a/sqlite-compress/src/functions/contains/matcher.rs b/sqlite-compress/src/functions/contains/matcher.rs new file mode 100644 index 0000000..2374fc0 --- /dev/null +++ b/sqlite-compress/src/functions/contains/matcher.rs @@ -0,0 +1,219 @@ +use std::{ + cell::RefCell, + io::{self, Read}, +}; + +use memchr::memmem::Finder; +use sqlite_ffi::SqliteError; + +use regex_automata::{ + Anchored, + hybrid::dfa::{Cache, DFA}, + nfa::thompson, + util::{start, syntax}, +}; + +use crate::functions::contains::utils::{like_to_regex, to_sqlite}; + +const STREAM_CHUNK: usize = 16 * 1024; + +/// Compiled `LIKE` pattern, cached across rows through SQLite auxdata. +pub(crate) enum Matcher { + /// `%`-only pattern: lowercase literals that must appear in order. + Literals(Vec>), + /// Pattern with `_`: lazy DFA driven byte by byte across chunks. + Dfa { dfa: DFA, cache: RefCell }, +} + +impl Matcher { + pub(crate) fn compile(pattern: &[u8]) -> Result { + if !pattern.contains(&b'_') { + let finders = pattern + .split(|&b| b == b'%') + .filter(|s| !s.is_empty()) + .map(|s| Finder::new(&s.to_ascii_lowercase()).into_owned()) + .collect(); + return Ok(Self::Literals(finders)); + } + + let dfa = DFA::builder() + .syntax(syntax::Config::new().utf8(false)) + .thompson(thompson::Config::new().utf8(false)) + .build(&like_to_regex(pattern)) + .map_err(to_sqlite)?; + let cache = RefCell::new(dfa.create_cache()); + Ok(Self::Dfa { dfa, cache }) + } + + pub(crate) fn find_in(&self, reader: R, skip: usize) -> io::Result { + match self { + Self::Literals(finders) => find_literals(reader, finders, skip), + Self::Dfa { dfa, cache } => find_dfa(reader, dfa, &mut cache.borrow_mut(), skip), + } + } +} + +/// Finds each literal after the end of the previous one, over a byte stream. +/// +/// Leftmost matching is optimal for `%`-separated literals, so no +/// backtracking is needed. The window keeps `len - 1` bytes of the current +/// literal's prefix so matches spanning chunks are found. +fn find_literals( + mut reader: R, + finders: &[Finder<'static>], + mut skip: usize, +) -> io::Result { + let Some(longest) = finders.iter().map(|f| f.needle().len()).max() else { + return Ok(true); + }; + + let mut buf = vec![0u8; longest - 1 + STREAM_CHUNK]; + let mut kept = 0; + let mut current = 0; + + loop { + let n = reader.read(&mut buf[kept..kept + STREAM_CHUNK])?; + if n == 0 { + return Ok(false); + } + let mut end = kept + n; + + if skip > 0 { + if skip >= n { + skip -= n; + continue; + } + buf.copy_within(kept + skip..end, kept); + end -= skip; + skip = 0; + } + + buf[kept..end].make_ascii_lowercase(); + + let mut pos = 0; + while let Some(i) = finders[current].find(&buf[pos..end]) { + pos += i + finders[current].needle().len(); + current += 1; + if current == finders.len() { + return Ok(true); + } + } + + let keep = (finders[current].needle().len() - 1).min(end - pos); + buf.copy_within(end - keep..end, 0); + kept = keep; + } +} + +fn find_dfa( + mut reader: R, + dfa: &DFA, + cache: &mut Cache, + mut skip: usize, +) -> io::Result { + let mut sid = dfa + .start_state(cache, &start::Config::new().anchored(Anchored::No)) + .map_err(io::Error::other)?; + let mut chunk = vec![0u8; STREAM_CHUNK]; + + loop { + let n = reader.read(&mut chunk)?; + if n == 0 { + break; + } + let mut data = &chunk[..n]; + + if skip > 0 { + if skip >= data.len() { + skip -= data.len(); + continue; + } + data = &data[skip..]; + skip = 0; + } + + for &b in data { + sid = dfa.next_state(cache, sid, b).map_err(io::Error::other)?; + if sid.is_tagged() { + if sid.is_match() { + return Ok(true); + } + if sid.is_dead() { + return Ok(false); + } + if sid.is_quit() { + return Err(io::Error::other("pattern DFA quit")); + } + } + } + } + + let sid = dfa.next_eoi_state(cache, sid).map_err(io::Error::other)?; + Ok(sid.is_match()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn matches(pattern: &str, haystack: &[u8]) -> bool { + Matcher::compile(pattern.as_bytes()) + .unwrap() + .find_in(haystack, 0) + .unwrap() + } + + /// Feeds one byte per `read` to exercise matches spanning chunks. + struct Trickle<'a>(&'a [u8]); + + impl Read for Trickle<'_> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let Some((&b, rest)) = self.0.split_first() else { + return Ok(0); + }; + buf[0] = b; + self.0 = rest; + Ok(1) + } + } + + #[test] + fn literals_are_case_insensitive_and_ordered() { + assert!(matches("hello", b"say HeLLo world")); + assert!(matches("say%WORLD", b"say hello world")); + assert!(!matches("world%say", b"say hello world")); + assert!(matches("", b"anything")); + assert!(matches("%%", b"")); + assert!(!matches("x", b"")); + } + + #[test] + fn underscore_matches_one_character() { + assert!(matches("h_llo", b"xx HALLO yy")); + assert!(matches("a_c", "aéc".as_bytes())); + assert!(!matches("a__c", "aéc".as_bytes())); + assert!(matches("a_c", b"a\xFFc")); + assert!(!matches("a_c", b"ac")); + assert!(matches("_", b"z")); + assert!(!matches("_", b"")); + assert!(matches("a_%z", b"ab----Z")); + } + + #[test] + fn matches_span_chunks() { + for pattern in ["needle%tail", "ne_dle%tail"] { + let m = Matcher::compile(pattern.as_bytes()).unwrap(); + assert!(m.find_in(Trickle(b"..NEEDLE..TAIL.."), 0).unwrap()); + assert!(!m.find_in(Trickle(b"..TAIL..NEEDLE.."), 0).unwrap()); + } + } + + #[test] + fn offset_skips_prefix() { + for pattern in ["abc", "a_c"] { + let m = Matcher::compile(pattern.as_bytes()).unwrap(); + assert!(m.find_in(&b"abc---abc"[..], 3).unwrap()); + assert!(!m.find_in(&b"abc------"[..], 1).unwrap()); + } + } +} diff --git a/sqlite-compress/src/functions/contains/mod.rs b/sqlite-compress/src/functions/contains/mod.rs new file mode 100644 index 0000000..80ca6a4 --- /dev/null +++ b/sqlite-compress/src/functions/contains/mod.rs @@ -0,0 +1,48 @@ +use sqlite_ffi::{Context, SqliteError, Value}; + +use crate::{functions::contains::{matcher::Matcher, search::search_blob}, state::ExtensionState}; + +const PATTERN_ARG: i32 = 4; + +mod matcher; +mod search; +mod utils; + +/// `decompress_cont(schema, table, column, rowid, pattern, offset) -> 0|1` +/// +/// `pattern` uses `LIKE` wildcards (`%`, `_`) and matches anywhere in the +/// value, ASCII case-insensitively. +pub fn sqlite_decompress_cont(mut context: Context, values: &[Value]) -> Result<(), SqliteError> { + if values.len() < 6 { + return Err(SqliteError::Message( + "decompress_cont expects 6 arguments: schema, table, column, rowid, pattern, offset" + .into(), + )); + } + + let schema = values[0].to_text()?; + let table = values[1].to_text()?; + let column = values[2].to_text()?; + let rowid = values[3].to_i64(); + let pattern = values[PATTERN_ARG as usize].to_blob(); + let offset = values[5].to_i64().max(0) as usize; + + let state = ExtensionState::from_context(&context)?; + let blob = state + .connection + .open_blob(schema, table, column, rowid, false)?; + + let aux = context.aux::(PATTERN_ARG); + let found = match aux.get() { + Ok(matcher) => search_blob(&state, schema, &blob, matcher, offset)?, + Err(_) => { + let matcher = Matcher::compile(pattern)?; + let found = search_blob(&state, schema, &blob, &matcher, offset)?; + aux.set(matcher); + found + } + }; + + context.result_bool(found); + Ok(()) +} diff --git a/sqlite-compress/src/functions/contains/search.rs b/sqlite-compress/src/functions/contains/search.rs new file mode 100644 index 0000000..6fd809d --- /dev/null +++ b/sqlite-compress/src/functions/contains/search.rs @@ -0,0 +1,61 @@ +use sqlite_ffi::SqliteBlob; +use sqlite_ffi::SqliteError; +use std::io::{self, BufReader}; +use zstd::stream::read::Decoder; + +use crate::{ + DictError, DictId, + cache::get_decoder_in_cache, + functions::contains::utils::to_sqlite, + functions::{ParsedBlob, contains::matcher::Matcher, header::Header}, + state::ExtensionState, +}; + +pub(crate) fn search_blob( + state: &ExtensionState, + schema: &str, + blob: &SqliteBlob<'_>, + matcher: &Matcher, + offset: usize, +) -> Result { + let total = blob.len(); + if total == 0 { + return matcher.find_in(io::empty(), offset).map_err(to_sqlite); + } + + // Peek just enough to parse the header (no full materialization). + let peek_len = Header::PREFIX_SIZE.min(total); + let mut peek = vec![0u8; peek_len]; + blob.read_at(0, &mut peek)?; + + match Header::parse_blob(&peek) { + Ok(ParsedBlob::Plain) | Err(_) if peek_len < Header::PREFIX_SIZE => { + matcher.find_in(blob.reader(0), offset).map_err(to_sqlite) + } + Ok(ParsedBlob::Plain) => matcher.find_in(blob.reader(0), offset).map_err(to_sqlite), + Ok(ParsedBlob::Framed(header, _)) => { + let header_len = header.header_len as usize; + + let dict_id = DictId::from(header.dict_id.get()); + let prepared = if dict_id.get() != 0 { + match get_decoder_in_cache(state, schema, dict_id) { + Ok(d) => Some(d), + Err(DictError::NotReady) => None, + Err(e) => return Err(SqliteError::Message(e.to_string())), + } + } else { + None + }; + + let reader = BufReader::with_capacity(64 * 1024, blob.reader(header_len)); + let decoder = match prepared.as_deref() { + Some(dict) => Decoder::with_prepared_dictionary(reader, dict), + None => Decoder::with_dictionary(reader, &[]), + } + .map_err(to_sqlite)?; + + matcher.find_in(decoder, offset).map_err(to_sqlite) + } + Err(e) => Err(SqliteError::Message(e.to_string())), + } +} diff --git a/sqlite-compress/src/functions/contains/utils.rs b/sqlite-compress/src/functions/contains/utils.rs new file mode 100644 index 0000000..0e1fe39 --- /dev/null +++ b/sqlite-compress/src/functions/contains/utils.rs @@ -0,0 +1,26 @@ +use sqlite_ffi::SqliteError; +use std::fmt::Write as _; + +pub(crate) fn to_sqlite(e: impl std::fmt::Display) -> SqliteError { + SqliteError::Message(e.to_string()) +} + +/// Translates a `LIKE` pattern into a byte-oriented regex. +/// +/// Literals are emitted as `\xHH` so ASCII folds case and other bytes match +/// verbatim. `_` is one UTF-8 character, or one byte that cannot start a +/// valid sequence so invalid UTF-8 still advances. +pub(crate) fn like_to_regex(pattern: &[u8]) -> String { + let mut re = String::with_capacity(8 + pattern.len() * 4); + re.push_str("(?is-u)"); + for &b in pattern { + match b { + b'%' => re.push_str(".*"), + b'_' => re.push_str(r"(?:(?u-i:.)|[\x80-\xBF\xC0\xC1\xF5-\xFF])"), + _ => { + let _ = write!(re, r"\x{b:02X}"); + } + } + } + re +} From 55abf970f317463d09fdcc321b68efd303b07ded Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Thu, 22 Oct 2026 22:14:21 +0200 Subject: [PATCH 13/17] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Clean=20the=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .mise/tasks/sqlite-compress/benches | 3 - Cargo.lock | 472 ------------------ crates/sqlite-ffi/src/blob.rs | 2 +- sqlite-compress/Cargo.toml | 7 - sqlite-compress/benches/scan.rs | 115 ----- .../src/functions/contains/contains.rs | 26 - .../functions/{contains => like}/matcher.rs | 15 +- .../src/functions/{contains => like}/mod.rs | 11 +- .../functions/{contains => like}/search.rs | 11 +- .../src/functions/{contains => like}/utils.rs | 3 +- sqlite-compress/src/functions/mod.rs | 4 +- sqlite-compress/src/lib.rs | 2 +- 12 files changed, 30 insertions(+), 641 deletions(-) delete mode 100755 .mise/tasks/sqlite-compress/benches delete mode 100644 sqlite-compress/benches/scan.rs delete mode 100644 sqlite-compress/src/functions/contains/contains.rs rename sqlite-compress/src/functions/{contains => like}/matcher.rs (96%) rename sqlite-compress/src/functions/{contains => like}/mod.rs (81%) rename sqlite-compress/src/functions/{contains => like}/search.rs (91%) rename sqlite-compress/src/functions/{contains => like}/utils.rs (99%) diff --git a/.mise/tasks/sqlite-compress/benches b/.mise/tasks/sqlite-compress/benches deleted file mode 100755 index 0b4eafb..0000000 --- a/.mise/tasks/sqlite-compress/benches +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -#MISE description="Benchmarks" -cargo bench -p sqlite-compress --bench scan \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 790a661..7c9d114 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,42 +11,12 @@ dependencies = [ "memchr", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.4.6" @@ -65,161 +35,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - -[[package]] -name = "clap" -version = "4.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" -dependencies = [ - "anstyle", - "clap_lex", -] - -[[package]] -name = "clap_lex" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" - -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "either" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" - [[package]] name = "find-msvc-tools" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" -[[package]] -name = "futures-core" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" - -[[package]] -name = "futures-task" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" - -[[package]] -name = "futures-util" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "slab", -] - [[package]] name = "getrandom" version = "0.4.3" @@ -231,49 +52,6 @@ dependencies = [ "r-efi", ] -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - -[[package]] -name = "hermit-abi" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" - -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - [[package]] name = "jobserver" version = "0.1.35" @@ -284,17 +62,6 @@ dependencies = [ "libc", ] -[[package]] -name = "js-sys" -version = "0.3.105" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - [[package]] name = "libc" version = "0.2.189" @@ -330,27 +97,6 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "parking_lot" version = "0.12.5" @@ -374,46 +120,12 @@ dependencies = [ "windows-link", ] -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - [[package]] name = "pkg-config" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -448,26 +160,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -517,82 +209,18 @@ dependencies = [ "thiserror", ] -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.5", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - [[package]] name = "smallvec" version = "1.16.1" @@ -603,7 +231,6 @@ checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" name = "sqlite-compress" version = "0.1.0" dependencies = [ - "criterion", "libsqlite3-sys", "memchr", "parking_lot", @@ -675,16 +302,6 @@ dependencies = [ "syn 3.0.5", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "unicode-ident" version = "1.0.24" @@ -697,95 +314,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.128" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.128" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.128" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 3.0.5", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.128" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.105" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys", -] - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - [[package]] name = "zerocopy" version = "0.8.57" @@ -806,12 +340,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" - [[package]] name = "zstd" version = "0.13.3" diff --git a/crates/sqlite-ffi/src/blob.rs b/crates/sqlite-ffi/src/blob.rs index 4ff2347..cf68a36 100644 --- a/crates/sqlite-ffi/src/blob.rs +++ b/crates/sqlite-ffi/src/blob.rs @@ -72,7 +72,7 @@ impl Read for BlobReader<'_> { let n = buf.len().min(remaining); self.blob .read_at(self.pos, &mut buf[..n]) - .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + .map_err(|e| io::Error::other(e.to_string()))?; self.pos += n; Ok(n) } diff --git a/sqlite-compress/Cargo.toml b/sqlite-compress/Cargo.toml index 9a7854d..cedf9fe 100644 --- a/sqlite-compress/Cargo.toml +++ b/sqlite-compress/Cargo.toml @@ -36,10 +36,3 @@ loadable_extension = [ "sqlite-ffi/loadable_extension", "libsqlite3-sys/loadable_extension", ] - -[dev-dependencies] -criterion = "0.5" - -[[bench]] -name = "scan" -harness = false diff --git a/sqlite-compress/benches/scan.rs b/sqlite-compress/benches/scan.rs deleted file mode 100644 index 853e090..0000000 --- a/sqlite-compress/benches/scan.rs +++ /dev/null @@ -1,115 +0,0 @@ -use std::{hint::black_box, iter::repeat_n, ptr, sync::Once}; - -use criterion::{criterion_group, criterion_main, Criterion}; -use libsqlite3_sys::{sqlite3, sqlite3_auto_extension, sqlite3_close, sqlite3_open, SQLITE_OK}; -use sqlite_compress::{sqlite3_compress_init, ExtensionState}; -use sqlite_ffi::{first_value, SqlValue}; - -struct BenchDb(*mut sqlite3); - -impl BenchDb { - fn open() -> Self { - static REGISTER: Once = Once::new(); - REGISTER.call_once(|| unsafe { - #[allow(clippy::missing_transmute_annotations)] - sqlite3_auto_extension(Some(std::mem::transmute( - sqlite3_compress_init as *const (), - ))); - }); - let mut db = ptr::null_mut(); - assert_eq!( - unsafe { sqlite3_open(c":memory:".as_ptr(), &mut db) }, - SQLITE_OK - ); - Self(db) - } - - fn state(&self) -> ExtensionState { - ExtensionState::from_db(self.0).unwrap() - } -} - -impl Drop for BenchDb { - fn drop(&mut self) { - unsafe { sqlite3_close(self.0) }; - } -} - -const BODY_LEN: usize = 8 * 1024 * 1024; - -fn needle(len: usize) -> String { - let mut s = String::from("NEEDLE"); - s.extend(repeat_n('X', len - s.len())); - s -} - -fn insert(conn: &ExtensionState, kind: &str, needle: &str, at: Option) { - let mut body = vec![b'a'; BODY_LEN]; - if let Some(at) = at { - body[at..at + needle.len()].copy_from_slice(needle.as_bytes()); - } - conn.as_ref() - .execute( - "INSERT INTO t(data, kind) VALUES (compress(?1, 'main', 't', 'data'), ?2)", - &[SqlValue::Blob(body), SqlValue::Text(kind.to_string())], - ) - .unwrap(); -} - -fn hit(conn: &ExtensionState, sql: &str) -> i64 { - let rows = conn.as_ref().query(sql, &[]).unwrap(); - first_value(&rows).and_then(SqlValue::as_i64).unwrap() -} - -fn bench_scan(c: &mut Criterion) { - let db = BenchDb::open(); - let conn = db.state(); - - conn.as_ref() - .batch_execute("CREATE TABLE t(id INTEGER PRIMARY KEY, data BLOB, kind TEXT)") - .unwrap(); - - let mut group = c.benchmark_group("clike_vs_like"); - group.sample_size(10); - - for len in [6, 64, 256, 1024] { - let needle = needle(len); - for (place, at) in [ - ("start", Some(0)), - ("middle", Some(BODY_LEN / 2)), - ("end", Some(BODY_LEN - needle.len())), - ("absent", None), - ] { - insert(&conn, &format!("{place}-{len}"), &needle, at); - } - - for (place, pattern, expect) in [ - ("start", format!("{needle}%"), 1), - ("middle", format!("%{needle}%"), 1), - ("end", format!("%{needle}"), 1), - ("absent", format!("%{needle}%"), 0), - ] { - let kind = format!("{place}-{len}"); - let like_sql = format!( - "SELECT decompress(data, 'main') LIKE '{pattern}' FROM t WHERE kind = '{kind}'" - ); - let clike_sql = format!( - "SELECT clike('main', 't', 'data', id, '{pattern}') FROM t WHERE kind = '{kind}'" - ); - assert_eq!(hit(&conn, &like_sql), expect, "{kind}"); - assert_eq!(hit(&conn, &clike_sql), expect, "{kind}"); - - group.bench_function(format!("decompress-like-{place}-{len}"), |b| { - b.iter(|| black_box(hit(&conn, &like_sql))) - }); - group.bench_function(format!("clike-{place}-{len}"), |b| { - b.iter(|| black_box(hit(&conn, &clike_sql))) - }); - } - } - - group.finish(); -} - -criterion_group!(benches, bench_scan); -criterion_main!(benches); diff --git a/sqlite-compress/src/functions/contains/contains.rs b/sqlite-compress/src/functions/contains/contains.rs deleted file mode 100644 index 2aac78d..0000000 --- a/sqlite-compress/src/functions/contains/contains.rs +++ /dev/null @@ -1,26 +0,0 @@ -use memchr::memmem::Finder; -use regex_automata::{ - Anchored, - hybrid::dfa::{Cache, DFA}, - nfa::thompson, - util::{start, syntax}, -}; -use sqlite_ffi::SqliteBlob; -use sqlite_ffi::{Context, SqliteError, Value}; -use std::cell::RefCell; -use std::fmt::Write as _; -use std::io::{self, BufReader, Read}; -use zstd::stream::read::Decoder; - -use crate::{ - DictError, DictId, - cache::get_decoder_in_cache, - functions::{ParsedBlob, header::Header}, - state::ExtensionState, -}; - -const STREAM_CHUNK: usize = 16 * 1024; - - - - diff --git a/sqlite-compress/src/functions/contains/matcher.rs b/sqlite-compress/src/functions/like/matcher.rs similarity index 96% rename from sqlite-compress/src/functions/contains/matcher.rs rename to sqlite-compress/src/functions/like/matcher.rs index 2374fc0..f2d77f1 100644 --- a/sqlite-compress/src/functions/contains/matcher.rs +++ b/sqlite-compress/src/functions/like/matcher.rs @@ -4,16 +4,15 @@ use std::{ }; use memchr::memmem::Finder; -use sqlite_ffi::SqliteError; - use regex_automata::{ Anchored, hybrid::dfa::{Cache, DFA}, nfa::thompson, util::{start, syntax}, }; +use sqlite_ffi::SqliteError; -use crate::functions::contains::utils::{like_to_regex, to_sqlite}; +use crate::functions::like::utils::{like_to_regex, to_sqlite}; const STREAM_CHUNK: usize = 16 * 1024; @@ -22,7 +21,10 @@ pub(crate) enum Matcher { /// `%`-only pattern: lowercase literals that must appear in order. Literals(Vec>), /// Pattern with `_`: lazy DFA driven byte by byte across chunks. - Dfa { dfa: DFA, cache: RefCell }, + Dfa { + dfa: Box, + cache: Box>, + }, } impl Matcher { @@ -42,7 +44,10 @@ impl Matcher { .build(&like_to_regex(pattern)) .map_err(to_sqlite)?; let cache = RefCell::new(dfa.create_cache()); - Ok(Self::Dfa { dfa, cache }) + Ok(Self::Dfa { + dfa: Box::new(dfa), + cache: Box::new(cache), + }) } pub(crate) fn find_in(&self, reader: R, skip: usize) -> io::Result { diff --git a/sqlite-compress/src/functions/contains/mod.rs b/sqlite-compress/src/functions/like/mod.rs similarity index 81% rename from sqlite-compress/src/functions/contains/mod.rs rename to sqlite-compress/src/functions/like/mod.rs index 80ca6a4..17afe7a 100644 --- a/sqlite-compress/src/functions/contains/mod.rs +++ b/sqlite-compress/src/functions/like/mod.rs @@ -1,6 +1,9 @@ use sqlite_ffi::{Context, SqliteError, Value}; -use crate::{functions::contains::{matcher::Matcher, search::search_blob}, state::ExtensionState}; +use crate::{ + functions::like::{matcher::Matcher, search::search_blob}, + state::ExtensionState, +}; const PATTERN_ARG: i32 = 4; @@ -8,14 +11,14 @@ mod matcher; mod search; mod utils; -/// `decompress_cont(schema, table, column, rowid, pattern, offset) -> 0|1` +/// `decompress_like(schema, table, column, rowid, pattern, offset) -> 0|1` /// /// `pattern` uses `LIKE` wildcards (`%`, `_`) and matches anywhere in the /// value, ASCII case-insensitively. -pub fn sqlite_decompress_cont(mut context: Context, values: &[Value]) -> Result<(), SqliteError> { +pub fn sqlite_decompress_like(mut context: Context, values: &[Value]) -> Result<(), SqliteError> { if values.len() < 6 { return Err(SqliteError::Message( - "decompress_cont expects 6 arguments: schema, table, column, rowid, pattern, offset" + "decompress_like expects 6 arguments: schema, table, column, rowid, pattern, offset" .into(), )); } diff --git a/sqlite-compress/src/functions/contains/search.rs b/sqlite-compress/src/functions/like/search.rs similarity index 91% rename from sqlite-compress/src/functions/contains/search.rs rename to sqlite-compress/src/functions/like/search.rs index 6fd809d..388de63 100644 --- a/sqlite-compress/src/functions/contains/search.rs +++ b/sqlite-compress/src/functions/like/search.rs @@ -1,13 +1,16 @@ -use sqlite_ffi::SqliteBlob; -use sqlite_ffi::SqliteError; use std::io::{self, BufReader}; + +use sqlite_ffi::{SqliteBlob, SqliteError}; use zstd::stream::read::Decoder; use crate::{ DictError, DictId, cache::get_decoder_in_cache, - functions::contains::utils::to_sqlite, - functions::{ParsedBlob, contains::matcher::Matcher, header::Header}, + functions::{ + ParsedBlob, + header::Header, + like::{matcher::Matcher, utils::to_sqlite}, + }, state::ExtensionState, }; diff --git a/sqlite-compress/src/functions/contains/utils.rs b/sqlite-compress/src/functions/like/utils.rs similarity index 99% rename from sqlite-compress/src/functions/contains/utils.rs rename to sqlite-compress/src/functions/like/utils.rs index 0e1fe39..7357a3d 100644 --- a/sqlite-compress/src/functions/contains/utils.rs +++ b/sqlite-compress/src/functions/like/utils.rs @@ -1,6 +1,7 @@ -use sqlite_ffi::SqliteError; use std::fmt::Write as _; +use sqlite_ffi::SqliteError; + pub(crate) fn to_sqlite(e: impl std::fmt::Display) -> SqliteError { SqliteError::Message(e.to_string()) } diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index 86c3171..130a08b 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -2,8 +2,8 @@ mod compress; mod decompress; mod errors; mod header; +mod like; mod types; -mod contains; /// Compression level used when configuration does not select another level. pub const DEFAULT_LEVEL: types::Level = types::Level::new(3); @@ -14,5 +14,5 @@ pub(crate) use decompress::{decompress_raw, decompress_with_decoder}; pub use errors::CodecError; pub use header::Header; pub(crate) use header::ParsedBlob; +pub use like::sqlite_decompress_like; pub use types::Level; -pub use contains::sqlite_decompress_cont; \ No newline at end of file diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index 1d0f00e..64339a1 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -34,7 +34,7 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> connection.create_function("decompress", 2, functions::sqlite_decompress)?; - connection.create_function("decompress_cont", 6, functions::sqlite_decompress_cont)?; + connection.create_function("decompress_like", 6, functions::sqlite_decompress_like)?; Ok(()) } From ec9f3d2f35fa55e7a14d465c19a00374f4a3d5bb Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Fri, 23 Oct 2026 09:58:46 +0200 Subject: [PATCH 14/17] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Turn=20matcher=20to?= =?UTF-8?q?=20a=20crate=20and=20apply=20pr=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/validate.yml | 6 +- .mise/tasks/matcher/build | 3 + .mise/tasks/matcher/clippy | 4 + .mise/tasks/matcher/test | 3 + .mise/tasks/matcher/validate | 3 + .mise/tasks/sqlite-compress/clippy | 2 +- .mise/tasks/sqlite-compress/test | 2 +- Cargo.lock | 12 +- Cargo.toml | 6 +- crates/matcher/Cargo.toml | 18 +++ crates/matcher/src/errors.rs | 12 ++ crates/matcher/src/lib.rs | 7 + .../like => crates/matcher/src}/matcher.rs | 29 ++-- crates/matcher/src/utils.rs | 125 ++++++++++++++++++ crates/sqlite-ffi/src/error.rs | 1 + sqlite-compress/Cargo.toml | 10 +- sqlite-compress/src/functions/like/mod.rs | 7 +- sqlite-compress/src/functions/like/search.rs | 20 +-- sqlite-compress/src/functions/like/utils.rs | 27 ---- sqlite-compress/src/functions/mod.rs | 2 + sqlite-compress/src/lib.rs | 1 + 21 files changed, 230 insertions(+), 70 deletions(-) create mode 100755 .mise/tasks/matcher/build create mode 100755 .mise/tasks/matcher/clippy create mode 100755 .mise/tasks/matcher/test create mode 100755 .mise/tasks/matcher/validate create mode 100644 crates/matcher/Cargo.toml create mode 100644 crates/matcher/src/errors.rs create mode 100644 crates/matcher/src/lib.rs rename {sqlite-compress/src/functions/like => crates/matcher/src}/matcher.rs (90%) create mode 100644 crates/matcher/src/utils.rs delete mode 100644 sqlite-compress/src/functions/like/utils.rs diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 83ca2d1..c226a37 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - component: [sqlite-compress, reglite] + component: [sqlite-compress, reglite, matcher] steps: - name: Checkout repository uses: actions/checkout@v6.0.2 @@ -64,7 +64,7 @@ jobs: strategy: fail-fast: false matrix: - component: [sqlite-compress, reglite] + component: [sqlite-compress, reglite, matcher] steps: - name: Checkout repository uses: actions/checkout@v6.0.2 @@ -98,4 +98,4 @@ jobs: log_level: debug install_args: --env ${{ env.MISE_ENV }} - name: Run Miri - run: mise run ${{ matrix.component }}:miri \ No newline at end of file + run: mise run ${{ matrix.component }}:miri diff --git a/.mise/tasks/matcher/build b/.mise/tasks/matcher/build new file mode 100755 index 0000000..d16c531 --- /dev/null +++ b/.mise/tasks/matcher/build @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +#MISE description="Build the project" +cargo build -p matcher \ No newline at end of file diff --git a/.mise/tasks/matcher/clippy b/.mise/tasks/matcher/clippy new file mode 100755 index 0000000..3f20524 --- /dev/null +++ b/.mise/tasks/matcher/clippy @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +#MISE description="Lint with clippy" + +cargo clippy -p matcher --all-targets -- -D warnings diff --git a/.mise/tasks/matcher/test b/.mise/tasks/matcher/test new file mode 100755 index 0000000..5dff5a2 --- /dev/null +++ b/.mise/tasks/matcher/test @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +#MISE description="Run tests" +cargo test -p matcher -- --test-threads=1 diff --git a/.mise/tasks/matcher/validate b/.mise/tasks/matcher/validate new file mode 100755 index 0000000..bfdeaf8 --- /dev/null +++ b/.mise/tasks/matcher/validate @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +#MISE description="Run all checks (fmt, clippy, test)" +#MISE depends=["fmt:dev", "matcher:clippy", "matcher:test"] \ No newline at end of file diff --git a/.mise/tasks/sqlite-compress/clippy b/.mise/tasks/sqlite-compress/clippy index 1d9659f..d721d67 100755 --- a/.mise/tasks/sqlite-compress/clippy +++ b/.mise/tasks/sqlite-compress/clippy @@ -1,4 +1,4 @@ #!/usr/bin/env bash #MISE description="Lint with clippy" -cargo clippy -p sqlite-compress --all-targets --no-default-features --features loadable_extension -- -D warnings +cargo clippy -p sqlite-compress --all-targets --no-default-features --features loadable_extension,like -- -D warnings diff --git a/.mise/tasks/sqlite-compress/test b/.mise/tasks/sqlite-compress/test index 12f2954..29b0714 100755 --- a/.mise/tasks/sqlite-compress/test +++ b/.mise/tasks/sqlite-compress/test @@ -1,3 +1,3 @@ #!/usr/bin/env bash #MISE description="Run tests" -cargo test -p sqlite-compress -- --test-threads=1 +cargo test -p sqlite-compress --features like -- --test-threads=1 diff --git a/Cargo.lock b/Cargo.lock index 7c9d114..a3bf66d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "matcher" +version = "0.1.0" +dependencies = [ + "memchr", + "regex-automata", + "thiserror", +] + [[package]] name = "memchr" version = "2.8.3" @@ -232,9 +241,8 @@ name = "sqlite-compress" version = "0.1.0" dependencies = [ "libsqlite3-sys", - "memchr", + "matcher", "parking_lot", - "regex-automata", "sqlite-ffi", "sqlite-ffi-macros", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 101dfd0..c9eedaa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,8 @@ members = [ "sqlite-compress", "reglite", "crates/sqlite-ffi", - "crates/sqlite-ffi-macros", + "crates/sqlite-ffi-macros", + "crates/matcher", ] resolver = "2" @@ -11,4 +12,5 @@ resolver = "2" thiserror = "2.0" libsqlite3-sys = { version = "=0.30.1", features = ["bundled"], package = "libsqlite3-sys" } sqlite-ffi = { version = "0.1.0", path = "crates/sqlite-ffi", default-features = false } -sqlite-ffi-macros = { version = "0.1.0", path = "crates/sqlite-ffi-macros" } \ No newline at end of file +sqlite-ffi-macros = { version = "0.1.0", path = "crates/sqlite-ffi-macros" } +matcher = { version = "0.1.0", path = "crates/matcher" } \ No newline at end of file diff --git a/crates/matcher/Cargo.toml b/crates/matcher/Cargo.toml new file mode 100644 index 0000000..bac6786 --- /dev/null +++ b/crates/matcher/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "matcher" +authors = ["Caido Labs Inc. "] +repository = "https://github.com/caido/sqlite-rs" +license = "MIT" +version = "0.1.0" +edition = "2024" + +[dependencies] +memchr = { version = "2.8.3", default-features = false, features = ["alloc"] } +regex-automata = { version = "0.4", default-features = false, features = [ + "std", + "syntax", + "unicode", + "hybrid", + "perf-inline", +] } +thiserror = { workspace = true } diff --git a/crates/matcher/src/errors.rs b/crates/matcher/src/errors.rs new file mode 100644 index 0000000..645de46 --- /dev/null +++ b/crates/matcher/src/errors.rs @@ -0,0 +1,12 @@ +use std::io; + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum MatcherError { + #[error("invalid LIKE pattern: {0}")] + Pattern(String), + + #[error(transparent)] + Io(#[from] io::Error), +} diff --git a/crates/matcher/src/lib.rs b/crates/matcher/src/lib.rs new file mode 100644 index 0000000..ecd27ad --- /dev/null +++ b/crates/matcher/src/lib.rs @@ -0,0 +1,7 @@ +mod errors; +mod matcher; +mod utils; + +pub use errors::MatcherError; +pub use matcher::Matcher; +pub use utils::{like_literal_segments, like_to_regex}; diff --git a/sqlite-compress/src/functions/like/matcher.rs b/crates/matcher/src/matcher.rs similarity index 90% rename from sqlite-compress/src/functions/like/matcher.rs rename to crates/matcher/src/matcher.rs index f2d77f1..884ffb9 100644 --- a/sqlite-compress/src/functions/like/matcher.rs +++ b/crates/matcher/src/matcher.rs @@ -10,14 +10,16 @@ use regex_automata::{ nfa::thompson, util::{start, syntax}, }; -use sqlite_ffi::SqliteError; -use crate::functions::like::utils::{like_to_regex, to_sqlite}; +use crate::{ + errors::MatcherError, + utils::{like_literal_segments, like_to_regex}, +}; const STREAM_CHUNK: usize = 16 * 1024; /// Compiled `LIKE` pattern, cached across rows through SQLite auxdata. -pub(crate) enum Matcher { +pub enum Matcher { /// `%`-only pattern: lowercase literals that must appear in order. Literals(Vec>), /// Pattern with `_`: lazy DFA driven byte by byte across chunks. @@ -28,11 +30,10 @@ pub(crate) enum Matcher { } impl Matcher { - pub(crate) fn compile(pattern: &[u8]) -> Result { - if !pattern.contains(&b'_') { - let finders = pattern - .split(|&b| b == b'%') - .filter(|s| !s.is_empty()) + pub fn compile(pattern: &[u8]) -> Result { + if let Some(segments) = like_literal_segments(pattern) { + let finders = segments + .into_iter() .map(|s| Finder::new(&s.to_ascii_lowercase()).into_owned()) .collect(); return Ok(Self::Literals(finders)); @@ -42,15 +43,17 @@ impl Matcher { .syntax(syntax::Config::new().utf8(false)) .thompson(thompson::Config::new().utf8(false)) .build(&like_to_regex(pattern)) - .map_err(to_sqlite)?; + .map_err(|e| MatcherError::Pattern(e.to_string()))?; + let cache = RefCell::new(dfa.create_cache()); + Ok(Self::Dfa { dfa: Box::new(dfa), cache: Box::new(cache), }) } - pub(crate) fn find_in(&self, reader: R, skip: usize) -> io::Result { + pub fn find_in(&self, reader: R, skip: usize) -> Result { match self { Self::Literals(finders) => find_literals(reader, finders, skip), Self::Dfa { dfa, cache } => find_dfa(reader, dfa, &mut cache.borrow_mut(), skip), @@ -67,7 +70,7 @@ fn find_literals( mut reader: R, finders: &[Finder<'static>], mut skip: usize, -) -> io::Result { +) -> Result { let Some(longest) = finders.iter().map(|f| f.needle().len()).max() else { return Ok(true); }; @@ -115,7 +118,7 @@ fn find_dfa( dfa: &DFA, cache: &mut Cache, mut skip: usize, -) -> io::Result { +) -> Result { let mut sid = dfa .start_state(cache, &start::Config::new().anchored(Anchored::No)) .map_err(io::Error::other)?; @@ -147,7 +150,7 @@ fn find_dfa( return Ok(false); } if sid.is_quit() { - return Err(io::Error::other("pattern DFA quit")); + return Err(MatcherError::Io(io::Error::other("pattern DFA quit"))); } } } diff --git a/crates/matcher/src/utils.rs b/crates/matcher/src/utils.rs new file mode 100644 index 0000000..c648e46 --- /dev/null +++ b/crates/matcher/src/utils.rs @@ -0,0 +1,125 @@ +use std::fmt::Write as _; + +/// Translates a `LIKE` pattern into a byte-oriented regex. +/// +/// Literals are emitted as `\xHH` so ASCII folds case and other bytes match +/// verbatim. `_` is one UTF-8 character, or one byte that cannot start a +/// valid sequence so invalid UTF-8 still advances. +pub fn like_to_regex(pattern: &[u8]) -> String { + let mut re = String::with_capacity(8 + pattern.len() * 4); + re.push_str("(?is-u)"); + let mut i = 0; + while i < pattern.len() { + match pattern[i] { + b'\\' if i + 1 < pattern.len() => { + let _ = write!(re, r"\x{:02X}", pattern[i + 1]); + i += 2; + } + b'\\' => { + let _ = write!(re, r"\x{:02X}", b'\\'); + i += 1; + } + b'%' => { + re.push_str(".*"); + i += 1; + } + b'_' => { + re.push_str(r"(?:(?u-i:.)|[\x80-\xBF\xC0\xC1\xF5-\xFF])"); + i += 1; + } + b => { + let _ = write!(re, r"\x{b:02X}"); + i += 1; + } + } + } + re +} + +pub fn like_literal_segments(pattern: &[u8]) -> Option>> { + let mut segments = Vec::new(); + let mut cur = Vec::new(); + let mut i = 0; + while i < pattern.len() { + match pattern[i] { + b'\\' if i + 1 < pattern.len() => { + cur.push(pattern[i + 1]); + i += 2; + } + b'\\' => { + cur.push(b'\\'); + i += 1; + } + b'%' => { + if !cur.is_empty() { + segments.push(std::mem::take(&mut cur)); + } + i += 1; + } + b'_' => return None, + b => { + cur.push(b); + i += 1; + } + } + } + if !cur.is_empty() { + segments.push(cur); + } + Some(segments) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn segs(p: &str) -> Option>> { + like_literal_segments(p.as_bytes()) + } + + #[test] + fn literal_segments_split_on_unescaped_percent() { + assert_eq!(segs("hello"), Some(vec![b"hello".to_vec()])); + assert_eq!( + segs("say%WORLD"), + Some(vec![b"say".to_vec(), b"WORLD".to_vec()]) + ); + assert_eq!(segs("%%"), Some(vec![])); + assert_eq!(segs("%abc%"), Some(vec![b"abc".to_vec()])); + assert_eq!(segs(""), Some(vec![])); + } + + #[test] + fn literal_segments_unescape() { + assert_eq!(segs(r"foo\%bar"), Some(vec![b"foo%bar".to_vec()])); + assert_eq!(segs(r"a\%b%c"), Some(vec![b"a%b".to_vec(), b"c".to_vec()])); + assert_eq!(segs(r"\_"), Some(vec![b"_".to_vec()])); // échappé → littéral, pas DFA + assert_eq!(segs(r"\\"), Some(vec![b"\\".to_vec()])); + assert_eq!(segs(r"\\%x"), Some(vec![b"\\".to_vec(), b"x".to_vec()])); + assert_eq!(segs(r"a\xb"), Some(vec![b"axb".to_vec()])); // `\x` → `x` + assert_eq!(segs(r"trail\"), Some(vec![b"trail\\".to_vec()])); + } + + #[test] + fn literal_segments_none_on_unescaped_underscore() { + assert_eq!(segs("a_c"), None); + assert_eq!(segs(r"a\_c_d"), None); // un `_` libre suffit + assert_eq!(segs(r"\\_"), None); // `\\` → `\`, puis `_` wildcard + } + + #[test] + fn like_to_regex_escapes() { + assert_eq!( + like_to_regex(br"foo\%bar"), + r"(?is-u)\x66\x6F\x6F\x25\x62\x61\x72" + ); + assert_eq!(like_to_regex(br"a\%b%c"), r"(?is-u)\x61\x25\x62.*\x63"); + assert_eq!(like_to_regex(br"\_"), r"(?is-u)\x5F"); + assert_eq!(like_to_regex(br"\\"), r"(?is-u)\x5C"); + assert_eq!(like_to_regex(br"\\%"), r"(?is-u)\x5C.*"); + let re = like_to_regex(br"\\_"); + assert!(re.starts_with(r"(?is-u)\x5C")); + assert!(re.contains(r"(?:(?u-i:.)|")); + assert!(!re.contains(r"\x5F")); // `_` n'est pas littéral + } +} diff --git a/crates/sqlite-ffi/src/error.rs b/crates/sqlite-ffi/src/error.rs index 412ea52..c6f63c0 100644 --- a/crates/sqlite-ffi/src/error.rs +++ b/crates/sqlite-ffi/src/error.rs @@ -37,6 +37,7 @@ impl SqliteError { } /// # Safety + /// `db` is the database connection pointer passed by SQLite into the C entrypoint #[cold] pub(crate) unsafe fn from_db(db: *mut sqlite3, operation: &'static str, code: i32) -> Self { let message = unsafe { diff --git a/sqlite-compress/Cargo.toml b/sqlite-compress/Cargo.toml index cedf9fe..97cb28c 100644 --- a/sqlite-compress/Cargo.toml +++ b/sqlite-compress/Cargo.toml @@ -14,20 +14,14 @@ thiserror = { workspace = true } libsqlite3-sys = { workspace = true } sqlite-ffi = { workspace = true } sqlite-ffi-macros = { workspace = true } -memchr = "2.8.3" -regex-automata = { version = "0.4", default-features = false, features = [ - "std", - "syntax", - "unicode", - "hybrid", - "perf-inline", -] } +matcher = { workspace = true, optional = true } [lib] crate-type = ["cdylib", "rlib"] [features] default = ["bundled"] +like = ["dep:matcher"] bundled = [ "sqlite-ffi/bundled", "libsqlite3-sys/bundled", diff --git a/sqlite-compress/src/functions/like/mod.rs b/sqlite-compress/src/functions/like/mod.rs index 17afe7a..2cd4e52 100644 --- a/sqlite-compress/src/functions/like/mod.rs +++ b/sqlite-compress/src/functions/like/mod.rs @@ -1,15 +1,14 @@ +use matcher::Matcher; use sqlite_ffi::{Context, SqliteError, Value}; use crate::{ - functions::like::{matcher::Matcher, search::search_blob}, + functions::like::search::{search_blob, to_sqlite}, state::ExtensionState, }; const PATTERN_ARG: i32 = 4; -mod matcher; mod search; -mod utils; /// `decompress_like(schema, table, column, rowid, pattern, offset) -> 0|1` /// @@ -39,7 +38,7 @@ pub fn sqlite_decompress_like(mut context: Context, values: &[Value]) -> Result< let found = match aux.get() { Ok(matcher) => search_blob(&state, schema, &blob, matcher, offset)?, Err(_) => { - let matcher = Matcher::compile(pattern)?; + let matcher = Matcher::compile(pattern).map_err(to_sqlite)?; let found = search_blob(&state, schema, &blob, &matcher, offset)?; aux.set(matcher); found diff --git a/sqlite-compress/src/functions/like/search.rs b/sqlite-compress/src/functions/like/search.rs index 388de63..494bb31 100644 --- a/sqlite-compress/src/functions/like/search.rs +++ b/sqlite-compress/src/functions/like/search.rs @@ -1,16 +1,13 @@ use std::io::{self, BufReader}; +use matcher::Matcher; use sqlite_ffi::{SqliteBlob, SqliteError}; -use zstd::stream::read::Decoder; +use zstd::{stream::read::Decoder, zstd_safe}; use crate::{ DictError, DictId, cache::get_decoder_in_cache, - functions::{ - ParsedBlob, - header::Header, - like::{matcher::Matcher, utils::to_sqlite}, - }, + functions::{ParsedBlob, header::Header}, state::ExtensionState, }; @@ -33,9 +30,9 @@ pub(crate) fn search_blob( match Header::parse_blob(&peek) { Ok(ParsedBlob::Plain) | Err(_) if peek_len < Header::PREFIX_SIZE => { - matcher.find_in(blob.reader(0), offset).map_err(to_sqlite) + matcher.find_in(blob.reader(offset), 0).map_err(to_sqlite) } - Ok(ParsedBlob::Plain) => matcher.find_in(blob.reader(0), offset).map_err(to_sqlite), + Ok(ParsedBlob::Plain) => matcher.find_in(blob.reader(offset), 0).map_err(to_sqlite), Ok(ParsedBlob::Framed(header, _)) => { let header_len = header.header_len as usize; @@ -50,7 +47,8 @@ pub(crate) fn search_blob( None }; - let reader = BufReader::with_capacity(64 * 1024, blob.reader(header_len)); + let reader = + BufReader::with_capacity(zstd_safe::DCtx::in_size(), blob.reader(header_len)); let decoder = match prepared.as_deref() { Some(dict) => Decoder::with_prepared_dictionary(reader, dict), None => Decoder::with_dictionary(reader, &[]), @@ -62,3 +60,7 @@ pub(crate) fn search_blob( Err(e) => Err(SqliteError::Message(e.to_string())), } } + +pub(crate) fn to_sqlite(e: impl std::fmt::Display) -> SqliteError { + SqliteError::Message(e.to_string()) +} diff --git a/sqlite-compress/src/functions/like/utils.rs b/sqlite-compress/src/functions/like/utils.rs deleted file mode 100644 index 7357a3d..0000000 --- a/sqlite-compress/src/functions/like/utils.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::fmt::Write as _; - -use sqlite_ffi::SqliteError; - -pub(crate) fn to_sqlite(e: impl std::fmt::Display) -> SqliteError { - SqliteError::Message(e.to_string()) -} - -/// Translates a `LIKE` pattern into a byte-oriented regex. -/// -/// Literals are emitted as `\xHH` so ASCII folds case and other bytes match -/// verbatim. `_` is one UTF-8 character, or one byte that cannot start a -/// valid sequence so invalid UTF-8 still advances. -pub(crate) fn like_to_regex(pattern: &[u8]) -> String { - let mut re = String::with_capacity(8 + pattern.len() * 4); - re.push_str("(?is-u)"); - for &b in pattern { - match b { - b'%' => re.push_str(".*"), - b'_' => re.push_str(r"(?:(?u-i:.)|[\x80-\xBF\xC0\xC1\xF5-\xFF])"), - _ => { - let _ = write!(re, r"\x{b:02X}"); - } - } - } - re -} diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index 130a08b..68689da 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -2,6 +2,7 @@ mod compress; mod decompress; mod errors; mod header; +#[cfg(feature = "like")] mod like; mod types; @@ -14,5 +15,6 @@ pub(crate) use decompress::{decompress_raw, decompress_with_decoder}; pub use errors::CodecError; pub use header::Header; pub(crate) use header::ParsedBlob; +#[cfg(feature = "like")] pub use like::sqlite_decompress_like; pub use types::Level; diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index 64339a1..056e337 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -34,6 +34,7 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> connection.create_function("decompress", 2, functions::sqlite_decompress)?; + #[cfg(feature = "like")] connection.create_function("decompress_like", 6, functions::sqlite_decompress_like)?; Ok(()) From 77194ea88ee018a4dcfdb41ba2f5d435a2423261 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Fri, 23 Oct 2026 10:36:03 +0200 Subject: [PATCH 15/17] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Turn=20dfa=20to=20a?= =?UTF-8?q?=20common=20struct=20with=20segment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 1 - crates/matcher/Cargo.toml | 7 - crates/matcher/src/errors.rs | 3 - crates/matcher/src/lib.rs | 1 - crates/matcher/src/matcher.rs | 244 +++++++++++++++------- crates/matcher/src/utils.rs | 146 ++++++------- sqlite-compress/src/functions/like/mod.rs | 7 +- 7 files changed, 240 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3bf66d..e178e41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,7 +96,6 @@ name = "matcher" version = "0.1.0" dependencies = [ "memchr", - "regex-automata", "thiserror", ] diff --git a/crates/matcher/Cargo.toml b/crates/matcher/Cargo.toml index bac6786..777cbeb 100644 --- a/crates/matcher/Cargo.toml +++ b/crates/matcher/Cargo.toml @@ -8,11 +8,4 @@ edition = "2024" [dependencies] memchr = { version = "2.8.3", default-features = false, features = ["alloc"] } -regex-automata = { version = "0.4", default-features = false, features = [ - "std", - "syntax", - "unicode", - "hybrid", - "perf-inline", -] } thiserror = { workspace = true } diff --git a/crates/matcher/src/errors.rs b/crates/matcher/src/errors.rs index 645de46..935cf4f 100644 --- a/crates/matcher/src/errors.rs +++ b/crates/matcher/src/errors.rs @@ -4,9 +4,6 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum MatcherError { - #[error("invalid LIKE pattern: {0}")] - Pattern(String), - #[error(transparent)] Io(#[from] io::Error), } diff --git a/crates/matcher/src/lib.rs b/crates/matcher/src/lib.rs index ecd27ad..c756394 100644 --- a/crates/matcher/src/lib.rs +++ b/crates/matcher/src/lib.rs @@ -4,4 +4,3 @@ mod utils; pub use errors::MatcherError; pub use matcher::Matcher; -pub use utils::{like_literal_segments, like_to_regex}; diff --git a/crates/matcher/src/matcher.rs b/crates/matcher/src/matcher.rs index 884ffb9..1e8c586 100644 --- a/crates/matcher/src/matcher.rs +++ b/crates/matcher/src/matcher.rs @@ -1,62 +1,43 @@ -use std::{ - cell::RefCell, - io::{self, Read}, -}; +use std::io::{self, Read}; use memchr::memmem::Finder; -use regex_automata::{ - Anchored, - hybrid::dfa::{Cache, DFA}, - nfa::thompson, - util::{start, syntax}, -}; use crate::{ errors::MatcherError, - utils::{like_literal_segments, like_to_regex}, + utils::{Segment, compile_segments}, }; const STREAM_CHUNK: usize = 16 * 1024; /// Compiled `LIKE` pattern, cached across rows through SQLite auxdata. pub enum Matcher { - /// `%`-only pattern: lowercase literals that must appear in order. - Literals(Vec>), - /// Pattern with `_`: lazy DFA driven byte by byte across chunks. - Dfa { - dfa: Box, - cache: Box>, - }, + /// No `_`: lowercase literals that must appear in order. + Ordered(Vec>), + /// At least one `_`: segments matched over the buffered stream. + WithAny(Vec), } impl Matcher { - pub fn compile(pattern: &[u8]) -> Result { - if let Some(segments) = like_literal_segments(pattern) { - let finders = segments - .into_iter() - .map(|s| Finder::new(&s.to_ascii_lowercase()).into_owned()) - .collect(); - return Ok(Self::Literals(finders)); + pub fn compile(pattern: &[u8]) -> Self { + let segments = compile_segments(pattern); + if segments.iter().any(|s| matches!(s, Segment::Any)) { + return Self::WithAny(segments); } - let dfa = DFA::builder() - .syntax(syntax::Config::new().utf8(false)) - .thompson(thompson::Config::new().utf8(false)) - .build(&like_to_regex(pattern)) - .map_err(|e| MatcherError::Pattern(e.to_string()))?; - - let cache = RefCell::new(dfa.create_cache()); - - Ok(Self::Dfa { - dfa: Box::new(dfa), - cache: Box::new(cache), - }) + let finders = segments + .into_iter() + .filter_map(|s| match s { + Segment::Literal(f) => Some(f), + _ => None, + }) + .collect(); + Self::Ordered(finders) } pub fn find_in(&self, reader: R, skip: usize) -> Result { match self { - Self::Literals(finders) => find_literals(reader, finders, skip), - Self::Dfa { dfa, cache } => find_dfa(reader, dfa, &mut cache.borrow_mut(), skip), + Self::Ordered(finders) => fast_path_ordered_segments(reader, finders, skip), + Self::WithAny(segments) => slow_path_with_any(reader, segments, skip), } } } @@ -66,7 +47,7 @@ impl Matcher { /// Leftmost matching is optimal for `%`-separated literals, so no /// backtracking is needed. The window keeps `len - 1` bytes of the current /// literal's prefix so matches spanning chunks are found. -fn find_literals( +fn fast_path_ordered_segments( mut reader: R, finders: &[Finder<'static>], mut skip: usize, @@ -113,51 +94,153 @@ fn find_literals( } } -fn find_dfa( +/// Matches each `%`-separated group at its leftmost position, over a buffer +/// holding everything read so far. +/// +/// The leftmost match of a group leaves the most room for the next ones, so +/// backtracking never crosses a `Skip`: on mismatch, only the group's start +/// moves forward, to the next hit of its first literal. +fn slow_path_with_any( mut reader: R, - dfa: &DFA, - cache: &mut Cache, + segments: &[Segment], mut skip: usize, ) -> Result { - let mut sid = dfa - .start_state(cache, &start::Config::new().anchored(Anchored::No)) - .map_err(io::Error::other)?; - let mut chunk = vec![0u8; STREAM_CHUNK]; + let mut buf = Vec::new(); + let mut eof = false; + let mut group_start = 0; + let mut pos = 0; loop { - let n = reader.read(&mut chunk)?; - if n == 0 { - break; - } - let mut data = &chunk[..n]; + let group_end = segments[group_start..] + .iter() + .position(|s| matches!(s, Segment::Skip)) + .map_or(segments.len(), |i| group_start + i); - if skip > 0 { - if skip >= data.len() { - skip -= data.len(); - continue; + match find_group(&segments[group_start..group_end], &buf, pos, eof) { + Group::Found(end) => { + if group_end == segments.len() { + return Ok(true); + } + group_start = group_end + 1; + pos = end; + } + Group::Missing => return Ok(false), + Group::Pending(from) => { + pos = from; + eof = !read_more(&mut reader, &mut buf, &mut skip)?; } - data = &data[skip..]; - skip = 0; } + } +} - for &b in data { - sid = dfa.next_state(cache, sid, b).map_err(io::Error::other)?; - if sid.is_tagged() { - if sid.is_match() { - return Ok(true); +/// Appends the next bytes past `skip`, lowercased; returns `false` at EOF. +fn read_more(reader: &mut R, buf: &mut Vec, skip: &mut usize) -> io::Result { + loop { + let len = buf.len(); + buf.resize(len + STREAM_CHUNK, 0); + let n = reader.read(&mut buf[len..])?; + let dropped = n.min(*skip); + *skip -= dropped; + buf.copy_within(len + dropped..len + n, len); + buf.truncate(len + n - dropped); + + if n == 0 { + return Ok(false); + } + if n > dropped { + buf[len..].make_ascii_lowercase(); + return Ok(true); + } + } +} + +enum Group { + /// Matched, ending at this offset. + Found(usize), + /// No match starts before this offset, more input is needed past it. + Pending(usize), + Missing, +} + +/// Finds the leftmost match of a `Skip`-free group starting at or after `from`. +fn find_group(group: &[Segment], hay: &[u8], mut from: usize, eof: bool) -> Group { + loop { + let start = match &group[0] { + Segment::Literal(f) => match f.find(&hay[from..]) { + Some(i) => from + i, + None if eof => return Group::Missing, + None => { + let tail = hay.len().saturating_sub(f.needle().len() - 1); + return Group::Pending(tail.max(from)); } - if sid.is_dead() { - return Ok(false); + }, + _ if from < hay.len() => from, + _ if eof => return Group::Missing, + _ => return Group::Pending(from), + }; + + match match_at(group, hay, start, eof) { + Step::Match(end) => return Group::Found(end), + Step::Mismatch => from = start + 1, + Step::NeedMore => return Group::Pending(start), + } + } +} + +enum Step { + Match(usize), + Mismatch, + NeedMore, +} + +/// Matches a `Skip`-free group anchored at `pos`. +fn match_at(group: &[Segment], hay: &[u8], mut pos: usize, eof: bool) -> Step { + for segment in group { + let rest = &hay[pos..]; + match segment { + Segment::Literal(f) => { + let needle = f.needle(); + if rest.len() < needle.len() { + return if !eof && needle.starts_with(rest) { + Step::NeedMore + } else { + Step::Mismatch + }; } - if sid.is_quit() { - return Err(MatcherError::Io(io::Error::other("pattern DFA quit"))); + if !rest.starts_with(needle) { + return Step::Mismatch; } + pos += needle.len(); } + Segment::Any => match any_len(rest, eof) { + Some(0) => return Step::Mismatch, + Some(n) => pos += n, + None => return Step::NeedMore, + }, + Segment::Skip => unreachable!("groups are split on Skip"), } } + Step::Match(pos) +} - let sid = dfa.next_eoi_state(cache, sid).map_err(io::Error::other)?; - Ok(sid.is_match()) +/// Bytes taken by one `_` at the start of `hay`: a whole UTF-8 character, or +/// a single byte when the sequence is invalid. `None` when more input could +/// change the answer. +fn any_len(hay: &[u8], eof: bool) -> Option { + let Some(&lead) = hay.first() else { + return if eof { Some(0) } else { None }; + }; + let width = match lead { + 0xC2..=0xDF => 2, + 0xE0..=0xEF => 3, + 0xF0..=0xF4 => 4, + _ => return Some(1), + }; + match std::str::from_utf8(&hay[..width.min(hay.len())]) { + Ok(_) => Some(width), + Err(e) if e.error_len().is_none() && !eof => None, + Err(_) => Some(1), + } } #[cfg(test)] @@ -166,7 +249,6 @@ mod tests { fn matches(pattern: &str, haystack: &[u8]) -> bool { Matcher::compile(pattern.as_bytes()) - .unwrap() .find_in(haystack, 0) .unwrap() } @@ -201,25 +283,43 @@ mod tests { assert!(matches("a_c", "aéc".as_bytes())); assert!(!matches("a__c", "aéc".as_bytes())); assert!(matches("a_c", b"a\xFFc")); + assert!(matches("a_c", b"a\xE2c")); assert!(!matches("a_c", b"ac")); assert!(matches("_", b"z")); assert!(!matches("_", b"")); assert!(matches("a_%z", b"ab----Z")); + assert!(matches("%_%", b"z")); + assert!(!matches("a_", b"a")); + } + + #[test] + fn backtracks_to_next_literal_hit() { + assert!(matches("test_oui", b"testXtestYoui")); + assert!(matches("te%test_oui", b"te---testX---te---testYoui")); + assert!(!matches("te%test_oui", b"te---testX---testYnon")); + assert!(matches("a_c", b"axxa_c")); + assert!(matches("aa_b", b"aaaxb")); + assert!(matches("_%z", b"..z")); + assert!(!matches("_%z", b"z")); + assert!(matches("_b", b"aab")); } #[test] fn matches_span_chunks() { for pattern in ["needle%tail", "ne_dle%tail"] { - let m = Matcher::compile(pattern.as_bytes()).unwrap(); + let m = Matcher::compile(pattern.as_bytes()); assert!(m.find_in(Trickle(b"..NEEDLE..TAIL.."), 0).unwrap()); assert!(!m.find_in(Trickle(b"..TAIL..NEEDLE.."), 0).unwrap()); } + let m = Matcher::compile(b"a_c"); + assert!(m.find_in(Trickle("xaéc".as_bytes()), 0).unwrap()); + assert!(!m.find_in(Trickle("xaé".as_bytes()), 0).unwrap()); } #[test] fn offset_skips_prefix() { for pattern in ["abc", "a_c"] { - let m = Matcher::compile(pattern.as_bytes()).unwrap(); + let m = Matcher::compile(pattern.as_bytes()); assert!(m.find_in(&b"abc---abc"[..], 3).unwrap()); assert!(!m.find_in(&b"abc------"[..], 1).unwrap()); } diff --git a/crates/matcher/src/utils.rs b/crates/matcher/src/utils.rs index c648e46..536126e 100644 --- a/crates/matcher/src/utils.rs +++ b/crates/matcher/src/utils.rs @@ -1,42 +1,20 @@ -use std::fmt::Write as _; +use memchr::memmem::Finder; -/// Translates a `LIKE` pattern into a byte-oriented regex. -/// -/// Literals are emitted as `\xHH` so ASCII folds case and other bytes match -/// verbatim. `_` is one UTF-8 character, or one byte that cannot start a -/// valid sequence so invalid UTF-8 still advances. -pub fn like_to_regex(pattern: &[u8]) -> String { - let mut re = String::with_capacity(8 + pattern.len() * 4); - re.push_str("(?is-u)"); - let mut i = 0; - while i < pattern.len() { - match pattern[i] { - b'\\' if i + 1 < pattern.len() => { - let _ = write!(re, r"\x{:02X}", pattern[i + 1]); - i += 2; - } - b'\\' => { - let _ = write!(re, r"\x{:02X}", b'\\'); - i += 1; - } - b'%' => { - re.push_str(".*"); - i += 1; - } - b'_' => { - re.push_str(r"(?:(?u-i:.)|[\x80-\xBF\xC0\xC1\xF5-\xFF])"); - i += 1; - } - b => { - let _ = write!(re, r"\x{b:02X}"); - i += 1; - } - } - } - re +/// One piece of a compiled `LIKE` pattern. +pub enum Segment { + /// ASCII-lowercased bytes that must appear verbatim. + Literal(Finder<'static>), + /// `%`: any run of bytes, possibly empty. + Skip, + /// `_`: one UTF-8 character, or one byte when the sequence is invalid. + Any, } -pub fn like_literal_segments(pattern: &[u8]) -> Option>> { +/// Splits a `LIKE` pattern into segments. +/// +/// Runs of `%` collapse into one `Skip`, and leading or trailing `Skip`s are +/// dropped since matching is unanchored on both ends. +pub(crate) fn compile_segments(pattern: &[u8]) -> Vec { let mut segments = Vec::new(); let mut cur = Vec::new(); let mut i = 0; @@ -46,80 +24,88 @@ pub fn like_literal_segments(pattern: &[u8]) -> Option>> { cur.push(pattern[i + 1]); i += 2; } - b'\\' => { - cur.push(b'\\'); - i += 1; - } b'%' => { - if !cur.is_empty() { - segments.push(std::mem::take(&mut cur)); + push_literal(&mut segments, &mut cur); + if !segments.is_empty() && !matches!(segments.last(), Some(Segment::Skip)) { + segments.push(Segment::Skip); } i += 1; } - b'_' => return None, + b'_' => { + push_literal(&mut segments, &mut cur); + segments.push(Segment::Any); + i += 1; + } b => { cur.push(b); i += 1; } } } - if !cur.is_empty() { - segments.push(cur); + push_literal(&mut segments, &mut cur); + if matches!(segments.last(), Some(Segment::Skip)) { + segments.pop(); } - Some(segments) + segments +} + +fn push_literal(segments: &mut Vec, cur: &mut Vec) { + if cur.is_empty() { + return; + } + cur.make_ascii_lowercase(); + segments.push(Segment::Literal(Finder::new(cur).into_owned())); + cur.clear(); } #[cfg(test)] mod tests { use super::*; - fn segs(p: &str) -> Option>> { - like_literal_segments(p.as_bytes()) + fn segs(p: &str) -> Vec { + compile_segments(p.as_bytes()) + .iter() + .map(|s| match s { + Segment::Literal(f) => String::from_utf8_lossy(f.needle()).into_owned(), + Segment::Skip => "".into(), + Segment::Any => "".into(), + }) + .collect() } #[test] - fn literal_segments_split_on_unescaped_percent() { - assert_eq!(segs("hello"), Some(vec![b"hello".to_vec()])); + fn splits_on_wildcards() { assert_eq!( - segs("say%WORLD"), - Some(vec![b"say".to_vec(), b"WORLD".to_vec()]) + segs("te%test_oui"), + ["te", "", "test", "", "oui"] ); - assert_eq!(segs("%%"), Some(vec![])); - assert_eq!(segs("%abc%"), Some(vec![b"abc".to_vec()])); - assert_eq!(segs(""), Some(vec![])); + assert_eq!(segs("a_%z"), ["a", "", "", "z"]); + assert_eq!(segs("__"), ["", ""]); } #[test] - fn literal_segments_unescape() { - assert_eq!(segs(r"foo\%bar"), Some(vec![b"foo%bar".to_vec()])); - assert_eq!(segs(r"a\%b%c"), Some(vec![b"a%b".to_vec(), b"c".to_vec()])); - assert_eq!(segs(r"\_"), Some(vec![b"_".to_vec()])); // échappé → littéral, pas DFA - assert_eq!(segs(r"\\"), Some(vec![b"\\".to_vec()])); - assert_eq!(segs(r"\\%x"), Some(vec![b"\\".to_vec(), b"x".to_vec()])); - assert_eq!(segs(r"a\xb"), Some(vec![b"axb".to_vec()])); // `\x` → `x` - assert_eq!(segs(r"trail\"), Some(vec![b"trail\\".to_vec()])); + fn normalizes_skips() { + assert_eq!(segs(""), [] as [&str; 0]); + assert_eq!(segs("%%"), [] as [&str; 0]); + assert_eq!(segs("%abc%"), ["abc"]); + assert_eq!(segs("a%%b"), ["a", "", "b"]); + assert_eq!(segs("_abc_%"), ["", "abc", ""]); } #[test] - fn literal_segments_none_on_unescaped_underscore() { - assert_eq!(segs("a_c"), None); - assert_eq!(segs(r"a\_c_d"), None); // un `_` libre suffit - assert_eq!(segs(r"\\_"), None); // `\\` → `\`, puis `_` wildcard + fn lowercases_ascii_literals() { + assert_eq!(segs("SaY%WÉrld"), ["say", "", "wÉrld"]); } #[test] - fn like_to_regex_escapes() { - assert_eq!( - like_to_regex(br"foo\%bar"), - r"(?is-u)\x66\x6F\x6F\x25\x62\x61\x72" - ); - assert_eq!(like_to_regex(br"a\%b%c"), r"(?is-u)\x61\x25\x62.*\x63"); - assert_eq!(like_to_regex(br"\_"), r"(?is-u)\x5F"); - assert_eq!(like_to_regex(br"\\"), r"(?is-u)\x5C"); - assert_eq!(like_to_regex(br"\\%"), r"(?is-u)\x5C.*"); - let re = like_to_regex(br"\\_"); - assert!(re.starts_with(r"(?is-u)\x5C")); - assert!(re.contains(r"(?:(?u-i:.)|")); - assert!(!re.contains(r"\x5F")); // `_` n'est pas littéral + fn unescapes() { + assert_eq!(segs(r"foo\%bar"), ["foo%bar"]); + assert_eq!(segs(r"a\%b%c"), ["a%b", "", "c"]); + assert_eq!(segs(r"\_"), ["_"]); + assert_eq!(segs(r"\\"), ["\\"]); + assert_eq!(segs(r"\\%x"), ["\\", "", "x"]); + assert_eq!(segs(r"\\_"), ["\\", ""]); + assert_eq!(segs(r"a\xb"), ["axb"]); + assert_eq!(segs(r"trail\"), ["trail\\"]); } } diff --git a/sqlite-compress/src/functions/like/mod.rs b/sqlite-compress/src/functions/like/mod.rs index 2cd4e52..85a2110 100644 --- a/sqlite-compress/src/functions/like/mod.rs +++ b/sqlite-compress/src/functions/like/mod.rs @@ -1,10 +1,7 @@ use matcher::Matcher; use sqlite_ffi::{Context, SqliteError, Value}; -use crate::{ - functions::like::search::{search_blob, to_sqlite}, - state::ExtensionState, -}; +use crate::{functions::like::search::search_blob, state::ExtensionState}; const PATTERN_ARG: i32 = 4; @@ -38,7 +35,7 @@ pub fn sqlite_decompress_like(mut context: Context, values: &[Value]) -> Result< let found = match aux.get() { Ok(matcher) => search_blob(&state, schema, &blob, matcher, offset)?, Err(_) => { - let matcher = Matcher::compile(pattern).map_err(to_sqlite)?; + let matcher = Matcher::compile(pattern); let found = search_blob(&state, schema, &blob, &matcher, offset)?; aux.set(matcher); found From ceb4c6a0fa15507b4cccdf964f048c7ce50c16e7 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Tue, 27 Oct 2026 12:33:32 +0100 Subject: [PATCH 16/17] :recycle: Set correct bound --- crates/matcher/src/matcher.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/matcher/src/matcher.rs b/crates/matcher/src/matcher.rs index 1e8c586..565bf83 100644 --- a/crates/matcher/src/matcher.rs +++ b/crates/matcher/src/matcher.rs @@ -94,8 +94,8 @@ fn fast_path_ordered_segments( } } -/// Matches each `%`-separated group at its leftmost position, over a buffer -/// holding everything read so far. +/// Matches each `%`-separated group at its leftmost position, using a bounded +/// buffer that retains only bytes needed by the current group. /// /// The leftmost match of a group leaves the most room for the next ones, so /// backtracking never crosses a `Skip`: on mismatch, only the group's start @@ -126,7 +126,8 @@ fn slow_path_with_any( } Group::Missing => return Ok(false), Group::Pending(from) => { - pos = from; + buf.drain(..from); + pos = 0; eof = !read_more(&mut reader, &mut buf, &mut skip)?; } } From 56b507b75a5d02b7784982549e640c8db2481c58 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Tue, 27 Oct 2026 12:44:28 +0100 Subject: [PATCH 17/17] Fix clippy --- crates/matcher/src/matcher.rs | 2 +- crates/matcher/src/utils.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/matcher/src/matcher.rs b/crates/matcher/src/matcher.rs index 565bf83..4d701fd 100644 --- a/crates/matcher/src/matcher.rs +++ b/crates/matcher/src/matcher.rs @@ -27,7 +27,7 @@ impl Matcher { let finders = segments .into_iter() .filter_map(|s| match s { - Segment::Literal(f) => Some(f), + Segment::Literal(f) => Some(*f), _ => None, }) .collect(); diff --git a/crates/matcher/src/utils.rs b/crates/matcher/src/utils.rs index 536126e..b82f337 100644 --- a/crates/matcher/src/utils.rs +++ b/crates/matcher/src/utils.rs @@ -3,7 +3,7 @@ use memchr::memmem::Finder; /// One piece of a compiled `LIKE` pattern. pub enum Segment { /// ASCII-lowercased bytes that must appear verbatim. - Literal(Finder<'static>), + Literal(Box>), /// `%`: any run of bytes, possibly empty. Skip, /// `_`: one UTF-8 character, or one byte when the sequence is invalid. @@ -54,7 +54,7 @@ fn push_literal(segments: &mut Vec, cur: &mut Vec) { return; } cur.make_ascii_lowercase(); - segments.push(Segment::Literal(Finder::new(cur).into_owned())); + segments.push(Segment::Literal(Box::new(Finder::new(cur).into_owned()))); cur.clear(); }