From 8d57890be424a2e27d23aa22db11834f06244549 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Tue, 27 Oct 2026 18:48:14 +0100 Subject: [PATCH 1/3] :sparkles: Add new function to do bool operation on dict_id header --- crates/sqlite-ffi/src/context/mod.rs | 7 +++ .../src/functions/compressed_dict_id.rs | 60 +++++++++++++++++++ sqlite-compress/src/functions/mod.rs | 2 + sqlite-compress/src/lib.rs | 6 ++ 4 files changed, 75 insertions(+) create mode 100644 sqlite-compress/src/functions/compressed_dict_id.rs diff --git a/crates/sqlite-ffi/src/context/mod.rs b/crates/sqlite-ffi/src/context/mod.rs index 7730d37..7c68c47 100644 --- a/crates/sqlite-ffi/src/context/mod.rs +++ b/crates/sqlite-ffi/src/context/mod.rs @@ -4,6 +4,7 @@ use std::{ffi::c_void, marker::PhantomData, ptr::NonNull}; use libsqlite3_sys::{ sqlite3_context, sqlite3_result_blob, sqlite3_result_error, sqlite3_result_int64, + sqlite3_result_null, }; pub use crate::context::aux::Aux; @@ -52,6 +53,12 @@ impl Context { pub fn result_int64(&self, value: i64) { unsafe { result_int_raw(self.ctx.as_ptr(), value) }; } + + pub fn result_null(&self) { + unsafe { + sqlite3_result_null(self.ctx.as_ptr()); + } + } } unsafe fn result_blob_raw(context: *mut sqlite3_context, blob: *const c_void, size: i32) { diff --git a/sqlite-compress/src/functions/compressed_dict_id.rs b/sqlite-compress/src/functions/compressed_dict_id.rs new file mode 100644 index 0000000..2d31ed6 --- /dev/null +++ b/sqlite-compress/src/functions/compressed_dict_id.rs @@ -0,0 +1,60 @@ +use sqlite_ffi::{Context, SqliteError, Value}; + +use crate::functions::{ParsedBlob, errors::CodecError, header::Header}; + +fn dict_id_from_blob(blob: &[u8]) -> Result, CodecError> { + match Header::parse_blob(blob)? { + ParsedBlob::Framed(header, _) => Ok(Some(header.dict_id.get())), + ParsedBlob::Plain => Ok(None), + } +} + +/// SQLite callback for `compressed_dict_id(blob)`. +/// +/// Returns the dictionary ID stored in the blob header, or NULL if the blob +/// is not in a recognized compressed format. +pub fn sqlite_compressed_dict_id(context: Context, values: &[Value]) -> Result<(), SqliteError> { + match dict_id_from_blob(values[0].to_blob()).map_err(|e| SqliteError::Message(e.to_string()))? { + Some(dict_id) => context.result_int64(i64::from(dict_id)), + None => context.result_null(), + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::dict_id_from_blob; + use crate::{DictId, functions::Header}; + + fn framed_blob(dict_id: u32) -> Vec { + let header = Header::new(DictId::from(dict_id), 0); + let mut blob = Vec::new(); + header.write(&mut blob); + blob + } + + #[test] + fn returns_dictionary_id_from_framed_blob() { + let blob = framed_blob(42); + + assert_eq!(dict_id_from_blob(&blob).unwrap(), Some(42)); + } + + #[test] + fn returns_zero_for_raw_zstd_blob() { + let blob = framed_blob(0); + + assert_eq!(dict_id_from_blob(&blob).unwrap(), Some(0)); + } + + #[test] + fn returns_none_for_plain_blob() { + assert_eq!(dict_id_from_blob(b"plain text").unwrap(), None); + } + + #[test] + fn returns_none_for_malformed_header() { + assert_eq!(dict_id_from_blob(&[0, 1, 2]).unwrap(), None); + } +} diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index 68689da..8111cfa 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -1,4 +1,5 @@ mod compress; +mod compressed_dict_id; mod decompress; mod errors; mod header; @@ -10,6 +11,7 @@ mod types; pub const DEFAULT_LEVEL: types::Level = types::Level::new(3); pub use compress::sqlite_compress; +pub use compressed_dict_id::sqlite_compressed_dict_id; pub use decompress::sqlite_decompress; pub(crate) use decompress::{decompress_raw, decompress_with_decoder}; pub use errors::CodecError; diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index a71648b..aad3177 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -37,5 +37,11 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> #[cfg(feature = "like")] connection.create_function("decompress_like", 6, functions::sqlite_decompress_like)?; + connection.create_function( + "compressed_dict_id", + 1, + functions::sqlite_compressed_dict_id, + )?; + Ok(()) } From 9b909ceb74e997940ba8fedb54ea0c61d0729842 Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Tue, 27 Oct 2026 20:42:44 +0100 Subject: [PATCH 2/3] :recycle: Set features dict_id_functions --- .mise/tasks/sqlite-compress/clippy | 2 +- .mise/tasks/sqlite-compress/test | 2 +- sqlite-compress/Cargo.toml | 1 + .../src/functions/compressed_dict_id.rs | 14 ++++++------ .../src/functions/latest_dict_id.rs | 22 +++++++++++++++++++ sqlite-compress/src/functions/mod.rs | 6 +++++ sqlite-compress/src/lib.rs | 4 ++++ sqlite-compress/tests/dict_train.rs | 9 ++++---- 8 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 sqlite-compress/src/functions/latest_dict_id.rs diff --git a/.mise/tasks/sqlite-compress/clippy b/.mise/tasks/sqlite-compress/clippy index d721d67..dfbb563 100755 --- a/.mise/tasks/sqlite-compress/clippy +++ b/.mise/tasks/sqlite-compress/clippy @@ -1,4 +1,4 @@ #!/usr/bin/env bash #MISE description="Lint with clippy" -cargo clippy -p sqlite-compress --all-targets --no-default-features --features loadable_extension,like -- -D warnings +cargo clippy -p sqlite-compress --all-targets --no-default-features --features loadable_extension,like,dict_id_functions -- -D warnings diff --git a/.mise/tasks/sqlite-compress/test b/.mise/tasks/sqlite-compress/test index 29b0714..91b10b1 100755 --- a/.mise/tasks/sqlite-compress/test +++ b/.mise/tasks/sqlite-compress/test @@ -1,3 +1,3 @@ #!/usr/bin/env bash #MISE description="Run tests" -cargo test -p sqlite-compress --features like -- --test-threads=1 +cargo test -p sqlite-compress --features like,dict_id_functions -- --test-threads=1 diff --git a/sqlite-compress/Cargo.toml b/sqlite-compress/Cargo.toml index 97cb28c..690ad92 100644 --- a/sqlite-compress/Cargo.toml +++ b/sqlite-compress/Cargo.toml @@ -21,6 +21,7 @@ crate-type = ["cdylib", "rlib"] [features] default = ["bundled"] +dict_id_functions = [] like = ["dep:matcher"] bundled = [ "sqlite-ffi/bundled", diff --git a/sqlite-compress/src/functions/compressed_dict_id.rs b/sqlite-compress/src/functions/compressed_dict_id.rs index 2d31ed6..60f6d42 100644 --- a/sqlite-compress/src/functions/compressed_dict_id.rs +++ b/sqlite-compress/src/functions/compressed_dict_id.rs @@ -2,13 +2,6 @@ use sqlite_ffi::{Context, SqliteError, Value}; use crate::functions::{ParsedBlob, errors::CodecError, header::Header}; -fn dict_id_from_blob(blob: &[u8]) -> Result, CodecError> { - match Header::parse_blob(blob)? { - ParsedBlob::Framed(header, _) => Ok(Some(header.dict_id.get())), - ParsedBlob::Plain => Ok(None), - } -} - /// SQLite callback for `compressed_dict_id(blob)`. /// /// Returns the dictionary ID stored in the blob header, or NULL if the blob @@ -22,6 +15,13 @@ pub fn sqlite_compressed_dict_id(context: Context, values: &[Value]) -> Result<( Ok(()) } +fn dict_id_from_blob(blob: &[u8]) -> Result, CodecError> { + match Header::parse_blob(blob)? { + ParsedBlob::Framed(header, _) => Ok(Some(header.dict_id.get())), + ParsedBlob::Plain => Ok(None), + } +} + #[cfg(test)] mod tests { use super::dict_id_from_blob; diff --git a/sqlite-compress/src/functions/latest_dict_id.rs b/sqlite-compress/src/functions/latest_dict_id.rs new file mode 100644 index 0000000..99cdc2f --- /dev/null +++ b/sqlite-compress/src/functions/latest_dict_id.rs @@ -0,0 +1,22 @@ +use sqlite_ffi::{Context, SqliteError, Value}; + +use crate::{ColumnKey, dict::read_current_id, state::ExtensionState}; + +pub fn sqlite_latest_dict_id(context: Context, values: &[Value]) -> Result<(), SqliteError> { + let schema = values[0].to_text()?; + let table = values[1].to_text()?; + let column = values[2].to_text()?; + + let state = ExtensionState::from_context(&context)?; + let key = ColumnKey::new(schema, table, column); + + let latest_id = + read_current_id(&state, &key).map_err(|error| SqliteError::Message(error.to_string()))?; + + match latest_id { + Some(dict_id) => context.result_int64(i64::from(dict_id.get())), + None => context.result_null(), + } + + Ok(()) +} diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index 8111cfa..4c7d9fe 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -1,8 +1,11 @@ mod compress; +#[cfg(feature = "dict_id_functions")] mod compressed_dict_id; mod decompress; mod errors; mod header; +#[cfg(feature = "dict_id_functions")] +mod latest_dict_id; #[cfg(feature = "like")] mod like; mod types; @@ -11,12 +14,15 @@ mod types; pub const DEFAULT_LEVEL: types::Level = types::Level::new(3); pub use compress::sqlite_compress; +#[cfg(feature = "dict_id_functions")] pub use compressed_dict_id::sqlite_compressed_dict_id; pub use decompress::sqlite_decompress; pub(crate) use decompress::{decompress_raw, decompress_with_decoder}; pub use errors::CodecError; pub use header::Header; pub(crate) use header::ParsedBlob; +#[cfg(feature = "dict_id_functions")] +pub use latest_dict_id::sqlite_latest_dict_id; #[cfg(feature = "like")] pub use like::sqlite_decompress_like; pub use types::Level; diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index aad3177..8e1fd4d 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -37,11 +37,15 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> #[cfg(feature = "like")] connection.create_function("decompress_like", 6, functions::sqlite_decompress_like)?; + #[cfg(feature = "dict_id_functions")] connection.create_function( "compressed_dict_id", 1, functions::sqlite_compressed_dict_id, )?; + #[cfg(feature = "dict_id_functions")] + connection.create_function("latest_dict_id", 3, functions::sqlite_latest_dict_id)?; + Ok(()) } diff --git a/sqlite-compress/tests/dict_train.rs b/sqlite-compress/tests/dict_train.rs index c808c95..f05c3e5 100644 --- a/sqlite-compress/tests/dict_train.rs +++ b/sqlite-compress/tests/dict_train.rs @@ -812,7 +812,7 @@ fn second_connection_compress_uses_trained_dict_id() { fn train_by_column_reports_progress() { let sample_count = 64; let max_samples = 64; - let progress_every = 10; + let progress_every = 1; let db = common::TestDb::open(); let state = db.state(); @@ -867,7 +867,7 @@ fn train_by_column_reports_progress() { }), }; - let dict_id = train_by_column( + train_by_column( &db, &table, &column, @@ -877,15 +877,14 @@ fn train_by_column_reports_progress() { ) .unwrap(); - assert!(dict_id.is_some()); assert!(!reports.is_empty()); - assert!(reports.iter().all(|&(_, target)| target == max_samples)); assert!( reports .iter() - .all(|&(done, _)| done > 0 && done.is_multiple_of(progress_every)) + .all(|&(done, target)| done > 0 && done <= target && target == max_samples) ); assert!(reports.windows(2).all(|w| w[0].0 <= w[1].0)); + assert_eq!(reports.last(), Some(&(max_samples, max_samples))); } #[test] From 6668642016568e8122b35efee1e66819cf3675cb Mon Sep 17 00:00:00 2001 From: Gauron Dorian Date: Wed, 28 Oct 2026 08:46:40 +0100 Subject: [PATCH 3/3] :recycle: Remove feature --- .mise/tasks/sqlite-compress/clippy | 2 +- .mise/tasks/sqlite-compress/test | 2 +- sqlite-compress/Cargo.toml | 1 - sqlite-compress/src/functions/mod.rs | 4 ---- sqlite-compress/src/lib.rs | 2 -- 5 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.mise/tasks/sqlite-compress/clippy b/.mise/tasks/sqlite-compress/clippy index dfbb563..d721d67 100755 --- a/.mise/tasks/sqlite-compress/clippy +++ b/.mise/tasks/sqlite-compress/clippy @@ -1,4 +1,4 @@ #!/usr/bin/env bash #MISE description="Lint with clippy" -cargo clippy -p sqlite-compress --all-targets --no-default-features --features loadable_extension,like,dict_id_functions -- -D warnings +cargo clippy -p sqlite-compress --all-targets --no-default-features --features loadable_extension,like -- -D warnings diff --git a/.mise/tasks/sqlite-compress/test b/.mise/tasks/sqlite-compress/test index 91b10b1..29b0714 100755 --- a/.mise/tasks/sqlite-compress/test +++ b/.mise/tasks/sqlite-compress/test @@ -1,3 +1,3 @@ #!/usr/bin/env bash #MISE description="Run tests" -cargo test -p sqlite-compress --features like,dict_id_functions -- --test-threads=1 +cargo test -p sqlite-compress --features like -- --test-threads=1 diff --git a/sqlite-compress/Cargo.toml b/sqlite-compress/Cargo.toml index 690ad92..97cb28c 100644 --- a/sqlite-compress/Cargo.toml +++ b/sqlite-compress/Cargo.toml @@ -21,7 +21,6 @@ crate-type = ["cdylib", "rlib"] [features] default = ["bundled"] -dict_id_functions = [] like = ["dep:matcher"] bundled = [ "sqlite-ffi/bundled", diff --git a/sqlite-compress/src/functions/mod.rs b/sqlite-compress/src/functions/mod.rs index 4c7d9fe..c1e5c2d 100644 --- a/sqlite-compress/src/functions/mod.rs +++ b/sqlite-compress/src/functions/mod.rs @@ -1,10 +1,8 @@ mod compress; -#[cfg(feature = "dict_id_functions")] mod compressed_dict_id; mod decompress; mod errors; mod header; -#[cfg(feature = "dict_id_functions")] mod latest_dict_id; #[cfg(feature = "like")] mod like; @@ -14,14 +12,12 @@ mod types; pub const DEFAULT_LEVEL: types::Level = types::Level::new(3); pub use compress::sqlite_compress; -#[cfg(feature = "dict_id_functions")] pub use compressed_dict_id::sqlite_compressed_dict_id; pub use decompress::sqlite_decompress; pub(crate) use decompress::{decompress_raw, decompress_with_decoder}; pub use errors::CodecError; pub use header::Header; pub(crate) use header::ParsedBlob; -#[cfg(feature = "dict_id_functions")] pub use latest_dict_id::sqlite_latest_dict_id; #[cfg(feature = "like")] pub use like::sqlite_decompress_like; diff --git a/sqlite-compress/src/lib.rs b/sqlite-compress/src/lib.rs index 8e1fd4d..eebd52b 100644 --- a/sqlite-compress/src/lib.rs +++ b/sqlite-compress/src/lib.rs @@ -37,14 +37,12 @@ pub fn sqlite3_compress_init(connection: Connection) -> Result<(), SqliteError> #[cfg(feature = "like")] connection.create_function("decompress_like", 6, functions::sqlite_decompress_like)?; - #[cfg(feature = "dict_id_functions")] connection.create_function( "compressed_dict_id", 1, functions::sqlite_compressed_dict_id, )?; - #[cfg(feature = "dict_id_functions")] connection.create_function("latest_dict_id", 3, functions::sqlite_latest_dict_id)?; Ok(())