diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c3d4448c..626aec2e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -126,6 +126,62 @@ jobs: EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" + run-integration-duckdb: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Build release (DuckDB backend) + run: cargo build --release -p extenddb --no-default-features --features duckdb + + - name: Initialize ExtendDB + id: init + run: | + output=$(./target/release/extenddb init --backend duckdb --config extenddb.toml 2>&1) + echo "$output" + password=$(echo "$output" | grep -oP 'Password: \K\S+') + echo "admin_password=$password" >> "$GITHUB_OUTPUT" + + - name: Start ExtendDB + run: | + # --write-pid-file so devtools/run-tests can restart the server with + # 'extenddb stop' when it needs to apply a config change. + ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & + for i in $(seq 1 30); do + if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then + echo "Server ready" + exit 0 + fi + sleep 1 + done + echo "Server failed to start" + exit 1 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Python dependencies + run: pip install -r requirements.txt + + - name: Run integration tests + env: + EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + # test_gsi_async drives the server out-of-band with `extenddb settings + # set` from a second process. DuckDB holds a process-exclusive lock on + # the database file, so a second process cannot open it while the + # server is running; the propagation paths it exercises are covered by + # the crate's unit tests instead. + run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export and not test_gsi_async" + run-integration-dev-mode: # dev-mode was shipped with no CI coverage at all, which is how the batch and # transaction authorization regression reached main: the build compiled, so @@ -321,6 +377,7 @@ jobs: [ run-integration, run-integration-sqlite, + run-integration-duckdb, run-integration-dev-mode, run-rust-integration, run-rust-integration-sqlite, @@ -330,6 +387,7 @@ jobs: - run: | if [ "${{ needs.run-integration.result }}" != "success" ] || \ [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \ + [ "${{ needs.run-integration-duckdb.result }}" != "success" ] || \ [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then diff --git a/Cargo.lock b/Cargo.lock index 5cd97d10..d992aed7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,6 +50,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -71,6 +72,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -127,6 +137,15 @@ version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.9.1" @@ -142,6 +161,169 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "arrow" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ord" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +dependencies = [ + "bitflags", +] + +[[package]] +name = "arrow-select" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "asn1-rs" version = "0.7.2" @@ -357,6 +539,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -504,6 +692,12 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.62" @@ -533,6 +727,17 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "cipher" version = "0.4.4" @@ -614,6 +819,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "comfy-table" +version = "7.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" +dependencies = [ + "crossterm", + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "compression-codecs" version = "0.4.38" @@ -797,6 +1013,28 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "crossterm_winapi", + "parking_lot", + "rustix 0.38.44", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -961,6 +1199,17 @@ dependencies = [ "syn", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1034,6 +1283,23 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "duckdb" +version = "1.10505.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970e05eedd3f55c435194d9104f90a9b4a79a80d6e73251bc9ff43e178130c4e" +dependencies = [ + "arrow", + "cast", + "comfy-table", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink 0.10.0", + "libduckdb-sys", + "num-integer", + "strum", +] + [[package]] name = "dunce" version = "1.0.5" @@ -1113,6 +1379,7 @@ dependencies = [ "extenddb-app", "extenddb-config", "extenddb-storage", + "extenddb-storage-duckdb", "extenddb-storage-mongodb", "extenddb-storage-postgres", "extenddb-storage-sqlite", @@ -1282,6 +1549,32 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "extenddb-storage-duckdb" +version = "0.1.10" +dependencies = [ + "aes-gcm", + "async-trait", + "base64 0.22.1", + "bcrypt", + "bigdecimal", + "crc32fast", + "duckdb", + "extenddb-auth", + "extenddb-core", + "extenddb-storage", + "futures", + "rand 0.9.4", + "serde", + "serde_json", + "time", + "tokio", + "toml", + "tracing", + "uuid", + "zeroize", +] + [[package]] name = "extenddb-storage-mongodb" version = "0.1.10" @@ -1365,12 +1658,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1385,6 +1700,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1620,6 +1936,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -1873,6 +2201,30 @@ dependencies = [ "tower-service", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -2140,12 +2492,86 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libduckdb-sys" +version = "1.10505.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb514dab5e271e849235c1cb98bd65a2ae107fbd619a6740219319c54a71d95" +dependencies = [ + "cc", + "flate2", + "pkg-config", + "serde", + "serde_json", + "tar", + "ureq", + "vcpkg", + "zip", +] + [[package]] name = "libm" version = "0.2.16" @@ -2175,6 +2601,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2481,6 +2913,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2959,6 +3400,18 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -3066,6 +3519,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3075,7 +3541,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -3639,6 +4105,27 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3723,6 +4210,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -3732,7 +4230,7 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -4128,6 +4626,12 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -4156,6 +4660,34 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64 0.23.1", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", + "webpki-roots 1.0.7", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -4174,6 +4706,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4378,6 +4916,22 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -4387,6 +4941,47 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -4706,6 +5301,16 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "yaml-rust2" version = "0.8.1" @@ -4844,8 +5449,40 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 157aae4f..6999aaa3 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/storage-postgres", "crates/storage-mongodb", "crates/storage-sqlite", + "crates/storage-duckdb", "crates/auth", "crates/server", "crates/app", @@ -33,6 +34,7 @@ extenddb-config = { path = "crates/config" } extenddb-storage-postgres = { path = "crates/storage-postgres" } extenddb-storage-mongodb = { path = "crates/storage-mongodb" } extenddb-storage-sqlite = { path = "crates/storage-sqlite" } +extenddb-storage-duckdb = { path = "crates/storage-duckdb" } extenddb-auth = { path = "crates/auth" } extenddb-server = { path = "crates/server" } extenddb-app = { path = "crates/app" } diff --git a/crates/app/src/cmd_init.rs b/crates/app/src/cmd_init.rs index 6cbe339f..2773af4a 100755 --- a/crates/app/src/cmd_init.rs +++ b/crates/app/src/cmd_init.rs @@ -35,6 +35,12 @@ pub struct InitArgs { #[arg(long)] sqlite_path: Option, + /// DuckDB database file path (default: extenddb.duckdb). The chosen path + /// is written to the generated config file, so `serve` finds it without + /// further flags. DuckDB backend only. + #[arg(long)] + duckdb_path: Option, + /// PostgreSQL host (hostname, IP address, or absolute Unix socket directory path) #[arg(long)] pg_host: Option, @@ -381,4 +387,14 @@ mod tests { fn init_rejects_unknown_flags() { assert!(parse(&["--sqlite-pathological", "/data/x.sqlite"]).is_err()); } + + /// `--duckdb-path` is the DuckDB backend's analogue of `--sqlite-path` and + /// is read by that backend's bootstrapper from the raw argv, so it must be + /// declared here or clap rejects it before the bootstrapper ever sees it. + #[test] + fn init_accepts_duckdb_path() { + let args = + parse(&["--backend", "duckdb", "--duckdb-path", "/data/x.duckdb"]).expect("parse"); + assert_eq!(args.duckdb_path.as_deref(), Some("/data/x.duckdb")); + } } diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index 5d6aa546..fa1ba3af 100755 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -17,10 +17,13 @@ postgres = ["extenddb-storage-postgres"] sqlite = ["extenddb-storage-sqlite"] # Build the SQLite backend in its zero-config, ephemeral in-memory mode. sqlite-memory = ["sqlite", "extenddb-storage-sqlite/memory"] +duckdb = ["extenddb-storage-duckdb"] +# Build the DuckDB backend in its zero-config, ephemeral in-memory mode. +duckdb-memory = ["duckdb", "extenddb-storage-duckdb/memory"] mongodb = ["extenddb-storage-mongodb"] # Developer mode: plain HTTP on loopback, open authorization (SigV4 still # enforced), seeded dev credential. A build-time profile for local/CI only; it -# requires a dev backend (`sqlite`/`sqlite-memory`) and fails to build with a +# requires a dev backend (`sqlite`/`sqlite-memory`, `duckdb`/`duckdb-memory`) and fails to build with a # production backend like `postgres` (enforced at compile time in `main.rs`). dev-mode = ["extenddb-app/dev-mode"] @@ -30,5 +33,6 @@ extenddb-storage = { workspace = true } extenddb-storage-postgres = { workspace = true, optional = true } extenddb-storage-mongodb = { workspace = true, optional = true } extenddb-storage-sqlite = { workspace = true, optional = true } +extenddb-storage-duckdb = { workspace = true, optional = true } anyhow = { workspace = true } extenddb-config = { workspace = true } diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index bd75de06..37fc4b4f 100755 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -11,8 +11,9 @@ //! //! In-tree backends are selected by mutually exclusive Cargo features: //! `postgres` (the default production backend), `mongodb` (production, built with -//! `--no-default-features --features mongodb`), and `sqlite`/`sqlite-memory` (the -//! dev/CI backend). Exactly one must be enabled: [`set_backend`] installs one +//! `--no-default-features --features mongodb`), `sqlite`/`sqlite-memory` (the +//! dev/CI backend), and `duckdb`/`duckdb-memory` (embedded, columnar). Exactly +//! one must be enabled: [`set_backend`] installs one //! backend per process, so a build with more than one would be ambiguous and is //! rejected at compile time. @@ -20,16 +21,24 @@ #[cfg(any( all(feature = "postgres", feature = "sqlite"), all(feature = "postgres", feature = "mongodb"), + all(feature = "postgres", feature = "duckdb"), all(feature = "sqlite", feature = "mongodb"), + all(feature = "sqlite", feature = "duckdb"), + all(feature = "mongodb", feature = "duckdb"), ))] compile_error!( - "the `postgres`, `mongodb`, and `sqlite` features are mutually exclusive: a \ + "the `postgres`, `mongodb`, `sqlite`, and `duckdb` features are mutually exclusive: a \ thin bin installs exactly one backend (e.g. build the MongoDB binary with \ `--no-default-features --features mongodb`)" ); -#[cfg(not(any(feature = "postgres", feature = "mongodb", feature = "sqlite")))] +#[cfg(not(any( + feature = "postgres", + feature = "mongodb", + feature = "sqlite", + feature = "duckdb" +)))] compile_error!( - "no backend selected: enable the `postgres` (default), `mongodb`, or `sqlite` feature" + "no backend selected: enable the `postgres` (default), `mongodb`, `sqlite`, or `duckdb` feature" ); // Developer mode relaxes the security posture (plain HTTP on loopback, open @@ -37,14 +46,16 @@ compile_error!( // dev/CI-suitable backend. Rather than denying each production backend by name // (every backend is a production backend unless proven otherwise, so a deny-list // would have to grow with each new one), require a known dev backend: dev-mode -// compiles only when `sqlite` is enabled. `sqlite-memory` enables `sqlite`, so it -// is covered too; postgres, mongodb — or any future production backend — fail the -// build, so there is no path by which a production deployment can serve in dev mode. -#[cfg(all(feature = "dev-mode", not(feature = "sqlite")))] +// compiles only when `sqlite` or `duckdb` is enabled (both embedded, both with an +// in-memory mode). `sqlite-memory` / `duckdb-memory` enable their base feature, +// so they are covered too; postgres, mongodb — or any future production backend — +// fail the build, so there is no path by which a production deployment can serve +// in dev mode. +#[cfg(all(feature = "dev-mode", not(any(feature = "sqlite", feature = "duckdb"))))] compile_error!( - "the `dev-mode` feature requires a dev/CI backend such as `sqlite`; it must \ - not be built with a production backend like `postgres` or `mongodb` (build \ - with `--no-default-features --features sqlite-memory,dev-mode`)" + "the `dev-mode` feature requires a dev/CI backend such as `sqlite` or `duckdb`; \ + it must not be built with a production backend like `postgres` or `mongodb` \ + (build with `--no-default-features --features sqlite-memory,dev-mode`)" ); fn main() -> anyhow::Result<()> { @@ -57,6 +68,8 @@ fn main() -> anyhow::Result<()> { extenddb_storage::set_backend(extenddb_storage_sqlite::backend())?; #[cfg(feature = "mongodb")] extenddb_storage::set_backend(extenddb_storage_mongodb::backend())?; + #[cfg(feature = "duckdb")] + extenddb_storage::set_backend(extenddb_storage_duckdb::backend())?; extenddb_app::run(extenddb_app::BuildInfo { // Read from the bin crate so the reported version is the deployed @@ -77,6 +90,8 @@ mod tests { let _ = extenddb_storage::set_backend(extenddb_storage_sqlite::backend()); #[cfg(feature = "mongodb")] let _ = extenddb_storage::set_backend(extenddb_storage_mongodb::backend()); + #[cfg(feature = "duckdb")] + let _ = extenddb_storage::set_backend(extenddb_storage_duckdb::backend()); } /// Zero-config serve contract: with the SQLite backend installed, @@ -104,6 +119,28 @@ mod tests { } } + /// Zero-config serve contract for the DuckDB backend, mirroring the SQLite + /// one: built-in defaults load with no config file, bind to loopback, and + /// select the backend's default database path. + #[cfg(feature = "duckdb")] + #[test] + fn builtin_defaults_load_for_duckdb_and_bind_loopback() { + install_backend(); + let cfg = extenddb_config::load_builtin_defaults() + .expect("duckdb storage config has no required fields"); + assert_eq!(cfg.server.bind_addr, "127.0.0.1"); + assert_eq!(cfg.server.port, 18443); + let path = cfg.storage.connection_config(); + if cfg!(feature = "duckdb-memory") { + assert_eq!(path, ":memory:"); + } else { + assert!( + path.ends_with("extenddb.duckdb"), + "default file path should be extenddb.duckdb, got: {path}" + ); + } + } + /// `load_builtin_defaults` is only reachable from dev-mode builds (a /// postgres + dev-mode binary is a compile error), but its defaults must /// never relax the production posture regardless of backend: loopback diff --git a/crates/storage-duckdb/.gitignore b/crates/storage-duckdb/.gitignore new file mode 100644 index 00000000..c9772db5 --- /dev/null +++ b/crates/storage-duckdb/.gitignore @@ -0,0 +1,4 @@ +/target +Cargo.lock +*.duckdb +*.duckdb.wal diff --git a/crates/storage-duckdb/Cargo.toml b/crates/storage-duckdb/Cargo.toml new file mode 100644 index 00000000..d4f9f271 --- /dev/null +++ b/crates/storage-duckdb/Cargo.toml @@ -0,0 +1,37 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +[package] +name = "extenddb-storage-duckdb" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[features] +# Compile the in-memory (`:memory:`) database in as the default storage path, +# yielding a zero-config, ephemeral, bootstrap-on-serve deployment (no `init`, +# no file on disk). Without this feature the backend is file-backed and the path +# must be configured as usual. +memory = [] + +[dependencies] +extenddb-core = { workspace = true } +extenddb-storage = { workspace = true } +extenddb-auth = { workspace = true } +async-trait = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +duckdb = { version = "1.10505.0", features = ["bundled", "json"] } +tokio = { workspace = true, features = ["sync"] } +toml = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } +time = { workspace = true } +base64 = { workspace = true } +bcrypt = { workspace = true } +aes-gcm = { workspace = true } +rand = { workspace = true } +bigdecimal = { workspace = true } +crc32fast = { workspace = true } +zeroize = { workspace = true } diff --git a/crates/storage-duckdb/README.md b/crates/storage-duckdb/README.md new file mode 100644 index 00000000..6bbfaffc --- /dev/null +++ b/crates/storage-duckdb/README.md @@ -0,0 +1,124 @@ +# extenddb-storage-duckdb + +DuckDB storage backend for [ExtendDB](https://github.com/ExtendDB/extenddb), an +in-tree workspace crate selected by Cargo feature. + +## Design + +Single-file (or in-memory) storage using the embedded [DuckDB](https://duckdb.org) +engine via the `duckdb` crate (statically linked; no server, no external +dependency at runtime). Targets: + +- Local development without PostgreSQL +- CI / integration tests (especially the ephemeral in-memory mode) +- Single-node, embedded, and edge deployments +- Anyone who has ever wanted to run `SELECT ... GROUP BY` directly against the + file their DynamoDB-compatible database lives in + +The crate is a port of `extenddb-storage-sqlite` — same catalog schema, same +per-table data layout, same order-preserving `N` sort-key encoding, same +persistent `gsi_pending` queue — with the storage engine swapped. The differences +that matter: + +- **No `sqlx` driver exists for DuckDB**, so `src/db.rs` provides a small + `sqlx`-shaped async facade over the synchronous `duckdb` crate: a connection + pool, positional `query` / `query_as` / `query_scalar` builders, and an + explicit-commit / rollback-on-drop `Transaction`. Every statement runs on a + `spawn_blocking` thread; the connection is moved there and back. +- **In-memory databases are shared across the pool.** All connections are + `try_clone()`s of one root connection onto the same database instance, so + `:memory:` gets a real read pool instead of SQLite's single pinned connection. +- **No foreign keys.** DuckDB cannot cascade deletes, so the catalog declares + none; `src/referential.rs` removes children explicitly inside the parent's + delete transaction and checks parents exist before child inserts. +- **MVCC inside the process.** Readers never block on the writer. Writers are + still serialized in-process by the engine's `write_lock`, which is what makes + condition-check-then-write atomic; DuckDB's optimistic conflict detection is a + backstop, not the mechanism. Across processes there is no sharing at all (see + Known limits). +- **64-bit integers everywhere.** DuckDB's `INTEGER` is 32-bit, so every catalog + integer column is `BIGINT`, and `REAL` (32-bit in DuckDB) is `DOUBLE`. + +## Building + +The backend is compiled into the `extenddb` binary via feature flags (Postgres +remains the default): + +```bash +# DuckDB only, no Postgres compiled in +cargo build -p extenddb --no-default-features --features duckdb + +# DuckDB in zero-config, ephemeral in-memory mode +cargo build -p extenddb --no-default-features --features duckdb-memory +``` + +The first build compiles DuckDB from source (`libduckdb-sys` with the `bundled` +feature). Budget several minutes and a cup of something; it is cached afterwards. + +## Configuration + +```toml +[storage] +backend = "duckdb" + +[storage.duckdb] +# Database file path, or ":memory:" for an ephemeral in-memory database. +path = "extenddb.duckdb" +# Connection pool size (writes are serialized regardless). +pool_size = 10 +``` + +`extenddb init --backend duckdb --duckdb-path ` writes the path into the +generated config file. + +### In-memory mode + +`path = ":memory:"` selects an ephemeral database that bootstraps on `serve` +(no `init`, no file on disk). The `memory` crate feature (exposed by the binary +as `duckdb-memory`) makes `:memory:` the compiled-in default path. + +## Developer mode + +`dev-mode` (plain HTTP on loopback, open authorization, seeded credential, SigV4 +still verified) builds with this backend exactly as it does with SQLite: + +```bash +cargo build -p extenddb --no-default-features --features duckdb-memory,dev-mode +extenddb serve --config extenddb.toml # bootstraps on serve; no init +``` + +## Conformance + +Per [RFC-0002](https://github.com/ExtendDB/extenddb/blob/main/docs/rfcs), this +backend implements the full storage trait surface. + +Mandatory traits: + +- `TableEngine`, `DataEngine` +- `ManagementStore`, `AdminStore`, `Bootstrapper` + +Optional traits (all implemented): + +- `MetadataEngine`, `StreamEngine`, `WorkerStore` +- `SettingsStore`, `MetricsStore`, `RateLimitStore`, `AuthorizationStore`, + `BackupEngine` + +Conformance is validated by the shared ExtendDB integration suite run against a +DuckDB-served instance (`run-integration-duckdb` in CI). + +## Known limits + +- **One process at a time.** DuckDB takes a process-exclusive lock on the database + file. While `extenddb serve` is running, no second process can open the same + file, so out-of-band tooling that connects directly (`extenddb settings get|set`, + `catalog-check`, `verify`) must run while the server is stopped or against an + `:memory:` server not at all. Everything that goes through the server's own + API is unaffected. +- **No TTL expression index.** DuckDB does not allow indexes over extension + functions, so the TTL sweep is a filtered scan of `item_data` rather than an + indexed lookup. +- **Cold builds are slow.** DuckDB is compiled from source the first time. + +## License + +Apache License 2.0 — see the workspace [LICENSE](../../LICENSE). diff --git a/crates/storage-duckdb/docs/design-decisions.md b/crates/storage-duckdb/docs/design-decisions.md new file mode 100644 index 00000000..6dad5796 --- /dev/null +++ b/crates/storage-duckdb/docs/design-decisions.md @@ -0,0 +1,154 @@ + +# extenddb-storage-duckdb — Design Decisions + +This memo records the DuckDB-specific decisions taken to reach behavioural +parity with the SQLite backend (`crates/storage-sqlite`), from which this crate +was ported, and through it with the reference PostgreSQL backend. Decisions the +SQLite backend already made and that carry over unchanged (the order-preserving +`N` sort-key TEXT encoding, the single logical database, the persistent +`gsi_pending` queue, synchronous LSI maintenance, `rowid`-based parallel scan +segments) are not repeated here; see that crate's `docs/design-decisions.md`. + +--- + +## D1 — Async execution model + +### Problem +`duckdb::Connection` is synchronous, `Send` but not `Sync`, and there is no +`sqlx` driver. The rest of the crate is async and written against `sqlx`'s +builder API (`query(...).bind(...).fetch_all(&pool)`), with ~600 call sites. + +### Options +- **(A) Call DuckDB inline on the async thread.** Simplest; blocks a tokio + worker for the duration of every statement. Point lookups are sub-millisecond + but a `Scan` or a backfill batch is not, and blocking the executor stalls + unrelated requests. +- **(B) `block_in_place`.** Panics on a current-thread runtime, which every + `#[tokio::test]` in the crate uses. +- **(C) Move the connection onto a `spawn_blocking` thread per statement.** + Each pooled connection lives in a `tokio::sync::Mutex>` + slot; a statement takes the connection out, runs on the blocking pool, and + puts it back. A transaction holds its slot for its whole lifetime so every + statement in it runs on the same connection. If the blocking task panics the + slot is left empty and re-cloned from the root connection on next use. + +### Decision +**(C)**, wrapped in `src/db.rs` as a facade that reproduces the subset of the +`sqlx` API the crate uses (`query`, `query_as`, `query_scalar`, `raw_sql`, +`Pool::begin`, `Transaction::commit`, rollback on drop, `rows_affected`). Rows +are materialised into `Vec` on the blocking side and +decoded into tuples or structs on the async side, so nothing borrowed from +DuckDB crosses a thread boundary. The port of the storage code is then mostly +`sqlx::` → `db::`. + +--- + +## D2 — One database instance per process, per file + +### Problem +DuckDB permits a single database instance per file per process and takes a +file lock. The SQLite backend opens *two* independent pools over the same file +at serve time (engine + catalog) and a third in the bootstrapper; a naive port +fails on the second `Connection::open`. + +### Decision +`db::Pool` keeps a process-wide registry of root connections keyed by canonical +path. Every pool over a path is a set of `try_clone()`s of that path's root, so +they share one instance; the catalog pool is opened as a `sibling` of the engine +pool. `drop_databases` forgets the registry entry before deleting the file so +the lock is released. In-memory databases are **not** registered: each +`Pool::open(":memory:")` is a private database (what the unit tests expect), and +connections cloned from it all see the same data — which is why the ephemeral +mode can run a real read pool where SQLite had to pin one connection. + +--- + +## D3 — Referential integrity without foreign keys + +### Problem +DuckDB accepts `FOREIGN KEY` clauses but does not implement `ON DELETE CASCADE`, +and it rejects updates to rows that a foreign key references. The SQLite catalog +relies on cascades in eleven parent-delete paths (tables → indexes / +vector_indexes / stream_shards / stream_records; accounts → every IAM table; +users → tags / access keys / memberships; groups → memberships; roles → tags / +sessions; backups → items) and on FK violations to report `NotFound` in seven +child-insert paths. + +### Options +- **(A) Keep the constraints, drop `ON DELETE CASCADE`.** Parent deletes then + fail while children exist, and every update to `tables` (status flips, item + counts) risks the referenced-row restriction. Rejected. +- **(B) Declare no foreign keys; enforce in code.** `src/referential.rs` holds + one function per parent that deletes its children, called inside the parent's + delete transaction, plus `ensure_*_exists` checks that produce the same + `NotFound` errors the constraints did. + +### Decision +**(B)**. The pool-based deletes (`delete_user`, `delete_group`, `delete_role`, +`delete_account`) become single transactions via `delete_with_children`. The +existence checks in autocommit paths are not atomic with the insert that follows; +a lost race surfaces as `NotFound` from a later read instead of from a +constraint, which is the same outcome the callers already handle. + +--- + +## D4 — Type widths and dialect + +- `INTEGER` → `BIGINT` throughout (DuckDB `INTEGER` is 32-bit; SQLite's is + 64-bit, and `table_size_bytes` alone would overflow it). `REAL` → `DOUBLE` for + the same reason (DuckDB `REAL` is `FLOAT`). +- Booleans stay 0/1 integers in the catalog; a Rust `bool` parameter binds as + `BIGINT` so `col = ?` compares integer to integer. +- `INTEGER PRIMARY KEY AUTOINCREMENT` → `BIGINT PRIMARY KEY DEFAULT + nextval('gsi_pending_seq')`. +- Timestamps: `strftime('%Y-%m-%dT%H:%M:%fZ','now')` → + `strftime(now()::TIMESTAMP, '%Y-%m-%dT%H:%M:%S.%gZ')`. Without the ICU + extension a `TIMESTAMPTZ` casts to `TIMESTAMP` as UTC, which is the stored + format. `strftime('%s','now')` → `epoch(now())`. +- `sqlite_master` → `duckdb_tables()`; `GLOB` is supported natively. +- Partial indexes (`CREATE INDEX ... WHERE`) are not supported; the two the + SQLite schema used become full indexes. +- `BEGIN IMMEDIATE` → `BEGIN TRANSACTION`. There is no reserved-lock concept; + the engine `write_lock` already serializes writers, and DuckDB's optimistic + `Conflict on update` error is classified `Transient` by `map_db_err` as a + backstop. +- The TTL sweep's `json_extract` → `json_extract_string` + `TRY_CAST` (DuckDB's + `json_extract` returns a quoted JSON scalar, and the sweep must not error on a + non-numeric TTL attribute). +- `rowid` starts at 0 in DuckDB (1 in SQLite); the vector-backfill cursor + starts at `-1`. +- Numbered parameters are `$1`, not `?1`. + +--- + +## D5 — Error classification + +`is_unique_violation` matches DuckDB's `Duplicate key ... violates primary key +constraint` / `violates unique constraint` messages. A failed statement aborts +a DuckDB transaction (unlike SQLite, where the transaction stays usable); every +path in the crate that catches a constraint error inside a transaction already +returns the error and lets the transaction drop, so the rollback-on-drop in +`db::Transaction` is what keeps pooled connections clean. + +--- + +## D6 — Out-of-process tooling and the file lock + +DuckDB takes a process-exclusive lock on a read-write database file. The +SQLite backend lets `extenddb settings`, `catalog-check`, and `verify` open the +file alongside a running server; here they must run while the server is stopped. +No workaround was attempted: a read-only open conflicts with a writer too, and +routing the CLI through the server's management API is a cross-backend change +outside this crate. Documented in the README; the integration module that +depends on it (`test_gsi_async`) is excluded from the DuckDB CI job. + +## D7 — TTL sweep without an expression index + +DuckDB rejects `CREATE INDEX ... (json_extract_string(...))` ("Cannot use +json_extract_string in this context"). Left as an error, the engine retries the +index forever and never marks `ttl_index_ready`, so nothing ever expires. The +backend therefore skips the index and flips the flag; the sweep is a filtered +scan bounded by `LIMIT`, which at the batch sizes the worker uses is acceptable. diff --git a/crates/storage-duckdb/src/admin_store.rs b/crates/storage-duckdb/src/admin_store.rs new file mode 100644 index 00000000..c062fa16 --- /dev/null +++ b/crates/storage-duckdb/src/admin_store.rs @@ -0,0 +1,126 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `AdminStore` implementation: admin-user management, separate from IAM users. + +use crate::db; +use extenddb_storage::management_store::{AdminEntry, AdminStore, OpError, OpResult}; +use futures::future::BoxFuture; + +use crate::catalog_store::DuckDbCatalogStore; +use crate::duckdb_util::{is_unique_violation, parse_timestamp}; + +impl AdminStore for DuckDbCatalogStore { + fn create_admin(&self, admin_name: &str, password_hash: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + db::query("INSERT INTO admin_users (admin_name, password_hash) VALUES (?, ?)") + .bind(&admin_name) + .bind(&password_hash) + .execute(self.pool()) + .await + .map_err(|e| { + if is_unique_violation(&e) { + OpError::AlreadyExists("Admin user already exists".to_owned()) + } else { + tracing::error!("create_admin: {e}"); + OpError::Internal("Database error".to_owned()) + } + })?; + Ok(()) + }) + } + + fn list_admins(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async move { + let rows: Vec<(String, String)> = + db::query_as("SELECT admin_name, created_at FROM admin_users ORDER BY admin_name") + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_admins: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + rows.into_iter() + .map(|(admin_name, created)| { + Ok(AdminEntry { + admin_name, + created_at: parse_timestamp(&created) + .map_err(|e| OpError::Internal(format!("parse created_at: {e}")))?, + }) + }) + .collect() + }) + } + + fn delete_admin(&self, admin_name: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + Box::pin(async move { + let result = db::query("DELETE FROM admin_users WHERE admin_name = ?") + .bind(&admin_name) + .execute(self.pool()) + .await + .map_err(|e| { + tracing::error!("delete_admin: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.rows_affected() == 0 { + return Err(OpError::NotFound("Admin user not found".to_owned())); + } + Ok(()) + }) + } + + fn change_admin_password( + &self, + admin_name: &str, + password_hash: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + let result = db::query("UPDATE admin_users SET password_hash = ? WHERE admin_name = ?") + .bind(&password_hash) + .bind(&admin_name) + .execute(self.pool()) + .await + .map_err(|e| { + tracing::error!("change_admin_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.rows_affected() == 0 { + return Err(OpError::NotFound("Admin user not found".to_owned())); + } + Ok(()) + }) + } + + fn verify_admin_password( + &self, + admin_name: &str, + password: &str, + ) -> BoxFuture<'_, OpResult>> { + let admin_name = admin_name.to_owned(); + let password = password.to_owned(); + Box::pin(async move { + let row: Option<(String,)> = + db::query_as("SELECT password_hash FROM admin_users WHERE admin_name = ?") + .bind(&admin_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("verify_admin_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let Some((hash,)) = row else { + return Ok(None); + }; + let verified = tokio::task::spawn_blocking(move || bcrypt::verify(&password, &hash)) + .await + .map_err(|e| OpError::Internal(format!("bcrypt task: {e}")))? + .unwrap_or(false); + Ok(Some(verified)) + }) + } +} diff --git a/crates/storage-duckdb/src/authorization_store.rs b/crates/storage-duckdb/src/authorization_store.rs new file mode 100644 index 00000000..13b6d3fb --- /dev/null +++ b/crates/storage-duckdb/src/authorization_store.rs @@ -0,0 +1,253 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `AuthorizationStore` implementation: read-only IAM policy, boundary, +//! session, and tag lookups used by the policy evaluator on every authorized +//! request. Policy documents are stored as JSON text and returned verbatim. + +use crate::db; +use extenddb_storage::authorization_store::{AuthorizationStore, SessionData}; +use extenddb_storage::management_store::{OpError, OpResult}; +use futures::future::BoxFuture; + +use crate::catalog_store::DuckDbCatalogStore; +use crate::duckdb_util::parse_timestamp; + +/// Map any query error to a sanitized internal error, logging the detail. +fn db_err(ctx: &str, e: db::Error) -> OpError { + tracing::error!("{ctx}: {e}"); + OpError::Internal("Database error".to_owned()) +} + +impl AuthorizationStore for DuckDbCatalogStore { + fn fetch_user_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let rows: Vec<(String,)> = db::query_as( + "SELECT policy_document FROM iam_policies \ + WHERE account_id = ? AND principal_type = 'user' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_user_policies", e))?; + Ok(rows.into_iter().map(|(d,)| d).collect()) + }) + } + + fn fetch_user_group_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let rows: Vec<(String,)> = db::query_as( + "SELECT p.policy_document FROM iam_policies p \ + JOIN iam_group_members m \ + ON m.account_id = p.account_id AND m.group_name = p.principal_name \ + WHERE p.account_id = ? AND p.principal_type = 'group' AND m.user_name = ?", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_user_group_policies", e))?; + Ok(rows.into_iter().map(|(d,)| d).collect()) + }) + } + + fn fetch_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let row: Option<(String,)> = db::query_as( + "SELECT policy_document FROM iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = 'user' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| db_err("fetch_user_boundary", e))?; + Ok(row.map(|(d,)| d)) + }) + } + + fn fetch_role_policies( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let rows: Vec<(String,)> = db::query_as( + "SELECT policy_document FROM iam_policies \ + WHERE account_id = ? AND principal_type = 'role' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&role_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_role_policies", e))?; + Ok(rows.into_iter().map(|(d,)| d).collect()) + }) + } + + fn fetch_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let row: Option<(String,)> = db::query_as( + "SELECT policy_document FROM iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = 'role' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&role_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| db_err("fetch_role_boundary", e))?; + Ok(row.map(|(d,)| d)) + }) + } + + fn fetch_session_data( + &self, + account_id: &str, + role_name: &str, + session_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let session_name = session_name.to_owned(); + Box::pin(async move { + let row: Option<(Option, Option, String)> = db::query_as( + "SELECT session_policy, session_tags, expires_at FROM iam_sessions \ + WHERE account_id = ? AND role_name = ? AND session_name = ?", + ) + .bind(&account_id) + .bind(&role_name) + .bind(&session_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| db_err("fetch_session_data", e))?; + + let Some((session_policy, tags_json, expires_at)) = row else { + return Ok(None); + }; + + // Treat an expired session as absent (parity with the Postgres + // expiry filter). Authentication already rejects expired sessions, + // so this is defense-in-depth; evaluate it with the same parser + // lookup_session uses rather than a format-fragile SQL comparison. + if let Ok(expires) = parse_timestamp(&expires_at) + && expires < time::OffsetDateTime::now_utc() + { + return Ok(None); + } + + // session_tags is stored as JSON: either an object {key: value} or + // an array [{"Key": ..., "Value": ...}] (the form AWS SDKs send). + // Handle both, matching the Postgres backend. + let session_tags: Vec<(String, String)> = tags_json + .as_deref() + .and_then(|s| serde_json::from_str::(s).ok()) + .map(|v| { + if let Some(arr) = v.as_array() { + arr.iter() + .filter_map(|tag| { + match ( + tag.get("Key").and_then(|k| k.as_str()), + tag.get("Value").and_then(|x| x.as_str()), + ) { + (Some(k), Some(val)) => Some((k.to_owned(), val.to_owned())), + _ => None, + } + }) + .collect() + } else if let Some(obj) = v.as_object() { + obj.iter() + .filter_map(|(k, val)| val.as_str().map(|s| (k.clone(), s.to_owned()))) + .collect() + } else { + Vec::new() + } + }) + .unwrap_or_default(); + + Ok(Some(SessionData { + session_policy, + session_tags, + })) + }) + } + + fn fetch_user_tags( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + db::query_as( + "SELECT tag_key, tag_value FROM iam_user_tags \ + WHERE account_id = ? AND user_name = ? ORDER BY tag_key", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_user_tags", e)) + }) + } + + fn fetch_role_tags( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + db::query_as( + "SELECT tag_key, tag_value FROM iam_role_tags \ + WHERE account_id = ? AND role_name = ? ORDER BY tag_key", + ) + .bind(&account_id) + .bind(&role_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_role_tags", e)) + }) + } + + fn fetch_resource_tags(&self, arn: &str) -> BoxFuture<'_, OpResult>> { + let arn = arn.to_owned(); + Box::pin(async move { + db::query_as( + "SELECT tag_key, tag_value FROM tags WHERE resource_arn = ? ORDER BY tag_key", + ) + .bind(&arn) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_resource_tags", e)) + }) + } +} diff --git a/crates/storage-duckdb/src/backup.rs b/crates/storage-duckdb/src/backup.rs new file mode 100644 index 00000000..447f33d9 --- /dev/null +++ b/crates/storage-duckdb/src/backup.rs @@ -0,0 +1,527 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `BackupEngine` for the DuckDB backend. +//! +//! A backup snapshots every item's `item_data` into `backup_items`. Restore +//! recreates the table via `create_table` and upserts the snapshot under the +//! engine write lock. `RestoreTableToPointInTime` is implemented as a +//! snapshot-then-restore (then discard the temporary backup), matching the +//! PostgreSQL backend's behaviour. + +use crate::db; +use extenddb_core::types::{ + AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, BillingMode, + ContinuousBackupsDescription, CreateTableInput, KeySchemaElement, + PointInTimeRecoveryDescription, ProvisionedThroughput, SourceTableDetails, TableDescription, + TableKeyInfo, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{BackupEngine, TableEngine}; +use futures::future::BoxFuture; + +use crate::data::{data_table_name, upsert_item_in_tx}; +use crate::duckdb_util::parse_timestamp; +use crate::store::DuckDbEngine; + +fn epoch_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +/// Backup id: creation timestamp plus an 8-hex random suffix, matching the +/// PostgreSQL backend. The suffix makes ARNs non-guessable and prevents two +/// backups created in the same millisecond from colliding. +fn backup_id() -> String { + use rand::Rng; + let suffix: u32 = rand::rng().random(); + format!("{ts}-{suffix:08x}", ts = epoch_millis()) +} + +#[allow(clippy::cast_precision_loss)] +fn ts_to_epoch(s: &str) -> f64 { + parse_timestamp(s) + .map(|dt| dt.unix_timestamp() as f64) + .unwrap_or(0.0) +} + +impl BackupEngine for DuckDbEngine { + fn create_backup( + &self, + account_id: &str, + table_name: &str, + backup_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + let backup_name = backup_name.to_owned(); + Box::pin(async move { + let row: Option<(String, String, String, String, i64)> = db::query_as( + "SELECT table_id, key_schema, attribute_definitions, billing_mode, table_size_bytes \ + FROM tables WHERE account_id = ? AND table_name = ? AND table_status = 'ACTIVE'", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id, key_schema, attr_defs, billing_mode, size_bytes) = + row.ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let backup_arn = format!( + "arn:aws:dynamodb:{region}:{account_id}:table/{table_name}/backup/{id}", + region = self.region, + id = backup_id() + ); + + let ddb_table = data_table_name(&table_id); + let items: Vec<(String,)> = db::query_as(&format!("SELECT item_data FROM {ddb_table}")) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let item_count = i64::try_from(items.len()).unwrap_or(i64::MAX); + + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + db::query( + "INSERT INTO backups (backup_arn, backup_name, table_id, table_name, account_id, \ + backup_status, backup_size_bytes, item_count, key_schema, attribute_definitions, \ + billing_mode) VALUES (?, ?, ?, ?, ?, 'AVAILABLE', ?, ?, ?, ?, ?)", + ) + .bind(&backup_arn) + .bind(&backup_name) + .bind(&table_id) + .bind(&table_name) + .bind(&account_id) + .bind(size_bytes) + .bind(item_count) + .bind(&key_schema) + .bind(&attr_defs) + .bind(&billing_mode) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + for (item_data,) in &items { + db::query( + "INSERT INTO backup_items (backup_arn, pk, sk, item_data) VALUES (?, '', NULL, ?)", + ) + .bind(&backup_arn) + .bind(item_data) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + let created_at: (String,) = + db::query_as("SELECT created_at FROM backups WHERE backup_arn = ?") + .bind(&backup_arn) + .fetch_one(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDetails { + backup_arn, + backup_name, + backup_status: "AVAILABLE".to_owned(), + backup_type: "USER".to_owned(), + backup_size_bytes: size_bytes, + backup_creation_date_time: ts_to_epoch(&created_at.0), + }) + }) + } + + fn describe_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let backup_arn = backup_arn.to_owned(); + Box::pin(async move { + #[allow(clippy::type_complexity)] + let row: Option<(String, String, String, String, i64, i64, String, String, String, String)> = + db::query_as( + "SELECT b.backup_name, b.backup_status, b.table_id, b.table_name, \ + b.backup_size_bytes, b.item_count, b.key_schema, b.billing_mode, \ + COALESCE(t.table_arn, \ + 'arn:aws:dynamodb:' || ? || ':' || b.account_id || ':table/' || b.table_name), \ + b.created_at \ + FROM backups b LEFT JOIN tables t ON t.table_id = b.table_id \ + WHERE b.backup_arn = ? AND b.account_id = ? \ + AND b.backup_status != 'DELETED'", + ) + .bind(&self.region) + .bind(&backup_arn) + .bind(&account_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (name, status, table_id, table_name, size, count, ks, billing, table_arn, created) = + row.ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + + let key_schema: Vec = + serde_json::from_str(&ks).map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_arn: backup_arn.clone(), + backup_name: name, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: ts_to_epoch(&created), + }, + source_table_details: SourceTableDetails { + table_name, + table_id, + table_arn, + key_schema, + item_count: count, + table_size_bytes: size, + billing_mode: Some(billing), + table_creation_date_time: ts_to_epoch(&created), + }, + }) + }) + } + + fn list_backups( + &self, + account_id: &str, + table_name: Option<&str>, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_owned(); + let table_name = table_name.map(str::to_owned); + Box::pin(async move { + let rows: Vec<(String, String, String, String, i64, String, String)> = + if let Some(tn) = table_name { + db::query_as( + "SELECT b.backup_arn, b.backup_name, b.table_name, b.backup_status, \ + b.backup_size_bytes, \ + COALESCE(t.table_arn, 'arn:aws:dynamodb:' || ? || ':' || b.account_id || ':table/' || b.table_name), \ + b.created_at FROM backups b LEFT JOIN tables t ON t.table_id = b.table_id \ + WHERE b.account_id = ? AND b.table_name = ? AND b.backup_status != 'DELETED' \ + ORDER BY b.created_at DESC", + ) + .bind(&self.region) + .bind(&account_id) + .bind(tn) + .fetch_all(&self.pool) + .await + } else { + db::query_as( + "SELECT b.backup_arn, b.backup_name, b.table_name, b.backup_status, \ + b.backup_size_bytes, \ + COALESCE(t.table_arn, 'arn:aws:dynamodb:' || ? || ':' || b.account_id || ':table/' || b.table_name), \ + b.created_at FROM backups b LEFT JOIN tables t ON t.table_id = b.table_id \ + WHERE b.account_id = ? AND b.backup_status != 'DELETED' \ + ORDER BY b.created_at DESC", + ) + .bind(&self.region) + .bind(&account_id) + .fetch_all(&self.pool) + .await + } + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(rows + .into_iter() + .map( + |(arn, name, tn, status, size, table_arn, created)| BackupSummary { + backup_arn: arn, + backup_name: name, + table_name: tn, + table_arn, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: ts_to_epoch(&created), + }, + ) + .collect()) + }) + } + + fn delete_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let backup_arn = backup_arn.to_owned(); + Box::pin(async move { + // Resolves account-scoped, so a backup owned by another account is + // reported missing here and the writes below never run. + let desc = self.describe_backup(&account_id, &backup_arn).await?; + + // The account predicate is repeated on both writes rather than + // relying on the lookup above, so the statements are correct on + // their own terms. + db::query( + "DELETE FROM backup_items WHERE backup_arn = $1 AND EXISTS (\ + SELECT 1 FROM backups b WHERE b.backup_arn = $1 AND b.account_id = $2)", + ) + .bind(&backup_arn) + .bind(&account_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + db::query( + "UPDATE backups SET backup_status = 'DELETED' \ + WHERE backup_arn = ? AND account_id = ?", + ) + .bind(&backup_arn) + .bind(&account_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(BackupDescription { + backup_details: BackupDetails { + backup_status: "DELETED".to_owned(), + ..desc.backup_details + }, + source_table_details: desc.source_table_details, + }) + }) + } + + fn restore_table_from_backup( + &self, + account_id: &str, + target_table_name: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let target_table_name = target_table_name.to_owned(); + let backup_arn = backup_arn.to_owned(); + Box::pin(async move { + let row: Option<(String, String, String)> = db::query_as( + "SELECT key_schema, attribute_definitions, billing_mode \ + FROM backups WHERE backup_arn = ? AND account_id = ? \ + AND backup_status = 'AVAILABLE'", + ) + .bind(&backup_arn) + .bind(&account_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (ks, ad, billing) = row.ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + let key_schema: Vec = + serde_json::from_str(&ks).map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs: Vec = + serde_json::from_str(&ad).map_err(|e| StorageError::Internal(e.to_string()))?; + let billing_mode = Some(if billing == "PAY_PER_REQUEST" { + BillingMode::PayPerRequest + } else { + BillingMode::Provisioned + }); + + let create_input = CreateTableInput { + table_name: target_table_name.clone(), + key_schema: key_schema.clone(), + attribute_definitions: attr_defs.clone(), + billing_mode, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + global_secondary_indexes: None, + local_secondary_indexes: None, + stream_specification: None, + tags: None, + deletion_protection_enabled: None, + sse_specification: None, + table_class: None, + on_demand_throughput: None, + ..Default::default() + }; + + let desc = self.create_table(&account_id, create_input).await?; + let key_info = TableKeyInfo { + table_name: target_table_name.clone(), + account_id: account_id.clone(), + table_id: desc.table_id.clone(), + base_key_schema: key_schema.clone(), + key_schema, + attribute_definitions: attr_defs, + has_lsi: false, + // The restored table is created above without secondary + // indexes, so there is no index metadata to carry. + global_secondary_indexes: Vec::new(), + local_secondary_indexes: Vec::new(), + stream_specification: None, + ..Default::default() + }; + + let items: Vec<(String,)> = + db::query_as("SELECT item_data FROM backup_items WHERE backup_arn = ?") + .bind(&backup_arn) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let item_count = i64::try_from(items.len()).unwrap_or(i64::MAX); + + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + for (item_json,) in &items { + let item: extenddb_core::types::Item = serde_json::from_str(item_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + upsert_item_in_tx(&mut tx, &key_info, &item).await?; + } + db::query("UPDATE tables SET item_count = ? WHERE account_id = ? AND table_name = ?") + .bind(item_count) + .bind(&account_id) + .bind(&target_table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + // Mark the restored table ACTIVE immediately: the data is fully + // populated and ready to serve. This matches the Postgres backend + // and real DynamoDB, where a restored table becomes ACTIVE once the + // restore completes (the CREATING status is transient) rather than + // waiting for the control-plane transition poller. + db::query( + "UPDATE tables SET table_status = 'ACTIVE', status_transition_at = NULL \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&target_table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(desc) + }) + } + + fn describe_continuous_backups( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + Box::pin(async move { + let exists: bool = db::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables WHERE account_id = ? AND table_name = ?)", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if !exists { + return Err(StorageError::TableNotFound(table_name)); + } + + let pitr: Option<(bool,)> = db::query_as( + "SELECT pitr_enabled FROM continuous_backups WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let enabled = pitr.is_some_and(|r| r.0); + + #[allow(clippy::cast_precision_loss)] + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as f64; + + Ok(ContinuousBackupsDescription { + continuous_backups_status: "ENABLED".to_owned(), + point_in_time_recovery_description: Some(PointInTimeRecoveryDescription { + point_in_time_recovery_status: if enabled { "ENABLED" } else { "DISABLED" } + .to_owned(), + earliest_restorable_date_time: enabled.then_some(now - 35.0 * 24.0 * 3600.0), + latest_restorable_date_time: enabled.then_some(now), + }), + }) + }) + } + + fn update_continuous_backups( + &self, + account_id: &str, + table_name: &str, + pitr_enabled: bool, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + Box::pin(async move { + let exists: bool = db::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables WHERE account_id = ? AND table_name = ?)", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if !exists { + return Err(StorageError::TableNotFound(table_name)); + } + db::query( + "INSERT INTO continuous_backups (account_id, table_name, pitr_enabled) \ + VALUES (?, ?, ?) \ + ON CONFLICT (account_id, table_name) DO UPDATE SET pitr_enabled = excluded.pitr_enabled", + ) + .bind(&account_id) + .bind(&table_name) + .bind(pitr_enabled) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + self.describe_continuous_backups(&account_id, &table_name) + .await + }) + } + + fn restore_table_to_point_in_time( + &self, + account_id: &str, + source_table_name: &str, + target_table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let source_table_name = source_table_name.to_owned(); + let target_table_name = target_table_name.to_owned(); + Box::pin(async move { + let backup = self + .create_backup(&account_id, &source_table_name, "__pitr_restore__") + .await?; + let desc = self + .restore_table_from_backup(&account_id, &target_table_name, &backup.backup_arn) + .await?; + let _ = self.delete_backup(&account_id, &backup.backup_arn).await; + Ok(desc) + }) + } +} diff --git a/crates/storage-duckdb/src/bootstrapper.rs b/crates/storage-duckdb/src/bootstrapper.rs new file mode 100644 index 00000000..178ce76d --- /dev/null +++ b/crates/storage-duckdb/src/bootstrapper.rs @@ -0,0 +1,502 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `Bootstrapper` implementation for the DuckDB backend. +//! +//! DuckDB has no server, users, or roles, and catalog and data live in one +//! file. Initialization therefore reduces to: create the file, apply the +//! catalog schema, and seed the encryption key, default account, and admin +//! user. Destruction removes the database file and its WAL/SHM sidecars. + +use crate::db; +use async_trait::async_trait; +use extenddb_core::types::{AttributeDefinition, KeySchemaElement, KeyType}; +use extenddb_storage::bootstrapper::{ + AdminBootstrapResult, Bootstrapper, KeyDefinitionRepair, helpers, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::management_store::{OpError, OpResult}; +use extenddb_storage::util::recover_sort_key_definitions; + +use crate::duckdb_util::duckdb_path; +use crate::schema::{self, CATALOG_VERSION}; + +/// DuckDB backend bootstrapper. +/// +/// Holds the database file path. Each operation opens a short-lived +/// single-connection pool, since `init`/`destroy`/`migrate` are one-shot CLI +/// paths, not the hot serving path. +pub struct DuckDbBootstrapper { + path: String, +} + +impl DuckDbBootstrapper { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + /// Build a `DuckDbBootstrapper` from the config file and CLI args. + /// + /// Resolution order for the database path (bootstrapper commands: init, + /// destroy, migrate): the `--duckdb-path

` CLI flag (declared on + /// `InitArgs`), then `[storage.duckdb].path` in the config file, then the + /// default `extenddb.duckdb`. `serve` does not use this path; it reads + /// the config (with `EXTENDDB__STORAGE__SQLITE__PATH` overriding). + pub async fn from_config(config_path: &str, cli_args: &[String]) -> Result { + if let Some(p) = helpers::extract_arg(cli_args, "--duckdb-path") { + return Ok(Self::new(p)); + } + + let path = if std::path::Path::new(config_path).exists() { + let content = std::fs::read_to_string(config_path) + .map_err(|e| StorageError::Internal(format!("read config {config_path}: {e}")))?; + let parsed: toml::Value = toml::from_str(&content) + .map_err(|e| StorageError::Internal(format!("parse config {config_path}: {e}")))?; + parsed + .get("storage") + .and_then(|s| s.get("duckdb")) + .and_then(|s| s.get("path")) + .and_then(|v| v.as_str()) + .unwrap_or("extenddb.duckdb") + .to_owned() + } else { + "extenddb.duckdb".to_owned() + }; + + Ok(Self::new(path)) + } + + fn is_memory(&self) -> bool { + self.path == ":memory:" || self.path.starts_with("file::memory:") + } + + /// Open a short-lived single-connection pool with the standard PRAGMAs. + /// For a real filesystem path, the parent directory is created first: + /// `init --duckdb-path` may point into a directory that does not exist + /// yet, and bootstrapping it is exactly init's job. + async fn pool(&self) -> OpResult { + if self.path != ":memory:" + && let Some(parent) = std::path::Path::new(&self.path).parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).map_err(|e| { + OpError::Internal(format!( + "create parent directory for DuckDB database '{}': {e}", + self.path + )) + })?; + } + let pool = db::Pool::open(&duckdb_path(&self.path), 1) + .await + .map_err(|e| OpError::Internal(format!("open DuckDB database '{}': {e}", self.path)))?; + // The database stores the encryption key next to the secrets it + // protects; keep the file and its WAL owner-only from the moment they + // exist rather than waiting for the first `serve`. + #[cfg(unix)] + if !self.is_memory() { + use std::os::unix::fs::PermissionsExt; + for suffix in ["", ".wal"] { + let f = format!("{}{suffix}", self.path); + if std::path::Path::new(&f).exists() { + let _ = std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o600)); + } + } + } + Ok(pool) + } +} + +#[async_trait] +impl Bootstrapper for DuckDbBootstrapper { + async fn ensure_app_user(&self) -> OpResult<()> { + Ok(()) // DuckDB has no user concept. + } + + async fn grant_app_role_to_admin(&self) -> OpResult<()> { + Ok(()) // DuckDB has no role concept. + } + + async fn create_catalog_db(&self) -> OpResult<()> { + // Connecting with mode=rwc creates the file; refuse to clobber an + // existing database so `init` is not silently destructive. + if !self.is_memory() && std::path::Path::new(&self.path).exists() { + return Err(OpError::AlreadyExists(format!( + "DuckDB database '{}' already exists. Run 'destroy' first, then 'init'.", + self.path + ))); + } + Ok(()) + } + + async fn create_data_db(&self) -> OpResult<()> { + Ok(()) // Catalog and data share one file. + } + + async fn run_catalog_migrations(&self) -> OpResult<()> { + let pool = self.pool().await?; + schema::apply(&pool).await + } + + async fn run_data_migrations(&self) -> OpResult<()> { + Ok(()) // Single file — schema applied in run_catalog_migrations. + } + + async fn pending_data_migrations(&self) -> OpResult> { + // DuckDB applies its complete schema in run_catalog_migrations (a single + // file), so there are no separately-tracked data migrations that can be + // pending. + Ok(Vec::new()) + } + + async fn record_data_connection(&self) -> OpResult<()> { + let pool = self.pool().await?; + db::query( + "INSERT INTO settings (key, value) VALUES ('data_database_name', ?) \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ) + .bind(&self.path) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("record data db name: {e}")))?; + Ok(()) + } + + async fn bootstrap_encryption_key(&self) -> OpResult<()> { + let pool = self.pool().await?; + let exists: bool = + db::query_scalar("SELECT EXISTS(SELECT 1 FROM settings WHERE key = 'encryption_key')") + .fetch_one(&pool) + .await + .map_err(|e| OpError::Internal(format!("check encryption key: {e}")))?; + if exists { + return Ok(()); + } + let key = helpers::generate_encryption_key(); + db::query("INSERT OR IGNORE INTO settings (key, value) VALUES ('encryption_key', ?)") + .bind(&key) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("store encryption key: {e}")))?; + Ok(()) + } + + async fn bootstrap_default_account(&self) -> OpResult<()> { + let pool = self.pool().await?; + // Reuse the existing account when already bootstrapped, otherwise create + // the single default account. Either way, record its id as the canonical + // default account so callers never infer it from list ordering. The + // settings write is idempotent (INSERT OR IGNORE), so it also backfills + // the marker for catalogs bootstrapped before it existed. + let account_id: String = match db::query_scalar( + "SELECT account_id FROM accounts ORDER BY account_id LIMIT 1", + ) + .fetch_optional(&pool) + .await + .map_err(|e| OpError::Internal(format!("check accounts: {e}")))? + { + Some(id) => id, + None => { + let id = helpers::generate_account_id(); + db::query( + "INSERT OR IGNORE INTO accounts (account_id, account_name) VALUES (?, 'default')", + ) + .bind(&id) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("create default account: {e}")))?; + id + } + }; + db::query("INSERT OR IGNORE INTO settings (key, value) VALUES ('default_account_id', ?)") + .bind(&account_id) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("record default account id: {e}")))?; + Ok(()) + } + + async fn bootstrap_admin_user( + &self, + env_user: Option<&str>, + env_password: Option<&str>, + ) -> OpResult { + let pool = self.pool().await?; + let username = env_user + .filter(|s| !s.is_empty()) + .unwrap_or("admin") + .to_owned(); + + let exists: bool = + db::query_scalar("SELECT EXISTS(SELECT 1 FROM admin_users WHERE admin_name = ?)") + .bind(&username) + .fetch_one(&pool) + .await + .map_err(|e| OpError::Internal(format!("check admin user: {e}")))?; + if exists { + return Ok(AdminBootstrapResult { + username, + generated_password: None, + already_existed: true, + from_env: env_user.is_some(), + }); + } + + let (password, from_env) = match env_password.filter(|s| !s.is_empty()) { + Some(p) => (p.to_owned(), true), + None => (helpers::generate_random_password(), false), + }; + let password_hash = helpers::hash_password_async(password.clone()).await?; + + db::query("INSERT INTO admin_users (admin_name, password_hash) VALUES (?, ?)") + .bind(&username) + .bind(&password_hash) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("create admin user: {e}")))?; + + Ok(AdminBootstrapResult { + username, + generated_password: if from_env { None } else { Some(password) }, + already_existed: false, + from_env, + }) + } + + async fn is_catalog_initialized(&self) -> OpResult { + if !self.is_memory() && !std::path::Path::new(&self.path).exists() { + return Ok(false); + } + let Ok(pool) = self.pool().await else { + return Ok(false); + }; + schema::table_exists(&pool, "settings").await + } + + /// Repair table metadata damaged by the pre-fix `UpdateTable` (#259). + /// + /// DuckDB keeps the catalog and the data tables in one file, so the physical + /// sort key columns are read with `PRAGMA table_info`. Otherwise identical to + /// the PostgreSQL implementation: for any base sort key with no attribute + /// definition, recover the type from its column name. + async fn repair_lost_sort_key_definitions(&self, apply: bool) -> OpResult { + let mut report = KeyDefinitionRepair::default(); + let Ok(pool) = self.pool().await else { + return Ok(report); + }; + // A never-initialised catalog has no `tables` table, and `pool()` opens + // with mode=rwc, which silently creates an empty database file, so the + // SELECT below would hard-error rather than find nothing. PostgreSQL + // degrades gracefully through its get_data_db_name() guard; this is the + // DuckDB equivalent. + if !schema::table_exists(&pool, "tables").await? { + return Ok(report); + } + + let rows: Vec<(String, String, String, String, String)> = db::query_as( + "SELECT account_id, table_name, table_id, key_schema, attribute_definitions \ + FROM tables ORDER BY table_name", + ) + .fetch_all(&pool) + .await + .map_err(|e| OpError::Internal(format!("Cannot read tables: {e}")))?; + + for (account_id, table_name, table_id, ks_json, ad_json) in rows { + let key_schema: Vec = match serde_json::from_str(&ks_json) { + Ok(v) => v, + Err(e) => { + report + .needs_attention + .push(format!("{table_name}: unreadable key_schema ({e})")); + continue; + } + }; + let attr_defs: Vec = + serde_json::from_str(&ad_json).unwrap_or_default(); + + // Reported for every table on every run, not only when a sort key is + // repaired: the partition key definition is dropped by the same write and + // is not recoverable from the schema, because the pk column is always TEXT. + // It is not needed for correctness either, since partition key values are + // encoded from the key schema alone, so it is reported rather than guessed + // and keeps being reported until a human restores it. + let pk_missing: Vec<&str> = key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Hash) + .filter(|ks| { + !attr_defs + .iter() + .any(|ad| ad.attribute_name == ks.attribute_name) + }) + .map(|ks| ks.attribute_name.as_str()) + .collect(); + if !pk_missing.is_empty() { + report.needs_attention.push(format!( + "{table_name}: partition key definition(s) [{}] are absent; reads and \ + writes are unaffected, but index key type validation cannot check them", + pk_missing.join(", ") + )); + } + + let missing: Vec<&KeySchemaElement> = key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Range) + .filter(|ks| { + !attr_defs + .iter() + .any(|ad| ad.attribute_name == ks.attribute_name) + }) + .collect(); + if missing.is_empty() { + continue; + } + + // PRIMARY KEY columns only, in key order. Every data table carries all + // three typed columns for each sort key position (sk_s, sk_n, sk_b), so + // column existence says nothing about the declared type: only the pk + // position reported by table_info does. `table_id` is interpolated + // because PRAGMA takes no bind parameters; it is checked as a UUID first + // so a malformed catalog row cannot reach the statement. + if uuid::Uuid::parse_str(&table_id).is_err() { + report + .needs_attention + .push(format!("{table_name}: table_id {table_id:?} is not a UUID")); + continue; + } + let mut key_columns: Vec<(i64, String)> = + db::query_as::<(i64, String, String, i64, Option, i64)>(&format!( + "PRAGMA table_info(\"_ddb_{table_id}\")" + )) + .fetch_all(&pool) + .await + .map_err(|e| { + OpError::Internal(format!("Cannot read columns of {table_name}: {e}")) + })? + .into_iter() + .filter(|(.., pk_pos)| *pk_pos > 0) + .map(|(_, name, _, _, _, pk_pos)| (pk_pos, name)) + .collect(); + key_columns.sort_unstable(); + let columns: Vec = key_columns.into_iter().map(|(_, name)| name).collect(); + + let recovered = recover_sort_key_definitions(&key_schema, &attr_defs, &columns); + if recovered.is_empty() { + report.needs_attention.push(format!( + "{table_name}: sort key(s) [{}] have no attribute definition and no \ + matching PRIMARY KEY column was found to recover the type from", + missing + .iter() + .map(|ks| ks.attribute_name.as_str()) + .collect::>() + .join(", ") + )); + continue; + } + + let mut merged = attr_defs; + merged.extend(recovered.iter().cloned()); + let merged_json = serde_json::to_string(&merged) + .map_err(|e| OpError::Internal(format!("Cannot serialize definitions: {e}")))?; + if apply { + db::query( + "UPDATE tables SET attribute_definitions = $1 \ + WHERE account_id = $2 AND table_name = $3", + ) + .bind(&merged_json) + .bind(&account_id) + .bind(&table_name) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("Cannot repair {table_name}: {e}")))?; + } + + for def in &recovered { + report.repaired.push(format!( + "{table_name}: {} ({:?})", + def.attribute_name, def.attribute_type + )); + } + } + + Ok(report) + } + + async fn list_table_names(&self) -> OpResult> { + let Ok(pool) = self.pool().await else { + return Ok(Vec::new()); + }; + let rows: Vec<(String,)> = + db::query_as("SELECT table_name FROM tables ORDER BY table_name") + .fetch_all(&pool) + .await + .unwrap_or_default(); + Ok(rows.into_iter().map(|(n,)| n).collect()) + } + + async fn get_data_db_name(&self) -> OpResult> { + let Ok(pool) = self.pool().await else { + return Ok(None); + }; + let row: Option<(String,)> = + db::query_as("SELECT value FROM settings WHERE key = 'data_database_name'") + .fetch_optional(&pool) + .await + .unwrap_or(None); + Ok(row.map(|(v,)| v)) + } + + async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { + if self.is_memory() { + return Ok(()); + } + // Release this process's handle on the database first: DuckDB holds a + // file lock for as long as an instance is open. + db::forget_path(&self.path); + if std::path::Path::new(&self.path).exists() { + std::fs::remove_file(&self.path) + .map_err(|e| OpError::Internal(format!("remove database file: {e}")))?; + } + // Remove the write-ahead log sidecar if present. + let _ = std::fs::remove_file(format!("{}.wal", self.path)); + Ok(()) + } + + async fn read_catalog_version(&self) -> OpResult> { + let Ok(pool) = self.pool().await else { + return Ok(None); + }; + if !schema::table_exists(&pool, "settings").await? { + return Ok(None); + } + let row: Option<(String,)> = + db::query_as("SELECT value FROM settings WHERE key = 'catalog_version'") + .fetch_optional(&pool) + .await + .map_err(|e| OpError::Internal(format!("read catalog version: {e}")))?; + Ok(row.map(|(v,)| v)) + } + + fn expected_catalog_version(&self) -> String { + CATALOG_VERSION.to_string() + } + + fn catalog_database_name(&self) -> String { + self.path.clone() + } + + fn endpoint_info(&self) -> String { + format!("duckdb:{}", self.path) + } + + fn catalog_connection_url(&self) -> String { + duckdb_path(&self.path) + } + + fn generate_backend_config_section(&self) -> String { + format!( + "[storage.duckdb]\n\ + path = \"{}\"\n\ + # pool_size = 10", + self.path + ) + } +} diff --git a/crates/storage-duckdb/src/catalog_store.rs b/crates/storage-duckdb/src/catalog_store.rs new file mode 100644 index 00000000..99cf8a37 --- /dev/null +++ b/crates/storage-duckdb/src/catalog_store.rs @@ -0,0 +1,367 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `DuckDbCatalogStore`: the catalog/management store backing the management +//! API and authorization. Implements `SettingsStore`, `MetricsStore`, +//! `RateLimitStore`, `DiagnosticsStore`, `AdminStore` (in `admin_store`), +//! `AuthorizationStore` (in `authorization_store`), `ManagementStore` (in the +//! `management_store` module), and the `CatalogStore` supertrait. + +use crate::db; +use std::sync::Arc; + +use extenddb_storage::CatalogStore; +use extenddb_storage::diagnostics::{DiagError, DiagResult, DiagnosticsStore}; +use extenddb_storage::management_store::{ + MetricsRow, MetricsStore, OpError, OpResult, RateLimitStore, SettingsStore, +}; +use futures::future::BoxFuture; + +use time::OffsetDateTime; + +use crate::duckdb_util::{format_timestamp, parse_timestamp}; + +/// Catalog store over the shared DuckDB pool. +/// +/// The encryption key is cached at construction (from the `settings` table) so +/// access-key creation/import can encrypt secrets without an extra query. +pub struct DuckDbCatalogStore { + pool: db::Pool, + encryption_key: Option>, +} + +impl DuckDbCatalogStore { + /// Construct without a cached encryption key (settings/diagnostics-only use). + pub fn new(pool: db::Pool) -> Self { + Self { + pool, + encryption_key: None, + } + } + + /// Construct with the cached AES-256-GCM encryption key (base64). + pub fn with_encryption_key(pool: db::Pool, encryption_key: String) -> Self { + Self { + pool, + encryption_key: Some(Arc::from(encryption_key.as_str())), + } + } + + pub(crate) fn pool(&self) -> &db::Pool { + &self.pool + } + + pub(crate) fn encryption_key(&self) -> Option<&Arc> { + self.encryption_key.as_ref() + } +} + +// ── SettingsStore ────────────────────────────────────────────────────── + +impl SettingsStore for DuckDbCatalogStore { + fn get_setting(&self, key: &str) -> BoxFuture<'_, OpResult>> { + let key = key.to_owned(); + Box::pin(async move { + let row: Option<(String,)> = db::query_as("SELECT value FROM settings WHERE key = ?") + .bind(&key) + .fetch_optional(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("get_setting: {e}")))?; + Ok(row.map(|(v,)| v)) + }) + } + + fn set_setting(&self, key: &str, value: &str) -> BoxFuture<'_, OpResult<()>> { + let key = key.to_owned(); + let value = value.to_owned(); + Box::pin(async move { + db::query( + "INSERT INTO settings (key, value) VALUES (?, ?) \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ) + .bind(&key) + .bind(&value) + .execute(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("set_setting: {e}")))?; + Ok(()) + }) + } + + fn list_settings(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async move { + db::query_as("SELECT key, value FROM settings ORDER BY key") + .fetch_all(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("list_settings: {e}"))) + }) + } + + fn cached_encryption_key(&self) -> Option { + self.encryption_key.as_ref().map(|k| k.to_string()) + } +} + +// ── MetricsStore ─────────────────────────────────────────────────────── + +struct DbMetricsRow { + bucket: String, + metric: String, + table_name: String, + index_name: String, + operation: String, + sum: f64, + count: i64, + min: f64, + max: f64, +} + +crate::impl_from_row!(DbMetricsRow { + bucket, + metric, + table_name, + index_name, + operation, + sum, + count, + min, + max +}); + +impl MetricsStore for DuckDbCatalogStore { + fn insert_metrics(&self, rows: &[MetricsRow]) -> BoxFuture<'_, OpResult<()>> { + let rows = rows.to_vec(); + Box::pin(async move { + for row in &rows { + let bucket = format_timestamp(row.bucket); + let result = db::query( + "INSERT INTO metrics \ + (bucket, metric, table_name, index_name, operation, sum, count, min, max) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT(bucket, metric, table_name, index_name, operation) DO UPDATE SET \ + sum = metrics.sum + excluded.sum, \ + count = metrics.count + excluded.count, \ + min = LEAST(metrics.min, excluded.min), \ + max = GREATEST(metrics.max, excluded.max)", + ) + .bind(&bucket) + .bind(&row.metric) + .bind(row.table_name.as_deref().unwrap_or("")) + .bind(row.index_name.as_deref().unwrap_or("")) + .bind(row.operation.as_deref().unwrap_or("")) + .bind(row.sum) + .bind(row.count) + .bind(row.min) + .bind(row.max) + .execute(&self.pool) + .await; + if let Err(e) = result { + tracing::warn!("insert_metrics row failed: {e}"); + } + } + Ok(()) + }) + } + + fn query_metrics( + &self, + start: OffsetDateTime, + end: OffsetDateTime, + table_name: Option<&str>, + metric: Option<&str>, + ) -> BoxFuture<'_, OpResult>> { + let table_name = table_name.map(str::to_owned); + let metric = metric.map(str::to_owned); + let start_str = format_timestamp(start); + let end_str = format_timestamp(end); + Box::pin(async move { + let mut sql = String::from( + "SELECT bucket, metric, table_name, index_name, operation, sum, count, min, max \ + FROM metrics WHERE bucket >= ? AND bucket <= ?", + ); + let table_filter = table_name.as_deref().filter(|s| !s.is_empty()); + if table_filter.is_some() { + sql.push_str(" AND table_name = ?"); + } + if metric.is_some() { + sql.push_str(" AND metric = ?"); + } + sql.push_str(" ORDER BY bucket"); + + let mut q = db::query_as::(&sql) + .bind(&start_str) + .bind(&end_str); + if let Some(tn) = table_filter { + q = q.bind(tn.to_owned()); + } + if let Some(m) = metric.as_deref() { + q = q.bind(m.to_owned()); + } + + let rows = q + .fetch_all(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("query_metrics: {e}")))?; + + Ok(rows + .into_iter() + .filter_map(|r| { + Some(MetricsRow { + bucket: parse_timestamp(&r.bucket).ok()?, + metric: r.metric, + table_name: (!r.table_name.is_empty()).then_some(r.table_name), + index_name: (!r.index_name.is_empty()).then_some(r.index_name), + operation: (!r.operation.is_empty()).then_some(r.operation), + sum: r.sum, + count: r.count, + min: r.min, + max: r.max, + }) + }) + .collect()) + }) + } + + fn prune_metrics(&self, retention: std::time::Duration) -> BoxFuture<'_, OpResult<()>> { + Box::pin(async move { + let cutoff = OffsetDateTime::now_utc() + - time::Duration::seconds(i64::try_from(retention.as_secs()).unwrap_or(i64::MAX)); + let cutoff_str = format_timestamp(cutoff); + db::query("DELETE FROM metrics WHERE bucket < ?") + .bind(&cutoff_str) + .execute(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("prune_metrics: {e}")))?; + Ok(()) + }) + } +} + +// ── RateLimitStore ───────────────────────────────────────────────────── + +impl RateLimitStore for DuckDbCatalogStore { + fn count_principal_failures( + &self, + principal: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let principal = principal.to_owned(); + Box::pin(async move { + let cutoff = format_timestamp( + OffsetDateTime::now_utc() - time::Duration::seconds(window_seconds), + ); + let row: (i64,) = db::query_as( + "SELECT COUNT(*) FROM login_attempts \ + WHERE principal = ? AND success = 0 AND attempted_at > ?", + ) + .bind(&principal) + .bind(&cutoff) + .fetch_one(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("count_principal_failures: {e}")))?; + Ok(row.0) + }) + } + + fn count_ip_failures( + &self, + source_ip: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let source_ip = source_ip.to_owned(); + Box::pin(async move { + let cutoff = format_timestamp( + OffsetDateTime::now_utc() - time::Duration::seconds(window_seconds), + ); + let row: (i64,) = db::query_as( + "SELECT COUNT(*) FROM login_attempts \ + WHERE source_ip = ? AND success = 0 AND attempted_at > ?", + ) + .bind(&source_ip) + .bind(&cutoff) + .fetch_one(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("count_ip_failures: {e}")))?; + Ok(row.0) + }) + } + + fn record_failed_login(&self, principal: &str, source_ip: Option<&str>) -> BoxFuture<'_, ()> { + let principal = principal.to_owned(); + let source_ip = source_ip.map(str::to_owned); + Box::pin(async move { + let now = format_timestamp(OffsetDateTime::now_utc()); + let result = db::query( + "INSERT INTO login_attempts (principal, attempted_at, success, source_ip) \ + VALUES (?, ?, 0, ?)", + ) + .bind(&principal) + .bind(&now) + .bind(source_ip.as_deref()) + .execute(&self.pool) + .await; + if let Err(e) = result { + tracing::error!("record_failed_login: {e}"); + } + }) + } + + fn cleanup_old_attempts(&self, max_age_seconds: i64) -> BoxFuture<'_, ()> { + Box::pin(async move { + let cutoff = format_timestamp( + OffsetDateTime::now_utc() - time::Duration::seconds(max_age_seconds), + ); + if let Err(e) = db::query("DELETE FROM login_attempts WHERE attempted_at < ?") + .bind(&cutoff) + .execute(&self.pool) + .await + { + tracing::error!("cleanup_old_attempts: {e}"); + } + }) + } +} + +// ── DiagnosticsStore ─────────────────────────────────────────────────── + +impl DiagnosticsStore for DuckDbCatalogStore { + fn count_tables(&self) -> BoxFuture<'_, DiagResult> { + Box::pin(async move { + let (count,): (i64,) = db::query_as("SELECT COUNT(*) FROM tables") + .fetch_one(&self.pool) + .await + .map_err(|e| DiagError::QueryFailed(e.to_string()))?; + Ok(count) + }) + } + + fn count_indexes(&self) -> BoxFuture<'_, DiagResult> { + Box::pin(async move { + let (count,): (i64,) = db::query_as("SELECT COUNT(*) FROM indexes") + .fetch_one(&self.pool) + .await + .map_err(|e| DiagError::QueryFailed(e.to_string()))?; + Ok(count) + }) + } + + fn test_data_database_connection(&self) -> BoxFuture<'_, DiagResult> { + Box::pin(async move { + // Catalog and data share one DuckDB file; report its recorded name. + let row: Option<(String,)> = + db::query_as("SELECT value FROM settings WHERE key = 'data_database_name'") + .fetch_optional(&self.pool) + .await + .map_err(|e| DiagError::QueryFailed(e.to_string()))?; + Ok(row.map_or_else(|| "duckdb (embedded)".to_owned(), |(n,)| n)) + }) + } +} + +// ── CatalogStore supertrait ──────────────────────────────────────────── + +impl CatalogStore for DuckDbCatalogStore { + fn cached_encryption_key(&self) -> Option { + self.encryption_key.as_ref().map(|k| k.to_string()) + } +} diff --git a/crates/storage-duckdb/src/config.rs b/crates/storage-duckdb/src/config.rs new file mode 100644 index 00000000..985690b7 --- /dev/null +++ b/crates/storage-duckdb/src/config.rs @@ -0,0 +1,144 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Storage configuration for the DuckDB backend (`[storage.duckdb]`). + +use std::any::Any; + +use extenddb_storage::config::StorageConfig; +use serde::Deserialize; + +/// DuckDB backend configuration. +/// +/// `path` is the database file location; `:memory:` selects an ephemeral +/// in-memory database. `pool_size` bounds the read connection pool (writes are +/// serialized by the engine regardless). +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DuckDbConfig { + #[serde(default = "default_path")] + pub path: String, + #[serde(default = "default_pool_size")] + pub pool_size: u32, +} + +impl Default for DuckDbConfig { + fn default() -> Self { + Self { + path: default_path(), + pool_size: default_pool_size(), + } + } +} + +/// The compiled-in default database location. +/// +/// With the `memory` feature the default is `:memory:`, producing an ephemeral, +/// bootstrap-on-serve deployment with no file on disk. Otherwise the default is +/// a file in the working directory. +fn default_path() -> String { + if cfg!(feature = "memory") { + ":memory:".to_owned() + } else { + "extenddb.duckdb".to_owned() + } +} + +fn default_pool_size() -> u32 { + 10 +} + +impl StorageConfig for DuckDbConfig { + fn connection_config(&self) -> &str { + &self.path + } + + fn max_connections(&self) -> u32 { + self.pool_size + } + + fn max_catalog_connections(&self) -> u32 { + // Single DuckDB file — catalog and data share one connection pool. + self.pool_size + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +/// Deserialize a `[storage.duckdb]` TOML table into a boxed `StorageConfig`. +pub fn deserialize_config(table: &toml::Table) -> Result, String> { + let mut config: DuckDbConfig = table + .clone() + .try_into() + .map_err(|e: toml::de::Error| format!("Failed to parse duckdb config: {e}"))?; + config.path = resolve_path(config.path); + Ok(Box::new(config)) +} + +/// Resolve a relative database file path to absolute, against the current +/// working directory. +/// +/// This runs at config load — before `serve` daemonizes. Daemonization changes +/// the process working directory, so a relative `path` would otherwise resolve +/// differently (or fail to open) in the daemonized child. In-memory and URI +/// paths are left untouched. +fn resolve_path(path: String) -> String { + if path.contains(":memory:") || path.starts_with("file:") || path.starts_with("duckdb:") { + return path; + } + let p = std::path::Path::new(&path); + if p.is_absolute() { + return path; + } + match std::env::current_dir() { + Ok(cwd) => cwd.join(p).to_string_lossy().into_owned(), + Err(_) => path, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults() { + let c = DuckDbConfig::default(); + #[cfg(not(feature = "memory"))] + assert_eq!(c.path, "extenddb.duckdb"); + #[cfg(feature = "memory")] + assert_eq!(c.path, ":memory:"); + assert_eq!(c.pool_size, 10); + assert_eq!(c.max_connections(), 10); + assert_eq!(c.max_catalog_connections(), 10); + } + + #[cfg(feature = "memory")] + #[test] + fn memory_feature_defaults_to_in_memory() { + assert_eq!(DuckDbConfig::default().path, ":memory:"); + assert_eq!(default_path(), ":memory:"); + } + + #[test] + fn deserialize_full() { + let mut t = toml::Table::new(); + t.insert("path".into(), toml::Value::String(":memory:".into())); + t.insert("pool_size".into(), toml::Value::Integer(4)); + let boxed = deserialize_config(&t).expect("parse"); + assert_eq!(boxed.connection_config(), ":memory:"); + assert_eq!(boxed.max_connections(), 4); + } + + #[test] + fn deserialize_rejects_unknown_field() { + let mut t = toml::Table::new(); + t.insert("bogus".into(), toml::Value::Integer(1)); + assert!(deserialize_config(&t).is_err()); + } +} diff --git a/crates/storage-duckdb/src/create_table.rs b/crates/storage-duckdb/src/create_table.rs new file mode 100644 index 00000000..6b962c0b --- /dev/null +++ b/crates/storage-duckdb/src/create_table.rs @@ -0,0 +1,518 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `create_table` implementation for `DuckDbEngine`. +//! +//! Catalog metadata (table row, indexes, tags, stream shards) is written in one +//! transaction; the per-table/index data tables are created in a second +//! transaction afterward, with catalog cleanup if the data DDL fails. The +//! control-plane delay (`control_plane_delay_seconds`) decides whether the +//! table starts ACTIVE (delay 0) or CREATING with a scheduled transition. + +use crate::db; +use extenddb_core::types::{ + BillingMode, BillingModeSummary, CreateTableInput, GsiDescription, LsiDescription, + ProvisionedThroughputDescription, SseDescription, SseType, TableDescription, TableStatus, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{index_arn, stream_arn, table_arn}; + +use crate::duckdb_util::{format_timestamp, is_unique_violation}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + pub(crate) async fn create_table_impl( + &self, + account_id: &str, + input: CreateTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + let table_id = uuid::Uuid::new_v4().to_string(); + let table_arn = table_arn(&self.region, account_id, &input.table_name); + let billing_mode = input.billing_mode.unwrap_or(BillingMode::Provisioned); + let billing_str = match billing_mode { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + }; + + let to_str = |v: &serde_json::Value| -> Result { + serde_json::to_string(v).map_err(|e| StorageError::Internal(e.to_string())) + }; + let key_schema_json = serde_json::to_string(&input.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs_json = serde_json::to_string(&input.attribute_definitions) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let pt_json = input + .provisioned_throughput + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let stream_json = input + .stream_specification + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let sse_json = input.sse_specification.as_ref().map(to_str).transpose()?; + let on_demand_json = input + .on_demand_throughput + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let deletion_protection = input.deletion_protection_enabled.unwrap_or(false); + + // Control-plane delay decides initial status and scheduled transition. + let delay_secs: f64 = db::query_scalar::( + "SELECT value FROM settings WHERE key = 'control_plane_delay_seconds'", + ) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.25); + + let now = time::OffsetDateTime::now_utc(); + let creation_ts = format_timestamp(now); + #[allow(clippy::cast_precision_loss)] + let creation_epoch = now.unix_timestamp() as f64; + let (initial_status, status_transition_at) = if delay_secs <= 0.0 { + ("ACTIVE", None) + } else { + ( + "CREATING", + Some(format_timestamp( + now + time::Duration::seconds_f64(delay_secs), + )), + ) + }; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + db::query( + "INSERT INTO tables \ + (account_id, table_name, key_schema, attribute_definitions, billing_mode, \ + provisioned_throughput, stream_specification, table_status, creation_date_time, \ + table_arn, table_id, deletion_protection_enabled, status_transition_at, \ + table_class, sse_specification, on_demand_throughput) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(account_id) + .bind(&input.table_name) + .bind(&key_schema_json) + .bind(&attr_defs_json) + .bind(billing_str) + .bind(&pt_json) + .bind(&stream_json) + .bind(initial_status) + .bind(&creation_ts) + .bind(&table_arn) + .bind(&table_id) + .bind(deletion_protection) + .bind(&status_transition_at) + .bind(&input.table_class) + .bind(&sse_json) + .bind(&on_demand_json) + .execute(&mut *tx) + .await + .map_err(|e| { + if is_unique_violation(&e) { + StorageError::TableAlreadyExists(input.table_name.clone()) + } else { + StorageError::Internal(e.to_string()) + } + })?; + + // GSI / LSI metadata. + let mut gsi_ids: Vec = Vec::new(); + if let Some(gsis) = &input.global_secondary_indexes { + for gsi in gsis { + let ks = serde_json::to_string(&gsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj = serde_json::to_string(&gsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let pt = gsi + .provisioned_throughput + .as_ref() + .map(|pt| { + serde_json::to_string(&ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }) + }) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let index_id = uuid::Uuid::new_v4().to_string(); + db::query( + "INSERT INTO indexes \ + (table_id, index_name, index_id, index_type, key_schema, projection, \ + index_status, provisioned_throughput) \ + VALUES (?, ?, ?, 'GSI', ?, ?, 'ACTIVE', ?)", + ) + .bind(&table_id) + .bind(&gsi.index_name) + .bind(&index_id) + .bind(&ks) + .bind(&proj) + .bind(&pt) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + gsi_ids.push(index_id); + } + } + let mut lsi_ids: Vec = Vec::new(); + if let Some(lsis) = &input.local_secondary_indexes { + for lsi in lsis { + let ks = serde_json::to_string(&lsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj = serde_json::to_string(&lsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let index_id = uuid::Uuid::new_v4().to_string(); + db::query( + "INSERT INTO indexes \ + (table_id, index_name, index_id, index_type, key_schema, projection, \ + index_status, provisioned_throughput) \ + VALUES (?, ?, ?, 'LSI', ?, ?, 'ACTIVE', NULL)", + ) + .bind(&table_id) + .bind(&lsi.index_name) + .bind(&index_id) + .bind(&ks) + .bind(&proj) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + lsi_ids.push(index_id); + } + } + + // Vector indexes. A CreateTable's table is empty, so there is nothing to + // backfill and no `backfilling` member is ever reported on this path. + // The index's status tracks the TABLE's: measured against the service + // (2026-08-21, eu-west-2, three runs polling at 250ms), an index created + // with its table reports CREATING while the table is CREATING and + // reaches ACTIVE in the same DescribeTable poll as the table, with no + // observable gap in either direction. The control-plane worker flips + // both in one pass; see `process_control_plane_transitions`. + let mut vector_ids: Vec = Vec::new(); + if let Some(vis) = &input.vector_indexes { + for vi in vis { + let vec_attr = serde_json::to_string(&vi.vector_attribute) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let search_schema = vi + .search_schema + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj = vi + .projection + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + // Core validation requires Projection, so reaching here + // means the request bypassed validation rather than that + // the caller omitted it. + StorageError::Internal( + "vector index reached storage without a projection".to_owned(), + ) + })?; + let distance = serde_json::to_string(&vi.distance_function) + .map_err(|e| StorageError::Internal(e.to_string()))? + .trim_matches('"') + .to_owned(); + let index_id = uuid::Uuid::new_v4().to_string(); + db::query( + "INSERT INTO vector_indexes \ + (table_id, index_name, index_id, dimensions, distance_function, \ + vector_attribute, search_schema, projection, index_status, backfilling) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)", + ) + .bind(&table_id) + .bind(&vi.index_name) + .bind(&index_id) + .bind(i64::from(vi.dimensions)) + .bind(&distance) + .bind(&vec_attr) + .bind(&search_schema) + .bind(&proj) + .bind(initial_status) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + vector_ids.push(index_id); + } + } + + // Tags. + if let Some(tags) = &input.tags { + for tag in tags { + db::query("INSERT INTO tags (resource_arn, tag_key, tag_value) VALUES (?, ?, ?)") + .bind(&table_arn) + .bind(&tag.key) + .bind(&tag.value) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + + // Stream shards (same transaction — single file). + let stream_label = if input + .stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled) + { + Some(Self::init_stream_shards(&mut tx, account_id, &input.table_name, &table_id).await?) + } else { + None + }; + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Data tables in a second transaction; clean up catalog on failure. + let data_result = async { + let mut data_tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Self::create_data_table( + &mut data_tx, + &table_id, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + if let Some(gsis) = &input.global_secondary_indexes { + for (i, gsi) in gsis.iter().enumerate() { + Self::create_index_data_table( + &mut data_tx, + &gsi_ids[i], + &gsi.key_schema, + &input.attribute_definitions, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } + if let Some(lsis) = &input.local_secondary_indexes { + for (i, lsi) in lsis.iter().enumerate() { + Self::create_index_data_table( + &mut data_tx, + &lsi_ids[i], + &lsi.key_schema, + &input.attribute_definitions, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } + if input.vector_indexes.is_some() { + for index_id in &vector_ids { + Self::create_vector_data_table( + &mut data_tx, + &table_id, + index_id, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok::<(), StorageError>(()) + } + .await; + + if let Err(e) = data_result { + tracing::error!( + "Failed to create data tables for '{}', cleaning up catalog: {e}", + input.table_name + ); + if let Ok(mut cleanup) = self.pool.begin().await { + let _ = crate::referential::delete_table_children(&mut cleanup, &table_id).await; + let _ = db::query("DELETE FROM tables WHERE account_id = ? AND table_name = ?") + .bind(account_id) + .bind(&input.table_name) + .execute(&mut *cleanup) + .await; + let _ = cleanup.commit().await; + } + return Err(e); + } + + self.control_plane_notify.notify_one(); + + // Build the response from in-scope data (avoids a post-commit read race). + let (rcu, wcu) = input.provisioned_throughput.as_ref().map_or((0, 0), |pt| { + (pt.read_capacity_units, pt.write_capacity_units) + }); + + let gsis = input.global_secondary_indexes.as_ref().map(|gs| { + gs.iter() + .map(|g| GsiDescription { + index_name: g.index_name.clone(), + key_schema: g.key_schema.clone(), + projection: g.projection.clone(), + index_status: "ACTIVE".to_owned(), + provisioned_throughput: Some(ProvisionedThroughputDescription { + read_capacity_units: g + .provisioned_throughput + .as_ref() + .map_or(0, |pt| pt.read_capacity_units), + write_capacity_units: g + .provisioned_throughput + .as_ref() + .map_or(0, |pt| pt.write_capacity_units), + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn( + &self.region, + account_id, + &input.table_name, + &g.index_name, + ), + }) + .collect() + }); + let lsis = input.local_secondary_indexes.as_ref().map(|ls| { + ls.iter() + .map(|l| LsiDescription { + index_name: l.index_name.clone(), + key_schema: l.key_schema.clone(), + projection: l.projection.clone(), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn( + &self.region, + account_id, + &input.table_name, + &l.index_name, + ), + }) + .collect() + }); + + let billing_mode_summary = (billing_mode == BillingMode::PayPerRequest).then_some({ + BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: Some(creation_epoch), + } + }); + let latest_stream_arn = stream_label + .as_ref() + .map(|label| stream_arn(&self.region, account_id, &input.table_name, label)); + let response_status = if initial_status == "ACTIVE" { + TableStatus::Active + } else { + TableStatus::Creating + }; + let sse_description = input.sse_specification.as_ref().and_then(|spec| { + let enabled = spec + .get("Enabled") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + enabled.then(|| SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(SseType::KMS), + kms_master_key_arn: Some(format!( + "arn:aws:kms:{}:{}:key/default", + self.region, account_id + )), + }) + }); + + // Echo the vector indexes we just created. Built from the request plus the + // ids assigned above rather than re-read from the catalog, which would add + // a round trip to say something already known. A CreateTable's table is + // empty, so each index is ACTIVE with no `backfilling` member. + let vector_index_descs: Option> = input + .vector_indexes + .as_ref() + .map(|vis| { + vis.iter() + .map(|vi| extenddb_core::types::VectorIndexDescription { + index_name: vi.index_name.clone(), + vector_attribute: vi.vector_attribute.clone(), + dimensions: vi.dimensions, + search_schema: vi.search_schema.clone(), + distance_function: vi.distance_function, + index_status: extenddb_core::types::IndexStatus::Active, + backfilling: None, + index_size_bytes: 0, + item_count: 0, + index_arn: extenddb_storage::util::index_arn( + &self.region, + account_id, + &input.table_name, + &vi.index_name, + ), + projection: vi.projection.clone(), + }) + .collect() + }) + .filter(|v: &Vec<_>| !v.is_empty()); + + Ok(TableDescription { + restore_summary: None, + table_name: input.table_name, + key_schema: input.key_schema, + attribute_definitions: input.attribute_definitions, + table_status: response_status, + creation_date_time: creation_epoch, + table_size_bytes: 0, + item_count: 0, + table_arn, + table_id, + provisioned_throughput: ProvisionedThroughputDescription { + read_capacity_units: rcu, + write_capacity_units: wcu, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + billing_mode_summary, + global_secondary_indexes: gsis, + local_secondary_indexes: lsis, + stream_specification: input.stream_specification, + latest_stream_arn, + latest_stream_label: stream_label, + deletion_protection_enabled: deletion_protection, + sse_description, + table_class_summary: input + .table_class + .as_ref() + .map(|tc| serde_json::json!({ "TableClass": tc })), + on_demand_throughput: input.on_demand_throughput, + // Every field is populated deliberately, with no `..Default::default()` + // spread. This response is the complete description of what was just + // created, so a new core field should break this site and force a + // decision about whether create must report it, rather than silently + // defaulting. Sites that legitimately opt out still use the spread. + vector_indexes: vector_index_descs, + }) + } +} diff --git a/crates/storage-duckdb/src/credential_store.rs b/crates/storage-duckdb/src/credential_store.rs new file mode 100644 index 00000000..8e9c7def --- /dev/null +++ b/crates/storage-duckdb/src/credential_store.rs @@ -0,0 +1,173 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! DuckDB-backed `CredentialStore` for SigV4 verification. +//! +//! `AKIA…` keys resolve to long-lived IAM user access keys; `ASIA…` keys +//! resolve to temporary `AssumeRole` sessions (with expiry enforcement). +//! Secrets are decrypted with AES-256-GCM; the `access_key_id` is the AAD, with +//! a no-AAD fallback for any legacy ciphertext. + +use crate::db; +use aes_gcm::aead::{Aead, Payload}; +use aes_gcm::{Aes256Gcm, Key, KeyInit, Nonce}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use extenddb_auth::{CredentialStore, StoredCredential}; +use extenddb_core::error::DynamoDbError; + +use zeroize::{Zeroize, ZeroizeOnDrop}; + +use crate::duckdb_util::parse_timestamp; + +/// Decrypt a `nonce(12) || ciphertext` secret with AES-256-GCM. +fn decrypt_secret(encrypted: &[u8], key_b64: &str, aad: &str) -> Result { + if encrypted.len() < 28 { + return Err("ciphertext too short (need 12-byte nonce + 16-byte tag)".to_owned()); + } + let key_bytes = BASE64 + .decode(key_b64) + .map_err(|e| format!("decode encryption key: {e}"))?; + if key_bytes.len() != 32 { + return Err("encryption key must be 32 bytes".to_owned()); + } + let cipher = Aes256Gcm::new(Key::::from_slice(&key_bytes)); + let nonce = Nonce::from_slice(&encrypted[..12]); + + if let Ok(plaintext) = cipher.decrypt( + nonce, + Payload { + msg: &encrypted[12..], + aad: aad.as_bytes(), + }, + ) { + return String::from_utf8(plaintext).map_err(|e| format!("secret not UTF-8: {e}")); + } + // Fallback: ciphertext written without AAD. + let plaintext = cipher + .decrypt(nonce, &encrypted[12..]) + .map_err(|e| format!("decrypt: {e}"))?; + String::from_utf8(plaintext).map_err(|e| format!("secret not UTF-8: {e}")) +} + +/// Credential store over the catalog DuckDB pool. +#[derive(Zeroize, ZeroizeOnDrop)] +pub struct DuckDbCredentialStore { + #[zeroize(skip)] + pool: db::Pool, + encryption_key: String, +} + +impl DuckDbCredentialStore { + pub fn new(pool: db::Pool, encryption_key: String) -> Self { + Self { + pool, + encryption_key, + } + } + + async fn lookup_user( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let row: Option<(Vec, String, String, bool)> = db::query_as( + "SELECT secret_key_encrypted, account_id, user_name, is_active \ + FROM access_keys WHERE access_key_id = ?", + ) + .bind(access_key_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + tracing::error!("credential lookup failed for {access_key_id}: {e}"); + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + + let Some((encrypted, account_id, user_name, is_active)) = row else { + return Ok(None); + }; + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!("secret decryption failed for {access_key_id}: {e}"); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: user_name, + session_name: None, + is_session: false, + session_token: None, + is_active, + expires_at: None, + })) + } + + async fn lookup_session( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let row: Option<(Vec, String, String, String, String, String)> = db::query_as( + "SELECT secret_key_encrypted, account_id, role_name, session_name, \ + session_token, expires_at \ + FROM iam_sessions WHERE access_key_id = ?", + ) + .bind(access_key_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + tracing::error!("session lookup failed for {access_key_id}: {e}"); + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + + let Some((encrypted, account_id, role_name, session_name, session_token, expires_at)) = row + else { + return Ok(None); + }; + + let expires = parse_timestamp(&expires_at).map_err(|e| { + tracing::error!("session expiry parse error: {e}"); + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + if expires < time::OffsetDateTime::now_utc() { + return Err(DynamoDbError::ExpiredTokenException( + "The security token included in the request is expired".to_owned(), + )); + } + + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!("session secret decryption failed for {access_key_id}: {e}"); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: role_name, + session_name: Some(session_name), + is_session: true, + session_token: Some(session_token), + is_active: true, + expires_at: Some(expires), + })) + } +} + +#[async_trait::async_trait] +impl CredentialStore for DuckDbCredentialStore { + async fn lookup_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + if access_key_id.starts_with("ASIA") { + self.lookup_session(access_key_id).await + } else if access_key_id.starts_with("AKIA") { + self.lookup_user(access_key_id).await + } else { + Ok(None) + } + } +} diff --git a/crates/storage-duckdb/src/data/data_engine.rs b/crates/storage-duckdb/src/data/data_engine.rs new file mode 100644 index 00000000..1679d94f --- /dev/null +++ b/crates/storage-duckdb/src/data/data_engine.rs @@ -0,0 +1,398 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `DataEngine` trait implementation for `DuckDbEngine`. +//! +//! Each method clones its borrowed arguments into owned values and delegates to +//! the corresponding `*_impl` method inside a boxed future. The transact-write +//! path rebuilds borrowed `TransactWriteOp`s from owned components inside the +//! future, since the trait's borrowed ops cannot outlive the call. + +use extenddb_core::expression::{Expr, ExpressionMaps, KeyCondition, UpdateAction}; +use extenddb_core::types::{Item, ReturnValuesOnConditionCheckFailure, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, IdempotencyKey, StreamCapture, TransactGetOp, TransactWriteOp}; +use futures::future::BoxFuture; + +use crate::store::DuckDbEngine; + +impl DataEngine for DuckDbEngine { + /// Declares vector support by handing over the implementation. `Some(self)` + /// only compiles because `DuckDbEngine` implements `VectorSearchEngine`, so + /// this cannot claim a capability the backend does not have. + fn as_vector_search(&self) -> Option<&dyn extenddb_storage::VectorSearchEngine> { + Some(self) + } + + fn put_item( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.put_item_impl( + &key_info, + item, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn get_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + Box::pin(async move { self.get_item_impl(&key_info, &key).await }) + } + + fn delete_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.delete_item_impl( + &key_info, + &key, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn update_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result<(Option, Option), StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + let actions = actions.to_vec(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.update_item_impl( + &key_info, + &key, + &actions, + return_old, + return_new, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn query( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> BoxFuture<'_, Result<(Vec, Option), StorageError>> { + let key_info = key_info.clone(); + let key_condition = key_condition.clone(); + let maps = maps.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(str::to_owned); + Box::pin(async move { + self.query_impl( + &key_info, + &key_condition, + &maps, + forward, + limit, + exclusive_start_key.as_ref(), + index_name.as_deref(), + ) + .await + }) + } + + fn scan( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> BoxFuture<'_, Result<(Vec, Option), StorageError>> { + let key_info = key_info.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(str::to_owned); + Box::pin(async move { + self.scan_impl( + &key_info, + limit, + exclusive_start_key.as_ref(), + segment, + total_segments, + index_name.as_deref(), + ) + .await + }) + } + + fn transact_get_items( + &self, + ops: &[TransactGetOp<'_>], + ) -> BoxFuture<'_, Result>, StorageError>> { + let owned: Vec<(TableKeyInfo, Item)> = ops + .iter() + .map(|op| (op.key_info.clone(), op.key.clone())) + .collect(); + Box::pin(async move { + let borrowed: Vec = owned + .iter() + .map(|(key_info, key)| TransactGetOp { key_info, key }) + .collect(); + self.transact_get_items_impl(&borrowed).await + }) + } + + fn transact_write_items( + &self, + ops: &[TransactWriteOp<'_>], + idempotency: Option>, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let owned: Vec = ops.iter().map(OwnedWriteOp::from_op).collect(); + let idempotency = idempotency.map(|k| { + ( + k.account_id.to_owned(), + k.token.to_owned(), + k.fingerprint.to_owned(), + ) + }); + Box::pin(async move { + let borrowed: Vec = owned.iter().map(OwnedWriteOp::as_op).collect(); + self.transact_write_items_impl( + &borrowed, + idempotency.as_ref().map(|(a, t, f)| IdempotencyKey { + account_id: a, + token: t, + fingerprint: f, + }), + ) + .await + }) + } + + fn cleanup_expired_idempotency_tokens( + &self, + max_age_seconds: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + self.cleanup_expired_idempotency_tokens_impl(max_age_seconds) + .await + }) + } +} + +/// Owned form of a `TransactWriteOp`, so the borrowed ops can be reconstructed +/// inside the `'static` future. +enum OwnedWriteOp { + Put { + key_info: TableKeyInfo, + item: Item, + condition: Option, + maps: ExpressionMaps, + rv: ReturnValuesOnConditionCheckFailure, + stream: Option, + }, + Delete { + key_info: TableKeyInfo, + key: Item, + condition: Option, + maps: ExpressionMaps, + rv: ReturnValuesOnConditionCheckFailure, + stream: Option, + }, + Update { + key_info: TableKeyInfo, + key: Item, + actions: Vec, + condition: Option, + maps: ExpressionMaps, + rv: ReturnValuesOnConditionCheckFailure, + stream: Option, + }, + ConditionCheck { + key_info: TableKeyInfo, + key: Item, + condition: Expr, + maps: ExpressionMaps, + rv: ReturnValuesOnConditionCheckFailure, + }, +} + +impl OwnedWriteOp { + fn from_op(op: &TransactWriteOp<'_>) -> Self { + match op { + TransactWriteOp::Put { + key_info, + item, + condition, + maps, + return_values_on_ccf, + stream, + } => Self::Put { + key_info: (*key_info).clone(), + item: (*item).clone(), + condition: condition.cloned(), + maps: (*maps).clone(), + rv: *return_values_on_ccf, + stream: stream.clone(), + }, + TransactWriteOp::Delete { + key_info, + key, + condition, + maps, + return_values_on_ccf, + stream, + } => Self::Delete { + key_info: (*key_info).clone(), + key: (*key).clone(), + condition: condition.cloned(), + maps: (*maps).clone(), + rv: *return_values_on_ccf, + stream: stream.clone(), + }, + TransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + return_values_on_ccf, + stream, + } => Self::Update { + key_info: (*key_info).clone(), + key: (*key).clone(), + actions: actions.to_vec(), + condition: condition.cloned(), + maps: (*maps).clone(), + rv: *return_values_on_ccf, + stream: stream.clone(), + }, + TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf, + } => Self::ConditionCheck { + key_info: (*key_info).clone(), + key: (*key).clone(), + condition: (*condition).clone(), + maps: (*maps).clone(), + rv: *return_values_on_ccf, + }, + } + } + + fn as_op(&self) -> TransactWriteOp<'_> { + match self { + Self::Put { + key_info, + item, + condition, + maps, + rv, + stream, + } => TransactWriteOp::Put { + key_info, + item, + condition: condition.as_ref(), + maps, + return_values_on_ccf: *rv, + stream: stream.clone(), + }, + Self::Delete { + key_info, + key, + condition, + maps, + rv, + stream, + } => TransactWriteOp::Delete { + key_info, + key, + condition: condition.as_ref(), + maps, + return_values_on_ccf: *rv, + stream: stream.clone(), + }, + Self::Update { + key_info, + key, + actions, + condition, + maps, + rv, + stream, + } => TransactWriteOp::Update { + key_info, + key, + actions, + condition: condition.as_ref(), + maps, + return_values_on_ccf: *rv, + stream: stream.clone(), + }, + Self::ConditionCheck { + key_info, + key, + condition, + maps, + rv, + } => TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf: *rv, + }, + } + } +} diff --git a/crates/storage-duckdb/src/data/ddl.rs b/crates/storage-duckdb/src/data/ddl.rs new file mode 100644 index 00000000..921cf809 --- /dev/null +++ b/crates/storage-duckdb/src/data/ddl.rs @@ -0,0 +1,744 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! DDL for per-DynamoDB-table and per-index data tables, plus catalog fetches +//! of `TableKeyInfo` / `IndexInfo`. +//! +//! Sort-key columns follow design decision D2: `sk_s`/`base_sk_s` are `TEXT`, +//! `sk_n`/`base_sk_n` are `TEXT` (order-preserving numeric encoding, never +//! `DOUBLE`), `sk_b`/`base_sk_b` are `BLOB`. + +use crate::db; +use extenddb_core::types::{ + AttributeDefinition, IndexInfo, IndexType, KeySchemaElement, Projection, StreamSpecification, + TableKeyInfo, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::sk_column_n; + +use super::{ + all_sort_key_info, data_table_name, index_table_name, vector_table_glob_pattern, + vector_table_name, +}; +use crate::store::DuckDbEngine; + +/// DuckDB column type for the Nth sort-key position and scalar type (D2). +fn sk_col_defs(index: usize) -> [String; 3] { + if index == 0 { + [ + "sk_s TEXT".to_owned(), + "sk_n TEXT".to_owned(), + "sk_b BLOB".to_owned(), + ] + } else { + let n = index + 1; + [ + format!("sk{n}_s TEXT"), + format!("sk{n}_n TEXT"), + format!("sk{n}_b BLOB"), + ] + } +} + +/// DuckDB column type for the Nth base-table sort-key position (D2). +fn base_sk_col_defs(index: usize) -> [String; 3] { + if index == 0 { + [ + "base_sk_s TEXT".to_owned(), + "base_sk_n TEXT".to_owned(), + "base_sk_b BLOB".to_owned(), + ] + } else { + let n = index + 1; + [ + format!("base_sk{n}_s TEXT"), + format!("base_sk{n}_n TEXT"), + format!("base_sk{n}_b BLOB"), + ] + } +} + +impl DuckDbEngine { + /// Create the per-DynamoDB-table data table. + /// + /// # Safety (SQL injection) + /// `table_id` is a server-generated UUID; column names are constants. No + /// user input is interpolated into the DDL. + pub(crate) async fn create_data_table( + tx: &mut db::Transaction, + table_id: &str, + key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let ddb_table = data_table_name(table_id); + let sk_infos = all_sort_key_info(key_schema, attr_defs); + + let ddl = if sk_infos.is_empty() { + format!( + "CREATE TABLE {ddb_table} (pk TEXT NOT NULL PRIMARY KEY, item_data TEXT NOT NULL)" + ) + } else { + let mut col_defs = vec!["pk TEXT NOT NULL".to_owned()]; + let mut pk_cols = vec!["pk".to_owned()]; + for (i, &(_, sk_type)) in sk_infos.iter().enumerate() { + col_defs.extend(sk_col_defs(i)); + pk_cols.push(sk_column_n(i, sk_type)); + } + col_defs.push("item_data TEXT NOT NULL".to_owned()); + format!( + "CREATE TABLE {ddb_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ) + }; + + db::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Create a vector-index data table: one row per indexed vector. + /// + /// `part` is the search-schema HASH value when one is declared, and a single + /// constant otherwise, so an unscoped index is one partition rather than a + /// separate code path. `nrm` is the vector's precomputed L2 norm, so cosine + /// costs one dot product at query time instead of two passes. + /// + /// # Safety (SQL injection) + /// `index_id` is a server-generated UUID and column names are constants, so + /// no user input reaches the DDL. Vector attribute names are stored as data, + /// never as identifiers. + pub(crate) async fn create_vector_data_table( + tx: &mut db::Transaction, + table_id: &str, + index_id: &str, + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, index_id); + let base_sks = all_sort_key_info(base_key_schema, base_attr_defs); + + let mut col_defs = vec![ + "part TEXT NOT NULL".to_owned(), + "base_pk TEXT NOT NULL".to_owned(), + ]; + for i in 0..base_sks.len() { + col_defs.extend(base_sk_col_defs(i)); + } + col_defs.push("vec BLOB NOT NULL".to_owned()); + col_defs.push("nrm DOUBLE NOT NULL".to_owned()); + col_defs.push("item_data TEXT NOT NULL".to_owned()); + + // Keyed by the base item, not by the partition, so one base item yields + // at most one vector row and a re-put replaces rather than duplicates. + let mut pk_cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + pk_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + + let ddl = format!( + "CREATE TABLE {vec_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ); + db::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // The scan is always partition-scoped, so this index is what keeps a + // search off the full table when a HASH element is declared. + let part_idx = + format!("CREATE INDEX \"_vidx_part_{table_id}_{index_id}\" ON {vec_table} (part)"); + db::query(&part_idx) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Drop every vector-index data table belonging to one DynamoDB table. + /// + /// Discovered from `duckdb_tables()` rather than from the catalog, because the + /// caller runs this after the catalog rows have been cascade-deleted in the + /// same transaction, so `vector_indexes` is already empty for this table. + /// Without this, dropping a table would leave its vector data tables behind + /// forever with nothing left pointing at them. + /// Drop one vector index's data table. + /// + /// The table-drop path sweeps `duckdb_tables()` instead, because by the time it + /// runs the catalog rows have already cascade-deleted and the index ids are + /// unreadable. Here the id is known, so the name is derived directly rather + /// than matched by pattern. + pub(crate) async fn drop_vector_data_table_by_id( + pool: &db::Pool, + table_id: &str, + index_id: &str, + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, index_id); + db::query(&format!("DROP TABLE IF EXISTS {vec_table}")) + .execute(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn drop_all_vector_data_tables( + tx: &mut db::Transaction, + table_id: &str, + ) -> Result<(), StorageError> { + let names: Vec = + db::query_scalar("SELECT table_name FROM duckdb_tables() WHERE table_name GLOB ?") + .bind(vector_table_glob_pattern(table_id)) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + for name in names { + // Names come from duckdb_tables(), not from user input, and are quoted. + db::query(&format!("DROP TABLE IF EXISTS \"{name}\"")) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + } + + /// Drop the per-DynamoDB-table data table. + pub(crate) async fn drop_data_table( + tx: &mut db::Transaction, + table_id: &str, + ) -> Result<(), StorageError> { + let ddb_table = data_table_name(table_id); + db::query(&format!("DROP TABLE IF EXISTS {ddb_table}")) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + // Vector data tables are keyed by index, not by table, so they are not + // reached by dropping the item table or by the catalog cascade. + Self::drop_all_vector_data_tables(tx, table_id).await?; + Ok(()) + } + + /// Create a GSI/LSI data table. GSI keys are not unique, so the primary key + /// is `(pk, base_pk, base_sk*)`: the base-table key is what makes a row + /// unique, keeping one row per base item per index partition. + /// + /// Two secondary indexes are created: + /// - `idx_order_*` on `(pk, sk*, base_pk, base_sk*)`, for sort-key ordering + /// within an index partition. Only when the index declares a sort key. + /// - `idx_base_key_*` on `(base_pk, base_sk*)`, for the reverse lookup that + /// index propagation uses to delete an item's old index row. Always. + pub(crate) async fn create_index_data_table( + tx: &mut db::Transaction, + index_id: &str, + index_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let idx_table = index_table_name(index_id); + let idx_sks = all_sort_key_info(index_key_schema, attr_defs); + let base_sks = all_sort_key_info(base_key_schema, base_attr_defs); + + let mut col_defs = vec!["pk TEXT NOT NULL".to_owned()]; + for i in 0..idx_sks.len() { + col_defs.extend(sk_col_defs(i)); + } + col_defs.push("base_pk TEXT NOT NULL".to_owned()); + for i in 0..base_sks.len() { + col_defs.extend(base_sk_col_defs(i)); + } + col_defs.push("item_data TEXT NOT NULL".to_owned()); + + let mut pk_cols = vec!["pk".to_owned(), "base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + pk_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + + let ddl = format!( + "CREATE TABLE {idx_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ); + db::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if !idx_sks.is_empty() { + let mut order_cols = vec!["pk".to_owned()]; + for (i, &(_, sk_type)) in idx_sks.iter().enumerate() { + order_cols.push(sk_column_n(i, sk_type)); + } + order_cols.push("base_pk".to_owned()); + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + order_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + let order_idx = format!( + "CREATE INDEX \"idx_order_{index_id}\" ON {idx_table} ({})", + order_cols.join(", ") + ); + db::query(&order_idx) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Index on the base-table key columns, for `delete_index_row_multi`. + // + // GSI/LSI propagation deletes the old index row by base-table key: + // `DELETE FROM WHERE base_pk = ? AND base_sk* = ?`, with no `pk` + // predicate. Both the PRIMARY KEY and the ordering index above lead + // with `pk`, so neither can serve that filter and DuckDB plans a full + // `SCAN` of the index table on every propagating write. Unconditional + // rather than gated on `idx_sks`, because the reverse lookup happens + // whether or not the index declares a sort key. + { + let mut base_key_cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + base_key_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + let base_key_idx = format!( + "CREATE INDEX \"idx_base_key_{index_id}\" ON {idx_table} ({})", + base_key_cols.join(", ") + ); + db::query(&base_key_idx) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + Ok(()) + } + + /// Drop a GSI/LSI data table. + pub(crate) async fn drop_index_data_table( + tx: &mut db::Transaction, + index_id: &str, + ) -> Result<(), StorageError> { + let idx_table = index_table_name(index_id); + db::query(&format!("DROP TABLE IF EXISTS {idx_table}")) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Fetch `TableKeyInfo` for an ACTIVE table from the catalog. + pub(crate) async fn fetch_table_key_info( + &self, + account_id: &str, + table_name: &str, + ) -> Result { + let row: Option<(String, String, String, String, Option)> = db::query_as( + "SELECT key_schema, attribute_definitions, table_status, table_id, \ + stream_specification \ + FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (ks_str, ad_str, status, table_id, stream_spec_str) = + row.ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; + + match status.as_str() { + "ACTIVE" => {} + // A table still being created, or being deleted, is not usable for + // data-plane operations; DynamoDB reports it as not found (not + // in-use). UPDATING keeps the not-active classification. + "CREATING" | "DELETING" => { + return Err(StorageError::TableNotFound(table_name.to_owned())); + } + _ => return Err(StorageError::TableNotActive(table_name.to_owned())), + } + + let key_schema: Vec = + serde_json::from_str(&ks_str).map_err(|e| StorageError::Internal(e.to_string()))?; + let attribute_definitions: Vec = + serde_json::from_str(&ad_str).map_err(|e| StorageError::Internal(e.to_string()))?; + let stream_specification: Option = stream_spec_str + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Full secondary-index metadata rides on the (cached) TableKeyInfo so + // per-index consumed-capacity accounting avoids a separate + // describe_table round-trip on every write; it also supplies `has_lsi`. + let (global_secondary_indexes, local_secondary_indexes) = + self.fetch_all_index_info(&table_id).await?; + let has_lsi = !local_secondary_indexes.is_empty(); + // Vector indexes ride on the cached key info too, so the write path can + // decide whether any maintenance is needed without a query, and so the + // engine can validate vector attributes on writes. + let vector_indexes = self.fetch_vector_index_key_info(&table_id).await?; + + let key_info = TableKeyInfo { + table_name: table_name.to_owned(), + account_id: account_id.to_owned(), + table_id, + base_key_schema: key_schema.clone(), + key_schema, + attribute_definitions, + has_lsi, + global_secondary_indexes, + local_secondary_indexes, + stream_specification, + // Every field is populated, with no `..Default::default()` spread: a + // new core field should break this site and force a decision about + // whether the write path needs it, rather than silently defaulting. + vector_indexes, + }; + // Catalog metadata that cannot describe its own sort key would make the + // keyed read paths fall back to a partition-only lookup and return the + // wrong item, so refuse it here rather than serve a wrong answer (#259). + key_info + .validate_sort_key_definitions() + .map_err(StorageError::Internal)?; + Ok(key_info) + } + + /// Fetch the vector indexes of a table in the shape the engine caches. + /// + /// Note what this cannot carry: `VectorIndexKeyInfo` has no distance + /// function, so a search still reads the catalog for it. Widening that + /// type would remove the last per-search catalog read. + async fn fetch_vector_index_key_info( + &self, + table_id: &str, + ) -> Result, StorageError> { + let rows: Vec<(String, i64, String, Option, String)> = db::query_as( + "SELECT index_name, dimensions, vector_attribute, search_schema, projection \ + FROM vector_indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut out = Vec::with_capacity(rows.len()); + for (index_name, dimensions, vector_attribute, search_schema, projection) in rows { + let attr: extenddb_core::types::VectorAttribute = + serde_json::from_str(&vector_attribute) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; + let search_schema = match search_schema.as_deref() { + Some(json) => serde_json::from_str(json) + .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?, + None => Vec::new(), + }; + let projection: extenddb_core::types::Projection = serde_json::from_str(&projection) + .map_err(|e| StorageError::Internal(format!("vector projection: {e}")))?; + out.push(extenddb_core::types::VectorIndexKeyInfo { + index_name, + dimensions: u32::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?, + vector_attribute_name: attr.attribute_name, + search_schema, + projection, + }); + } + Ok(out) + } + + /// Fetch every secondary index defined on a table, split into + /// `(global_secondary_indexes, local_secondary_indexes)`. + async fn fetch_all_index_info( + &self, + table_id: &str, + ) -> Result<(Vec, Vec), StorageError> { + let rows: Vec<(String, String, String, String, String)> = db::query_as( + "SELECT index_name, index_type, index_id, key_schema, projection \ + FROM indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut infos = Vec::new(); + for (index_name, idx_type_str, index_id, ks_json, proj_json) in rows { + let index_type = match idx_type_str.as_str() { + "GSI" => IndexType::Gsi, + "LSI" => IndexType::Lsi, + other => { + return Err(StorageError::Internal(format!( + "unknown index type in database: {other}" + ))); + } + }; + let key_schema: Vec = serde_json::from_str(&ks_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection: Projection = serde_json::from_str(&proj_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let info = IndexInfo { + index_name, + index_id, + index_type, + key_schema, + projection, + }; + infos.push(info); + } + // Grouped by core rather than matched here, so a new IndexType variant + // does not break this backend. The string parse above already rejects + // any kind this backend cannot have created. + let grouped = extenddb_core::types::partition_indexes(infos); + Ok((grouped.gsis, grouped.lsis)) + } + + /// Fetch `IndexInfo` for a secondary index, validating the table is ACTIVE. + pub(crate) async fn fetch_index_info( + &self, + account_id: &str, + table_name: &str, + index_name: &str, + ) -> Result { + let row: Option<(String, String)> = db::query_as( + "SELECT table_id, table_status FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id, status) = + row.ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; + match status.as_str() { + "ACTIVE" => {} + // A table still being created, or being deleted, is not usable for + // data-plane operations; DynamoDB reports it as not found (not + // in-use). UPDATING keeps the not-active classification. + "CREATING" | "DELETING" => { + return Err(StorageError::TableNotFound(table_name.to_owned())); + } + _ => return Err(StorageError::TableNotActive(table_name.to_owned())), + } + self.fetch_index_info_by_table_id(&table_id, index_name) + .await + } + + /// Fetch `IndexInfo` using a known `table_id`. + pub(crate) async fn fetch_index_info_by_table_id( + &self, + table_id: &str, + index_name: &str, + ) -> Result { + let row: Option<(String, String, String, String)> = db::query_as( + "SELECT index_type, index_id, key_schema, projection \ + FROM indexes WHERE table_id = ? AND index_name = ?", + ) + .bind(table_id) + .bind(index_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (idx_type_str, index_id, ks_str, proj_str) = + row.ok_or_else(|| StorageError::IndexNotFound(index_name.to_owned()))?; + + let index_type = match idx_type_str.as_str() { + "GSI" => IndexType::Gsi, + "LSI" => IndexType::Lsi, + other => { + return Err(StorageError::Internal(format!( + "unknown index type in database: {other}" + ))); + } + }; + let key_schema: Vec = + serde_json::from_str(&ks_str).map_err(|e| StorageError::Internal(e.to_string()))?; + let projection: Projection = + serde_json::from_str(&proj_str).map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(IndexInfo { + index_name: index_name.to_owned(), + index_id, + index_type, + key_schema, + projection, + }) + } +} + +#[cfg(test)] +mod index_data_table_tests { + use crate::db; + use crate::store::DuckDbEngine; + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + + fn hash(name: &str) -> KeySchemaElement { + KeySchemaElement { + attribute_name: name.to_owned(), + key_type: KeyType::Hash, + } + } + + fn range(name: &str) -> KeySchemaElement { + KeySchemaElement { + attribute_name: name.to_owned(), + key_type: KeyType::Range, + } + } + + fn attr(name: &str, t: ScalarAttributeType) -> AttributeDefinition { + AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type: t, + } + } + + /// Create an index data table through the real DDL path. + async fn make_index_table( + index_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> db::Pool { + let pool = db::Pool::open(":memory:", 1).await.unwrap(); + let mut tx = pool.begin().await.unwrap(); + DuckDbEngine::create_index_data_table( + &mut tx, + "probe", + index_key_schema, + attr_defs, + base_key_schema, + base_attr_defs, + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + pool + } + + /// Column list of a named index, in definition order, read from DuckDB's + /// catalog. The SQLite backend asserts on `EXPLAIN QUERY PLAN` here; DuckDB's + /// optimizer chooses a sequential scan below a row-count threshold whatever + /// indexes exist, so on a probe table the plan says nothing about the index. + /// The index definition does. + async fn index_columns(pool: &db::Pool, index_name: &str) -> Vec { + let row: Option<(String,)> = db::query_as( + "SELECT CAST(expressions AS VARCHAR) FROM duckdb_indexes() WHERE index_name = ?", + ) + .bind(index_name) + .fetch_optional(pool) + .await + .unwrap(); + let Some((expr,)) = row else { + return Vec::new(); + }; + expr.trim_matches(|c| c == '[' || c == ']') + .split(',') + .map(|c| c.trim().trim_matches('"').to_owned()) + .filter(|c| !c.is_empty()) + .collect() + } + + #[tokio::test] + async fn reverse_lookup_delete_has_an_index_composite_base_key() { + let pool = make_index_table( + &[hash("gsi_pk"), range("gsi_sk")], + &[ + attr("gsi_pk", ScalarAttributeType::S), + attr("gsi_sk", ScalarAttributeType::S), + ], + &[hash("id"), range("ts")], + &[ + attr("id", ScalarAttributeType::S), + attr("ts", ScalarAttributeType::S), + ], + ) + .await; + + assert_eq!( + index_columns(&pool, "idx_base_key_probe").await, + vec!["base_pk".to_owned(), "base_sk_s".to_owned()], + "reverse-lookup DELETE must be served by the base-key index" + ); + } + + #[tokio::test] + async fn reverse_lookup_delete_has_an_index_hash_only_base_key() { + let pool = make_index_table( + &[hash("gsi_pk"), range("gsi_sk")], + &[ + attr("gsi_pk", ScalarAttributeType::S), + attr("gsi_sk", ScalarAttributeType::S), + ], + &[hash("id")], + &[attr("id", ScalarAttributeType::S)], + ) + .await; + + assert_eq!( + index_columns(&pool, "idx_base_key_probe").await, + vec!["base_pk".to_owned()], + "hash-only base key must still index the reverse lookup" + ); + } + + /// The ordering index is only created when the index declares a sort key, so + /// a sort-key-less index is the case where the base-key index is the *only* + /// secondary index. It must still be created, which is why that block is + /// unconditional. + #[tokio::test] + async fn reverse_lookup_delete_has_an_index_when_index_has_no_sort_key() { + let pool = make_index_table( + &[hash("gsi_pk")], + &[attr("gsi_pk", ScalarAttributeType::S)], + &[hash("id"), range("ts")], + &[ + attr("id", ScalarAttributeType::S), + attr("ts", ScalarAttributeType::S), + ], + ) + .await; + + assert_eq!( + index_columns(&pool, "idx_base_key_probe").await, + vec!["base_pk".to_owned(), "base_sk_s".to_owned()], + "an index with no sort key still needs the base-key index" + ); + assert!( + index_columns(&pool, "idx_order_probe").await.is_empty(), + "no ordering index without a sort key" + ); + } + + /// Guards the column order. An index on `(base_sk_s, base_pk)` would also + /// satisfy a composite lookup, but would not serve a `base_pk`-only lookup, + /// so pin that the index leads with `base_pk`. + #[tokio::test] + async fn base_key_index_leads_with_base_pk() { + let pool = make_index_table( + &[hash("gsi_pk"), range("gsi_sk")], + &[ + attr("gsi_pk", ScalarAttributeType::S), + attr("gsi_sk", ScalarAttributeType::S), + ], + &[hash("id"), range("ts")], + &[ + attr("id", ScalarAttributeType::S), + attr("ts", ScalarAttributeType::S), + ], + ) + .await; + + assert_eq!( + index_columns(&pool, "idx_base_key_probe") + .await + .first() + .map(String::as_str), + Some("base_pk"), + "base-key index must lead with base_pk" + ); + } +} diff --git a/crates/storage-duckdb/src/data/delete_item.rs b/crates/storage-duckdb/src/data/delete_item.rs new file mode 100644 index 00000000..22707600 --- /dev/null +++ b/crates/storage-duckdb/src/data/delete_item.rs @@ -0,0 +1,120 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `delete_item` for the DuckDB backend. + +use extenddb_core::expression::{Expr, ExpressionMaps}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; + +use super::index::{enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::query::check_condition; +use super::tx_helpers::{delete_item_in_tx, fetch_item_in_tx, write_stream_record_in_tx}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + pub(crate) async fn delete_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + // Read the propagation delay BEFORE taking the write lock. It is a + // runtime setting, not an invariant of this write, so it does not need + // to be read under the lock, and the lock serialises every write in the + // process: work done inside it is the backend's throughput bottleneck. + let system_delay = self.index_propagation_delay().await; + let _writer = self.write_lock.lock().await; + // Read the index set after acquiring the write lock so a concurrently + // added GSI (UpdateTable holds the same lock) is not missed. + let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old = fetch_item_in_tx(&mut tx, key_info, key).await?; + + if condition.is_some() { + let empty = Item::new(); + let target = old.as_ref().unwrap_or(&empty); + if let Err(e) = check_condition(condition, target, maps) { + return match e { + StorageError::ConditionFailed(_) => Err(StorageError::ConditionFailed(old)), + other => Err(other), + }; + } + } + + // Only mutate when an item actually exists; deleting a missing item is + // a no-op (no row removed, no index change, no stream record). + let mut enqueued = false; + if old.is_some() { + delete_item_in_tx(&mut tx, key_info, key).await?; + + if !indexes.is_empty() { + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old.as_ref(), + None, + system_delay, + ) + .await?; + } + // Vector rows for this base item are removed too. `new_item` is None, + // so this is a pure removal, applied in this transaction when the + // propagation delay is 0 and enqueued otherwise. + if !key_info.vector_indexes.is_empty() + && crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + None, + system_delay, + ) + .await? + > 0 + { + enqueued = true; + } + if enqueue_async_indexes( + &mut tx, + key_info, + &indexes, + old.as_ref(), + None, + system_delay, + ) + .await? + > 0 + { + enqueued = true; + } + + if let Some(capture) = stream { + write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), None).await?; + } + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if enqueued { + self.gsi_notify.notify_waiters(); + } + + Ok(if return_old { old } else { None }) + } +} diff --git a/crates/storage-duckdb/src/data/index.rs b/crates/storage-duckdb/src/data/index.rs new file mode 100644 index 00000000..af8ffe83 --- /dev/null +++ b/crates/storage-duckdb/src/data/index.rs @@ -0,0 +1,813 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! GSI/LSI index maintenance for the DuckDB backend. +//! +//! Synchronous indexes (LSIs and zero-delay GSIs) are reconciled in the base +//! write transaction. Async GSIs are deferred via the `gsi_pending` queue — +//! **one self-describing row per index**: each row snapshots the base key +//! schema, attribute definitions, and its single target index definition, so +//! the worker applies with zero catalog reads. Each row carries its index's own +//! (jittered) propagation delay, and `ready_at` is kept monotonic within the +//! base key's `worker_partition` so jitter cannot reorder updates to one item. +//! Index key columns follow D2 — `N` keys are stored as the order-preserving +//! TEXT encoding, `S` as TEXT, `B` as BLOB — via [`sk_bound`]. + +use crate::db; +use extenddb_core::types::{ + AttributeDefinition, Item, KeySchemaElement, Projection, ProjectionType, ScalarAttributeType, + TableKeyInfo, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{composite_pk_to_text, parse_sk, sk_column, sk_column_n}; +use serde::{Deserialize, Serialize}; + +use super::{BoundValue, all_sort_key_info, index_table_name, sk_bound}; + +/// Number of partitions a base key can hash to. Updates to one base item share +/// a partition; `ready_at` is clamped monotonic within a partition so the +/// single drain worker (which orders by `id`) applies them in order. More +/// partitions reduce false serialization between unrelated keys; there is no +/// concurrency cost because DuckDB has a single writer. +const NUM_PARTITIONS: u64 = 16; + +/// Stable partition for a base-table key (FNV-1a, mapped to a partition). A +/// local hash keeps the mapping fixed for a key across builds (`std`'s hasher +/// is not guaranteed stable). +fn partition_for(base_pk_text: &str) -> i64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; // FNV-1a offset basis + for byte in base_pk_text.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); // FNV prime + } + (hash % NUM_PARTITIONS) as i64 +} + +/// Jittered propagation delay (ms), uniform in `[delay_ms / 2, delay_ms]`. +/// +/// Simulates DynamoDB's eventual-consistency variability (a fixed delay is +/// perfectly predictable) while keeping the configured delay a meaningful lower +/// bound — jittering all the way to ~0 would make the delay almost meaningless. +/// `delay_ms <= 1` is returned unchanged. Callers only enqueue async indexes, +/// so `delay_ms >= 1`. +fn jitter_delay_ms(delay_ms: u64) -> u64 { + if delay_ms <= 1 { + delay_ms + } else { + use rand::Rng; + rand::rng().random_range(delay_ms / 2 + 1..=delay_ms) + } +} + +/// A single target index definition, snapshotted at enqueue time. The +/// propagation delay is not stored — it is already encoded in the row's +/// `ready_at`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct GsiIndexDef { + pub(crate) index_id: String, + pub(crate) key_schema: Vec, + pub(crate) projection: Projection, +} + +/// Self-describing apply context serialized into `gsi_pending.index_context`. +/// One per row → exactly one target index, so the worker needs no catalog read. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct GsiApplyContext { + pub(crate) base_key_schema: Vec, + pub(crate) attribute_definitions: Vec, + pub(crate) index: GsiIndexDef, +} + +/// What one `gsi_pending` row asks the worker to do: maintain a GSI, or maintain a +/// vector index. One queue serves both so that updates to a single base item stay +/// ordered across index kinds, which two queues drained independently could not +/// guarantee. +/// +/// `untagged` is load-bearing, not stylistic. A GSI context serializes to exactly +/// the bytes it did before this enum existed, so rows already on disk from an +/// earlier version still deserialize, and rows written now are still readable by +/// one. That matters more than it looks: an unparseable `index_context` is treated +/// as a poison row and *dropped*, so a tagged representation would silently discard +/// every in-flight GSI update across an upgrade. +/// +/// The variants are unambiguous by shape rather than by tag: a GSI context requires +/// `index` and a vector context requires `vector`, and no writer emits the other's +/// field, so for any context this code produces exactly one variant matches. +/// `pending_context_tests` pins both directions, including a verbatim legacy payload. +/// +/// To be exact rather than reassuring: a hand-corrupted payload carrying BOTH fields +/// would match `Gsi`, because untagged tries variants in declaration order and +/// ignores unknown fields. That is unreachable from any serializer here, and a +/// genuinely malformed context is already dropped as a poison row, so it is a +/// property of the representation worth knowing rather than a case to defend. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum PendingApplyContext { + Gsi(GsiApplyContext), + Vector(super::vector_index::VectorApplyContext), +} + +impl PendingApplyContext { + /// The base table's key schema, which both kinds carry and the queue needs in + /// order to place the row in its base key's partition. + fn base_key_schema(&self) -> &[KeySchemaElement] { + match self { + Self::Gsi(c) => &c.base_key_schema, + Self::Vector(c) => &c.base_key_schema, + } + } +} + +/// Apply one claimed row to whichever index kind it describes. +/// +/// Both arms tolerate a data table that no longer exists, so a base table or index +/// dropped while a row was in flight is applied-as-skip rather than logged and +/// dropped as unprocessable. +pub(crate) async fn apply_pending_context( + tx: &mut db::Transaction, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &PendingApplyContext, +) -> Result<(), StorageError> { + match context { + PendingApplyContext::Gsi(c) => apply_claimed_row(tx, old_item, new_item, c).await, + PendingApplyContext::Vector(c) => { + super::vector_index::apply_vector_context(tx, old_item, new_item, c).await + } + } +} + +/// Metadata for a single index, used on the write path and by the GSI worker. +pub(crate) struct IndexMeta { + pub(super) index_id: String, + pub(super) index_name: String, + pub(super) index_type: String, + pub(super) key_schema: Vec, + pub(super) projection: Projection, + /// Per-GSI propagation delay (ms). `None` = use system default; `Some(0)` = + /// synchronous. + pub(super) propagation_delay_ms: Option, +} + +/// Fetch all index metadata for a table from the catalog. +pub(crate) async fn fetch_indexes_for_table( + table_id: &str, + pool: &db::Pool, +) -> Result, StorageError> { + let rows: Vec<(String, String, String, String, String, Option)> = db::query_as( + "SELECT index_id, index_name, index_type, key_schema, projection, propagation_delay_ms \ + FROM indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(pool) + .await + .map_err(crate::duckdb_util::map_db_err)?; + + rows.into_iter() + .map(|(index_id, index_name, index_type, ks, proj, delay)| { + Ok(IndexMeta { + index_id, + index_name, + index_type, + key_schema: serde_json::from_str(&ks) + .map_err(|e| StorageError::Internal(e.to_string()))?, + projection: serde_json::from_str(&proj) + .map_err(|e| StorageError::Internal(e.to_string()))?, + propagation_delay_ms: delay, + }) + }) + .collect() +} + +/// Effective GSI propagation delay (ms): per-GSI override, else system default. +/// `Some(0)` forces synchronous; negative is treated as "use system default". +pub(crate) fn effective_delay(idx: &IndexMeta, system_default: u64) -> u64 { + match idx.propagation_delay_ms { + Some(0) => 0, + Some(ms) if ms > 0 => ms as u64, + _ => system_default, + } +} + +/// Enqueue async GSI propagation for a write: **one self-describing +/// `gsi_pending` row per async index**, each honoring its own effective delay. +/// +/// Returns the number of rows enqueued so callers can skip waking the worker +/// when nothing was queued. Must run inside the base write transaction so the +/// pending rows commit atomically with the item mutation. +pub(crate) async fn enqueue_async_indexes( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + indexes: &[IndexMeta], + old_item: Option<&Item>, + new_item: Option<&Item>, + system_default_delay: u64, +) -> Result { + let mut enqueued = 0usize; + for idx in indexes { + if idx.index_type == "LSI" { + continue; // LSIs are always synchronous. + } + let delay = effective_delay(idx, system_default_delay); + if delay == 0 { + continue; // Synchronous GSI — handled in-txn by sync_indexes. + } + let context = GsiApplyContext { + base_key_schema: key_info.key_schema.clone(), + attribute_definitions: key_info.attribute_definitions.clone(), + index: GsiIndexDef { + index_id: idx.index_id.clone(), + key_schema: idx.key_schema.clone(), + projection: idx.projection.clone(), + }, + }; + enqueue_pending_row( + tx, + &key_info.table_id, + old_item, + new_item, + delay, + &PendingApplyContext::Gsi(context), + ) + .await?; + enqueued += 1; + } + Ok(enqueued) +} + +/// Insert one self-describing `gsi_pending` row inside the base write transaction +/// (zero crash window). Shared by the GSI and vector write paths, which is what puts +/// both index kinds in one totally ordered queue. +/// +/// `delay_ms` is the effective delay; a jitter in `[delay/2, delay]` is applied. +/// `ready_at` is clamped to `max(now + jitter, MAX(ready_at) in the base key's +/// partition)` so a later write that draws a smaller jitter can never become +/// eligible before an earlier one — preserving per-key FIFO when the worker drains +/// the partition in `id` order. +pub(crate) async fn enqueue_pending_row( + tx: &mut db::Transaction, + table_id: &str, + old_item: Option<&Item>, + new_item: Option<&Item>, + delay_ms: u64, + context: &PendingApplyContext, +) -> Result<(), StorageError> { + let old_json = old_item + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let new_json = new_item + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let context_json = + serde_json::to_string(context).map_err(|e| StorageError::Internal(e.to_string()))?; + + // Route all updates for one base item to a single partition (per-key FIFO). + // The base key is immutable: `new_item` carries it for puts/updates, + // `old_item` for deletes. Both context kinds describe the same base table, so + // a GSI row and a vector row for one item hash to the same partition and stay + // mutually ordered. + let worker_partition = match new_item.or(old_item) { + Some(item) => partition_for(&composite_pk_to_text(item, context.base_key_schema())?), + None => 0, + }; + + // ready_at as RFC 3339 so it compares correctly (lexically) against the + // worker's RFC 3339 `now` cutoff and against other rows' ready_at. + let jittered = jitter_delay_ms(delay_ms); + let candidate = crate::duckdb_util::format_timestamp( + time::OffsetDateTime::now_utc() + + time::Duration::milliseconds(i64::try_from(jittered).unwrap_or(i64::MAX)), + ); + // Monotonic clamp within the partition (RFC 3339 strings sort lexically). + let part_max: Option = + db::query_scalar("SELECT MAX(ready_at) FROM gsi_pending WHERE worker_partition = ?") + .bind(worker_partition) + .fetch_one(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + let ready_at = match part_max { + Some(prev) if prev > candidate => prev, + _ => candidate, + }; + + db::query( + "INSERT INTO gsi_pending \ + (table_id, worker_partition, old_item, new_item, index_context, ready_at) \ + VALUES (?, ?, ?, ?, ?, ?)", + ) + .bind(table_id) + .bind(worker_partition) + .bind(&old_json) + .bind(&new_json) + .bind(&context_json) + .bind(&ready_at) + .execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + Ok(()) +} + +/// Project an item according to an index's projection configuration. +pub(crate) fn project_item_for_index( + item: &Item, + index_ks: &[KeySchemaElement], + base_ks: &[KeySchemaElement], + projection: &Projection, +) -> Item { + match projection.projection_type { + ProjectionType::All => item.clone(), + ProjectionType::KeysOnly | ProjectionType::Include => { + let mut projected = Item::new(); + for ks in base_ks.iter().chain(index_ks.iter()) { + if let Some(v) = item.get(&ks.attribute_name) { + projected.insert(ks.attribute_name.clone(), v.clone()); + } + } + if projection.projection_type == ProjectionType::Include + && let Some(attrs) = &projection.non_key_attributes + { + for attr in attrs { + if let Some(v) = item.get(attr) { + projected.insert(attr.clone(), v.clone()); + } + } + } + projected + } + } +} + +/// Whether an item carries every key attribute an index requires. +pub(crate) fn item_has_index_keys(item: &Item, index_ks: &[KeySchemaElement]) -> bool { + index_ks + .iter() + .all(|ks| item.contains_key(&ks.attribute_name)) +} + +/// Synchronously reconcile index tables for a base-item change — but only for +/// indexes that are synchronous: LSIs (always) and GSIs whose effective delay +/// is 0. GSIs with a non-zero delay are deferred via `gsi_pending` and applied +/// by the worker, so they are skipped here. +pub(crate) async fn sync_indexes( + tx: &mut db::Transaction, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + indexes: &[IndexMeta], + old_item: Option<&Item>, + new_item: Option<&Item>, + system_default_delay: u64, +) -> Result<(), StorageError> { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + for idx in indexes { + if idx.index_type != "LSI" && effective_delay(idx, system_default_delay) != 0 { + continue; // Async GSI — applied later by the propagation worker. + } + let idx_table = index_table_name(&idx.index_id); + let idx_sks = all_sort_key_info(&idx.key_schema, attr_defs); + + if let Some(old) = old_item + && item_has_index_keys(old, &idx.key_schema) + { + delete_index_row_multi(tx, &idx_table, old, base_key_schema, &base_sks).await?; + } + + if let Some(new) = new_item + && item_has_index_keys(new, &idx.key_schema) + { + let projected = + project_item_for_index(new, &idx.key_schema, base_key_schema, &idx.projection); + insert_index_row_multi( + tx, + &idx_table, + new, + &projected, + &idx.key_schema, + base_key_schema, + &idx_sks, + &base_sks, + ) + .await?; + } + } + Ok(()) +} + +/// Delete an index row identified by its base-table key columns. +pub(crate) async fn delete_index_row_multi( + tx: &mut db::Transaction, + idx_table: &str, + item: &Item, + base_ks: &[KeySchemaElement], + base_sks: &[(&str, ScalarAttributeType)], +) -> Result<(), StorageError> { + let base_pk_text = composite_pk_to_text(item, base_ks)?; + + let mut where_parts = vec!["base_pk = ?".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + where_parts.push(format!("base_{} = ?", sk_column_n(i, sk_type))); + } + let sql = format!( + "DELETE FROM {idx_table} WHERE {}", + where_parts.join(" AND ") + ); + + let mut query = db::query(&sql).bind(base_pk_text); + for &(sk_name, sk_type) in base_sks { + // Bind a value for every placeholder, mirroring insert_index_row_multi: + // a missing sort-key attribute binds an empty string rather than + // skipping the bind (which would desynchronise placeholders and binds). + let bound = match item.get(sk_name) { + Some(v) => sk_bound(&parse_sk(v, sk_type)?), + None => BoundValue::Text(String::new()), + }; + query = super::bind_bound!(query, bound); + } + query + .execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + Ok(()) +} + +/// Insert (or replace) an index row for a base item. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn insert_index_row_multi( + tx: &mut db::Transaction, + idx_table: &str, + item: &Item, + projected: &Item, + index_ks: &[KeySchemaElement], + base_ks: &[KeySchemaElement], + idx_sks: &[(&str, ScalarAttributeType)], + base_sks: &[(&str, ScalarAttributeType)], +) -> Result<(), StorageError> { + let idx_pk_text = composite_pk_to_text(item, index_ks)?; + let base_pk_text = composite_pk_to_text(item, base_ks)?; + let item_json = + serde_json::to_string(projected).map_err(|e| StorageError::Internal(e.to_string()))?; + + // Columns and the matching bound values, in column order. + let mut cols = vec!["pk".to_owned()]; + let mut values: Vec = vec![BoundValue::Text(idx_pk_text)]; + + for (i, &(sk_name, sk_type)) in idx_sks.iter().enumerate() { + cols.push(sk_column_n(i, sk_type)); + if let Some(v) = item.get(sk_name) { + values.push(sk_bound(&parse_sk(v, sk_type)?)); + } else { + values.push(BoundValue::Text(String::new())); + } + } + + cols.push("base_pk".to_owned()); + values.push(BoundValue::Text(base_pk_text)); + + for (i, &(sk_name, sk_type)) in base_sks.iter().enumerate() { + cols.push(format!("base_{}", sk_column_n(i, sk_type))); + if let Some(v) = item.get(sk_name) { + values.push(sk_bound(&parse_sk(v, sk_type)?)); + } else { + values.push(BoundValue::Text(String::new())); + } + } + + cols.push("item_data".to_owned()); + values.push(BoundValue::Text(item_json)); + + let placeholders = vec!["?"; cols.len()].join(", "); + let sql = format!( + "INSERT OR REPLACE INTO {idx_table} ({}) VALUES ({placeholders})", + cols.join(", ") + ); + + let mut query = db::query(&sql); + for v in values { + query = super::bind_bound!(query, v); + } + query + .execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + Ok(()) +} + +/// Helper: derive the sort-key column name used by an index data table for the +/// Nth index sort key. Exposed for query routing. +#[allow(dead_code)] +pub(crate) fn index_sk_column(index: usize, sk_type: ScalarAttributeType) -> String { + if index == 0 { + sk_column(sk_type).to_owned() + } else { + sk_column_n(index, sk_type) + } +} + +/// True if a storage error is a "missing table" error (the `_ddb_*` index table +/// or a `_vidx_*` vector table was dropped, e.g. the base table was deleted +/// while a `gsi_pending` row was in flight). Such rows are benignly skipped. +pub(crate) fn is_no_such_table(e: &StorageError) -> bool { + matches!(e, StorageError::Internal(msg) if msg.contains("no such table")) +} + +/// Apply one claimed `gsi_pending` row to its single target index, within the +/// worker's transaction, using the row's self-describing `index_context` — no +/// catalog read. A missing index data table (the base table was deleted while +/// the row was in flight) is skipped benignly. +pub(crate) async fn apply_claimed_row( + tx: &mut db::Transaction, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &GsiApplyContext, +) -> Result<(), StorageError> { + let base_sks = all_sort_key_info(&context.base_key_schema, &context.attribute_definitions); + let idx = &context.index; + let idx_table = index_table_name(&idx.index_id); + let idx_sks = all_sort_key_info(&idx.key_schema, &context.attribute_definitions); + + if let Some(old) = old_item + && item_has_index_keys(old, &idx.key_schema) + { + delete_index_row_multi(tx, &idx_table, old, &context.base_key_schema, &base_sks) + .await + .or_else(|e| if is_no_such_table(&e) { Ok(()) } else { Err(e) })?; + } + if let Some(new) = new_item + && item_has_index_keys(new, &idx.key_schema) + { + let projected = project_item_for_index( + new, + &idx.key_schema, + &context.base_key_schema, + &idx.projection, + ); + insert_index_row_multi( + tx, + &idx_table, + new, + &projected, + &idx.key_schema, + &context.base_key_schema, + &idx_sks, + &base_sks, + ) + .await + .or_else(|e| if is_no_such_table(&e) { Ok(()) } else { Err(e) })?; + } + Ok(()) +} + +#[cfg(test)] +mod pending_context_tests { + use super::{GsiApplyContext, GsiIndexDef, PendingApplyContext}; + use crate::data::vector_index::{VectorApplyContext, VectorIndexMeta}; + use crate::db; + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, Projection, ProjectionType, + ScalarAttributeType, + }; + + /// A context written by a build that predates vector rows, verbatim. It must + /// still deserialize. + /// + /// This is the test that protects an upgrade. A row whose `index_context` fails + /// to parse is treated as poison and DROPPED, so if the representation had + /// changed incompatibly, every GSI update in flight at the moment of the upgrade + /// would have been discarded silently, with the item written and its index never + /// catching up. + const LEGACY_GSI_CONTEXT: &str = r#"{ + "base_key_schema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "attribute_definitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "index": { + "index_id": "idx-1", + "key_schema": [{"AttributeName": "gsipk", "KeyType": "HASH"}], + "projection": {"ProjectionType": "ALL"} + } + }"#; + + fn base_ks() -> Vec { + vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }] + } + + fn base_ad() -> Vec { + vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }] + } + + fn gsi_context() -> GsiApplyContext { + GsiApplyContext { + base_key_schema: base_ks(), + attribute_definitions: base_ad(), + index: GsiIndexDef { + index_id: "idx-1".to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "gsipk".to_owned(), + key_type: KeyType::Hash, + }], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + }, + } + } + + fn vector_context() -> VectorApplyContext { + VectorApplyContext { + base_key_schema: base_ks(), + attribute_definitions: base_ad(), + table_id: "t-1".to_owned(), + vector: VectorIndexMeta { + index_id: "vidx-1".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + hash_attribute_name: Some("tenant".to_owned()), + search_schema_attribute_names: vec!["tenant".to_owned()], + }, + } + } + + #[test] + fn a_legacy_gsi_context_still_deserializes_as_a_gsi_row() { + let parsed: PendingApplyContext = + serde_json::from_str(LEGACY_GSI_CONTEXT).expect("legacy context must still parse"); + match parsed { + PendingApplyContext::Gsi(c) => assert_eq!(c.index.index_id, "idx-1"), + PendingApplyContext::Vector(_) => { + panic!("a legacy GSI context must not be read as a vector row") + } + } + } + + /// The bytes a GSI row writes today are the bytes it wrote before the enum + /// existed, so a row written by this build is still readable by the previous + /// one. `untagged` is what buys this, and this test is what keeps it. + #[test] + fn wrapping_a_gsi_context_does_not_change_its_serialized_form() { + let bare = serde_json::to_string(&gsi_context()).expect("bare"); + let wrapped = + serde_json::to_string(&PendingApplyContext::Gsi(gsi_context())).expect("wrapped"); + assert_eq!( + bare, wrapped, + "the queue's on-disk format must not change for GSI rows" + ); + } + + /// A vector context round-trips and carries no GSI discriminant field. + /// + /// Note what this does NOT guard: it still passes if `untagged` is removed, so it + /// is not the protection for on-disk compatibility. The two tests above are, and + /// both fail without `untagged`. This one pins the shape contract that makes the + /// discrimination possible in the first place. + #[test] + fn a_vector_context_round_trips_and_carries_no_gsi_discriminant() { + let json = serde_json::to_string(&PendingApplyContext::Vector(vector_context())) + .expect("serialize"); + assert!( + !json.contains("\"index\":"), + "a vector context must not carry the GSI discriminant field: {json}" + ); + let parsed: PendingApplyContext = serde_json::from_str(&json).expect("deserialize"); + match parsed { + PendingApplyContext::Vector(c) => { + assert_eq!(c.vector.index_id, "vidx-1"); + assert_eq!(c.table_id, "t-1"); + assert_eq!(c.vector.dimensions, 2); + assert_eq!(c.vector.hash_attribute_name.as_deref(), Some("tenant")); + assert_eq!(c.vector.search_schema_attribute_names, ["tenant"]); + assert_eq!( + c.vector.projection.projection_type, + ProjectionType::KeysOnly + ); + } + PendingApplyContext::Gsi(_) => panic!("a vector context must not be read as a GSI row"), + } + } + + /// A GSI row and a vector row for the SAME base item must land in the same queue + /// partition, which is what makes them mutually ordered. + /// + /// This is the claim that reusing one queue buys ordering ACROSS index kinds, and + /// it holds only because both contexts hash the same base key rather than + /// anything index-specific. Asserted on the partition because the partition is the + /// mechanism: two kinds hashing differently would land in separate partitions, + /// each monotonic on its own, leaving the relative order of a GSI and a vector + /// update to one item unconstrained. + /// + /// Driven through the real `enqueue_pending_row` rather than a test shim, so it is + /// the production path being measured. + #[tokio::test] + async fn a_gsi_row_and_a_vector_row_for_one_item_share_a_partition() { + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let the_item: extenddb_core::types::Item = + serde_json::from_str(r#"{"pk":{"S":"shared-key"},"emb":{"L":[{"N":"1"}]}}"#) + .expect("item"); + let mut tx = engine.pool.begin().await.expect("tx"); + for context in [ + PendingApplyContext::Gsi(gsi_context()), + PendingApplyContext::Vector(vector_context()), + ] { + super::enqueue_pending_row(&mut tx, "t-1", None, Some(&the_item), 60_000, &context) + .await + .expect("enqueue"); + } + tx.commit().await.expect("commit"); + + let partitions: Vec<(i64,)> = + db::query_as("SELECT DISTINCT worker_partition FROM gsi_pending") + .fetch_all(&engine.pool) + .await + .expect("partitions"); + assert_eq!( + partitions.len(), + 1, + "a GSI row and a vector row for one base item must share a partition, or \ + their relative order is unconstrained" + ); + let (depth,): (i64,) = db::query_as("SELECT COUNT(*) FROM gsi_pending") + .fetch_one(&engine.pool) + .await + .expect("depth"); + assert_eq!( + depth, 2, + "both kinds must have enqueued, so one partition is not an artefact of a \ + missing row" + ); + } + + /// A catalog created before the rename must keep honouring its operator's value. + /// + /// This is the compatibility property the whole fallback exists for, and the cost + /// of getting it wrong is not cosmetic: the server refuses to start on a + /// catalog-version mismatch rather than migrating, so nothing ever rewrites the old + /// row. Reading past it would silently reset a configured delay to the default, and + /// since 0 means synchronous, the silent change would be from strict to eventually + /// consistent, which is exactly the direction that turns a passing test suite into + /// a flaky one somewhere else. + #[tokio::test] + async fn a_pre_rename_catalog_still_honours_its_configured_delay() { + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + // Reshape the catalog to look as it did before the rename: the legacy key + // only, carrying a deliberately non-default value. + db::query("DELETE FROM settings WHERE key = 'index_propagation_delay_ms'") + .execute(&engine.pool) + .await + .expect("drop canonical row"); + db::query("INSERT INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '0')") + .execute(&engine.pool) + .await + .expect("seed legacy row"); + + assert_eq!( + engine.index_propagation_delay().await, + 0, + "a value set under the pre-rename key must still be honoured, or an \ + operator's synchronous setting silently becomes asynchronous" + ); + } + + /// With both rows present the canonical one wins, deterministically. + /// + /// Reachable if an operator sets the legacy key on a build that predates the + /// canonicalising write path and then upgrades. Without the explicit ordering the + /// winner would be whichever row DuckDB happened to return first. + #[tokio::test] + async fn the_canonical_key_wins_when_both_are_present() { + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + db::query( + "INSERT OR REPLACE INTO settings (key, value) \ + VALUES ('index_propagation_delay_ms', '7'), ('gsi_propagation_delay_ms', '999')", + ) + .execute(&engine.pool) + .await + .expect("seed both rows"); + + assert_eq!( + engine.index_propagation_delay().await, + 7, + "the canonical key must take precedence over the deprecated alias" + ); + } +} diff --git a/crates/storage-duckdb/src/data/mod.rs b/crates/storage-duckdb/src/data/mod.rs new file mode 100644 index 00000000..5bed28c2 --- /dev/null +++ b/crates/storage-duckdb/src/data/mod.rs @@ -0,0 +1,187 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Per-DynamoDB-table data storage for the DuckDB backend. +//! +//! Each virtual DynamoDB table maps to a DuckDB table named `_ddb_`; +//! each secondary index maps to `_ddb_`. The full item is stored as +//! JSON `TEXT` in `item_data`; key columns exist only for lookup and ordering. +//! +//! # Key column types (design decision D2) +//! +//! - Partition key (`pk`): always `TEXT` via the shared `pk_to_text` (equality +//! only) — identical to the PostgreSQL backend. +//! - Sort key, by attribute type: +//! - `S` → `sk_s TEXT` (DuckDB BINARY collation = DynamoDB UTF-8 byte order). +//! - `N` → `sk_n TEXT` holding [`encode_orderable_number`], so lexicographic +//! order equals numeric order with full 38-digit precision (never `DOUBLE`). +//! - `B` → `sk_b BLOB` (memcmp = DynamoDB unsigned byte order). +//! +//! The exact, full-precision value is always preserved in `item_data` JSON, so +//! reads lose nothing; the typed key columns are used only for +//! lookup/range/ordering. + +use extenddb_core::types::{ + AttributeDefinition, Item, KeySchemaElement, KeyType, ScalarAttributeType, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::SortKeyValue; + +use crate::number_key::encode_orderable_number; + +mod data_engine; +mod ddl; +mod delete_item; +mod index; +mod put_item; +mod query; +mod query_scan; +mod transactions; +mod tx_helpers; +mod update_item; +pub(crate) mod vector_index; + +pub(crate) use index::{ + PendingApplyContext, apply_pending_context, insert_index_row_multi, project_item_for_index, +}; +pub(crate) use tx_helpers::upsert_item_in_tx; + +/// Quoted SQL identifier for a virtual DynamoDB table's data table. +pub(crate) fn data_table_name(table_id: &str) -> String { + format!("\"_ddb_{table_id}\"") +} + +/// Quoted SQL identifier for a GSI/LSI data table. +pub(crate) fn index_table_name(index_id: &str) -> String { + format!("\"_ddb_{index_id}\"") +} + +/// Quoted SQL identifier for a vector-index data table. +/// +/// The name carries the base `table_id` as well as the `index_id` so every vector +/// table belonging to a DynamoDB table is discoverable from the table alone. That +/// is what lets `drop_data_table` clean them up: by the time it runs, the catalog +/// rows have already been cascade-deleted inside the same transaction, so the +/// index ids can no longer be read from `vector_indexes`. +/// +/// One row per vector rather than a packed blob per partition. Measured +/// 2026-08-06: a packed blob reads 2 to 4x faster but makes every write +/// O(partition), since inserting one vector rewrites the whole blob (390 MB for +/// a 100k-vector partition at 1024 dimensions). Vector indexes are maintained on +/// every write to an indexed attribute, so that trade is not available. +pub(crate) fn vector_table_name(table_id: &str, index_id: &str) -> String { + format!("\"_vidx_{table_id}_{index_id}\"") +} + +/// `GLOB` pattern matching every vector data table of one DynamoDB table. +/// +/// `GLOB` rather than `LIKE` because the pattern itself is full of underscores, and +/// in `LIKE` an underscore is a single-character wildcard: `_vidx__%` would match +/// far more than it appears to. It can only ever over-match, since a wildcard also +/// matches a literal underscore, and no other table could share the UUID, so `LIKE` +/// was safe in practice. It was not safe for the stated reason, though, and a comment +/// that justifies the wrong half is worse than none. +/// +/// `GLOB` treats `_` literally and uses `*` for the wildcard, so the pattern means +/// what it reads. `table_id` is a server-generated UUID, which contains no `GLOB` +/// metacharacters either. +pub(crate) fn vector_table_glob_pattern(table_id: &str) -> String { + format!("_vidx_{table_id}_*") +} + +/// All RANGE key attributes in key-schema order, paired with their scalar type. +pub(crate) fn all_sort_key_info<'a>( + key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], +) -> Vec<(&'a str, ScalarAttributeType)> { + key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Range) + .filter_map(|ks| { + attr_defs + .iter() + .find(|ad| ad.attribute_name == ks.attribute_name) + .map(|ad| (ks.attribute_name.as_str(), ad.attribute_type)) + }) + .collect() +} + +/// Deserialize an `item_data` JSON value into an `Item`. +pub(crate) fn json_to_item(v: serde_json::Value) -> Result { + serde_json::from_value(v).map_err(|e| StorageError::Internal(e.to_string())) +} + +/// A value bound to a sort-key column, already mapped to its DuckDB storage +/// representation per D2. +#[derive(Debug, Clone)] +pub(crate) enum BoundValue { + /// `TEXT` — used for `S` (raw string) and `N` (order-preserving encoding). + Text(String), + /// `BLOB` — used for `B`. + Blob(Vec), +} + +/// Map a parsed sort-key value to its DuckDB storage representation (D2): +/// `S` → TEXT, `N` → order-preserving TEXT, `B` → BLOB. +pub(crate) fn sk_bound(sk: &SortKeyValue) -> BoundValue { + match sk { + SortKeyValue::S(s) => BoundValue::Text(s.clone()), + SortKeyValue::N(n) => BoundValue::Text(encode_orderable_number(n)), + SortKeyValue::B(b) => BoundValue::Blob(b.clone()), + } +} + +/// Bind a [`BoundValue`] onto a positional `sqlx` query. +macro_rules! bind_bound { + ($query:expr, $bound:expr) => { + match $bound { + crate::data::BoundValue::Text(s) => $query.bind(s), + crate::data::BoundValue::Blob(b) => $query.bind(b), + } + }; +} + +/// Bind `pk` then a sort-key value, then `fetch_optional` a single JSON column. +macro_rules! bind_sk_fetch_optional { + ($sql:expr, $pk:expr, $sk:expr, $executor:expr) => {{ + let __q = crate::db::query_as::<(serde_json::Value,)>($sql).bind($pk); + let __q = match crate::data::sk_bound($sk) { + crate::data::BoundValue::Text(s) => __q.bind(s), + crate::data::BoundValue::Blob(b) => __q.bind(b), + }; + __q.fetch_optional($executor) + .await + .map_err(|e| extenddb_storage::error::StorageError::Internal(e.to_string())) + }}; +} + +/// Bind `pk`, a sort-key value, then `item_json`, and `execute`. +macro_rules! bind_sk_execute { + ($sql:expr, $pk:expr, $sk:expr, $item_json:expr, $executor:expr) => {{ + let __q = crate::db::query($sql).bind($pk); + let __q = match crate::data::sk_bound($sk) { + crate::data::BoundValue::Text(s) => __q.bind(s), + crate::data::BoundValue::Blob(b) => __q.bind(b), + }; + __q.bind($item_json) + .execute($executor) + .await + .map_err(|e| extenddb_storage::error::StorageError::Internal(e.to_string())) + }}; +} + +/// Bind `pk` then a sort-key value, and `execute` (no item payload). +macro_rules! bind_sk_only_execute { + ($sql:expr, $pk:expr, $sk:expr, $executor:expr) => {{ + let __q = crate::db::query($sql).bind($pk); + let __q = match crate::data::sk_bound($sk) { + crate::data::BoundValue::Text(s) => __q.bind(s), + crate::data::BoundValue::Blob(b) => __q.bind(b), + }; + __q.execute($executor) + .await + .map_err(|e| extenddb_storage::error::StorageError::Internal(e.to_string())) + }}; +} + +pub(crate) use {bind_bound, bind_sk_execute, bind_sk_fetch_optional, bind_sk_only_execute}; diff --git a/crates/storage-duckdb/src/data/put_item.rs b/crates/storage-duckdb/src/data/put_item.rs new file mode 100644 index 00000000..0817ac75 --- /dev/null +++ b/crates/storage-duckdb/src/data/put_item.rs @@ -0,0 +1,190 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `put_item` and `get_item` for the DuckDB backend. +//! +//! Writes acquire the engine write lock (D1) and run in a transaction, so the +//! condition check, the write, index sync, and stream capture are one atomic +//! unit with no competing writer — no `INSERT ... ON CONFLICT DO NOTHING` race +//! dance is needed. + +use crate::db; +use extenddb_core::expression::{Expr, ExpressionMaps}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{pk_to_text, sk_column, sk_info}; + +use super::index::{enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::query::check_condition; +use super::tx_helpers::{fetch_item_in_tx, upsert_item_in_tx, write_stream_record_in_tx}; +use super::{bind_sk_fetch_optional, data_table_name, json_to_item}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + pub(crate) async fn put_item_impl( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + // Read the index set only after acquiring the write lock, so a GSI added + // by a concurrent UpdateTable (which holds the same lock) cannot be missed + // and left unmaintained by this write. + // Read the propagation delay BEFORE taking the write lock. It is a + // runtime setting, not an invariant of this write, so it does not need + // to be read under the lock, and the lock serialises every write in the + // process: work done inside it is the backend's throughput bottleneck. + let system_delay = self.index_propagation_delay().await; + let _writer = self.write_lock.lock().await; + let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + + // Index key attributes present in the item must match their declared + // scalar type and be non-empty, matching real DynamoDB. This is up-front + // input validation (a top-level ValidationException), so it runs before + // any write work. Mirrors the PostgreSQL backend. + if !indexes.is_empty() { + let index_refs: Vec> = indexes + .iter() + .map(|idx| extenddb_core::validation::IndexKeyRef { + index_name: &idx.index_name, + key_schema: &idx.key_schema, + }) + .collect(); + extenddb_core::validation::validate_index_keys( + &item, + &index_refs, + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; + } + + let need_old = condition.is_some() || return_old || !indexes.is_empty() || stream.is_some(); + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old = if need_old { + fetch_item_in_tx(&mut tx, key_info, &item).await? + } else { + None + }; + + if condition.is_some() { + let empty = Item::new(); + let target = old.as_ref().unwrap_or(&empty); + if let Err(e) = check_condition(condition, target, maps) { + return match e { + StorageError::ConditionFailed(_) => Err(StorageError::ConditionFailed(old)), + other => Err(other), + }; + } + } + + upsert_item_in_tx(&mut tx, key_info, &item).await?; + + // Synchronous indexes (LSIs + zero-delay GSIs) are applied in-txn; + // async GSIs are enqueued into gsi_pending within the same txn. + if !indexes.is_empty() { + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old.as_ref(), + Some(&item), + system_delay, + ) + .await?; + } + let enqueued_gsi = enqueue_async_indexes( + &mut tx, + key_info, + &indexes, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0; + // Vector indexes: applied in this transaction when the propagation delay is + // 0, otherwise enqueued alongside the async GSI work. Gated on the cached + // key info so a table without them costs no extra query, and deliberately + // outside the `indexes` guard above: a table may have a vector index and no + // GSI or LSI at all. + let enqueued_vector = if key_info.vector_indexes.is_empty() { + false + } else { + crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0 + }; + let enqueued = enqueued_gsi || enqueued_vector; + + if let Some(capture) = stream { + write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) + .await?; + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if enqueued { + self.gsi_notify.notify_waiters(); + } + + Ok(if return_old { old } else { None }) + } + + pub(crate) async fn get_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> Result, StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + let json_opt = if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = extenddb_storage::util::parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + let row: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&sql, pk_text.as_ref(), &sk, &self.pool)?; + row.map(|(v,)| v) + } else { + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let row: Option<(serde_json::Value,)> = db::query_as(&sql) + .bind(pk_text.as_ref()) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + row.map(|(v,)| v) + }; + + json_opt.map(json_to_item).transpose() + } +} diff --git a/crates/storage-duckdb/src/data/query.rs b/crates/storage-duckdb/src/data/query.rs new file mode 100644 index 00000000..c37941cb --- /dev/null +++ b/crates/storage-duckdb/src/data/query.rs @@ -0,0 +1,233 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared query helpers: condition evaluation, key-condition SQL fragments, +//! and dynamic query execution. +//! +//! Sort-key comparisons bind through [`super::sk_bound`], so `N` keys are +//! compared as their order-preserving TEXT encoding (D2): `>`, `<`, `BETWEEN` +//! all remain numerically correct. `begins_with` applies only to `S` (string +//! prefix via the maximal code point `char(1114111)`) and `B` (byte-range via +//! an incremented upper bound); DynamoDB rejects `begins_with` on `N`. + +use crate::db; +use extenddb_core::expression::{self, CompareOp, Expr, ExpressionMaps, SortKeyCondition}; +use extenddb_core::types::{ + AttributeValue, Item, KeySchemaElement, ScalarAttributeType, extract_key, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{SortKeyValue, parse_sk}; + +use super::BoundValue; + +/// Evaluate an optional condition expression against an item. +/// Returns `ConditionFailed(None)` when the condition evaluates to false. +pub(super) fn check_condition( + condition: Option<&Expr>, + item: &Item, + maps: &ExpressionMaps, +) -> Result<(), StorageError> { + if let Some(cond) = condition { + let passed = expression::evaluate_condition(cond, item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + return Err(StorageError::ConditionFailed(None)); + } + } + Ok(()) +} + +/// Resolve a key-condition placeholder expression to its `AttributeValue`. +pub(super) fn resolve_expr_to_av( + expr: &Expr, + maps: &ExpressionMaps, +) -> Result { + match expr { + Expr::Placeholder(name) => maps + .resolve_value(name) + .cloned() + .map_err(|e| StorageError::Validation(e.to_string())), + _ => Err(StorageError::Internal( + "expected placeholder in key condition".to_owned(), + )), + } +} + +/// SQL `WHERE` fragment for a sort-key condition on `sk_col`, together with the +/// values to bind for it. +/// +/// The fragment and its bind list are returned from one function on purpose. +/// `begins_with` on a binary key sometimes has no upper bound at all (see +/// [`binary_prefix_upper_bound`]), so the placeholder count is not fixed by the +/// condition kind alone. Building the SQL in one place and the binds in another +/// meant the two could disagree about how many placeholders exist, which is a +/// bind-offset corruption rather than a visible error. +pub(super) fn build_sk_sql_and_binds( + sk_cond: &SortKeyCondition, + sk_col: &str, + sk_type: ScalarAttributeType, + maps: &ExpressionMaps, +) -> Result<(String, Vec), StorageError> { + match sk_cond { + SortKeyCondition::Compare { op, value, .. } => { + let sql_op = match op { + CompareOp::Eq => "=", + CompareOp::Ne => "<>", + CompareOp::Lt => "<", + CompareOp::Le => "<=", + CompareOp::Gt => ">", + CompareOp::Ge => ">=", + }; + Ok(( + format!(" AND {sk_col} {sql_op} ?"), + vec![parse_sk(&resolve_expr_to_av(value, maps)?, sk_type)?], + )) + } + SortKeyCondition::Between { low, high, .. } => Ok(( + format!(" AND {sk_col} BETWEEN ? AND ?"), + vec![ + parse_sk(&resolve_expr_to_av(low, maps)?, sk_type)?, + parse_sk(&resolve_expr_to_av(high, maps)?, sk_type)?, + ], + )), + SortKeyCondition::BeginsWith { prefix, .. } => { + let sk = parse_sk(&resolve_expr_to_av(prefix, maps)?, sk_type)?; + match sk { + SortKeyValue::B(b) => match binary_prefix_upper_bound(&b) { + Some(upper) => Ok(( + format!(" AND {sk_col} >= ? AND {sk_col} < ?"), + vec![SortKeyValue::B(b), SortKeyValue::B(upper)], + )), + // Every byte string with this prefix is in range and there is + // no finite value above them all, so the only correct upper + // bound is none. Emitting one anyway is what silently dropped + // rows. + None => Ok((format!(" AND {sk_col} >= ?"), vec![SortKeyValue::B(b)])), + }, + // For strings the SQL upper bound is `? || chr(1114111)`, + // so the same prefix is bound twice. + SortKeyValue::S(s) => Ok(( + format!(" AND {sk_col} >= ? AND {sk_col} < (? || chr(1114111))"), + vec![SortKeyValue::S(s.clone()), SortKeyValue::S(s)], + )), + SortKeyValue::N(_) => Err(StorageError::Validation( + "begins_with is not supported on numeric sort keys".to_owned(), + )), + } + } + } +} + +/// Exclusive upper bound for a binary prefix range, or `None` when the range is +/// unbounded above. +/// +/// The bound is the smallest byte string greater than every string having +/// `prefix` as a prefix. Found by incrementing the last byte below `0xFF` and +/// discarding everything after it: `[1, 2]` yields `[1, 3]`, so `[1, 2, 9]` is +/// still included but `[1, 3]` is not. +/// +/// `None` when every byte is `0xFF`, which includes the empty prefix. No finite +/// bound exists in that case, because a longer all-`0xFF` string always sorts +/// after any candidate: with `[0xFF]` the strings `[0xFF, 0xFF]`, +/// `[0xFF, 0xFF, 0xFF]` and so on continue without end. The previous code +/// returned `vec![0xFF; prefix.len() + 1]` here, which is a value inside the +/// matching set rather than above it, so rows were silently dropped: for an +/// empty prefix it produced `[0xFF]`, which excluded every key sorting at or +/// after it, and `begins_with([])` then returned part of the partition while +/// reporting success. Returning `None` and omitting the predicate is the only +/// correct answer that does not depend on a maximum key length, which is +/// operator-configurable via `max_sort_key_size_bytes`. +fn binary_prefix_upper_bound(prefix: &[u8]) -> Option> { + let mut out = prefix.to_vec(); + while let Some(last) = out.last_mut() { + if *last < 0xFF { + *last += 1; + return Some(out); + } + out.pop(); + } + None +} + +/// Build a `LastEvaluatedKey` from an item's key attributes. +pub(super) fn build_key(item: &Item, key_schema: &[KeySchemaElement]) -> Item { + extract_key(item, key_schema) +} + +/// Execute a dynamically-built query, binding `BoundValue`s positionally. +pub(super) async fn execute_dynamic_query( + sql: &str, + values: Vec, + pool: &db::Pool, +) -> Result, StorageError> { + let mut query = db::query_as::<(serde_json::Value,)>(sql); + for v in values { + query = match v { + BoundValue::Text(s) => query.bind(s), + BoundValue::Blob(b) => query.bind(b), + }; + } + let rows: Vec<(serde_json::Value,)> = query + .fetch_all(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(rows.into_iter().map(|(v,)| v).collect()) +} + +#[cfg(test)] +mod tests { + use super::binary_prefix_upper_bound; + + /// Every byte string having `prefix` as a prefix must sort below the bound, + /// and the bound itself must not. Asserted against explicit witnesses rather + /// than trusting the arithmetic, since the defect this replaces was an + /// off-by-domain error that looked arithmetically reasonable. + #[test] + fn the_bound_excludes_itself_and_includes_every_extension() { + let upper = binary_prefix_upper_bound(&[1, 2]).expect("finite bound exists"); + assert_eq!(upper, vec![1, 3]); + for witness in [ + vec![1, 2], + vec![1, 2, 0], + vec![1, 2, 0xFF], + vec![1, 2, 9, 9], + ] { + assert!(witness < upper, "{witness:?} must be inside the range"); + } + assert!(vec![1, 3] >= upper, "the bound must be exclusive"); + } + + /// A trailing `0xFF` carries: the last byte below `0xFF` is incremented and + /// everything after it is discarded. + #[test] + fn a_trailing_all_ones_byte_carries_into_the_previous_byte() { + assert_eq!(binary_prefix_upper_bound(&[1, 0xFF]), Some(vec![2])); + assert_eq!(binary_prefix_upper_bound(&[1, 0xFF, 0xFF]), Some(vec![2])); + let upper = binary_prefix_upper_bound(&[1, 0xFF]).expect("finite bound exists"); + assert!(vec![1, 0xFF, 0xFF] < upper, "extension must be included"); + } + + /// The empty prefix matches everything, so no upper bound can exist. This is + /// the case that silently dropped rows: the old code returned `[0xFF]`, which + /// excluded every key sorting at or after it, so `begins_with([])` returned + /// part of the partition and reported success. + #[test] + fn an_empty_prefix_has_no_upper_bound() { + assert_eq!(binary_prefix_upper_bound(&[]), None); + } + + /// Same defect one length up, and not only for the empty prefix: an all-`0xFF` + /// prefix of any length has no finite bound, because a longer all-`0xFF` + /// string always sorts after any candidate. The old code returned + /// `[0xFF, 0xFF]` for `[0xFF]`, which wrongly excluded `[0xFF, 0xFF, 0xFF]`. + #[test] + fn an_all_ones_prefix_has_no_upper_bound_at_any_length() { + for prefix in [vec![0xFF], vec![0xFF, 0xFF], vec![0xFF; 8]] { + assert_eq!( + binary_prefix_upper_bound(&prefix), + None, + "no finite bound exists above {prefix:?}" + ); + } + } +} diff --git a/crates/storage-duckdb/src/data/query_scan.rs b/crates/storage-duckdb/src/data/query_scan.rs new file mode 100644 index 00000000..9562fb48 --- /dev/null +++ b/crates/storage-duckdb/src/data/query_scan.rs @@ -0,0 +1,465 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `query` and `scan` for the DuckDB backend. +//! +//! Mirrors the PostgreSQL backend's index routing and pagination. For index +//! operations the engine passes `key_info.key_schema` = index key schema and +//! `key_info.base_key_schema` = base table key schema, so the index `sk_*` +//! columns are addressed by the index sort key and `base_*` columns provide +//! tie-breakers. DuckDB differences: positional `?` placeholders, no `COLLATE` +//! (the default BINARY collation already matches DynamoDB byte order, and `N` +//! keys are stored as the order-preserving TEXT encoding per D2), and +//! `rowid % total_segments` for parallel scan. + +use std::fmt::Write; + +use extenddb_core::expression::{ExpressionMaps, KeyCondition, PathElement}; +use extenddb_core::types::{Item, ScalarAttributeType, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{ + encode_netstring_composite, parse_sk, pk_to_text, sk_column, sk_column_n, sk_info, +}; + +use super::query::{build_key, build_sk_sql_and_binds, execute_dynamic_query, resolve_expr_to_av}; +use super::{ + BoundValue, all_sort_key_info, data_table_name, index_table_name, json_to_item, sk_bound, +}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + #[allow(clippy::too_many_arguments)] + pub(crate) async fn query_impl( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + let (ddb_table, is_lsi) = if let Some(idx_name) = index_name { + let info = self + .fetch_index_info_by_table_id(&key_info.table_id, idx_name) + .await?; + ( + index_table_name(&info.index_id), + info.index_type == extenddb_core::types::IndexType::Lsi, + ) + } else { + (data_table_name(&key_info.table_id), false) + }; + + // Partition key (composite via netstring when multi-HASH). + let pk_text = if key_condition.extra_pk_conditions.is_empty() { + pk_to_text(&resolve_expr_to_av(&key_condition.pk_value, maps)?)?.into_owned() + } else { + let mut parts = + vec![pk_to_text(&resolve_expr_to_av(&key_condition.pk_value, maps)?)?.into_owned()]; + for (_, value) in &key_condition.extra_pk_conditions { + parts.push(pk_to_text(&resolve_expr_to_av(value, maps)?)?.into_owned()); + } + encode_netstring_composite(&parts) + }; + + let sk_info_val = sk_info(&key_info.key_schema, &key_info.attribute_definitions); + let all_sks = all_sort_key_info(&key_info.key_schema, &key_info.attribute_definitions); + let base_sk_info: Option<(String, ScalarAttributeType)> = if index_name.is_some() { + sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) + .map(|(n, t)| (n.to_owned(), t)) + } else { + None + }; + + let mut sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let mut binds: Vec = vec![BoundValue::Text(pk_text)]; + + // Primary sort-key condition. + if let (Some(sk_cond), Some((_, sk_type))) = (&key_condition.sk_condition, sk_info_val) { + let (sk_sql, sk_binds) = + build_sk_sql_and_binds(sk_cond, sk_column(sk_type), sk_type, maps)?; + sql.push_str(&sk_sql); + for v in &sk_binds { + binds.push(sk_bound(v)); + } + } + + // Extra RANGE-key equality conditions (multi-RANGE schemas). + for (path, value) in &key_condition.extra_sk_conditions { + let Some(attr_name) = resolve_attr_name(path, maps) else { + continue; + }; + if let Some(pos) = all_sks.iter().position(|(n, _)| *n == attr_name) + && pos > 0 + { + let (_, sk_type) = all_sks[pos]; + let _ = write!(sql, " AND {} = ?", sk_column_n(pos, sk_type)); + binds.push(sk_bound(&parse_sk( + &resolve_expr_to_av(value, maps)?, + sk_type, + )?)); + } + } + + // Pagination. + if exclusive_start_key.is_some() && sk_info_val.is_none() && index_name.is_none() { + return Ok((Vec::new(), None)); + } + if let Some(start_key) = exclusive_start_key { + append_query_pagination( + &mut sql, + &mut binds, + start_key, + sk_info_val, + base_sk_info.as_ref(), + key_info, + index_name.is_some(), + is_lsi, + forward, + )?; + } + + // ORDER BY. + let dir = if forward { "ASC" } else { "DESC" }; + if let Some((_, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + if let Some((_, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + if is_lsi { + let _ = write!(sql, " ORDER BY {sk_col} {dir}, {base_col} {dir}"); + } else { + // GSI: order by the full base primary key after the index SK + // so the ordering matches the pagination tie-breaker exactly. + // Ordering by base SK alone leaves rows that share an index SK + // and a base SK in an arbitrary order, which no + // ExclusiveStartKey can resume from deterministically. + let _ = write!(sql, " ORDER BY {sk_col} {dir}, base_pk ASC, {base_col} ASC"); + } + } else if index_name.is_some() { + let _ = write!(sql, " ORDER BY {sk_col} {dir}, base_pk ASC"); + } else { + let _ = write!(sql, " ORDER BY {sk_col} {dir}"); + } + } else if index_name.is_some() { + if let Some((_, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + let _ = write!(sql, " ORDER BY base_pk {dir}, {base_col} {dir}"); + } else { + let _ = write!(sql, " ORDER BY base_pk {dir}"); + } + } + + let fetch_limit = limit.map_or(1_000_001, |l| l + 1); + let _ = write!(sql, " LIMIT {fetch_limit}"); + + let rows = execute_dynamic_query(&sql, binds, &self.pool).await?; + finalize(rows, limit, &key_info.key_schema) + } + + pub(crate) async fn scan_impl( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + let ddb_table = if let Some(idx_name) = index_name { + let info = self + .fetch_index_info_by_table_id(&key_info.table_id, idx_name) + .await?; + index_table_name(&info.index_id) + } else { + data_table_name(&key_info.table_id) + }; + + let sk_info_val = sk_info(&key_info.key_schema, &key_info.attribute_definitions); + let base_sk_info: Option<(String, ScalarAttributeType)> = if index_name.is_some() { + sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) + .map(|(n, t)| (n.to_owned(), t)) + } else { + None + }; + + let mut sql = format!("SELECT item_data FROM {ddb_table}"); + let mut conditions: Vec = Vec::new(); + let mut binds: Vec = Vec::new(); + + // Parallel scan: disjoint rowid partitioning. seg/total are validated + // non-negative integers at the engine layer, safe to interpolate. + if let (Some(seg), Some(total)) = (segment, total_segments) { + conditions.push(format!("(rowid % {total}) = {seg}")); + } + + if let Some(start_key) = exclusive_start_key { + let pk_name = &key_info.key_schema[0].attribute_name; + if !start_key.contains_key(pk_name) { + return Err(StorageError::Validation( + "The provided starting key is invalid: The provided key element does not match the schema".to_owned(), + )); + } + let pk_text = pk_to_text(start_key.get(pk_name).unwrap())?.into_owned(); + + if index_name.is_some() { + if let Some((sk_name, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + let sk_bv = start_key + .get(sk_name) + .map(|v| parse_sk(v, sk_type)) + .transpose()? + .map(|s| sk_bound(&s)); + let base_pk_text = base_pk_from_start_key(start_key, key_info)?; + if let Some((base_name, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + conditions.push(format!( + "(pk, {sk_col}, base_pk, {base_col}) > (?, ?, ?, ?)" + )); + binds.push(BoundValue::Text(pk_text)); + binds.push(sk_bv.unwrap_or(BoundValue::Text(String::new()))); + binds.push(BoundValue::Text(base_pk_text)); + if let Some(v) = start_key.get(base_name.as_str()) { + binds.push(sk_bound(&parse_sk(v, *base_type)?)); + } else { + binds.push(BoundValue::Text(String::new())); + } + } else { + conditions.push(format!("(pk, {sk_col}, base_pk) > (?, ?, ?)")); + binds.push(BoundValue::Text(pk_text)); + binds.push(sk_bv.unwrap_or(BoundValue::Text(String::new()))); + binds.push(BoundValue::Text(base_pk_text)); + } + } else { + // Hash-only GSI. Include the base sort key in the + // pagination predicate when the base table has one: the + // index PRIMARY KEY is (pk, base_pk, base_sk*), so (pk, + // base_pk) alone is not a total order and would skip rows + // sharing a (pk, base_pk) across a page boundary. + let base_pk_text = base_pk_from_start_key(start_key, key_info)?; + if let Some((base_name, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + conditions.push(format!("(pk, base_pk, {base_col}) > (?, ?, ?)")); + binds.push(BoundValue::Text(pk_text)); + binds.push(BoundValue::Text(base_pk_text)); + if let Some(v) = start_key.get(base_name.as_str()) { + binds.push(sk_bound(&parse_sk(v, *base_type)?)); + } else { + binds.push(BoundValue::Text(String::new())); + } + } else { + conditions.push("(pk, base_pk) > (?, ?)".to_owned()); + binds.push(BoundValue::Text(pk_text)); + binds.push(BoundValue::Text(base_pk_text)); + } + } + } else if let Some((sk_name, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + conditions.push(format!("(pk, {sk_col}) > (?, ?)")); + binds.push(BoundValue::Text(pk_text)); + if let Some(v) = start_key.get(sk_name) { + binds.push(sk_bound(&parse_sk(v, sk_type)?)); + } else { + binds.push(BoundValue::Text(String::new())); + } + } else { + conditions.push("pk > ?".to_owned()); + binds.push(BoundValue::Text(pk_text)); + } + } + + if !conditions.is_empty() { + sql.push_str(" WHERE "); + sql.push_str(&conditions.join(" AND ")); + } + + // Deterministic ordering for pagination. + if index_name.is_some() { + if let Some((_, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + if let Some((_, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + let _ = write!(sql, " ORDER BY pk, {sk_col}, base_pk, {base_col}"); + } else { + let _ = write!(sql, " ORDER BY pk, {sk_col}, base_pk"); + } + } else if let Some((_, base_type)) = &base_sk_info { + // Hash-only GSI on a composite-key base table: order by the full + // index key so it is a total order matching the pagination + // predicate above. + let base_col = format!("base_{}", sk_column(*base_type)); + let _ = write!(sql, " ORDER BY pk, base_pk, {base_col}"); + } else { + let _ = write!(sql, " ORDER BY pk, base_pk"); + } + } else if let Some((_, sk_type)) = sk_info_val { + let _ = write!(sql, " ORDER BY pk, {}", sk_column(sk_type)); + } else { + sql.push_str(" ORDER BY pk"); + } + + let fetch_limit = limit.map_or(1_000_001, |l| l + 1); + let _ = write!(sql, " LIMIT {fetch_limit}"); + + let rows = execute_dynamic_query(&sql, binds, &self.pool).await?; + finalize(rows, limit, &key_info.key_schema) + } +} + +/// Resolve a key-condition path's attribute name, handling `#name` references. +fn resolve_attr_name(path: &[PathElement], maps: &ExpressionMaps) -> Option { + match path.first() { + Some(PathElement::Attribute(name)) => { + if let Some(reference) = name.strip_prefix('#') { + maps.names.get(reference).cloned() + } else { + Some(name.clone()) + } + } + _ => None, + } +} + +/// Extract the base-table partition key text from a (combined) start key. +fn base_pk_from_start_key( + start_key: &Item, + key_info: &TableKeyInfo, +) -> Result { + let base_pk_attr = &key_info.base_key_schema[0].attribute_name; + start_key + .get(base_pk_attr) + .map(pk_to_text) + .transpose()? + .map(|c| c.into_owned()) + .ok_or_else(|| { + StorageError::Validation( + "The provided starting key is invalid: missing base table partition key".to_owned(), + ) + }) +} + +/// Append the query pagination predicate and its binds, mirroring the +/// PostgreSQL `build_pagination_where` cases. +#[allow(clippy::too_many_arguments)] +fn append_query_pagination( + sql: &mut String, + binds: &mut Vec, + start_key: &Item, + sk_info_val: Option<(&str, ScalarAttributeType)>, + base_sk_info: Option<&(String, ScalarAttributeType)>, + key_info: &TableKeyInfo, + is_index: bool, + is_lsi: bool, + forward: bool, +) -> Result<(), StorageError> { + let cmp = if forward { ">" } else { "<" }; + + if let Some((sk_name, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + let sk_bv = start_key + .get(sk_name) + .map(|v| parse_sk(v, sk_type)) + .transpose()? + .map(|s| sk_bound(&s)); + + if let Some((base_name, base_type)) = base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + let sk_bv = sk_bv.unwrap_or(BoundValue::Text(String::new())); + let base_sk_bv = if let Some(v) = start_key.get(base_name.as_str()) { + sk_bound(&parse_sk(v, *base_type)?) + } else { + BoundValue::Text(String::new()) + }; + + if is_lsi { + // LSI: every row shares the queried partition key, so the base + // table's sort key alone identifies a row uniquely, and it is a + // user-visible sort dimension so it follows ScanIndexForward. + let _ = write!( + sql, + " AND ({sk_col} {cmp} ? OR ({sk_col} = ? AND {base_col} {cmp} ?))" + ); + binds.push(sk_bv.clone()); + binds.push(sk_bv); + binds.push(base_sk_bv); + } else { + // GSI: the tie-breaker must be the FULL base primary key. Rows + // in a GSI partition are unique on (index SK, base PK, base SK), + // not on (index SK, base SK): many base partitions can project + // the same index SK and the same base SK. Comparing base SK + // alone made a page-two query return nothing whenever the rows + // sharing an index SK also shared a base SK, so a paginating + // client silently stopped after page one. + let base_pk_bv = BoundValue::Text(base_pk_from_start_key(start_key, key_info)?); + let _ = write!( + sql, + " AND ({sk_col} {cmp} ? OR ({sk_col} = ? AND (base_pk > ? \ + OR (base_pk = ? AND {base_col} > ?))))" + ); + binds.push(sk_bv.clone()); + binds.push(sk_bv); + binds.push(base_pk_bv.clone()); + binds.push(base_pk_bv); + binds.push(base_sk_bv); + } + } else if is_index { + let _ = write!( + sql, + " AND ({sk_col} {cmp} ? OR ({sk_col} = ? AND base_pk > ?))" + ); + let sk_bv = sk_bv.unwrap_or(BoundValue::Text(String::new())); + binds.push(sk_bv.clone()); + binds.push(sk_bv); + binds.push(BoundValue::Text(base_pk_from_start_key( + start_key, key_info, + )?)); + } else { + let _ = write!(sql, " AND {sk_col} {cmp} ?"); + binds.push(sk_bv.unwrap_or(BoundValue::Text(String::new()))); + } + } else if is_index { + let base_pk_text = base_pk_from_start_key(start_key, key_info)?; + if let Some((base_name, base_type)) = base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + let _ = write!( + sql, + " AND (base_pk > ? OR (base_pk = ? AND {base_col} > ?))" + ); + binds.push(BoundValue::Text(base_pk_text.clone())); + binds.push(BoundValue::Text(base_pk_text)); + if let Some(v) = start_key.get(base_name.as_str()) { + binds.push(sk_bound(&parse_sk(v, *base_type)?)); + } else { + binds.push(BoundValue::Text(String::new())); + } + } else { + let _ = write!(sql, " AND base_pk > ?"); + binds.push(BoundValue::Text(base_pk_text)); + } + } + Ok(()) +} + +/// Trim the over-fetched extra row, deserialize items, and derive the +/// `LastEvaluatedKey` (storage-side: the queried table's own key; the engine +/// enriches index LEKs with base-table key attributes). +fn finalize( + rows: Vec, + limit: Option, + key_schema: &[extenddb_core::types::KeySchemaElement], +) -> Result<(Vec, Option), StorageError> { + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let actual_limit = limit.map_or(1_000_000_usize, |l| l.max(0) as usize); + let has_more = rows.len() > actual_limit; + let items: Vec = rows + .into_iter() + .take(actual_limit) + .map(json_to_item) + .collect::, _>>()?; + let last_key = if has_more { + items.last().map(|item| build_key(item, key_schema)) + } else { + None + }; + Ok((items, last_key)) +} diff --git a/crates/storage-duckdb/src/data/transactions.rs b/crates/storage-duckdb/src/data/transactions.rs new file mode 100644 index 00000000..9018398b --- /dev/null +++ b/crates/storage-duckdb/src/data/transactions.rs @@ -0,0 +1,506 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `TransactGetItems` / `TransactWriteItems` and idempotency-token cleanup. +//! +//! Writes run under the engine write lock (D1) in a single transaction, so all +//! operations, the idempotency check, and stream capture commit atomically or +//! roll back together. Reads run in one transaction for a consistent snapshot. + +use crate::db; +use std::collections::HashMap; + +use extenddb_core::expression::{self, ExpressionMaps}; +use extenddb_core::types::{CancellationReason, Item, ReturnValuesOnConditionCheckFailure}; +use extenddb_core::validation; +use extenddb_storage::error::StorageError; +use extenddb_storage::{IdempotencyKey, TransactGetOp, TransactWriteOp}; + +use super::index::{IndexMeta, enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::tx_helpers::{ + check_idempotency_token_in_tx, delete_item_in_tx, fetch_item_for_update, fetch_item_in_tx, + upsert_item_in_tx, write_stream_record_in_tx, +}; +use crate::duckdb_util::format_timestamp; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + pub(crate) async fn transact_get_items_impl( + &self, + ops: &[TransactGetOp<'_>], + ) -> Result>, StorageError> { + // Validate keys first, collecting per-item reasons (all-or-nothing). + let mut reasons = Vec::with_capacity(ops.len()); + let mut any_failed = false; + for op in ops { + match validation::validate_key_only( + op.key, + &op.key_info.key_schema, + &op.key_info.attribute_definitions, + ) { + Ok(()) => reasons.push(CancellationReason::none()), + Err(e) => { + any_failed = true; + reasons.push(CancellationReason::validation_error(e.to_string())); + } + } + } + if any_failed { + return Err(StorageError::TransactionCanceled(reasons)); + } + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let mut results = Vec::with_capacity(ops.len()); + for op in ops { + results.push(fetch_item_in_tx(&mut tx, op.key_info, op.key).await?); + } + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(results) + } + + pub(crate) async fn transact_write_items_impl( + &self, + ops: &[TransactWriteOp<'_>], + idempotency: Option>, + ) -> Result<(), StorageError> { + // Read the propagation delay BEFORE taking the write lock. It is a + // runtime setting, not an invariant of this write, so it does not need + // to be read under the lock, and the lock serialises every write in the + // process: work done inside it is the backend's throughput bottleneck. + let system_delay = self.index_propagation_delay().await; + let _writer = self.write_lock.lock().await; + // Fetch index metadata per distinct table AFTER acquiring the write lock, + // so a GSI added by a concurrent UpdateTable (same lock) is not missed and + // left unmaintained by these writes. + let mut table_indexes: HashMap> = HashMap::new(); + for op in ops { + let name = op_table_name(op); + if !table_indexes.contains_key(name) { + let indexes = fetch_indexes_for_table(op_table_id(op), &self.pool).await?; + table_indexes.insert(name.to_owned(), indexes); + } + } + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some(key) = idempotency { + // Idempotency is per-account: the same ClientRequestToken in two + // accounts must not collide. The engine passes the caller's + // account explicitly. + check_idempotency_token_in_tx(&mut tx, key.account_id, key.token, key.fingerprint) + .await?; + } + + let mut reasons: Vec = Vec::with_capacity(ops.len()); + let mut op_items: Vec<(Option, Option)> = Vec::with_capacity(ops.len()); + let mut any_failed = false; + + for op in ops { + let indexes = &table_indexes[op_table_name(op)]; + match execute_transact_write_op( + &mut tx, + op, + indexes, + self.max_item_size_bytes, + system_delay, + ) + .await + { + Ok(items) => { + op_items.push(items); + reasons.push(CancellationReason::none()); + } + Err(TxnOpError::Cancel(r)) => { + op_items.push((None, None)); + any_failed = true; + reasons.push(r); + } + Err(TxnOpError::Validation(msg)) => { + // Up-front input validation (e.g. empty secondary-index key): + // abort the whole transaction with a top-level + // ValidationException, not a per-item cancellation reason. + return Err(StorageError::Validation(msg)); + } + Err(TxnOpError::Storage(e)) => return Err(e), + } + } + + if any_failed { + // Dropping `tx` without commit rolls back all writes. + return Err(StorageError::TransactionCanceled(reasons)); + } + + // Capture stream records after all writes are staged. + for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { + let capture = match op { + TransactWriteOp::Put { stream, .. } + | TransactWriteOp::Delete { stream, .. } + | TransactWriteOp::Update { stream, .. } => stream.as_ref(), + TransactWriteOp::ConditionCheck { .. } => None, + }; + if let Some(capture) = capture { + write_stream_record_in_tx( + &mut tx, + op_key_info(op), + capture, + old_item.as_ref(), + new_item.as_ref(), + ) + .await?; + } + } + + // Persist index work for each op inside the same transaction: async GSI + // rows, and vector maintenance which is applied here when the propagation + // delay is 0 and enqueued otherwise. + // + // Deliberately after every op is staged rather than inside each op, so + // there is one call site instead of three. Both paths stay atomic with the + // item writes either way: the synchronous apply runs in this transaction, + // and a pending row is inserted into it, so a cancelled transaction leaves + // neither behind. The vector apply reads only the op's own before/after + // items, never the base table, so its position relative to the other ops' + // staged writes cannot change its result. + let mut needs_notify = false; + for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { + let indexes = &table_indexes[op_table_name(op)]; + if old_item.is_some() || new_item.is_some() { + let n = enqueue_async_indexes( + &mut tx, + op_key_info(op), + indexes, + old_item.as_ref(), + new_item.as_ref(), + system_delay, + ) + .await?; + if n > 0 { + needs_notify = true; + } + let key_info = op_key_info(op); + if !key_info.vector_indexes.is_empty() { + let n = crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old_item.as_ref(), + new_item.as_ref(), + system_delay, + ) + .await?; + if n > 0 { + needs_notify = true; + } + } + } + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if needs_notify { + self.gsi_notify.notify_waiters(); + } + Ok(()) + } + + pub(crate) async fn cleanup_expired_idempotency_tokens_impl( + &self, + max_age_seconds: i64, + ) -> Result { + let cutoff = format_timestamp( + time::OffsetDateTime::now_utc() - time::Duration::seconds(max_age_seconds), + ); + let result = db::query("DELETE FROM idempotency_tokens WHERE created_at < ?") + .bind(&cutoff) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(result.rows_affected()) + } +} + +fn op_table_name<'a>(op: &'a TransactWriteOp<'_>) -> &'a str { + &op_key_info(op).table_name +} + +fn op_table_id<'a>(op: &'a TransactWriteOp<'_>) -> &'a str { + &op_key_info(op).table_id +} + +fn op_key_info<'a>(op: &'a TransactWriteOp<'_>) -> &'a extenddb_core::types::TableKeyInfo { + match op { + TransactWriteOp::Put { key_info, .. } + | TransactWriteOp::Delete { key_info, .. } + | TransactWriteOp::Update { key_info, .. } + | TransactWriteOp::ConditionCheck { key_info, .. } => key_info, + } +} + +enum TxnOpError { + Cancel(CancellationReason), + /// Up-front input validation failure — aborts the whole transaction with a + /// top-level `ValidationException` (not a per-item cancellation reason). + Validation(String), + Storage(StorageError), +} + +/// Build [`validation::IndexKeyRef`] views over the table's indexes for +/// secondary-index key validation. +pub(crate) fn index_key_refs(indexes: &[IndexMeta]) -> Vec> { + indexes + .iter() + .map(|idx| validation::IndexKeyRef { + index_name: &idx.index_name, + key_schema: &idx.key_schema, + }) + .collect() +} + +/// Execute one transact-write op, returning `(old, new)` images for stream +/// capture, or a cancellation reason on a failed condition / validation. +async fn execute_transact_write_op( + tx: &mut db::Transaction, + op: &TransactWriteOp<'_>, + indexes: &[IndexMeta], + max_item_size_bytes: usize, + system_delay: u64, +) -> Result<(Option, Option), TxnOpError> { + match op { + TransactWriteOp::Put { + key_info, + item, + condition, + maps, + return_values_on_ccf, + .. + } => { + validation::validate_item_keys( + item, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + // Secondary-index key faults split by kind: a type mismatch is a + // per-item cancellation reason; an empty index key is up-front input + // validation (a top-level ValidationException). + let idx_refs = index_key_refs(indexes); + validation::validate_index_key_types(item, &idx_refs, &key_info.attribute_definitions) + .map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + validation::validate_index_key_not_empty( + item, + &idx_refs, + validation::SecondaryIndexEmptyContext::Item, + ) + .map_err(|e| TxnOpError::Validation(e.to_string()))?; + let existing = fetch_item_for_update(tx, key_info, item) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + *condition, + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + upsert_item_in_tx(tx, key_info, item) + .await + .map_err(TxnOpError::Storage)?; + if !indexes.is_empty() { + sync_indexes( + tx, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + existing.as_ref(), + Some(item), + system_delay, + ) + .await + .map_err(TxnOpError::Storage)?; + } + Ok((existing, Some((*item).clone()))) + } + TransactWriteOp::Delete { + key_info, + key, + condition, + maps, + return_values_on_ccf, + .. + } => { + validation::validate_batch_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + *condition, + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + delete_item_in_tx(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + if !indexes.is_empty() { + sync_indexes( + tx, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + existing.as_ref(), + None, + system_delay, + ) + .await + .map_err(TxnOpError::Storage)?; + } + Ok((existing, None)) + } + TransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + return_values_on_ccf, + .. + } => { + validation::validate_batch_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + *condition, + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + let mut item = existing.clone().unwrap_or_else(|| (*key).clone()); + expression::apply_update_validated( + actions, + &mut item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + validation::validate_item_size(&item, max_item_size_bytes).map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + // Secondary-index key validation on the post-update item: a type + // mismatch is a cancellation reason; setting an index key to an + // empty value is a top-level ValidationException. + let idx_refs = index_key_refs(indexes); + validation::validate_index_key_types(&item, &idx_refs, &key_info.attribute_definitions) + .map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + validation::validate_index_key_not_empty( + &item, + &idx_refs, + validation::SecondaryIndexEmptyContext::UpdateExpression, + ) + .map_err(|e| TxnOpError::Validation(e.to_string()))?; + upsert_item_in_tx(tx, key_info, &item) + .await + .map_err(TxnOpError::Storage)?; + if !indexes.is_empty() { + sync_indexes( + tx, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + existing.as_ref(), + Some(&item), + system_delay, + ) + .await + .map_err(TxnOpError::Storage)?; + } + Ok((existing, Some(item))) + } + TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf, + } => { + validation::validate_batch_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + Some(condition), + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + Ok((None, None)) + } + } +} + +/// Evaluate a transaction op's condition, producing a `CancellationReason` on +/// failure (attaching the old item when `ReturnValuesOnConditionCheckFailure` +/// is `AllOld`). +fn eval_condition( + condition: Option<&expression::Expr>, + item: &Item, + maps: &ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + existing: Option<&Item>, +) -> Result<(), TxnOpError> { + if let Some(cond) = condition { + let passed = expression::evaluate_condition(cond, item, maps) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + if !passed { + let returned = if return_values_on_ccf == ReturnValuesOnConditionCheckFailure::AllOld { + existing.cloned() + } else { + None + }; + return Err(TxnOpError::Cancel( + CancellationReason::condition_check_failed_with_item(returned), + )); + } + } + Ok(()) +} diff --git a/crates/storage-duckdb/src/data/tx_helpers.rs b/crates/storage-duckdb/src/data/tx_helpers.rs new file mode 100644 index 00000000..83798a52 --- /dev/null +++ b/crates/storage-duckdb/src/data/tx_helpers.rs @@ -0,0 +1,379 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Transaction helpers shared by the write path and transactions module: +//! in-transaction item fetch/upsert/delete, monotonic stream sequencing, +//! atomic stream-record capture, and idempotency-token checks. +//! +//! There is no `SELECT ... FOR UPDATE`: the engine's `write_lock` serializes +//! all writers (design decision D1), so an in-transaction read followed by a +//! write is already atomic with respect to other writers. + +use crate::db; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use extenddb_core::types::{ + AttributeValue, Item, StreamEventName, StreamRecord, StreamRecordData, StreamViewType, + TableKeyInfo, item_size_bytes, +}; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{pk_to_text, sk_column, sk_info}; + +use super::{ + bind_sk_execute, bind_sk_fetch_optional, bind_sk_only_execute, data_table_name, json_to_item, +}; +use crate::duckdb_util::format_timestamp; + +/// Fetch a single item within a transaction by primary key. +pub(super) async fn fetch_item_in_tx( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + key: &Item, +) -> Result, StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + let json_opt = if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = extenddb_storage::util::parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + let row: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&sql, pk_text.as_ref(), &sk, &mut **tx)?; + row.map(|(v,)| v) + } else { + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let row: Option<(serde_json::Value,)> = db::query_as(&sql) + .bind(pk_text.as_ref()) + .fetch_optional(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + row.map(|(v,)| v) + }; + + json_opt.map(json_to_item).transpose() +} + +/// Fetch for write. DuckDB has no `FOR UPDATE`; the engine write lock provides +/// the serialization, so this is the same as `fetch_item_in_tx`. +pub(super) async fn fetch_item_for_update( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + key: &Item, +) -> Result, StorageError> { + fetch_item_in_tx(tx, key_info, key).await +} + +/// Insert or replace an item within a transaction. +pub(crate) async fn upsert_item_in_tx( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + item: &Item, +) -> Result<(), StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = item + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + let item_json = + serde_json::to_string(item).map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((sk_name, sk_type)) = sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = item + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = extenddb_storage::util::parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!( + "INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES (?, ?, ?) \ + ON CONFLICT (pk, {sk_col}) DO UPDATE SET item_data = excluded.item_data" + ); + bind_sk_execute!(&sql, pk_text.as_ref(), &sk, &item_json, &mut **tx)?; + } else { + let sql = format!( + "INSERT INTO {ddb_table} (pk, item_data) VALUES (?, ?) \ + ON CONFLICT (pk) DO UPDATE SET item_data = excluded.item_data" + ); + db::query(&sql) + .bind(pk_text.as_ref()) + .bind(&item_json) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) +} + +/// Delete an item by key within a transaction. +pub(super) async fn delete_item_in_tx( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + key: &Item, +) -> Result<(), StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + if let Some((sk_name, sk_type)) = sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = extenddb_storage::util::parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!("DELETE FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + bind_sk_only_execute!(&sql, pk_text.as_ref(), &sk, &mut **tx)?; + } else { + let sql = format!("DELETE FROM {ddb_table} WHERE pk = ?"); + db::query(&sql) + .bind(pk_text.as_ref()) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) +} + +/// Allocate the next monotonic stream sequence number within a transaction. +async fn next_stream_seq(tx: &mut db::Transaction) -> Result { + db::query("UPDATE seq_counters SET value = value + 1 WHERE name = 'stream'") + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let value: i64 = db::query_scalar("SELECT value FROM seq_counters WHERE name = 'stream'") + .fetch_one(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(value) +} + +/// Write a stream record in the same transaction as the data write, so capture +/// is atomic with the mutation (a hard requirement for correct streams). +pub(super) async fn write_stream_record_in_tx( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + capture: &StreamCapture, + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + let event_name = match (old_item, new_item) { + (None, Some(_)) => StreamEventName::Insert, + (Some(_), Some(_)) => StreamEventName::Modify, + (Some(_), None) => StreamEventName::Remove, + (None, None) => return Ok(()), + }; + let source = new_item.or(old_item).expect("one image present"); + + // Keys are the primary-key attributes from whichever image is present. + let keys: Item = key_info + .key_schema + .iter() + .filter_map(|ks| { + source + .get(&ks.attribute_name) + .map(|v| (ks.attribute_name.clone(), v.clone())) + }) + .collect(); + + let new_image = matches!( + capture.view_type, + StreamViewType::NewImage | StreamViewType::NewAndOldImages + ) + .then(|| new_item.cloned()) + .flatten(); + let old_image = matches!( + capture.view_type, + StreamViewType::OldImage | StreamViewType::NewAndOldImages + ) + .then(|| old_item.cloned()) + .flatten(); + + let size_bytes = i64::try_from(item_size_bytes(source)).unwrap_or(i64::MAX); + + // Hash the partition key to a shard. + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_str = source + .get(pk_name) + .map(|v| match v { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => BASE64.encode(b), + _ => String::new(), + }) + .unwrap_or_default(); + + let shards: Vec<(String,)> = + db::query_as("SELECT shard_id FROM stream_shards WHERE table_id = ? ORDER BY shard_id") + .bind(&key_info.table_id) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if shards.is_empty() { + return Ok(()); // Streams not enabled for this table. + } + let idx = (crc32fast::hash(pk_str.as_bytes()) as usize) % shards.len(); + let shard_id = shards[idx].0.clone(); + + let seq = format!("{:021}", next_stream_seq(tx).await?); + let approximate_creation_date_time = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + ) + .unwrap_or(i64::MAX); + + let record = StreamRecord { + event_id: uuid::Uuid::new_v4().to_string(), + event_name, + event_version: "1.1".to_owned(), + event_source: "aws:dynamodb".to_owned(), + aws_region: capture.region.to_string(), + dynamodb: StreamRecordData { + approximate_creation_date_time, + keys, + new_image, + old_image, + sequence_number: seq.clone(), + size_bytes, + stream_view_type: capture.view_type, + }, + user_identity: capture.user_identity.clone(), + }; + let record_json = + serde_json::to_string(&record).map_err(|e| StorageError::Internal(e.to_string()))?; + + db::query( + "INSERT INTO stream_records (shard_id, sequence_number, table_id, event_name, record_data) \ + VALUES (?, ?, ?, ?, ?)", + ) + .bind(&shard_id) + .bind(&seq) + .bind(&key_info.table_id) + .bind(format!("{:?}", record.event_name)) + .bind(&record_json) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +/// Check (and record) an idempotency token within a transaction. Tokens are +/// valid for 10 minutes; the cutoff is computed in Rust as RFC 3339 so it +/// compares correctly against the stored RFC 3339 `created_at`. +pub(super) async fn check_idempotency_token_in_tx( + tx: &mut db::Transaction, + account_id: &str, + token: &str, + fingerprint: &str, +) -> Result<(), StorageError> { + let cutoff = format_timestamp(time::OffsetDateTime::now_utc() - time::Duration::minutes(10)); + let existing: Option<(String,)> = db::query_as( + "SELECT fingerprint FROM idempotency_tokens \ + WHERE account_id = ? AND token = ? AND created_at > ?", + ) + .bind(account_id) + .bind(token) + .bind(&cutoff) + .fetch_optional(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((stored_fp,)) = existing { + return if stored_fp == fingerprint { + Err(StorageError::IdempotentReplay) + } else { + Err(StorageError::IdempotentMismatch) + }; + } + + let now = format_timestamp(time::OffsetDateTime::now_utc()); + db::query( + "INSERT INTO idempotency_tokens (account_id, token, fingerprint, created_at) \ + VALUES (?, ?, ?, ?) \ + ON CONFLICT (account_id, token) DO UPDATE SET fingerprint = excluded.fingerprint, \ + created_at = excluded.created_at", + ) + .bind(account_id) + .bind(token) + .bind(fingerprint) + .bind(&now) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +#[cfg(test)] +mod idempotency_tests { + use super::check_idempotency_token_in_tx; + use crate::db; + use extenddb_storage::error::StorageError; + + async fn pool_with_schema() -> db::Pool { + let pool = db::Pool::open(":memory:", 1).await.unwrap(); + db::query( + "CREATE TABLE idempotency_tokens (\ + account_id TEXT NOT NULL,\ + token TEXT NOT NULL,\ + fingerprint TEXT NOT NULL,\ + created_at TEXT NOT NULL DEFAULT (strftime(now()::TIMESTAMP, '%Y-%m-%dT%H:%M:%S.%gZ')),\ + PRIMARY KEY (account_id, token)\ + )", + ) + .execute(&pool) + .await + .unwrap(); + pool + } + + async fn check( + pool: &db::Pool, + account: &str, + token: &str, + fp: &str, + ) -> Result<(), StorageError> { + let mut tx = pool.begin().await.unwrap(); + let r = check_idempotency_token_in_tx(&mut tx, account, token, fp).await; + tx.commit().await.unwrap(); + r + } + + #[tokio::test] + async fn idempotency_is_scoped_per_account() { + let pool = pool_with_schema().await; + + // First use of (acctA, tok) records it. + check(&pool, "acctA", "tok", "fpX").await.unwrap(); + + // Same account + token + fingerprint = idempotent replay. + assert!(matches!( + check(&pool, "acctA", "tok", "fpX").await, + Err(StorageError::IdempotentReplay) + )); + + // Same account + token, different fingerprint = mismatch. + assert!(matches!( + check(&pool, "acctA", "tok", "fpY").await, + Err(StorageError::IdempotentMismatch) + )); + + // Regression: a DIFFERENT account reusing the same token value must be + // independent — not a replay and not a mismatch. + check(&pool, "acctB", "tok", "fpX").await.unwrap(); + } +} diff --git a/crates/storage-duckdb/src/data/update_item.rs b/crates/storage-duckdb/src/data/update_item.rs new file mode 100644 index 00000000..8018957f --- /dev/null +++ b/crates/storage-duckdb/src/data/update_item.rs @@ -0,0 +1,155 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `update_item` for the DuckDB backend. +//! +//! UpdateItem is an upsert: when the item is absent a new one is created from +//! the key plus the applied actions. The condition is evaluated against the +//! pre-update image (or an empty item), then update actions are applied and the +//! result is validated and written — all under the engine write lock (D1). + +use extenddb_core::expression::{self, Expr, ExpressionMaps, UpdateAction}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_core::validation; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; + +use super::index::{enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::query::check_condition; +use super::transactions::index_key_refs; +use super::tx_helpers::{fetch_item_in_tx, upsert_item_in_tx, write_stream_record_in_tx}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + #[allow(clippy::too_many_arguments)] + pub(crate) async fn update_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result<(Option, Option), StorageError> { + // Read the propagation delay BEFORE taking the write lock. It is a + // runtime setting, not an invariant of this write, so it does not need + // to be read under the lock, and the lock serialises every write in the + // process: work done inside it is the backend's throughput bottleneck. + let system_delay = self.index_propagation_delay().await; + let _writer = self.write_lock.lock().await; + // Read the index set after acquiring the write lock so a concurrently + // added GSI (UpdateTable holds the same lock) is not missed. + let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old = fetch_item_in_tx(&mut tx, key_info, key).await?; + + if condition.is_some() { + let empty = Item::new(); + let target = old.as_ref().unwrap_or(&empty); + if let Err(e) = check_condition(condition, target, maps) { + return match e { + StorageError::ConditionFailed(_) => Err(StorageError::ConditionFailed(old)), + other => Err(other), + }; + } + } + + // Start from the existing image, or from the key for a fresh upsert. + let mut item = old.clone().unwrap_or_else(|| key.clone()); + expression::apply_update_validated( + actions, + &mut item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; + validation::validate_item_size(&item, self.max_item_size_bytes) + .map_err(|e| StorageError::Validation(e.to_string()))?; + // Secondary-index key validation on the post-update item, matching the + // TransactWriteItems update path: a wrong-typed index key attribute or + // an index key set to an empty value is a ValidationException up front, + // rather than silently producing a malformed / unmatchable index row. + if !indexes.is_empty() { + let idx_refs = index_key_refs(&indexes); + validation::validate_index_key_types(&item, &idx_refs, &key_info.attribute_definitions) + .map_err(|e| StorageError::Validation(e.to_string()))?; + validation::validate_index_key_not_empty( + &item, + &idx_refs, + validation::SecondaryIndexEmptyContext::UpdateExpression, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; + } + + upsert_item_in_tx(&mut tx, key_info, &item).await?; + + if !indexes.is_empty() { + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old.as_ref(), + Some(&item), + system_delay, + ) + .await?; + } + let enqueued_gsi = enqueue_async_indexes( + &mut tx, + key_info, + &indexes, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0; + // Vector indexes: applied in this transaction when the propagation delay is + // 0, otherwise enqueued alongside the async GSI work. Gated on the cached + // key info so a table without them costs no extra query, and kept outside + // the `indexes` guard because a table may have a vector index and no GSI. + let enqueued_vector = if key_info.vector_indexes.is_empty() { + false + } else { + crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0 + }; + let enqueued = enqueued_gsi || enqueued_vector; + + if let Some(capture) = stream { + write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) + .await?; + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if enqueued { + self.gsi_notify.notify_waiters(); + } + + let old_ret = if return_old { old } else { None }; + let new_ret = if return_new { Some(item) } else { None }; + Ok((old_ret, new_ret)) + } +} diff --git a/crates/storage-duckdb/src/data/vector_index.rs b/crates/storage-duckdb/src/data/vector_index.rs new file mode 100644 index 00000000..1b3f5971 --- /dev/null +++ b/crates/storage-duckdb/src/data/vector_index.rs @@ -0,0 +1,1078 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Vector index maintenance on the write path. +//! +//! Vector indexes are **eventually consistent**, the same model the service gives +//! them and the same model a GSI has here, so maintenance runs on the existing +//! `gsi_pending` queue rather than in the base write transaction. [`maintain_vector_indexes`] +//! is the single entry point and owns that choice: a propagation delay of 0 keeps +//! the work in the caller's transaction, any other delay enqueues it. +//! +//! Reusing the GSI queue is what makes the asynchronous path correct rather than +//! merely deferred, and none of these properties would come free from a queue of +//! its own: +//! +//! * **Crash safety.** The pending row is inserted in the base write transaction, +//! so there is no window in which the item is committed and the index work is +//! not yet durable. The worker claims and applies in one transaction, so a crash +//! mid-apply rolls back and the row is retried. At-least-once delivery is safe +//! because applying a row is idempotent: it deletes the base key's row and +//! reinserts it from the snapshotted item. +//! * **Per-key ordering, across index kinds.** The row's partition is a hash of the +//! *base* key, so a vector row and a GSI row for the same item share a partition, +//! `ready_at` is clamped monotonic within it, and the worker drains in `id` order. +//! Two writes to one item therefore reach both index kinds in write order even +//! though the delay is jittered. +//! * **Snapshot semantics.** The row carries its own [`VectorApplyContext`], so the +//! worker needs no catalog read and an index dropped, or redefined, between +//! enqueue and apply cannot make a queued write unapplicable or retroactively +//! change how it was indexed. +//! +//! A write to a table whose items do not carry the vector still enqueues, because +//! the removal is the point: an item that loses its vector attribute must leave the +//! index, and skipping the enqueue would leave the stale row in place forever. + +use crate::db; +use serde::{Deserialize, Serialize}; + +use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement, SearchSchemaElementType}; +use extenddb_core::validation::{vector_components, vector_norm}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::pk_to_text; + +use super::{BoundValue, all_sort_key_info, sk_bound, vector_table_name}; +use crate::vector_search::partition_value; + +/// A vector index as the write path needs it. +/// +/// Serializable because the asynchronous path snapshots it verbatim into the +/// pending row's [`VectorApplyContext`]. The write path and the worker therefore +/// apply from the *same* description of the index, which is the property that stops +/// a queued write from being reinterpreted under a later definition. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct VectorIndexMeta { + pub index_id: String, + pub dimensions: usize, + pub vector_attribute_name: String, + /// The index's projection, applied to the stored row exactly as the GSI path + /// applies its own. Not applying it was an unexplained divergence from the + /// sibling, and it made a search return attributes the index does not project. + pub projection: extenddb_core::types::Projection, + /// The single HASH element's attribute name, when the index declares one. + /// `None` means the index is unscoped and every row shares one partition. + pub hash_attribute_name: Option, + /// Every attribute named by the SearchSchema, HASH and INLINE_FILTER alike. + /// + /// These are projected regardless of `ProjectionType`, which is the documented + /// rule for a vector index and is NOT GSI `KEYS_ONLY` semantics: `KEYS_ONLY` + /// on a vector index projects the base primary key, the vector attribute and + /// the inline filter attributes. Withholding them is not merely a reporting + /// difference, it breaks search: the filter is evaluated against the stored + /// payload, so a missing filter attribute makes every row fail the predicate + /// and a filtered search match nothing. + pub search_schema_attribute_names: Vec, +} + +/// Load the vector indexes of a table. +/// +/// Read inside the write transaction rather than taken from the cached +/// `TableKeyInfo`, because the cache carries the search schema but not the index +/// id, and the id is what names the data table. +pub(crate) async fn fetch_vector_indexes_for_table( + tx: &mut db::Transaction, + table_id: &str, +) -> Result, StorageError> { + let rows: Vec<(String, i64, String, Option, String)> = db::query_as( + "SELECT index_id, dimensions, vector_attribute, search_schema, projection \ + FROM vector_indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + + let mut out = Vec::with_capacity(rows.len()); + for (index_id, dimensions, vector_attribute, search_schema, projection) in rows { + let attr: extenddb_core::types::VectorAttribute = + serde_json::from_str(&vector_attribute) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; + let (hash_attribute_name, search_schema_attribute_names) = match search_schema.as_deref() { + Some(json) => { + let elements: Vec = + serde_json::from_str(json) + .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?; + let hash = elements + .iter() + .find(|e| e.element_type == SearchSchemaElementType::Hash) + .map(|e| e.attribute_name.clone()); + let all = elements + .into_iter() + .map(|e| e.attribute_name) + .collect::>(); + (hash, all) + } + None => (None, Vec::new()), + }; + let projection: extenddb_core::types::Projection = serde_json::from_str(&projection) + .map_err(|e| StorageError::Internal(format!("vector projection: {e}")))?; + out.push(VectorIndexMeta { + index_id, + dimensions: usize::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?, + vector_attribute_name: attr.attribute_name, + hash_attribute_name, + search_schema_attribute_names, + projection, + }); + } + Ok(out) +} + +/// Whether an item belongs in a vector index. +/// +/// It must carry the vector attribute, and the HASH attribute when the index +/// declares one: without the latter the row could not be placed in a partition, +/// and putting it in the unscoped partition would make it visible to searches of +/// every other partition. Not an error, exactly as a GSI silently omits an item +/// missing its index key. +fn item_is_indexable(item: &Item, meta: &VectorIndexMeta) -> bool { + if !item.contains_key(&meta.vector_attribute_name) { + return false; + } + match &meta.hash_attribute_name { + Some(name) => item.contains_key(name), + None => true, + } +} + +/// The partition column value for an item under one index. +fn item_partition(item: &Item, meta: &VectorIndexMeta) -> Result { + match &meta.hash_attribute_name { + Some(name) => { + let value = item.get(name).ok_or_else(|| { + StorageError::Internal( + "indexable check passed but the hash attribute is absent".to_owned(), + ) + })?; + partition_value(Some((name.as_str(), value))) + } + None => partition_value(None), + } +} + +/// Base-key bind values for a row, in key-schema order. +fn base_key_binds( + item: &Item, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Result, StorageError> { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let pk_attr = &base_key_schema[0].attribute_name; + let pk = item.get(pk_attr).ok_or_else(|| { + StorageError::Internal("item written without its partition key".to_owned()) + })?; + let mut binds = vec![BoundValue::Text(pk_to_text(pk)?.into_owned())]; + for &(name, sk_type) in &base_sks { + // Sort keys use the same storage representation as the GSI/LSI tables: + // order-preserving text for numbers and a BLOB for binary. Encoding them + // any other way would still be self-consistent here but would diverge + // from every other index table for the same item. + match item.get(name) { + Some(value) => binds.push(sk_bound(&extenddb_storage::util::parse_sk(value, sk_type)?)), + None => binds.push(BoundValue::Text(String::new())), + } + } + Ok(binds) +} + +/// Column names for the base key, in key-schema order. +fn base_key_columns( + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Vec { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let mut cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + cols.push(format!( + "base_{}", + extenddb_storage::util::sk_column_n(i, sk_type) + )); + } + cols +} + +/// Everything the propagation worker needs to apply one vector index update, +/// serialized into `gsi_pending.index_context`. +/// +/// `table_id` is carried here even though the queue row has a `table_id` column of +/// its own, because a vector data table is named from the table id *and* the index +/// id. Reading it from the context preserves the invariant that the context alone +/// is sufficient, rather than splitting one apply's inputs across a column and a +/// JSON blob. Both are written from the same variable in the same statement, so +/// they cannot disagree. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct VectorApplyContext { + pub(crate) base_key_schema: Vec, + pub(crate) attribute_definitions: Vec, + pub(crate) table_id: String, + /// Deliberately named `vector` rather than `index`: it is the field whose + /// presence lets the untagged `PendingApplyContext` tell a vector row from a + /// GSI row by shape alone. See that type for why the discriminant is a shape + /// and not a tag. + pub(crate) vector: VectorIndexMeta, +} + +/// Maintain every vector index on a table for one item write. +/// +/// The single entry point for the write path, and the one place that decides +/// between synchronous and asynchronous. `delay_ms` of 0 applies in the caller's +/// transaction; anything else enqueues one `gsi_pending` row per index. Returns the +/// number of rows enqueued, so the caller knows whether to wake the worker, and +/// returns 0 for the synchronous path because there is nothing to wake. +/// +/// Keeping the branch here rather than at each call site matters: there are seven +/// write paths, and a single one that enqueued while also applying inline would +/// double-apply, while one that did neither would silently stop indexing. +/// +/// `old_item` and `new_item` follow the same convention as `sync_indexes`: a put +/// supplies both when replacing, a delete supplies only the old. +pub(crate) async fn maintain_vector_indexes( + tx: &mut db::Transaction, + table_id: &str, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + old_item: Option<&Item>, + new_item: Option<&Item>, + delay_ms: u64, +) -> Result { + // Read inside the transaction rather than from the cached `TableKeyInfo`: the + // cache carries the search schema but not the index id, and the id is what + // names the data table. + let metas = fetch_vector_indexes_for_table(tx, table_id).await?; + if metas.is_empty() { + return Ok(0); + } + let key_cols = base_key_columns(base_key_schema, attr_defs); + + if delay_ms == 0 { + for meta in &metas { + apply_vector_index( + tx, + table_id, + meta, + base_key_schema, + attr_defs, + &key_cols, + old_item, + new_item, + ) + .await?; + } + return Ok(0); + } + + let mut enqueued = 0usize; + for meta in metas { + // Enqueued even when the new item carries no vector: the removal is the + // work in that case, and skipping it would leave a stale row indexed. + let context = super::index::PendingApplyContext::Vector(VectorApplyContext { + base_key_schema: base_key_schema.to_vec(), + attribute_definitions: attr_defs.to_vec(), + table_id: table_id.to_owned(), + vector: meta, + }); + super::index::enqueue_pending_row(tx, table_id, old_item, new_item, delay_ms, &context) + .await?; + enqueued += 1; + } + Ok(enqueued) +} + +/// Apply one claimed vector pending row, from its self-describing context. +/// +/// A missing data table is skipped rather than treated as a failure: the base table +/// or the index itself can be dropped while a row is in flight, which is a routine +/// race and not a defect. +/// +/// This is log hygiene rather than data safety, and worth being exact about. The +/// batch already guards every row with a savepoint, so without this the row would be +/// rolled back and dropped, reaching the same end state by a noisier route. What the +/// tolerance changes is that an expected race stops emitting an ERROR line, which +/// otherwise trains operators to ignore the one signal that says a row was thrown +/// away. Matches the GSI sibling, so both arms of the dispatcher behave alike. +pub(crate) async fn apply_vector_context( + tx: &mut db::Transaction, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &VectorApplyContext, +) -> Result<(), StorageError> { + let key_cols = base_key_columns(&context.base_key_schema, &context.attribute_definitions); + apply_vector_index( + tx, + &context.table_id, + &context.vector, + &context.base_key_schema, + &context.attribute_definitions, + &key_cols, + old_item, + new_item, + ) + .await + .or_else(|e| { + if super::index::is_no_such_table(&e) { + Ok(()) + } else { + Err(e) + } + }) +} + +/// Apply an item write to a single vector index. +/// +/// The delete-then-insert shape matters. An item can move between partitions when +/// its HASH attribute changes, and the row is keyed by the base item rather than by +/// the partition, so an insert alone would leave the old partition's row in place +/// and the item would be findable in two partitions at once. +/// +/// The delete keys off `old_item.or(new_item)` because the base key is immutable, so +/// either carries it. That is what lets a put whose caller had no reason to read the +/// old item still displace the row it replaces. +#[allow(clippy::too_many_arguments)] +async fn apply_vector_index( + tx: &mut db::Transaction, + table_id: &str, + meta: &VectorIndexMeta, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + key_cols: &[String], + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, &meta.index_id); + let where_clause = key_cols + .iter() + .map(|c| format!("{c} = ?")) + .collect::>() + .join(" AND "); + + // Remove any existing row for this base item first, whatever partition it + // was in. + if let Some(source) = old_item.or(new_item) { + let binds = base_key_binds(source, base_key_schema, attr_defs)?; + let sql = format!("DELETE FROM {vec_table} WHERE {where_clause}"); + let mut q = db::query(&sql); + for b in binds { + q = super::bind_bound!(q, b); + } + q.execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + } + + let Some(new_item) = new_item else { + return Ok(()); // A delete: removal above is the whole of the work. + }; + insert_vector_row( + tx, + table_id, + meta, + new_item, + base_key_schema, + attr_defs, + key_cols, + VectorRowConflict::Fail, + ) + .await +} + +/// Write one item's row into one vector index. +/// +/// Shared by the write path and by backfill deliberately. These are the only two +/// producers of a vector row, and a second copy of this logic would be free to +/// drift: a backfilled row shaped differently from a live-written one would search +/// correctly right up until the difference mattered, with nothing to catch it. +/// +/// A non-indexable item is a no-op rather than an error, which is what makes a +/// backfill over a table where only some items carry the vector work. +/// How [`insert_vector_row`] treats a primary-key conflict. +/// +/// `Fail` is the write path's contract: every apply reaches the insert through +/// `apply_vector_index`, which deletes the base key's row first, so a conflict +/// there means a broken invariant and must be loud. `KeepExisting` is the +/// BACKFILL's contract: in synchronous-visibility mode +/// (`index_propagation_delay_ms == 0`) a base write landing mid-backfill is +/// applied to the CREATING index inline rather than queued, and each backfill +/// batch reads the base table LIVE under the write lock, so a row already +/// present was written from the same or a newer base image than the one the +/// backfill just read; keeping it is correct and inserting over it would only +/// fail the whole build. Without this, that collision wedged the index in +/// CREATING permanently (proven by test below). +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum VectorRowConflict { + Fail, + KeepExisting, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn insert_vector_row( + tx: &mut db::Transaction, + table_id: &str, + meta: &VectorIndexMeta, + item: &Item, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + key_cols: &[String], + on_conflict: VectorRowConflict, +) -> Result<(), StorageError> { + if !item_is_indexable(item, meta) { + return Ok(()); + } + let vec_table = vector_table_name(table_id, &meta.index_id); + + let value = item.get(&meta.vector_attribute_name).ok_or_else(|| { + StorageError::Internal("indexable check passed but the vector is absent".to_owned()) + })?; + let components = vector_components(value).ok_or_else(|| { + // Core validates the write before it reaches storage, so a malformed + // vector here means validation was bypassed rather than that a caller + // sent bad input. + StorageError::Internal( + "vector attribute reached storage without passing validation".to_owned(), + ) + })?; + if components.len() != meta.dimensions { + return Err(StorageError::Internal(format!( + "vector has {} components, index declares {}", + components.len(), + meta.dimensions + ))); + } + + let mut blob = Vec::with_capacity(components.len() * 4); + for x in &components { + blob.extend_from_slice(&x.to_le_bytes()); + } + let norm = vector_norm(&components); + let part = item_partition(item, meta)?; + // Projected exactly as the GSI sibling projects, so a search returns what + // the index declares and no more. + let mut projected = + super::index::project_item_for_index(item, &[], base_key_schema, &meta.projection); + // The SearchSchema attributes are always projected, whatever the + // ProjectionType. See `search_schema_attribute_names` for why: the inline + // filter is evaluated against this payload, so dropping the attribute would + // silently turn every filtered search into a zero-result search. + for name in &meta.search_schema_attribute_names { + if !projected.contains_key(name) + && let Some(v) = item.get(name) + { + projected.insert(name.clone(), v.clone()); + } + } + // The vector itself is not kept in the payload: it is already in the `vec` + // column as `f32`, which is the width the service validates against, and the + // search path rebuilds the attribute from those bits. Keeping a verbatim + // decimal copy here duplicated 10 to 15 KB per row at 1024 dimensions and + // would have returned the client's original precision where the service + // returns the narrowed value. + projected.remove(&meta.vector_attribute_name); + let item_json = serde_json::to_string(&projected) + .map_err(|e| StorageError::Internal(format!("serialize item: {e}")))?; + + // On the write path this stays a plain INSERT, deliberately, where the GSI + // sibling uses INSERT OR REPLACE: every apply reaches here through + // `apply_vector_index`, which unconditionally deletes the base key's row + // first, so a conflict means a broken invariant and must fail loudly. The + // backfill passes `KeepExisting` instead; see [`VectorRowConflict`] for + // why an existing row is the same-or-newer generation there and must win. + let cols = std::iter::once("part".to_owned()) + .chain(key_cols.iter().cloned()) + .chain(["vec".to_owned(), "nrm".to_owned(), "item_data".to_owned()]) + .collect::>(); + let placeholders = vec!["?"; cols.len()].join(", "); + let verb = match on_conflict { + VectorRowConflict::Fail => "INSERT", + VectorRowConflict::KeepExisting => "INSERT OR IGNORE", + }; + let sql = format!( + "{verb} INTO {vec_table} ({}) VALUES ({placeholders})", + cols.join(", ") + ); + let key_binds = base_key_binds(item, base_key_schema, attr_defs)?; + let mut q = db::query(&sql).bind(part); + for b in key_binds { + q = super::bind_bound!(q, b); + } + q.bind(blob) + .bind(f64::from(norm)) + .bind(item_json) + .execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + Ok(()) +} + +/// Populate a newly created vector index from the base table. +/// +/// Batched by offset exactly as `backfill_gsi` is, and for the same reason: a table +/// large enough to be worth indexing is too large to hold in memory. Returns the +/// number of rows written, which is what distinguishes "backfilled nothing because +/// no item carries the vector" from "backfilled nothing because the scan is broken". +/// Everything a backfill needs that does not change between batches. +/// +/// Bundled because the alternative was an eight-argument function threaded through two +/// drivers, where the only per-batch values are the page size and the cursor. +struct BackfillPlan<'a> { + table_id: &'a str, + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], + key_cols: Vec, +} + +impl<'a> BackfillPlan<'a> { + fn new( + table_id: &'a str, + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], + ) -> Self { + Self { + table_id, + meta, + base_key_schema, + attr_defs, + key_cols: base_key_columns(base_key_schema, attr_defs), + } + } +} + +/// Backfill one batch of existing rows into the vector index. +/// +/// Returns `(written, fetched, last_rowid)`. `fetched` distinguishes a short read +/// (the end) from a full one, and `last_rowid` is the cursor to resume from. +/// +/// Pagination is by KEY, not by `OFFSET`. Offset anchors on a position, so removing any +/// already-scanned row shifts every later position by one and the next batch skips a +/// row entirely. That row is then missing from the index permanently, and no queue +/// entry can repair it, because the skipped row was never written to: only the removed +/// one was. Reproduced before this change: one removal during a backfill left the row +/// at the batch boundary absent from the index. +/// +/// The cursor is `rowid`, not `pk`, and the difference is a correctness matter rather +/// than a preference. Composite-key base tables have `PRIMARY KEY (pk, sk*)`, so `pk` +/// alone is not unique; a `pk > last_pk` cursor whose batch boundary fell inside one +/// partition's sort-key group excluded every remaining row sharing that `pk`, +/// permanently, on both the UpdateTable path and startup reconciliation (which share +/// this function). Reproduced: a five-row partition scanned with a batch of three +/// indexed four rows and skipped two. `rowid` is unique regardless of the key layout, +/// so one query shape serves both. It is a valid cursor because the base tables are +/// ordinary rowid tables and every write is `INSERT ... ON CONFLICT DO UPDATE` +/// (`tx_helpers.rs`), which updates in place and never reassigns a rowid; rows +/// inserted after a batch passed their rowid position are the concurrent writes the +/// queue hold already captures and replays after ACTIVE. +/// +/// It was unreachable while the whole backfill ran in one transaction, and became +/// reachable the moment batches started committing independently. +/// One batch's outcome: rows indexed, rows skipped as poison, rows fetched +/// (for termination), and the cursor for the next batch. +struct BatchOutcome { + written: usize, + skipped: usize, + fetched: i64, + last_rowid: i64, +} + +async fn backfill_vector_batch( + tx: &mut db::Transaction, + plan: &BackfillPlan<'_>, + limit: i64, + after_rowid: i64, +) -> Result { + let base_table = super::data_table_name(plan.table_id); + // `rowid > ?` with -1 as the initial cursor: DuckDB rowids start at 0, so + // the first batch needs no separate query shape. + let sql = + format!("SELECT rowid, item_data FROM {base_table} WHERE rowid > ? ORDER BY rowid LIMIT ?"); + let rows: Vec<(i64, String)> = db::query_as(&sql) + .bind(after_rowid) + .bind(limit) + .fetch_all(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + let fetched = i64::try_from(rows.len()).unwrap_or(limit); + let last_rowid = rows.last().map_or(after_rowid, |(rid, _)| *rid); + let mut written = 0usize; + let mut skipped = 0usize; + for (rowid, item_json) in rows { + // Poison classification. The live write path treats a malformed vector + // as an invariant violation and errors loudly, because core validation + // ran before storage was reached. That reasoning is FALSE here: rows + // written before the index existed never passed vector validation, so + // a malformed or wrong-dimension vector in the base table is expected + // input for a backfill, not a bug. Propagating it wedged the build in + // an infinite recovery loop: the error left the index CREATING, the + // watchdog re-ran the rebuild, and the same row failed again, forever, + // while the CREATING hold also froze every queued index write for the + // table. A row whose stored bytes cannot enter the index is skipped + // and counted instead, exactly as a GSI omits an item whose key + // attribute has the wrong type. Transient failures (the INSERT itself + // erroring) still propagate: those are retryable and must not drop + // rows. + let Ok(item) = serde_json::from_str::(&item_json) else { + tracing::warn!( + rowid, + index = %plan.meta.index_id, + "backfill: stored item is unparseable; skipping row" + ); + skipped += 1; + continue; + }; + if !item_is_indexable(&item, plan.meta) { + continue; + } + let vector_ok = item + .get(&plan.meta.vector_attribute_name) + .and_then(vector_components) + .is_some_and(|c| c.len() == plan.meta.dimensions); + if !vector_ok { + tracing::warn!( + rowid, + index = %plan.meta.index_id, + "backfill: vector attribute malformed or wrong dimension; skipping row" + ); + skipped += 1; + continue; + } + insert_vector_row( + tx, + plan.table_id, + plan.meta, + &item, + plan.base_key_schema, + plan.attr_defs, + &plan.key_cols, + VectorRowConflict::KeepExisting, + ) + .await?; + written += 1; + } + Ok(BatchOutcome { + written, + skipped, + fetched, + last_rowid, + }) +} + +/// A completed backfill: rows indexed and rows skipped as poison. `skipped` +/// is recorded on the catalog row so an ACTIVE index that deliberately omits +/// rows says so, rather than the omission being indistinguishable from a bug. +pub(crate) struct BackfillOutcome { + pub(crate) written: usize, + pub(crate) skipped: usize, +} + +/// Backfill the index in independently committed batches, releasing DuckDB's write +/// lock between them. +/// +/// This is what lets the base table stay writable while an index builds, which is how +/// the service behaves: the table remains ACTIVE and accepts writes throughout, and +/// only the index reports CREATING. Holding one transaction for the whole backfill +/// would block every write until it finished. +/// +/// Releasing the lock is also what creates the ordering hazard this design has to +/// answer. A write landing mid-backfill is enqueued, and if it were applied before the +/// backfill wrote its older snapshot of the same item, the index would converge on the +/// stale generation. The queue worker therefore refuses to claim any row for a table +/// whose vector index is still CREATING, so those writes accumulate and are applied +/// only after this returns and the index flips to ACTIVE. +/// +/// A crash part-way leaves the index in CREATING with some rows written, which +/// `reconcile_incomplete_vector_indexes` repairs at startup by rebuilding it. +pub(crate) async fn backfill_vector_index_in_batches( + pool: &db::Pool, + write_lock: &tokio::sync::Mutex<()>, + table_id: &str, + meta: &VectorIndexMeta, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + batch_delay: std::time::Duration, +) -> Result { + const BATCH: i64 = 500; + let plan = BackfillPlan::new(table_id, meta, base_key_schema, attr_defs); + let mut cursor: i64 = -1; + let mut written = 0usize; + let mut skipped = 0usize; + loop { + let outcome = { + let _writer = write_lock.lock().await; + let mut tx = pool.begin().await.map_err(crate::duckdb_util::map_db_err)?; + let result = backfill_vector_batch(&mut tx, &plan, BATCH, cursor).await?; + tx.commit().await.map_err(crate::duckdb_util::map_db_err)?; + result + }; + written += outcome.written; + skipped += outcome.skipped; + if outcome.fetched < BATCH { + break; + } + cursor = outcome.last_rowid; + // Outside the lock, so a write can actually proceed during the pause. Zero in + // production; a test sets it so a write is guaranteed to land mid-backfill. + if !batch_delay.is_zero() { + tokio::time::sleep(batch_delay).await; + } + } + Ok(BackfillOutcome { written, skipped }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db; + + /// A composite-key table whose partition spans a batch boundary must still + /// backfill every row. + /// + /// This is the regression test for a silent data-loss defect: the cursor was + /// `pk > last_pk`, and on a table with `PRIMARY KEY (pk, sk_s)` a batch ending + /// inside one partition's sort-key group excluded every remaining row sharing + /// that `pk`. Five rows in one partition scanned with a batch of three indexed + /// four and skipped two, permanently, on both the UpdateTable path and startup + /// reconciliation, which share `backfill_vector_batch`. The rowid cursor cannot + /// lose rows because rowid is unique whatever the key layout. + /// + /// Driven through `backfill_vector_batch` directly with a batch of 3 rather + /// than through the drivers, because they hardcode a 500-row batch and seeding + /// 501 rows would test the same lines slower. + #[tokio::test] + async fn a_composite_key_partition_straddling_a_batch_boundary_is_fully_backfilled() { + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let ks = vec![ + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }, + ]; + let ad = vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ]; + + let table_id = "t-composite"; + let mut tx = engine.pool.begin().await.expect("tx"); + crate::DuckDbEngine::create_data_table(&mut tx, table_id, &ks, &ad) + .await + .expect("base table"); + crate::DuckDbEngine::create_vector_data_table(&mut tx, table_id, "vidx-1", &ks, &ad) + .await + .expect("vector table"); + + // One partition with five sort keys plus a second partition, so the batch + // of three ends INSIDE partition "a": exactly the boundary that lost rows. + let base_table = super::super::data_table_name(table_id); + for (pk, sk) in [ + ("a", "1"), + ("a", "2"), + ("a", "3"), + ("a", "4"), + ("a", "5"), + ("b", "1"), + ] { + db::query(&format!( + "INSERT INTO {base_table} (pk, sk_s, item_data) VALUES (?, ?, ?)" + )) + .bind(pk) + .bind(sk) + .bind(format!( + r#"{{"pk":{{"S":"{pk}"}},"sk":{{"S":"{sk}"}},"emb":{{"L":[{{"N":"1"}},{{"N":"0"}}]}}}}"# + )) + .execute(&mut *tx) + .await + .expect("seed"); + } + + let meta = VectorIndexMeta { + index_id: "vidx-1".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: extenddb_core::types::Projection { + projection_type: extenddb_core::types::ProjectionType::All, + non_key_attributes: None, + }, + hash_attribute_name: None, + search_schema_attribute_names: Vec::new(), + }; + let plan = BackfillPlan::new(table_id, &meta, &ks, &ad); + + let mut cursor: i64 = -1; + let mut written = 0usize; + loop { + let outcome = backfill_vector_batch(&mut tx, &plan, 3, cursor) + .await + .expect("batch"); + written += outcome.written; + if outcome.fetched < 3 { + break; + } + cursor = outcome.last_rowid; + } + tx.commit().await.expect("commit"); + + assert_eq!( + written, 6, + "every row must be indexed; the pk-only cursor wrote 4 and skipped 2" + ); + let vec_table = super::super::vector_table_name(table_id, "vidx-1"); + let (rows,): (i64,) = db::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 6, "the index must hold all six rows"); + } + + /// A row whose stored bytes cannot enter the index is skipped and counted, + /// not propagated. Propagating it wedged the build permanently: the error + /// left the index CREATING, the watchdog re-ran the rebuild, the same row + /// failed again, and the CREATING hold froze every queued index write for + /// the table. This is the review finding on this file: "non-conformant + /// items keep failing and the index creation will be stuck in recovery + /// loop forever". + /// + /// Discriminating by construction: the poison rows (a wrong-dimension + /// vector, a non-list vector, and unparseable item bytes) sit BETWEEN good + /// rows, so the pre-fix behaviour (error on first poison row, nothing + /// after it indexed) cannot produce these counts. + #[tokio::test] + async fn poison_rows_are_skipped_and_counted_rather_than_wedging_the_backfill() { + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let ad = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + + let table_id = "t-poison"; + let mut tx = engine.pool.begin().await.expect("tx"); + crate::DuckDbEngine::create_data_table(&mut tx, table_id, &ks, &ad) + .await + .expect("base table"); + crate::DuckDbEngine::create_vector_data_table(&mut tx, table_id, "vidx-p", &ks, &ad) + .await + .expect("vector table"); + + let base_table = super::super::data_table_name(table_id); + // good, wrong-dimension, good, non-list vector, unparseable, good. + let rows: [(&str, String); 6] = [ + ( + "g1", + r#"{"pk":{"S":"g1"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#.to_owned(), + ), + ( + "p1", + r#"{"pk":{"S":"p1"},"emb":{"L":[{"N":"1"}]}}"#.to_owned(), + ), + ( + "g2", + r#"{"pk":{"S":"g2"},"emb":{"L":[{"N":"0"},{"N":"1"}]}}"#.to_owned(), + ), + ( + "p2", + r#"{"pk":{"S":"p2"},"emb":{"S":"not-a-vector"}}"#.to_owned(), + ), + ("p3", "{not json".to_owned()), + ( + "g3", + r#"{"pk":{"S":"g3"},"emb":{"L":[{"N":"1"},{"N":"1"}]}}"#.to_owned(), + ), + ]; + for (pk, item) in &rows { + db::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES (?, ?)" + )) + .bind(pk) + .bind(item) + .execute(&mut *tx) + .await + .expect("seed"); + } + + let meta = VectorIndexMeta { + index_id: "vidx-p".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: extenddb_core::types::Projection { + projection_type: extenddb_core::types::ProjectionType::All, + non_key_attributes: None, + }, + hash_attribute_name: None, + search_schema_attribute_names: Vec::new(), + }; + let plan = BackfillPlan::new(table_id, &meta, &ks, &ad); + + let outcome = backfill_vector_batch(&mut tx, &plan, 100, 0) + .await + .expect("a batch containing poison rows must still complete"); + tx.commit().await.expect("commit"); + + assert_eq!(outcome.written, 3, "the three good rows are indexed"); + assert_eq!(outcome.skipped, 3, "the three poison rows are counted"); + + // The good rows AFTER the poison rows made it in, which is the part the + // pre-fix behaviour cannot do: it stopped at p1. + let vec_table = super::super::vector_table_name(table_id, "vidx-p"); + let (rows,): (i64,) = db::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 3); + } + /// A synchronous-visibility write (`index_propagation_delay_ms == 0`) + /// landing mid-backfill is applied to the CREATING index inline, bypassing + /// the queue's CREATING-hold. When the backfill later reaches that base + /// key it must keep the row the inline write produced rather than fail the + /// whole build on the primary-key conflict: each batch reads the base + /// table live under the write lock, so an existing row was written from + /// the same or a newer base image than the one the batch just read. Before + /// `VectorRowConflict::KeepExisting`, this collision errored the backfill + /// and wedged the index in CREATING permanently. + #[tokio::test] + async fn a_synchronous_write_landing_mid_backfill_does_not_wedge_the_build() { + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let ad = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + + let table_id = "t-inline-collision"; + let mut tx = engine.pool.begin().await.expect("tx"); + crate::DuckDbEngine::create_data_table(&mut tx, table_id, &ks, &ad) + .await + .expect("base table"); + crate::DuckDbEngine::create_vector_data_table(&mut tx, table_id, "vidx-c", &ks, &ad) + .await + .expect("vector table"); + // Catalog rows: a table and its index still CREATING with a build in + // flight, which is the state a mid-backfill write observes. + db::query("INSERT INTO accounts (account_id, account_name) VALUES ('a', 'a')") + .execute(&mut *tx) + .await + .expect("account row"); + db::query( + "INSERT INTO tables (account_id, table_name, key_schema, attribute_definitions, \ + billing_mode, table_status, creation_date_time, table_arn, table_id) \ + VALUES ('a', 't', '[]', '[]', 'PAY_PER_REQUEST', 'ACTIVE', 'now', 'arn', ?)", + ) + .bind(table_id) + .execute(&mut *tx) + .await + .expect("table row"); + db::query( + "INSERT INTO vector_indexes (table_id, index_name, index_id, dimensions, \ + distance_function, vector_attribute, search_schema, projection, index_status, \ + backfilling) \ + VALUES (?, 'vidx-c', 'vidx-c', 2, 'COSINE', ?, NULL, ?, 'CREATING', 1)", + ) + .bind(table_id) + .bind(r#"{"AttributeName":"emb"}"#) + .bind(r#"{"ProjectionType":"ALL"}"#) + .execute(&mut *tx) + .await + .expect("index row"); + let base_table = super::super::data_table_name(table_id); + db::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES ('z', ?)" + )) + .bind(r#"{"pk":{"S":"z"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#) + .execute(&mut *tx) + .await + .expect("seed base row"); + + // The inline write: at delay 0 maintenance applies to the CREATING + // index directly rather than enqueueing. + let item: extenddb_core::types::Item = + serde_json::from_str(r#"{"pk":{"S":"z"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#) + .expect("item"); + let enqueued = maintain_vector_indexes(&mut tx, table_id, &ks, &ad, None, Some(&item), 0) + .await + .expect("inline maintenance"); + assert_eq!( + enqueued, 0, + "delay 0 must take the inline arm, not the queue" + ); + tx.commit().await.expect("commit"); + + // The backfill now reaches the same base key and must complete rather + // than error on the conflict, leaving exactly one row for the key. + let write_lock = tokio::sync::Mutex::new(()); + let meta = + fetch_vector_indexes_for_table(&mut engine.pool.begin().await.expect("tx"), table_id) + .await + .expect("metas") + .into_iter() + .find(|m| m.index_id == "vidx-c") + .expect("meta"); + let outcome = backfill_vector_index_in_batches( + &engine.pool, + &write_lock, + table_id, + &meta, + &ks, + &ad, + std::time::Duration::ZERO, + ) + .await + .expect("backfill must not wedge on the inline write's row"); + assert_eq!(outcome.skipped, 0); + let vec_table = super::super::vector_table_name(table_id, "vidx-c"); + let (rows,): (i64,) = db::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!( + rows, 1, + "one base key must yield one index row, not a duplicate" + ); + } +} diff --git a/crates/storage-duckdb/src/db.rs b/crates/storage-duckdb/src/db.rs new file mode 100644 index 00000000..f02eddb8 --- /dev/null +++ b/crates/storage-duckdb/src/db.rs @@ -0,0 +1,1074 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A thin asynchronous facade over the synchronous `duckdb` crate. +//! +//! DuckDB has no `sqlx` driver, so this module supplies the small `sqlx`-shaped +//! surface the rest of the crate uses: a connection [`Pool`], positional +//! [`query`] / [`query_as`] / [`query_scalar`] builders with `.bind()`, and a +//! [`Transaction`] that commits explicitly and rolls back on drop. Everything +//! else in the crate is written against this module rather than against +//! `duckdb` directly, which keeps the storage code free of blocking calls. +//! +//! # Execution model +//! +//! `duckdb::Connection` is `Send` but not `Sync`, and every call on it blocks. +//! Each pooled connection therefore lives in a slot (`tokio::sync::Mutex