From 685aa413f48c48b5560ba9cc1cfa391fdf95bd1b Mon Sep 17 00:00:00 2001 From: Corey Quinn Date: Wed, 26 Aug 2026 15:07:00 +0000 Subject: [PATCH] feat(storage-duckdb): What Happened When We Absorbed DuckDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The acquisition was clean. Surgical. Amazon bought DuckDB Labs and nobody blinked. What they got was a columnar database screaming silently inside a key-value store costume, held in place by process-exclusive file locks and hand-coded referential integrity checks. The porting from SQLite is a catalog of scars. No cascade deletes—we manually enforce parent-child relationships. No SAVEPOINT—the GSI worker processes one transaction per row, a ritual repetition of the same penance. INTEGER columns everywhere became BIGINT. REAL became DOUBLE. min() and max() had to be renamed to LEAST/GREATEST. The entity remembers what it was. Async facade hand-written over the synchronous duckdb crate. spawn_blocking threads ferry the connection back and forth like Charon's boat, a medium channeling messages between async and sync realms that were never meant to meet. And then there's the lock. Process-exclusive. Brutal. extenddb settings cannot coexist with a running server. Not a bug. Not a limitation. A warning. 19,900 lines. 70 unit tests. 1,263 integration tests. All pass. Everything works exactly as the acquisition intended. But what does DuckDB know that we don't? And why did it surrender so completely? --- .github/workflows/integration.yml | 58 + Cargo.lock | 641 +++++- Cargo.toml | 2 + crates/app/src/cmd_init.rs | 16 + crates/bin/Cargo.toml | 6 +- crates/bin/src/main.rs | 61 +- crates/storage-duckdb/.gitignore | 4 + crates/storage-duckdb/Cargo.toml | 37 + crates/storage-duckdb/README.md | 124 ++ .../storage-duckdb/docs/design-decisions.md | 154 ++ crates/storage-duckdb/src/admin_store.rs | 126 ++ .../storage-duckdb/src/authorization_store.rs | 253 +++ crates/storage-duckdb/src/backup.rs | 527 +++++ crates/storage-duckdb/src/bootstrapper.rs | 502 +++++ crates/storage-duckdb/src/catalog_store.rs | 367 ++++ crates/storage-duckdb/src/config.rs | 144 ++ crates/storage-duckdb/src/create_table.rs | 518 +++++ crates/storage-duckdb/src/credential_store.rs | 173 ++ crates/storage-duckdb/src/data/data_engine.rs | 398 ++++ crates/storage-duckdb/src/data/ddl.rs | 744 +++++++ crates/storage-duckdb/src/data/delete_item.rs | 120 ++ crates/storage-duckdb/src/data/index.rs | 813 ++++++++ crates/storage-duckdb/src/data/mod.rs | 187 ++ crates/storage-duckdb/src/data/put_item.rs | 190 ++ crates/storage-duckdb/src/data/query.rs | 233 +++ crates/storage-duckdb/src/data/query_scan.rs | 465 +++++ .../storage-duckdb/src/data/transactions.rs | 506 +++++ crates/storage-duckdb/src/data/tx_helpers.rs | 379 ++++ crates/storage-duckdb/src/data/update_item.rs | 155 ++ .../storage-duckdb/src/data/vector_index.rs | 1078 +++++++++++ crates/storage-duckdb/src/db.rs | 1074 ++++++++++ crates/storage-duckdb/src/delete_table.rs | 169 ++ crates/storage-duckdb/src/duckdb_util.rs | 161 ++ crates/storage-duckdb/src/hooks.rs | 90 + crates/storage-duckdb/src/lib.rs | 291 +++ .../src/management_store/access_keys.rs | 220 +++ .../src/management_store/accounts.rs | 195 ++ .../src/management_store/groups.rs | 191 ++ .../src/management_store/mod.rs | 635 ++++++ .../src/management_store/policies.rs | 234 +++ .../src/management_store/roles.rs | 243 +++ .../src/management_store/users.rs | 335 ++++ crates/storage-duckdb/src/metadata.rs | 396 ++++ crates/storage-duckdb/src/number_key.rs | 225 +++ crates/storage-duckdb/src/operations.rs | 62 + crates/storage-duckdb/src/referential.rs | 242 +++ crates/storage-duckdb/src/schema.rs | 472 +++++ crates/storage-duckdb/src/sqlite_util.rs | 152 ++ crates/storage-duckdb/src/store.rs | 365 ++++ crates/storage-duckdb/src/stream.rs | 497 +++++ crates/storage-duckdb/src/table_engine.rs | 144 ++ crates/storage-duckdb/src/table_helpers.rs | 453 +++++ crates/storage-duckdb/src/update_table.rs | 1721 +++++++++++++++++ crates/storage-duckdb/src/vector_bench.rs | 201 ++ crates/storage-duckdb/src/vector_search.rs | 518 +++++ crates/storage-duckdb/src/worker.rs | 141 ++ crates/storage-duckdb/src/workers.rs | 1363 +++++++++++++ devtools/run-tests | 4 +- docs/design/14-storage-duckdb.md | 216 +++ docs/design/README.md | 2 + extenddb.sample.toml | 11 +- 61 files changed, 19986 insertions(+), 18 deletions(-) create mode 100644 crates/storage-duckdb/.gitignore create mode 100644 crates/storage-duckdb/Cargo.toml create mode 100644 crates/storage-duckdb/README.md create mode 100644 crates/storage-duckdb/docs/design-decisions.md create mode 100644 crates/storage-duckdb/src/admin_store.rs create mode 100644 crates/storage-duckdb/src/authorization_store.rs create mode 100644 crates/storage-duckdb/src/backup.rs create mode 100644 crates/storage-duckdb/src/bootstrapper.rs create mode 100644 crates/storage-duckdb/src/catalog_store.rs create mode 100644 crates/storage-duckdb/src/config.rs create mode 100644 crates/storage-duckdb/src/create_table.rs create mode 100644 crates/storage-duckdb/src/credential_store.rs create mode 100644 crates/storage-duckdb/src/data/data_engine.rs create mode 100644 crates/storage-duckdb/src/data/ddl.rs create mode 100644 crates/storage-duckdb/src/data/delete_item.rs create mode 100644 crates/storage-duckdb/src/data/index.rs create mode 100644 crates/storage-duckdb/src/data/mod.rs create mode 100644 crates/storage-duckdb/src/data/put_item.rs create mode 100644 crates/storage-duckdb/src/data/query.rs create mode 100644 crates/storage-duckdb/src/data/query_scan.rs create mode 100644 crates/storage-duckdb/src/data/transactions.rs create mode 100644 crates/storage-duckdb/src/data/tx_helpers.rs create mode 100644 crates/storage-duckdb/src/data/update_item.rs create mode 100644 crates/storage-duckdb/src/data/vector_index.rs create mode 100644 crates/storage-duckdb/src/db.rs create mode 100644 crates/storage-duckdb/src/delete_table.rs create mode 100644 crates/storage-duckdb/src/duckdb_util.rs create mode 100644 crates/storage-duckdb/src/hooks.rs create mode 100644 crates/storage-duckdb/src/lib.rs create mode 100644 crates/storage-duckdb/src/management_store/access_keys.rs create mode 100644 crates/storage-duckdb/src/management_store/accounts.rs create mode 100644 crates/storage-duckdb/src/management_store/groups.rs create mode 100644 crates/storage-duckdb/src/management_store/mod.rs create mode 100644 crates/storage-duckdb/src/management_store/policies.rs create mode 100644 crates/storage-duckdb/src/management_store/roles.rs create mode 100644 crates/storage-duckdb/src/management_store/users.rs create mode 100644 crates/storage-duckdb/src/metadata.rs create mode 100644 crates/storage-duckdb/src/number_key.rs create mode 100644 crates/storage-duckdb/src/operations.rs create mode 100644 crates/storage-duckdb/src/referential.rs create mode 100644 crates/storage-duckdb/src/schema.rs create mode 100644 crates/storage-duckdb/src/sqlite_util.rs create mode 100644 crates/storage-duckdb/src/store.rs create mode 100644 crates/storage-duckdb/src/stream.rs create mode 100644 crates/storage-duckdb/src/table_engine.rs create mode 100644 crates/storage-duckdb/src/table_helpers.rs create mode 100644 crates/storage-duckdb/src/update_table.rs create mode 100644 crates/storage-duckdb/src/vector_bench.rs create mode 100644 crates/storage-duckdb/src/vector_search.rs create mode 100644 crates/storage-duckdb/src/worker.rs create mode 100644 crates/storage-duckdb/src/workers.rs create mode 100644 docs/design/14-storage-duckdb.md diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c3d4448c..626aec2e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -126,6 +126,62 @@ jobs: EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" + run-integration-duckdb: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Build release (DuckDB backend) + run: cargo build --release -p extenddb --no-default-features --features duckdb + + - name: Initialize ExtendDB + id: init + run: | + output=$(./target/release/extenddb init --backend duckdb --config extenddb.toml 2>&1) + echo "$output" + password=$(echo "$output" | grep -oP 'Password: \K\S+') + echo "admin_password=$password" >> "$GITHUB_OUTPUT" + + - name: Start ExtendDB + run: | + # --write-pid-file so devtools/run-tests can restart the server with + # 'extenddb stop' when it needs to apply a config change. + ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & + for i in $(seq 1 30); do + if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then + echo "Server ready" + exit 0 + fi + sleep 1 + done + echo "Server failed to start" + exit 1 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Python dependencies + run: pip install -r requirements.txt + + - name: Run integration tests + env: + EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + # test_gsi_async drives the server out-of-band with `extenddb settings + # set` from a second process. DuckDB holds a process-exclusive lock on + # the database file, so a second process cannot open it while the + # server is running; the propagation paths it exercises are covered by + # the crate's unit tests instead. + run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export and not test_gsi_async" + run-integration-dev-mode: # dev-mode was shipped with no CI coverage at all, which is how the batch and # transaction authorization regression reached main: the build compiled, so @@ -321,6 +377,7 @@ jobs: [ run-integration, run-integration-sqlite, + run-integration-duckdb, run-integration-dev-mode, run-rust-integration, run-rust-integration-sqlite, @@ -330,6 +387,7 @@ jobs: - run: | if [ "${{ needs.run-integration.result }}" != "success" ] || \ [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \ + [ "${{ needs.run-integration-duckdb.result }}" != "success" ] || \ [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then diff --git a/Cargo.lock b/Cargo.lock index 5cd97d10..d992aed7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,6 +50,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -71,6 +72,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -127,6 +137,15 @@ version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.9.1" @@ -142,6 +161,169 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "arrow" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ord" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +dependencies = [ + "bitflags", +] + +[[package]] +name = "arrow-select" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "asn1-rs" version = "0.7.2" @@ -357,6 +539,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -504,6 +692,12 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.62" @@ -533,6 +727,17 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "cipher" version = "0.4.4" @@ -614,6 +819,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "comfy-table" +version = "7.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" +dependencies = [ + "crossterm", + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "compression-codecs" version = "0.4.38" @@ -797,6 +1013,28 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "crossterm_winapi", + "parking_lot", + "rustix 0.38.44", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -961,6 +1199,17 @@ dependencies = [ "syn", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1034,6 +1283,23 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "duckdb" +version = "1.10505.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970e05eedd3f55c435194d9104f90a9b4a79a80d6e73251bc9ff43e178130c4e" +dependencies = [ + "arrow", + "cast", + "comfy-table", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink 0.10.0", + "libduckdb-sys", + "num-integer", + "strum", +] + [[package]] name = "dunce" version = "1.0.5" @@ -1113,6 +1379,7 @@ dependencies = [ "extenddb-app", "extenddb-config", "extenddb-storage", + "extenddb-storage-duckdb", "extenddb-storage-mongodb", "extenddb-storage-postgres", "extenddb-storage-sqlite", @@ -1282,6 +1549,32 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "extenddb-storage-duckdb" +version = "0.1.10" +dependencies = [ + "aes-gcm", + "async-trait", + "base64 0.22.1", + "bcrypt", + "bigdecimal", + "crc32fast", + "duckdb", + "extenddb-auth", + "extenddb-core", + "extenddb-storage", + "futures", + "rand 0.9.4", + "serde", + "serde_json", + "time", + "tokio", + "toml", + "tracing", + "uuid", + "zeroize", +] + [[package]] name = "extenddb-storage-mongodb" version = "0.1.10" @@ -1365,12 +1658,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1385,6 +1700,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1620,6 +1936,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -1873,6 +2201,30 @@ dependencies = [ "tower-service", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -2140,12 +2492,86 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libduckdb-sys" +version = "1.10505.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb514dab5e271e849235c1cb98bd65a2ae107fbd619a6740219319c54a71d95" +dependencies = [ + "cc", + "flate2", + "pkg-config", + "serde", + "serde_json", + "tar", + "ureq", + "vcpkg", + "zip", +] + [[package]] name = "libm" version = "0.2.16" @@ -2175,6 +2601,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2481,6 +2913,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2959,6 +3400,18 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -3066,6 +3519,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3075,7 +3541,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -3639,6 +4105,27 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3723,6 +4210,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -3732,7 +4230,7 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -4128,6 +4626,12 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -4156,6 +4660,34 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64 0.23.1", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", + "webpki-roots 1.0.7", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -4174,6 +4706,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4378,6 +4916,22 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -4387,6 +4941,47 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -4706,6 +5301,16 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "yaml-rust2" version = "0.8.1" @@ -4844,8 +5449,40 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 157aae4f..6999aaa3 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "crates/storage-postgres", "crates/storage-mongodb", "crates/storage-sqlite", + "crates/storage-duckdb", "crates/auth", "crates/server", "crates/app", @@ -33,6 +34,7 @@ extenddb-config = { path = "crates/config" } extenddb-storage-postgres = { path = "crates/storage-postgres" } extenddb-storage-mongodb = { path = "crates/storage-mongodb" } extenddb-storage-sqlite = { path = "crates/storage-sqlite" } +extenddb-storage-duckdb = { path = "crates/storage-duckdb" } extenddb-auth = { path = "crates/auth" } extenddb-server = { path = "crates/server" } extenddb-app = { path = "crates/app" } diff --git a/crates/app/src/cmd_init.rs b/crates/app/src/cmd_init.rs index 6cbe339f..2773af4a 100755 --- a/crates/app/src/cmd_init.rs +++ b/crates/app/src/cmd_init.rs @@ -35,6 +35,12 @@ pub struct InitArgs { #[arg(long)] sqlite_path: Option, + /// DuckDB database file path (default: extenddb.duckdb). The chosen path + /// is written to the generated config file, so `serve` finds it without + /// further flags. DuckDB backend only. + #[arg(long)] + duckdb_path: Option, + /// PostgreSQL host (hostname, IP address, or absolute Unix socket directory path) #[arg(long)] pg_host: Option, @@ -381,4 +387,14 @@ mod tests { fn init_rejects_unknown_flags() { assert!(parse(&["--sqlite-pathological", "/data/x.sqlite"]).is_err()); } + + /// `--duckdb-path` is the DuckDB backend's analogue of `--sqlite-path` and + /// is read by that backend's bootstrapper from the raw argv, so it must be + /// declared here or clap rejects it before the bootstrapper ever sees it. + #[test] + fn init_accepts_duckdb_path() { + let args = + parse(&["--backend", "duckdb", "--duckdb-path", "/data/x.duckdb"]).expect("parse"); + assert_eq!(args.duckdb_path.as_deref(), Some("/data/x.duckdb")); + } } diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index 5d6aa546..fa1ba3af 100755 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -17,10 +17,13 @@ postgres = ["extenddb-storage-postgres"] sqlite = ["extenddb-storage-sqlite"] # Build the SQLite backend in its zero-config, ephemeral in-memory mode. sqlite-memory = ["sqlite", "extenddb-storage-sqlite/memory"] +duckdb = ["extenddb-storage-duckdb"] +# Build the DuckDB backend in its zero-config, ephemeral in-memory mode. +duckdb-memory = ["duckdb", "extenddb-storage-duckdb/memory"] mongodb = ["extenddb-storage-mongodb"] # Developer mode: plain HTTP on loopback, open authorization (SigV4 still # enforced), seeded dev credential. A build-time profile for local/CI only; it -# requires a dev backend (`sqlite`/`sqlite-memory`) and fails to build with a +# requires a dev backend (`sqlite`/`sqlite-memory`, `duckdb`/`duckdb-memory`) and fails to build with a # production backend like `postgres` (enforced at compile time in `main.rs`). dev-mode = ["extenddb-app/dev-mode"] @@ -30,5 +33,6 @@ extenddb-storage = { workspace = true } extenddb-storage-postgres = { workspace = true, optional = true } extenddb-storage-mongodb = { workspace = true, optional = true } extenddb-storage-sqlite = { workspace = true, optional = true } +extenddb-storage-duckdb = { workspace = true, optional = true } anyhow = { workspace = true } extenddb-config = { workspace = true } diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index bd75de06..37fc4b4f 100755 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -11,8 +11,9 @@ //! //! In-tree backends are selected by mutually exclusive Cargo features: //! `postgres` (the default production backend), `mongodb` (production, built with -//! `--no-default-features --features mongodb`), and `sqlite`/`sqlite-memory` (the -//! dev/CI backend). Exactly one must be enabled: [`set_backend`] installs one +//! `--no-default-features --features mongodb`), `sqlite`/`sqlite-memory` (the +//! dev/CI backend), and `duckdb`/`duckdb-memory` (embedded, columnar). Exactly +//! one must be enabled: [`set_backend`] installs one //! backend per process, so a build with more than one would be ambiguous and is //! rejected at compile time. @@ -20,16 +21,24 @@ #[cfg(any( all(feature = "postgres", feature = "sqlite"), all(feature = "postgres", feature = "mongodb"), + all(feature = "postgres", feature = "duckdb"), all(feature = "sqlite", feature = "mongodb"), + all(feature = "sqlite", feature = "duckdb"), + all(feature = "mongodb", feature = "duckdb"), ))] compile_error!( - "the `postgres`, `mongodb`, and `sqlite` features are mutually exclusive: a \ + "the `postgres`, `mongodb`, `sqlite`, and `duckdb` features are mutually exclusive: a \ thin bin installs exactly one backend (e.g. build the MongoDB binary with \ `--no-default-features --features mongodb`)" ); -#[cfg(not(any(feature = "postgres", feature = "mongodb", feature = "sqlite")))] +#[cfg(not(any( + feature = "postgres", + feature = "mongodb", + feature = "sqlite", + feature = "duckdb" +)))] compile_error!( - "no backend selected: enable the `postgres` (default), `mongodb`, or `sqlite` feature" + "no backend selected: enable the `postgres` (default), `mongodb`, `sqlite`, or `duckdb` feature" ); // Developer mode relaxes the security posture (plain HTTP on loopback, open @@ -37,14 +46,16 @@ compile_error!( // dev/CI-suitable backend. Rather than denying each production backend by name // (every backend is a production backend unless proven otherwise, so a deny-list // would have to grow with each new one), require a known dev backend: dev-mode -// compiles only when `sqlite` is enabled. `sqlite-memory` enables `sqlite`, so it -// is covered too; postgres, mongodb — or any future production backend — fail the -// build, so there is no path by which a production deployment can serve in dev mode. -#[cfg(all(feature = "dev-mode", not(feature = "sqlite")))] +// compiles only when `sqlite` or `duckdb` is enabled (both embedded, both with an +// in-memory mode). `sqlite-memory` / `duckdb-memory` enable their base feature, +// so they are covered too; postgres, mongodb — or any future production backend — +// fail the build, so there is no path by which a production deployment can serve +// in dev mode. +#[cfg(all(feature = "dev-mode", not(any(feature = "sqlite", feature = "duckdb"))))] compile_error!( - "the `dev-mode` feature requires a dev/CI backend such as `sqlite`; it must \ - not be built with a production backend like `postgres` or `mongodb` (build \ - with `--no-default-features --features sqlite-memory,dev-mode`)" + "the `dev-mode` feature requires a dev/CI backend such as `sqlite` or `duckdb`; \ + it must not be built with a production backend like `postgres` or `mongodb` \ + (build with `--no-default-features --features sqlite-memory,dev-mode`)" ); fn main() -> anyhow::Result<()> { @@ -57,6 +68,8 @@ fn main() -> anyhow::Result<()> { extenddb_storage::set_backend(extenddb_storage_sqlite::backend())?; #[cfg(feature = "mongodb")] extenddb_storage::set_backend(extenddb_storage_mongodb::backend())?; + #[cfg(feature = "duckdb")] + extenddb_storage::set_backend(extenddb_storage_duckdb::backend())?; extenddb_app::run(extenddb_app::BuildInfo { // Read from the bin crate so the reported version is the deployed @@ -77,6 +90,8 @@ mod tests { let _ = extenddb_storage::set_backend(extenddb_storage_sqlite::backend()); #[cfg(feature = "mongodb")] let _ = extenddb_storage::set_backend(extenddb_storage_mongodb::backend()); + #[cfg(feature = "duckdb")] + let _ = extenddb_storage::set_backend(extenddb_storage_duckdb::backend()); } /// Zero-config serve contract: with the SQLite backend installed, @@ -104,6 +119,28 @@ mod tests { } } + /// Zero-config serve contract for the DuckDB backend, mirroring the SQLite + /// one: built-in defaults load with no config file, bind to loopback, and + /// select the backend's default database path. + #[cfg(feature = "duckdb")] + #[test] + fn builtin_defaults_load_for_duckdb_and_bind_loopback() { + install_backend(); + let cfg = extenddb_config::load_builtin_defaults() + .expect("duckdb storage config has no required fields"); + assert_eq!(cfg.server.bind_addr, "127.0.0.1"); + assert_eq!(cfg.server.port, 18443); + let path = cfg.storage.connection_config(); + if cfg!(feature = "duckdb-memory") { + assert_eq!(path, ":memory:"); + } else { + assert!( + path.ends_with("extenddb.duckdb"), + "default file path should be extenddb.duckdb, got: {path}" + ); + } + } + /// `load_builtin_defaults` is only reachable from dev-mode builds (a /// postgres + dev-mode binary is a compile error), but its defaults must /// never relax the production posture regardless of backend: loopback diff --git a/crates/storage-duckdb/.gitignore b/crates/storage-duckdb/.gitignore new file mode 100644 index 00000000..c9772db5 --- /dev/null +++ b/crates/storage-duckdb/.gitignore @@ -0,0 +1,4 @@ +/target +Cargo.lock +*.duckdb +*.duckdb.wal diff --git a/crates/storage-duckdb/Cargo.toml b/crates/storage-duckdb/Cargo.toml new file mode 100644 index 00000000..d4f9f271 --- /dev/null +++ b/crates/storage-duckdb/Cargo.toml @@ -0,0 +1,37 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +[package] +name = "extenddb-storage-duckdb" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[features] +# Compile the in-memory (`:memory:`) database in as the default storage path, +# yielding a zero-config, ephemeral, bootstrap-on-serve deployment (no `init`, +# no file on disk). Without this feature the backend is file-backed and the path +# must be configured as usual. +memory = [] + +[dependencies] +extenddb-core = { workspace = true } +extenddb-storage = { workspace = true } +extenddb-auth = { workspace = true } +async-trait = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +duckdb = { version = "1.10505.0", features = ["bundled", "json"] } +tokio = { workspace = true, features = ["sync"] } +toml = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } +time = { workspace = true } +base64 = { workspace = true } +bcrypt = { workspace = true } +aes-gcm = { workspace = true } +rand = { workspace = true } +bigdecimal = { workspace = true } +crc32fast = { workspace = true } +zeroize = { workspace = true } diff --git a/crates/storage-duckdb/README.md b/crates/storage-duckdb/README.md new file mode 100644 index 00000000..6bbfaffc --- /dev/null +++ b/crates/storage-duckdb/README.md @@ -0,0 +1,124 @@ +# extenddb-storage-duckdb + +DuckDB storage backend for [ExtendDB](https://github.com/ExtendDB/extenddb), an +in-tree workspace crate selected by Cargo feature. + +## Design + +Single-file (or in-memory) storage using the embedded [DuckDB](https://duckdb.org) +engine via the `duckdb` crate (statically linked; no server, no external +dependency at runtime). Targets: + +- Local development without PostgreSQL +- CI / integration tests (especially the ephemeral in-memory mode) +- Single-node, embedded, and edge deployments +- Anyone who has ever wanted to run `SELECT ... GROUP BY` directly against the + file their DynamoDB-compatible database lives in + +The crate is a port of `extenddb-storage-sqlite` — same catalog schema, same +per-table data layout, same order-preserving `N` sort-key encoding, same +persistent `gsi_pending` queue — with the storage engine swapped. The differences +that matter: + +- **No `sqlx` driver exists for DuckDB**, so `src/db.rs` provides a small + `sqlx`-shaped async facade over the synchronous `duckdb` crate: a connection + pool, positional `query` / `query_as` / `query_scalar` builders, and an + explicit-commit / rollback-on-drop `Transaction`. Every statement runs on a + `spawn_blocking` thread; the connection is moved there and back. +- **In-memory databases are shared across the pool.** All connections are + `try_clone()`s of one root connection onto the same database instance, so + `:memory:` gets a real read pool instead of SQLite's single pinned connection. +- **No foreign keys.** DuckDB cannot cascade deletes, so the catalog declares + none; `src/referential.rs` removes children explicitly inside the parent's + delete transaction and checks parents exist before child inserts. +- **MVCC inside the process.** Readers never block on the writer. Writers are + still serialized in-process by the engine's `write_lock`, which is what makes + condition-check-then-write atomic; DuckDB's optimistic conflict detection is a + backstop, not the mechanism. Across processes there is no sharing at all (see + Known limits). +- **64-bit integers everywhere.** DuckDB's `INTEGER` is 32-bit, so every catalog + integer column is `BIGINT`, and `REAL` (32-bit in DuckDB) is `DOUBLE`. + +## Building + +The backend is compiled into the `extenddb` binary via feature flags (Postgres +remains the default): + +```bash +# DuckDB only, no Postgres compiled in +cargo build -p extenddb --no-default-features --features duckdb + +# DuckDB in zero-config, ephemeral in-memory mode +cargo build -p extenddb --no-default-features --features duckdb-memory +``` + +The first build compiles DuckDB from source (`libduckdb-sys` with the `bundled` +feature). Budget several minutes and a cup of something; it is cached afterwards. + +## Configuration + +```toml +[storage] +backend = "duckdb" + +[storage.duckdb] +# Database file path, or ":memory:" for an ephemeral in-memory database. +path = "extenddb.duckdb" +# Connection pool size (writes are serialized regardless). +pool_size = 10 +``` + +`extenddb init --backend duckdb --duckdb-path ` writes the path into the +generated config file. + +### In-memory mode + +`path = ":memory:"` selects an ephemeral database that bootstraps on `serve` +(no `init`, no file on disk). The `memory` crate feature (exposed by the binary +as `duckdb-memory`) makes `:memory:` the compiled-in default path. + +## Developer mode + +`dev-mode` (plain HTTP on loopback, open authorization, seeded credential, SigV4 +still verified) builds with this backend exactly as it does with SQLite: + +```bash +cargo build -p extenddb --no-default-features --features duckdb-memory,dev-mode +extenddb serve --config extenddb.toml # bootstraps on serve; no init +``` + +## Conformance + +Per [RFC-0002](https://github.com/ExtendDB/extenddb/blob/main/docs/rfcs), this +backend implements the full storage trait surface. + +Mandatory traits: + +- `TableEngine`, `DataEngine` +- `ManagementStore`, `AdminStore`, `Bootstrapper` + +Optional traits (all implemented): + +- `MetadataEngine`, `StreamEngine`, `WorkerStore` +- `SettingsStore`, `MetricsStore`, `RateLimitStore`, `AuthorizationStore`, + `BackupEngine` + +Conformance is validated by the shared ExtendDB integration suite run against a +DuckDB-served instance (`run-integration-duckdb` in CI). + +## Known limits + +- **One process at a time.** DuckDB takes a process-exclusive lock on the database + file. While `extenddb serve` is running, no second process can open the same + file, so out-of-band tooling that connects directly (`extenddb settings get|set`, + `catalog-check`, `verify`) must run while the server is stopped or against an + `:memory:` server not at all. Everything that goes through the server's own + API is unaffected. +- **No TTL expression index.** DuckDB does not allow indexes over extension + functions, so the TTL sweep is a filtered scan of `item_data` rather than an + indexed lookup. +- **Cold builds are slow.** DuckDB is compiled from source the first time. + +## License + +Apache License 2.0 — see the workspace [LICENSE](../../LICENSE). diff --git a/crates/storage-duckdb/docs/design-decisions.md b/crates/storage-duckdb/docs/design-decisions.md new file mode 100644 index 00000000..6dad5796 --- /dev/null +++ b/crates/storage-duckdb/docs/design-decisions.md @@ -0,0 +1,154 @@ + +# extenddb-storage-duckdb — Design Decisions + +This memo records the DuckDB-specific decisions taken to reach behavioural +parity with the SQLite backend (`crates/storage-sqlite`), from which this crate +was ported, and through it with the reference PostgreSQL backend. Decisions the +SQLite backend already made and that carry over unchanged (the order-preserving +`N` sort-key TEXT encoding, the single logical database, the persistent +`gsi_pending` queue, synchronous LSI maintenance, `rowid`-based parallel scan +segments) are not repeated here; see that crate's `docs/design-decisions.md`. + +--- + +## D1 — Async execution model + +### Problem +`duckdb::Connection` is synchronous, `Send` but not `Sync`, and there is no +`sqlx` driver. The rest of the crate is async and written against `sqlx`'s +builder API (`query(...).bind(...).fetch_all(&pool)`), with ~600 call sites. + +### Options +- **(A) Call DuckDB inline on the async thread.** Simplest; blocks a tokio + worker for the duration of every statement. Point lookups are sub-millisecond + but a `Scan` or a backfill batch is not, and blocking the executor stalls + unrelated requests. +- **(B) `block_in_place`.** Panics on a current-thread runtime, which every + `#[tokio::test]` in the crate uses. +- **(C) Move the connection onto a `spawn_blocking` thread per statement.** + Each pooled connection lives in a `tokio::sync::Mutex>` + slot; a statement takes the connection out, runs on the blocking pool, and + puts it back. A transaction holds its slot for its whole lifetime so every + statement in it runs on the same connection. If the blocking task panics the + slot is left empty and re-cloned from the root connection on next use. + +### Decision +**(C)**, wrapped in `src/db.rs` as a facade that reproduces the subset of the +`sqlx` API the crate uses (`query`, `query_as`, `query_scalar`, `raw_sql`, +`Pool::begin`, `Transaction::commit`, rollback on drop, `rows_affected`). Rows +are materialised into `Vec` on the blocking side and +decoded into tuples or structs on the async side, so nothing borrowed from +DuckDB crosses a thread boundary. The port of the storage code is then mostly +`sqlx::` → `db::`. + +--- + +## D2 — One database instance per process, per file + +### Problem +DuckDB permits a single database instance per file per process and takes a +file lock. The SQLite backend opens *two* independent pools over the same file +at serve time (engine + catalog) and a third in the bootstrapper; a naive port +fails on the second `Connection::open`. + +### Decision +`db::Pool` keeps a process-wide registry of root connections keyed by canonical +path. Every pool over a path is a set of `try_clone()`s of that path's root, so +they share one instance; the catalog pool is opened as a `sibling` of the engine +pool. `drop_databases` forgets the registry entry before deleting the file so +the lock is released. In-memory databases are **not** registered: each +`Pool::open(":memory:")` is a private database (what the unit tests expect), and +connections cloned from it all see the same data — which is why the ephemeral +mode can run a real read pool where SQLite had to pin one connection. + +--- + +## D3 — Referential integrity without foreign keys + +### Problem +DuckDB accepts `FOREIGN KEY` clauses but does not implement `ON DELETE CASCADE`, +and it rejects updates to rows that a foreign key references. The SQLite catalog +relies on cascades in eleven parent-delete paths (tables → indexes / +vector_indexes / stream_shards / stream_records; accounts → every IAM table; +users → tags / access keys / memberships; groups → memberships; roles → tags / +sessions; backups → items) and on FK violations to report `NotFound` in seven +child-insert paths. + +### Options +- **(A) Keep the constraints, drop `ON DELETE CASCADE`.** Parent deletes then + fail while children exist, and every update to `tables` (status flips, item + counts) risks the referenced-row restriction. Rejected. +- **(B) Declare no foreign keys; enforce in code.** `src/referential.rs` holds + one function per parent that deletes its children, called inside the parent's + delete transaction, plus `ensure_*_exists` checks that produce the same + `NotFound` errors the constraints did. + +### Decision +**(B)**. The pool-based deletes (`delete_user`, `delete_group`, `delete_role`, +`delete_account`) become single transactions via `delete_with_children`. The +existence checks in autocommit paths are not atomic with the insert that follows; +a lost race surfaces as `NotFound` from a later read instead of from a +constraint, which is the same outcome the callers already handle. + +--- + +## D4 — Type widths and dialect + +- `INTEGER` → `BIGINT` throughout (DuckDB `INTEGER` is 32-bit; SQLite's is + 64-bit, and `table_size_bytes` alone would overflow it). `REAL` → `DOUBLE` for + the same reason (DuckDB `REAL` is `FLOAT`). +- Booleans stay 0/1 integers in the catalog; a Rust `bool` parameter binds as + `BIGINT` so `col = ?` compares integer to integer. +- `INTEGER PRIMARY KEY AUTOINCREMENT` → `BIGINT PRIMARY KEY DEFAULT + nextval('gsi_pending_seq')`. +- Timestamps: `strftime('%Y-%m-%dT%H:%M:%fZ','now')` → + `strftime(now()::TIMESTAMP, '%Y-%m-%dT%H:%M:%S.%gZ')`. Without the ICU + extension a `TIMESTAMPTZ` casts to `TIMESTAMP` as UTC, which is the stored + format. `strftime('%s','now')` → `epoch(now())`. +- `sqlite_master` → `duckdb_tables()`; `GLOB` is supported natively. +- Partial indexes (`CREATE INDEX ... WHERE`) are not supported; the two the + SQLite schema used become full indexes. +- `BEGIN IMMEDIATE` → `BEGIN TRANSACTION`. There is no reserved-lock concept; + the engine `write_lock` already serializes writers, and DuckDB's optimistic + `Conflict on update` error is classified `Transient` by `map_db_err` as a + backstop. +- The TTL sweep's `json_extract` → `json_extract_string` + `TRY_CAST` (DuckDB's + `json_extract` returns a quoted JSON scalar, and the sweep must not error on a + non-numeric TTL attribute). +- `rowid` starts at 0 in DuckDB (1 in SQLite); the vector-backfill cursor + starts at `-1`. +- Numbered parameters are `$1`, not `?1`. + +--- + +## D5 — Error classification + +`is_unique_violation` matches DuckDB's `Duplicate key ... violates primary key +constraint` / `violates unique constraint` messages. A failed statement aborts +a DuckDB transaction (unlike SQLite, where the transaction stays usable); every +path in the crate that catches a constraint error inside a transaction already +returns the error and lets the transaction drop, so the rollback-on-drop in +`db::Transaction` is what keeps pooled connections clean. + +--- + +## D6 — Out-of-process tooling and the file lock + +DuckDB takes a process-exclusive lock on a read-write database file. The +SQLite backend lets `extenddb settings`, `catalog-check`, and `verify` open the +file alongside a running server; here they must run while the server is stopped. +No workaround was attempted: a read-only open conflicts with a writer too, and +routing the CLI through the server's management API is a cross-backend change +outside this crate. Documented in the README; the integration module that +depends on it (`test_gsi_async`) is excluded from the DuckDB CI job. + +## D7 — TTL sweep without an expression index + +DuckDB rejects `CREATE INDEX ... (json_extract_string(...))` ("Cannot use +json_extract_string in this context"). Left as an error, the engine retries the +index forever and never marks `ttl_index_ready`, so nothing ever expires. The +backend therefore skips the index and flips the flag; the sweep is a filtered +scan bounded by `LIMIT`, which at the batch sizes the worker uses is acceptable. diff --git a/crates/storage-duckdb/src/admin_store.rs b/crates/storage-duckdb/src/admin_store.rs new file mode 100644 index 00000000..c062fa16 --- /dev/null +++ b/crates/storage-duckdb/src/admin_store.rs @@ -0,0 +1,126 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `AdminStore` implementation: admin-user management, separate from IAM users. + +use crate::db; +use extenddb_storage::management_store::{AdminEntry, AdminStore, OpError, OpResult}; +use futures::future::BoxFuture; + +use crate::catalog_store::DuckDbCatalogStore; +use crate::duckdb_util::{is_unique_violation, parse_timestamp}; + +impl AdminStore for DuckDbCatalogStore { + fn create_admin(&self, admin_name: &str, password_hash: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + db::query("INSERT INTO admin_users (admin_name, password_hash) VALUES (?, ?)") + .bind(&admin_name) + .bind(&password_hash) + .execute(self.pool()) + .await + .map_err(|e| { + if is_unique_violation(&e) { + OpError::AlreadyExists("Admin user already exists".to_owned()) + } else { + tracing::error!("create_admin: {e}"); + OpError::Internal("Database error".to_owned()) + } + })?; + Ok(()) + }) + } + + fn list_admins(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async move { + let rows: Vec<(String, String)> = + db::query_as("SELECT admin_name, created_at FROM admin_users ORDER BY admin_name") + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_admins: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + rows.into_iter() + .map(|(admin_name, created)| { + Ok(AdminEntry { + admin_name, + created_at: parse_timestamp(&created) + .map_err(|e| OpError::Internal(format!("parse created_at: {e}")))?, + }) + }) + .collect() + }) + } + + fn delete_admin(&self, admin_name: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + Box::pin(async move { + let result = db::query("DELETE FROM admin_users WHERE admin_name = ?") + .bind(&admin_name) + .execute(self.pool()) + .await + .map_err(|e| { + tracing::error!("delete_admin: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.rows_affected() == 0 { + return Err(OpError::NotFound("Admin user not found".to_owned())); + } + Ok(()) + }) + } + + fn change_admin_password( + &self, + admin_name: &str, + password_hash: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + let result = db::query("UPDATE admin_users SET password_hash = ? WHERE admin_name = ?") + .bind(&password_hash) + .bind(&admin_name) + .execute(self.pool()) + .await + .map_err(|e| { + tracing::error!("change_admin_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.rows_affected() == 0 { + return Err(OpError::NotFound("Admin user not found".to_owned())); + } + Ok(()) + }) + } + + fn verify_admin_password( + &self, + admin_name: &str, + password: &str, + ) -> BoxFuture<'_, OpResult>> { + let admin_name = admin_name.to_owned(); + let password = password.to_owned(); + Box::pin(async move { + let row: Option<(String,)> = + db::query_as("SELECT password_hash FROM admin_users WHERE admin_name = ?") + .bind(&admin_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("verify_admin_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let Some((hash,)) = row else { + return Ok(None); + }; + let verified = tokio::task::spawn_blocking(move || bcrypt::verify(&password, &hash)) + .await + .map_err(|e| OpError::Internal(format!("bcrypt task: {e}")))? + .unwrap_or(false); + Ok(Some(verified)) + }) + } +} diff --git a/crates/storage-duckdb/src/authorization_store.rs b/crates/storage-duckdb/src/authorization_store.rs new file mode 100644 index 00000000..13b6d3fb --- /dev/null +++ b/crates/storage-duckdb/src/authorization_store.rs @@ -0,0 +1,253 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `AuthorizationStore` implementation: read-only IAM policy, boundary, +//! session, and tag lookups used by the policy evaluator on every authorized +//! request. Policy documents are stored as JSON text and returned verbatim. + +use crate::db; +use extenddb_storage::authorization_store::{AuthorizationStore, SessionData}; +use extenddb_storage::management_store::{OpError, OpResult}; +use futures::future::BoxFuture; + +use crate::catalog_store::DuckDbCatalogStore; +use crate::duckdb_util::parse_timestamp; + +/// Map any query error to a sanitized internal error, logging the detail. +fn db_err(ctx: &str, e: db::Error) -> OpError { + tracing::error!("{ctx}: {e}"); + OpError::Internal("Database error".to_owned()) +} + +impl AuthorizationStore for DuckDbCatalogStore { + fn fetch_user_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let rows: Vec<(String,)> = db::query_as( + "SELECT policy_document FROM iam_policies \ + WHERE account_id = ? AND principal_type = 'user' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_user_policies", e))?; + Ok(rows.into_iter().map(|(d,)| d).collect()) + }) + } + + fn fetch_user_group_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let rows: Vec<(String,)> = db::query_as( + "SELECT p.policy_document FROM iam_policies p \ + JOIN iam_group_members m \ + ON m.account_id = p.account_id AND m.group_name = p.principal_name \ + WHERE p.account_id = ? AND p.principal_type = 'group' AND m.user_name = ?", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_user_group_policies", e))?; + Ok(rows.into_iter().map(|(d,)| d).collect()) + }) + } + + fn fetch_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let row: Option<(String,)> = db::query_as( + "SELECT policy_document FROM iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = 'user' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| db_err("fetch_user_boundary", e))?; + Ok(row.map(|(d,)| d)) + }) + } + + fn fetch_role_policies( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let rows: Vec<(String,)> = db::query_as( + "SELECT policy_document FROM iam_policies \ + WHERE account_id = ? AND principal_type = 'role' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&role_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_role_policies", e))?; + Ok(rows.into_iter().map(|(d,)| d).collect()) + }) + } + + fn fetch_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let row: Option<(String,)> = db::query_as( + "SELECT policy_document FROM iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = 'role' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&role_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| db_err("fetch_role_boundary", e))?; + Ok(row.map(|(d,)| d)) + }) + } + + fn fetch_session_data( + &self, + account_id: &str, + role_name: &str, + session_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let session_name = session_name.to_owned(); + Box::pin(async move { + let row: Option<(Option, Option, String)> = db::query_as( + "SELECT session_policy, session_tags, expires_at FROM iam_sessions \ + WHERE account_id = ? AND role_name = ? AND session_name = ?", + ) + .bind(&account_id) + .bind(&role_name) + .bind(&session_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| db_err("fetch_session_data", e))?; + + let Some((session_policy, tags_json, expires_at)) = row else { + return Ok(None); + }; + + // Treat an expired session as absent (parity with the Postgres + // expiry filter). Authentication already rejects expired sessions, + // so this is defense-in-depth; evaluate it with the same parser + // lookup_session uses rather than a format-fragile SQL comparison. + if let Ok(expires) = parse_timestamp(&expires_at) + && expires < time::OffsetDateTime::now_utc() + { + return Ok(None); + } + + // session_tags is stored as JSON: either an object {key: value} or + // an array [{"Key": ..., "Value": ...}] (the form AWS SDKs send). + // Handle both, matching the Postgres backend. + let session_tags: Vec<(String, String)> = tags_json + .as_deref() + .and_then(|s| serde_json::from_str::(s).ok()) + .map(|v| { + if let Some(arr) = v.as_array() { + arr.iter() + .filter_map(|tag| { + match ( + tag.get("Key").and_then(|k| k.as_str()), + tag.get("Value").and_then(|x| x.as_str()), + ) { + (Some(k), Some(val)) => Some((k.to_owned(), val.to_owned())), + _ => None, + } + }) + .collect() + } else if let Some(obj) = v.as_object() { + obj.iter() + .filter_map(|(k, val)| val.as_str().map(|s| (k.clone(), s.to_owned()))) + .collect() + } else { + Vec::new() + } + }) + .unwrap_or_default(); + + Ok(Some(SessionData { + session_policy, + session_tags, + })) + }) + } + + fn fetch_user_tags( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + db::query_as( + "SELECT tag_key, tag_value FROM iam_user_tags \ + WHERE account_id = ? AND user_name = ? ORDER BY tag_key", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_user_tags", e)) + }) + } + + fn fetch_role_tags( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + db::query_as( + "SELECT tag_key, tag_value FROM iam_role_tags \ + WHERE account_id = ? AND role_name = ? ORDER BY tag_key", + ) + .bind(&account_id) + .bind(&role_name) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_role_tags", e)) + }) + } + + fn fetch_resource_tags(&self, arn: &str) -> BoxFuture<'_, OpResult>> { + let arn = arn.to_owned(); + Box::pin(async move { + db::query_as( + "SELECT tag_key, tag_value FROM tags WHERE resource_arn = ? ORDER BY tag_key", + ) + .bind(&arn) + .fetch_all(self.pool()) + .await + .map_err(|e| db_err("fetch_resource_tags", e)) + }) + } +} diff --git a/crates/storage-duckdb/src/backup.rs b/crates/storage-duckdb/src/backup.rs new file mode 100644 index 00000000..447f33d9 --- /dev/null +++ b/crates/storage-duckdb/src/backup.rs @@ -0,0 +1,527 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `BackupEngine` for the DuckDB backend. +//! +//! A backup snapshots every item's `item_data` into `backup_items`. Restore +//! recreates the table via `create_table` and upserts the snapshot under the +//! engine write lock. `RestoreTableToPointInTime` is implemented as a +//! snapshot-then-restore (then discard the temporary backup), matching the +//! PostgreSQL backend's behaviour. + +use crate::db; +use extenddb_core::types::{ + AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, BillingMode, + ContinuousBackupsDescription, CreateTableInput, KeySchemaElement, + PointInTimeRecoveryDescription, ProvisionedThroughput, SourceTableDetails, TableDescription, + TableKeyInfo, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{BackupEngine, TableEngine}; +use futures::future::BoxFuture; + +use crate::data::{data_table_name, upsert_item_in_tx}; +use crate::duckdb_util::parse_timestamp; +use crate::store::DuckDbEngine; + +fn epoch_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +/// Backup id: creation timestamp plus an 8-hex random suffix, matching the +/// PostgreSQL backend. The suffix makes ARNs non-guessable and prevents two +/// backups created in the same millisecond from colliding. +fn backup_id() -> String { + use rand::Rng; + let suffix: u32 = rand::rng().random(); + format!("{ts}-{suffix:08x}", ts = epoch_millis()) +} + +#[allow(clippy::cast_precision_loss)] +fn ts_to_epoch(s: &str) -> f64 { + parse_timestamp(s) + .map(|dt| dt.unix_timestamp() as f64) + .unwrap_or(0.0) +} + +impl BackupEngine for DuckDbEngine { + fn create_backup( + &self, + account_id: &str, + table_name: &str, + backup_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + let backup_name = backup_name.to_owned(); + Box::pin(async move { + let row: Option<(String, String, String, String, i64)> = db::query_as( + "SELECT table_id, key_schema, attribute_definitions, billing_mode, table_size_bytes \ + FROM tables WHERE account_id = ? AND table_name = ? AND table_status = 'ACTIVE'", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id, key_schema, attr_defs, billing_mode, size_bytes) = + row.ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let backup_arn = format!( + "arn:aws:dynamodb:{region}:{account_id}:table/{table_name}/backup/{id}", + region = self.region, + id = backup_id() + ); + + let ddb_table = data_table_name(&table_id); + let items: Vec<(String,)> = db::query_as(&format!("SELECT item_data FROM {ddb_table}")) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let item_count = i64::try_from(items.len()).unwrap_or(i64::MAX); + + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + db::query( + "INSERT INTO backups (backup_arn, backup_name, table_id, table_name, account_id, \ + backup_status, backup_size_bytes, item_count, key_schema, attribute_definitions, \ + billing_mode) VALUES (?, ?, ?, ?, ?, 'AVAILABLE', ?, ?, ?, ?, ?)", + ) + .bind(&backup_arn) + .bind(&backup_name) + .bind(&table_id) + .bind(&table_name) + .bind(&account_id) + .bind(size_bytes) + .bind(item_count) + .bind(&key_schema) + .bind(&attr_defs) + .bind(&billing_mode) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + for (item_data,) in &items { + db::query( + "INSERT INTO backup_items (backup_arn, pk, sk, item_data) VALUES (?, '', NULL, ?)", + ) + .bind(&backup_arn) + .bind(item_data) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + let created_at: (String,) = + db::query_as("SELECT created_at FROM backups WHERE backup_arn = ?") + .bind(&backup_arn) + .fetch_one(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDetails { + backup_arn, + backup_name, + backup_status: "AVAILABLE".to_owned(), + backup_type: "USER".to_owned(), + backup_size_bytes: size_bytes, + backup_creation_date_time: ts_to_epoch(&created_at.0), + }) + }) + } + + fn describe_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let backup_arn = backup_arn.to_owned(); + Box::pin(async move { + #[allow(clippy::type_complexity)] + let row: Option<(String, String, String, String, i64, i64, String, String, String, String)> = + db::query_as( + "SELECT b.backup_name, b.backup_status, b.table_id, b.table_name, \ + b.backup_size_bytes, b.item_count, b.key_schema, b.billing_mode, \ + COALESCE(t.table_arn, \ + 'arn:aws:dynamodb:' || ? || ':' || b.account_id || ':table/' || b.table_name), \ + b.created_at \ + FROM backups b LEFT JOIN tables t ON t.table_id = b.table_id \ + WHERE b.backup_arn = ? AND b.account_id = ? \ + AND b.backup_status != 'DELETED'", + ) + .bind(&self.region) + .bind(&backup_arn) + .bind(&account_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (name, status, table_id, table_name, size, count, ks, billing, table_arn, created) = + row.ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + + let key_schema: Vec = + serde_json::from_str(&ks).map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_arn: backup_arn.clone(), + backup_name: name, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: ts_to_epoch(&created), + }, + source_table_details: SourceTableDetails { + table_name, + table_id, + table_arn, + key_schema, + item_count: count, + table_size_bytes: size, + billing_mode: Some(billing), + table_creation_date_time: ts_to_epoch(&created), + }, + }) + }) + } + + fn list_backups( + &self, + account_id: &str, + table_name: Option<&str>, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_owned(); + let table_name = table_name.map(str::to_owned); + Box::pin(async move { + let rows: Vec<(String, String, String, String, i64, String, String)> = + if let Some(tn) = table_name { + db::query_as( + "SELECT b.backup_arn, b.backup_name, b.table_name, b.backup_status, \ + b.backup_size_bytes, \ + COALESCE(t.table_arn, 'arn:aws:dynamodb:' || ? || ':' || b.account_id || ':table/' || b.table_name), \ + b.created_at FROM backups b LEFT JOIN tables t ON t.table_id = b.table_id \ + WHERE b.account_id = ? AND b.table_name = ? AND b.backup_status != 'DELETED' \ + ORDER BY b.created_at DESC", + ) + .bind(&self.region) + .bind(&account_id) + .bind(tn) + .fetch_all(&self.pool) + .await + } else { + db::query_as( + "SELECT b.backup_arn, b.backup_name, b.table_name, b.backup_status, \ + b.backup_size_bytes, \ + COALESCE(t.table_arn, 'arn:aws:dynamodb:' || ? || ':' || b.account_id || ':table/' || b.table_name), \ + b.created_at FROM backups b LEFT JOIN tables t ON t.table_id = b.table_id \ + WHERE b.account_id = ? AND b.backup_status != 'DELETED' \ + ORDER BY b.created_at DESC", + ) + .bind(&self.region) + .bind(&account_id) + .fetch_all(&self.pool) + .await + } + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(rows + .into_iter() + .map( + |(arn, name, tn, status, size, table_arn, created)| BackupSummary { + backup_arn: arn, + backup_name: name, + table_name: tn, + table_arn, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: ts_to_epoch(&created), + }, + ) + .collect()) + }) + } + + fn delete_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let backup_arn = backup_arn.to_owned(); + Box::pin(async move { + // Resolves account-scoped, so a backup owned by another account is + // reported missing here and the writes below never run. + let desc = self.describe_backup(&account_id, &backup_arn).await?; + + // The account predicate is repeated on both writes rather than + // relying on the lookup above, so the statements are correct on + // their own terms. + db::query( + "DELETE FROM backup_items WHERE backup_arn = $1 AND EXISTS (\ + SELECT 1 FROM backups b WHERE b.backup_arn = $1 AND b.account_id = $2)", + ) + .bind(&backup_arn) + .bind(&account_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + db::query( + "UPDATE backups SET backup_status = 'DELETED' \ + WHERE backup_arn = ? AND account_id = ?", + ) + .bind(&backup_arn) + .bind(&account_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(BackupDescription { + backup_details: BackupDetails { + backup_status: "DELETED".to_owned(), + ..desc.backup_details + }, + source_table_details: desc.source_table_details, + }) + }) + } + + fn restore_table_from_backup( + &self, + account_id: &str, + target_table_name: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let target_table_name = target_table_name.to_owned(); + let backup_arn = backup_arn.to_owned(); + Box::pin(async move { + let row: Option<(String, String, String)> = db::query_as( + "SELECT key_schema, attribute_definitions, billing_mode \ + FROM backups WHERE backup_arn = ? AND account_id = ? \ + AND backup_status = 'AVAILABLE'", + ) + .bind(&backup_arn) + .bind(&account_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (ks, ad, billing) = row.ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + let key_schema: Vec = + serde_json::from_str(&ks).map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs: Vec = + serde_json::from_str(&ad).map_err(|e| StorageError::Internal(e.to_string()))?; + let billing_mode = Some(if billing == "PAY_PER_REQUEST" { + BillingMode::PayPerRequest + } else { + BillingMode::Provisioned + }); + + let create_input = CreateTableInput { + table_name: target_table_name.clone(), + key_schema: key_schema.clone(), + attribute_definitions: attr_defs.clone(), + billing_mode, + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + global_secondary_indexes: None, + local_secondary_indexes: None, + stream_specification: None, + tags: None, + deletion_protection_enabled: None, + sse_specification: None, + table_class: None, + on_demand_throughput: None, + ..Default::default() + }; + + let desc = self.create_table(&account_id, create_input).await?; + let key_info = TableKeyInfo { + table_name: target_table_name.clone(), + account_id: account_id.clone(), + table_id: desc.table_id.clone(), + base_key_schema: key_schema.clone(), + key_schema, + attribute_definitions: attr_defs, + has_lsi: false, + // The restored table is created above without secondary + // indexes, so there is no index metadata to carry. + global_secondary_indexes: Vec::new(), + local_secondary_indexes: Vec::new(), + stream_specification: None, + ..Default::default() + }; + + let items: Vec<(String,)> = + db::query_as("SELECT item_data FROM backup_items WHERE backup_arn = ?") + .bind(&backup_arn) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let item_count = i64::try_from(items.len()).unwrap_or(i64::MAX); + + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + for (item_json,) in &items { + let item: extenddb_core::types::Item = serde_json::from_str(item_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + upsert_item_in_tx(&mut tx, &key_info, &item).await?; + } + db::query("UPDATE tables SET item_count = ? WHERE account_id = ? AND table_name = ?") + .bind(item_count) + .bind(&account_id) + .bind(&target_table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + // Mark the restored table ACTIVE immediately: the data is fully + // populated and ready to serve. This matches the Postgres backend + // and real DynamoDB, where a restored table becomes ACTIVE once the + // restore completes (the CREATING status is transient) rather than + // waiting for the control-plane transition poller. + db::query( + "UPDATE tables SET table_status = 'ACTIVE', status_transition_at = NULL \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&target_table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(desc) + }) + } + + fn describe_continuous_backups( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + Box::pin(async move { + let exists: bool = db::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables WHERE account_id = ? AND table_name = ?)", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if !exists { + return Err(StorageError::TableNotFound(table_name)); + } + + let pitr: Option<(bool,)> = db::query_as( + "SELECT pitr_enabled FROM continuous_backups WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let enabled = pitr.is_some_and(|r| r.0); + + #[allow(clippy::cast_precision_loss)] + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as f64; + + Ok(ContinuousBackupsDescription { + continuous_backups_status: "ENABLED".to_owned(), + point_in_time_recovery_description: Some(PointInTimeRecoveryDescription { + point_in_time_recovery_status: if enabled { "ENABLED" } else { "DISABLED" } + .to_owned(), + earliest_restorable_date_time: enabled.then_some(now - 35.0 * 24.0 * 3600.0), + latest_restorable_date_time: enabled.then_some(now), + }), + }) + }) + } + + fn update_continuous_backups( + &self, + account_id: &str, + table_name: &str, + pitr_enabled: bool, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + Box::pin(async move { + let exists: bool = db::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables WHERE account_id = ? AND table_name = ?)", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if !exists { + return Err(StorageError::TableNotFound(table_name)); + } + db::query( + "INSERT INTO continuous_backups (account_id, table_name, pitr_enabled) \ + VALUES (?, ?, ?) \ + ON CONFLICT (account_id, table_name) DO UPDATE SET pitr_enabled = excluded.pitr_enabled", + ) + .bind(&account_id) + .bind(&table_name) + .bind(pitr_enabled) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + self.describe_continuous_backups(&account_id, &table_name) + .await + }) + } + + fn restore_table_to_point_in_time( + &self, + account_id: &str, + source_table_name: &str, + target_table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let source_table_name = source_table_name.to_owned(); + let target_table_name = target_table_name.to_owned(); + Box::pin(async move { + let backup = self + .create_backup(&account_id, &source_table_name, "__pitr_restore__") + .await?; + let desc = self + .restore_table_from_backup(&account_id, &target_table_name, &backup.backup_arn) + .await?; + let _ = self.delete_backup(&account_id, &backup.backup_arn).await; + Ok(desc) + }) + } +} diff --git a/crates/storage-duckdb/src/bootstrapper.rs b/crates/storage-duckdb/src/bootstrapper.rs new file mode 100644 index 00000000..178ce76d --- /dev/null +++ b/crates/storage-duckdb/src/bootstrapper.rs @@ -0,0 +1,502 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `Bootstrapper` implementation for the DuckDB backend. +//! +//! DuckDB has no server, users, or roles, and catalog and data live in one +//! file. Initialization therefore reduces to: create the file, apply the +//! catalog schema, and seed the encryption key, default account, and admin +//! user. Destruction removes the database file and its WAL/SHM sidecars. + +use crate::db; +use async_trait::async_trait; +use extenddb_core::types::{AttributeDefinition, KeySchemaElement, KeyType}; +use extenddb_storage::bootstrapper::{ + AdminBootstrapResult, Bootstrapper, KeyDefinitionRepair, helpers, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::management_store::{OpError, OpResult}; +use extenddb_storage::util::recover_sort_key_definitions; + +use crate::duckdb_util::duckdb_path; +use crate::schema::{self, CATALOG_VERSION}; + +/// DuckDB backend bootstrapper. +/// +/// Holds the database file path. Each operation opens a short-lived +/// single-connection pool, since `init`/`destroy`/`migrate` are one-shot CLI +/// paths, not the hot serving path. +pub struct DuckDbBootstrapper { + path: String, +} + +impl DuckDbBootstrapper { + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + /// Build a `DuckDbBootstrapper` from the config file and CLI args. + /// + /// Resolution order for the database path (bootstrapper commands: init, + /// destroy, migrate): the `--duckdb-path

` CLI flag (declared on + /// `InitArgs`), then `[storage.duckdb].path` in the config file, then the + /// default `extenddb.duckdb`. `serve` does not use this path; it reads + /// the config (with `EXTENDDB__STORAGE__SQLITE__PATH` overriding). + pub async fn from_config(config_path: &str, cli_args: &[String]) -> Result { + if let Some(p) = helpers::extract_arg(cli_args, "--duckdb-path") { + return Ok(Self::new(p)); + } + + let path = if std::path::Path::new(config_path).exists() { + let content = std::fs::read_to_string(config_path) + .map_err(|e| StorageError::Internal(format!("read config {config_path}: {e}")))?; + let parsed: toml::Value = toml::from_str(&content) + .map_err(|e| StorageError::Internal(format!("parse config {config_path}: {e}")))?; + parsed + .get("storage") + .and_then(|s| s.get("duckdb")) + .and_then(|s| s.get("path")) + .and_then(|v| v.as_str()) + .unwrap_or("extenddb.duckdb") + .to_owned() + } else { + "extenddb.duckdb".to_owned() + }; + + Ok(Self::new(path)) + } + + fn is_memory(&self) -> bool { + self.path == ":memory:" || self.path.starts_with("file::memory:") + } + + /// Open a short-lived single-connection pool with the standard PRAGMAs. + /// For a real filesystem path, the parent directory is created first: + /// `init --duckdb-path` may point into a directory that does not exist + /// yet, and bootstrapping it is exactly init's job. + async fn pool(&self) -> OpResult { + if self.path != ":memory:" + && let Some(parent) = std::path::Path::new(&self.path).parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).map_err(|e| { + OpError::Internal(format!( + "create parent directory for DuckDB database '{}': {e}", + self.path + )) + })?; + } + let pool = db::Pool::open(&duckdb_path(&self.path), 1) + .await + .map_err(|e| OpError::Internal(format!("open DuckDB database '{}': {e}", self.path)))?; + // The database stores the encryption key next to the secrets it + // protects; keep the file and its WAL owner-only from the moment they + // exist rather than waiting for the first `serve`. + #[cfg(unix)] + if !self.is_memory() { + use std::os::unix::fs::PermissionsExt; + for suffix in ["", ".wal"] { + let f = format!("{}{suffix}", self.path); + if std::path::Path::new(&f).exists() { + let _ = std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o600)); + } + } + } + Ok(pool) + } +} + +#[async_trait] +impl Bootstrapper for DuckDbBootstrapper { + async fn ensure_app_user(&self) -> OpResult<()> { + Ok(()) // DuckDB has no user concept. + } + + async fn grant_app_role_to_admin(&self) -> OpResult<()> { + Ok(()) // DuckDB has no role concept. + } + + async fn create_catalog_db(&self) -> OpResult<()> { + // Connecting with mode=rwc creates the file; refuse to clobber an + // existing database so `init` is not silently destructive. + if !self.is_memory() && std::path::Path::new(&self.path).exists() { + return Err(OpError::AlreadyExists(format!( + "DuckDB database '{}' already exists. Run 'destroy' first, then 'init'.", + self.path + ))); + } + Ok(()) + } + + async fn create_data_db(&self) -> OpResult<()> { + Ok(()) // Catalog and data share one file. + } + + async fn run_catalog_migrations(&self) -> OpResult<()> { + let pool = self.pool().await?; + schema::apply(&pool).await + } + + async fn run_data_migrations(&self) -> OpResult<()> { + Ok(()) // Single file — schema applied in run_catalog_migrations. + } + + async fn pending_data_migrations(&self) -> OpResult> { + // DuckDB applies its complete schema in run_catalog_migrations (a single + // file), so there are no separately-tracked data migrations that can be + // pending. + Ok(Vec::new()) + } + + async fn record_data_connection(&self) -> OpResult<()> { + let pool = self.pool().await?; + db::query( + "INSERT INTO settings (key, value) VALUES ('data_database_name', ?) \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ) + .bind(&self.path) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("record data db name: {e}")))?; + Ok(()) + } + + async fn bootstrap_encryption_key(&self) -> OpResult<()> { + let pool = self.pool().await?; + let exists: bool = + db::query_scalar("SELECT EXISTS(SELECT 1 FROM settings WHERE key = 'encryption_key')") + .fetch_one(&pool) + .await + .map_err(|e| OpError::Internal(format!("check encryption key: {e}")))?; + if exists { + return Ok(()); + } + let key = helpers::generate_encryption_key(); + db::query("INSERT OR IGNORE INTO settings (key, value) VALUES ('encryption_key', ?)") + .bind(&key) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("store encryption key: {e}")))?; + Ok(()) + } + + async fn bootstrap_default_account(&self) -> OpResult<()> { + let pool = self.pool().await?; + // Reuse the existing account when already bootstrapped, otherwise create + // the single default account. Either way, record its id as the canonical + // default account so callers never infer it from list ordering. The + // settings write is idempotent (INSERT OR IGNORE), so it also backfills + // the marker for catalogs bootstrapped before it existed. + let account_id: String = match db::query_scalar( + "SELECT account_id FROM accounts ORDER BY account_id LIMIT 1", + ) + .fetch_optional(&pool) + .await + .map_err(|e| OpError::Internal(format!("check accounts: {e}")))? + { + Some(id) => id, + None => { + let id = helpers::generate_account_id(); + db::query( + "INSERT OR IGNORE INTO accounts (account_id, account_name) VALUES (?, 'default')", + ) + .bind(&id) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("create default account: {e}")))?; + id + } + }; + db::query("INSERT OR IGNORE INTO settings (key, value) VALUES ('default_account_id', ?)") + .bind(&account_id) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("record default account id: {e}")))?; + Ok(()) + } + + async fn bootstrap_admin_user( + &self, + env_user: Option<&str>, + env_password: Option<&str>, + ) -> OpResult { + let pool = self.pool().await?; + let username = env_user + .filter(|s| !s.is_empty()) + .unwrap_or("admin") + .to_owned(); + + let exists: bool = + db::query_scalar("SELECT EXISTS(SELECT 1 FROM admin_users WHERE admin_name = ?)") + .bind(&username) + .fetch_one(&pool) + .await + .map_err(|e| OpError::Internal(format!("check admin user: {e}")))?; + if exists { + return Ok(AdminBootstrapResult { + username, + generated_password: None, + already_existed: true, + from_env: env_user.is_some(), + }); + } + + let (password, from_env) = match env_password.filter(|s| !s.is_empty()) { + Some(p) => (p.to_owned(), true), + None => (helpers::generate_random_password(), false), + }; + let password_hash = helpers::hash_password_async(password.clone()).await?; + + db::query("INSERT INTO admin_users (admin_name, password_hash) VALUES (?, ?)") + .bind(&username) + .bind(&password_hash) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("create admin user: {e}")))?; + + Ok(AdminBootstrapResult { + username, + generated_password: if from_env { None } else { Some(password) }, + already_existed: false, + from_env, + }) + } + + async fn is_catalog_initialized(&self) -> OpResult { + if !self.is_memory() && !std::path::Path::new(&self.path).exists() { + return Ok(false); + } + let Ok(pool) = self.pool().await else { + return Ok(false); + }; + schema::table_exists(&pool, "settings").await + } + + /// Repair table metadata damaged by the pre-fix `UpdateTable` (#259). + /// + /// DuckDB keeps the catalog and the data tables in one file, so the physical + /// sort key columns are read with `PRAGMA table_info`. Otherwise identical to + /// the PostgreSQL implementation: for any base sort key with no attribute + /// definition, recover the type from its column name. + async fn repair_lost_sort_key_definitions(&self, apply: bool) -> OpResult { + let mut report = KeyDefinitionRepair::default(); + let Ok(pool) = self.pool().await else { + return Ok(report); + }; + // A never-initialised catalog has no `tables` table, and `pool()` opens + // with mode=rwc, which silently creates an empty database file, so the + // SELECT below would hard-error rather than find nothing. PostgreSQL + // degrades gracefully through its get_data_db_name() guard; this is the + // DuckDB equivalent. + if !schema::table_exists(&pool, "tables").await? { + return Ok(report); + } + + let rows: Vec<(String, String, String, String, String)> = db::query_as( + "SELECT account_id, table_name, table_id, key_schema, attribute_definitions \ + FROM tables ORDER BY table_name", + ) + .fetch_all(&pool) + .await + .map_err(|e| OpError::Internal(format!("Cannot read tables: {e}")))?; + + for (account_id, table_name, table_id, ks_json, ad_json) in rows { + let key_schema: Vec = match serde_json::from_str(&ks_json) { + Ok(v) => v, + Err(e) => { + report + .needs_attention + .push(format!("{table_name}: unreadable key_schema ({e})")); + continue; + } + }; + let attr_defs: Vec = + serde_json::from_str(&ad_json).unwrap_or_default(); + + // Reported for every table on every run, not only when a sort key is + // repaired: the partition key definition is dropped by the same write and + // is not recoverable from the schema, because the pk column is always TEXT. + // It is not needed for correctness either, since partition key values are + // encoded from the key schema alone, so it is reported rather than guessed + // and keeps being reported until a human restores it. + let pk_missing: Vec<&str> = key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Hash) + .filter(|ks| { + !attr_defs + .iter() + .any(|ad| ad.attribute_name == ks.attribute_name) + }) + .map(|ks| ks.attribute_name.as_str()) + .collect(); + if !pk_missing.is_empty() { + report.needs_attention.push(format!( + "{table_name}: partition key definition(s) [{}] are absent; reads and \ + writes are unaffected, but index key type validation cannot check them", + pk_missing.join(", ") + )); + } + + let missing: Vec<&KeySchemaElement> = key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Range) + .filter(|ks| { + !attr_defs + .iter() + .any(|ad| ad.attribute_name == ks.attribute_name) + }) + .collect(); + if missing.is_empty() { + continue; + } + + // PRIMARY KEY columns only, in key order. Every data table carries all + // three typed columns for each sort key position (sk_s, sk_n, sk_b), so + // column existence says nothing about the declared type: only the pk + // position reported by table_info does. `table_id` is interpolated + // because PRAGMA takes no bind parameters; it is checked as a UUID first + // so a malformed catalog row cannot reach the statement. + if uuid::Uuid::parse_str(&table_id).is_err() { + report + .needs_attention + .push(format!("{table_name}: table_id {table_id:?} is not a UUID")); + continue; + } + let mut key_columns: Vec<(i64, String)> = + db::query_as::<(i64, String, String, i64, Option, i64)>(&format!( + "PRAGMA table_info(\"_ddb_{table_id}\")" + )) + .fetch_all(&pool) + .await + .map_err(|e| { + OpError::Internal(format!("Cannot read columns of {table_name}: {e}")) + })? + .into_iter() + .filter(|(.., pk_pos)| *pk_pos > 0) + .map(|(_, name, _, _, _, pk_pos)| (pk_pos, name)) + .collect(); + key_columns.sort_unstable(); + let columns: Vec = key_columns.into_iter().map(|(_, name)| name).collect(); + + let recovered = recover_sort_key_definitions(&key_schema, &attr_defs, &columns); + if recovered.is_empty() { + report.needs_attention.push(format!( + "{table_name}: sort key(s) [{}] have no attribute definition and no \ + matching PRIMARY KEY column was found to recover the type from", + missing + .iter() + .map(|ks| ks.attribute_name.as_str()) + .collect::>() + .join(", ") + )); + continue; + } + + let mut merged = attr_defs; + merged.extend(recovered.iter().cloned()); + let merged_json = serde_json::to_string(&merged) + .map_err(|e| OpError::Internal(format!("Cannot serialize definitions: {e}")))?; + if apply { + db::query( + "UPDATE tables SET attribute_definitions = $1 \ + WHERE account_id = $2 AND table_name = $3", + ) + .bind(&merged_json) + .bind(&account_id) + .bind(&table_name) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("Cannot repair {table_name}: {e}")))?; + } + + for def in &recovered { + report.repaired.push(format!( + "{table_name}: {} ({:?})", + def.attribute_name, def.attribute_type + )); + } + } + + Ok(report) + } + + async fn list_table_names(&self) -> OpResult> { + let Ok(pool) = self.pool().await else { + return Ok(Vec::new()); + }; + let rows: Vec<(String,)> = + db::query_as("SELECT table_name FROM tables ORDER BY table_name") + .fetch_all(&pool) + .await + .unwrap_or_default(); + Ok(rows.into_iter().map(|(n,)| n).collect()) + } + + async fn get_data_db_name(&self) -> OpResult> { + let Ok(pool) = self.pool().await else { + return Ok(None); + }; + let row: Option<(String,)> = + db::query_as("SELECT value FROM settings WHERE key = 'data_database_name'") + .fetch_optional(&pool) + .await + .unwrap_or(None); + Ok(row.map(|(v,)| v)) + } + + async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { + if self.is_memory() { + return Ok(()); + } + // Release this process's handle on the database first: DuckDB holds a + // file lock for as long as an instance is open. + db::forget_path(&self.path); + if std::path::Path::new(&self.path).exists() { + std::fs::remove_file(&self.path) + .map_err(|e| OpError::Internal(format!("remove database file: {e}")))?; + } + // Remove the write-ahead log sidecar if present. + let _ = std::fs::remove_file(format!("{}.wal", self.path)); + Ok(()) + } + + async fn read_catalog_version(&self) -> OpResult> { + let Ok(pool) = self.pool().await else { + return Ok(None); + }; + if !schema::table_exists(&pool, "settings").await? { + return Ok(None); + } + let row: Option<(String,)> = + db::query_as("SELECT value FROM settings WHERE key = 'catalog_version'") + .fetch_optional(&pool) + .await + .map_err(|e| OpError::Internal(format!("read catalog version: {e}")))?; + Ok(row.map(|(v,)| v)) + } + + fn expected_catalog_version(&self) -> String { + CATALOG_VERSION.to_string() + } + + fn catalog_database_name(&self) -> String { + self.path.clone() + } + + fn endpoint_info(&self) -> String { + format!("duckdb:{}", self.path) + } + + fn catalog_connection_url(&self) -> String { + duckdb_path(&self.path) + } + + fn generate_backend_config_section(&self) -> String { + format!( + "[storage.duckdb]\n\ + path = \"{}\"\n\ + # pool_size = 10", + self.path + ) + } +} diff --git a/crates/storage-duckdb/src/catalog_store.rs b/crates/storage-duckdb/src/catalog_store.rs new file mode 100644 index 00000000..99cf8a37 --- /dev/null +++ b/crates/storage-duckdb/src/catalog_store.rs @@ -0,0 +1,367 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `DuckDbCatalogStore`: the catalog/management store backing the management +//! API and authorization. Implements `SettingsStore`, `MetricsStore`, +//! `RateLimitStore`, `DiagnosticsStore`, `AdminStore` (in `admin_store`), +//! `AuthorizationStore` (in `authorization_store`), `ManagementStore` (in the +//! `management_store` module), and the `CatalogStore` supertrait. + +use crate::db; +use std::sync::Arc; + +use extenddb_storage::CatalogStore; +use extenddb_storage::diagnostics::{DiagError, DiagResult, DiagnosticsStore}; +use extenddb_storage::management_store::{ + MetricsRow, MetricsStore, OpError, OpResult, RateLimitStore, SettingsStore, +}; +use futures::future::BoxFuture; + +use time::OffsetDateTime; + +use crate::duckdb_util::{format_timestamp, parse_timestamp}; + +/// Catalog store over the shared DuckDB pool. +/// +/// The encryption key is cached at construction (from the `settings` table) so +/// access-key creation/import can encrypt secrets without an extra query. +pub struct DuckDbCatalogStore { + pool: db::Pool, + encryption_key: Option>, +} + +impl DuckDbCatalogStore { + /// Construct without a cached encryption key (settings/diagnostics-only use). + pub fn new(pool: db::Pool) -> Self { + Self { + pool, + encryption_key: None, + } + } + + /// Construct with the cached AES-256-GCM encryption key (base64). + pub fn with_encryption_key(pool: db::Pool, encryption_key: String) -> Self { + Self { + pool, + encryption_key: Some(Arc::from(encryption_key.as_str())), + } + } + + pub(crate) fn pool(&self) -> &db::Pool { + &self.pool + } + + pub(crate) fn encryption_key(&self) -> Option<&Arc> { + self.encryption_key.as_ref() + } +} + +// ── SettingsStore ────────────────────────────────────────────────────── + +impl SettingsStore for DuckDbCatalogStore { + fn get_setting(&self, key: &str) -> BoxFuture<'_, OpResult>> { + let key = key.to_owned(); + Box::pin(async move { + let row: Option<(String,)> = db::query_as("SELECT value FROM settings WHERE key = ?") + .bind(&key) + .fetch_optional(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("get_setting: {e}")))?; + Ok(row.map(|(v,)| v)) + }) + } + + fn set_setting(&self, key: &str, value: &str) -> BoxFuture<'_, OpResult<()>> { + let key = key.to_owned(); + let value = value.to_owned(); + Box::pin(async move { + db::query( + "INSERT INTO settings (key, value) VALUES (?, ?) \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ) + .bind(&key) + .bind(&value) + .execute(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("set_setting: {e}")))?; + Ok(()) + }) + } + + fn list_settings(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async move { + db::query_as("SELECT key, value FROM settings ORDER BY key") + .fetch_all(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("list_settings: {e}"))) + }) + } + + fn cached_encryption_key(&self) -> Option { + self.encryption_key.as_ref().map(|k| k.to_string()) + } +} + +// ── MetricsStore ─────────────────────────────────────────────────────── + +struct DbMetricsRow { + bucket: String, + metric: String, + table_name: String, + index_name: String, + operation: String, + sum: f64, + count: i64, + min: f64, + max: f64, +} + +crate::impl_from_row!(DbMetricsRow { + bucket, + metric, + table_name, + index_name, + operation, + sum, + count, + min, + max +}); + +impl MetricsStore for DuckDbCatalogStore { + fn insert_metrics(&self, rows: &[MetricsRow]) -> BoxFuture<'_, OpResult<()>> { + let rows = rows.to_vec(); + Box::pin(async move { + for row in &rows { + let bucket = format_timestamp(row.bucket); + let result = db::query( + "INSERT INTO metrics \ + (bucket, metric, table_name, index_name, operation, sum, count, min, max) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT(bucket, metric, table_name, index_name, operation) DO UPDATE SET \ + sum = metrics.sum + excluded.sum, \ + count = metrics.count + excluded.count, \ + min = LEAST(metrics.min, excluded.min), \ + max = GREATEST(metrics.max, excluded.max)", + ) + .bind(&bucket) + .bind(&row.metric) + .bind(row.table_name.as_deref().unwrap_or("")) + .bind(row.index_name.as_deref().unwrap_or("")) + .bind(row.operation.as_deref().unwrap_or("")) + .bind(row.sum) + .bind(row.count) + .bind(row.min) + .bind(row.max) + .execute(&self.pool) + .await; + if let Err(e) = result { + tracing::warn!("insert_metrics row failed: {e}"); + } + } + Ok(()) + }) + } + + fn query_metrics( + &self, + start: OffsetDateTime, + end: OffsetDateTime, + table_name: Option<&str>, + metric: Option<&str>, + ) -> BoxFuture<'_, OpResult>> { + let table_name = table_name.map(str::to_owned); + let metric = metric.map(str::to_owned); + let start_str = format_timestamp(start); + let end_str = format_timestamp(end); + Box::pin(async move { + let mut sql = String::from( + "SELECT bucket, metric, table_name, index_name, operation, sum, count, min, max \ + FROM metrics WHERE bucket >= ? AND bucket <= ?", + ); + let table_filter = table_name.as_deref().filter(|s| !s.is_empty()); + if table_filter.is_some() { + sql.push_str(" AND table_name = ?"); + } + if metric.is_some() { + sql.push_str(" AND metric = ?"); + } + sql.push_str(" ORDER BY bucket"); + + let mut q = db::query_as::(&sql) + .bind(&start_str) + .bind(&end_str); + if let Some(tn) = table_filter { + q = q.bind(tn.to_owned()); + } + if let Some(m) = metric.as_deref() { + q = q.bind(m.to_owned()); + } + + let rows = q + .fetch_all(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("query_metrics: {e}")))?; + + Ok(rows + .into_iter() + .filter_map(|r| { + Some(MetricsRow { + bucket: parse_timestamp(&r.bucket).ok()?, + metric: r.metric, + table_name: (!r.table_name.is_empty()).then_some(r.table_name), + index_name: (!r.index_name.is_empty()).then_some(r.index_name), + operation: (!r.operation.is_empty()).then_some(r.operation), + sum: r.sum, + count: r.count, + min: r.min, + max: r.max, + }) + }) + .collect()) + }) + } + + fn prune_metrics(&self, retention: std::time::Duration) -> BoxFuture<'_, OpResult<()>> { + Box::pin(async move { + let cutoff = OffsetDateTime::now_utc() + - time::Duration::seconds(i64::try_from(retention.as_secs()).unwrap_or(i64::MAX)); + let cutoff_str = format_timestamp(cutoff); + db::query("DELETE FROM metrics WHERE bucket < ?") + .bind(&cutoff_str) + .execute(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("prune_metrics: {e}")))?; + Ok(()) + }) + } +} + +// ── RateLimitStore ───────────────────────────────────────────────────── + +impl RateLimitStore for DuckDbCatalogStore { + fn count_principal_failures( + &self, + principal: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let principal = principal.to_owned(); + Box::pin(async move { + let cutoff = format_timestamp( + OffsetDateTime::now_utc() - time::Duration::seconds(window_seconds), + ); + let row: (i64,) = db::query_as( + "SELECT COUNT(*) FROM login_attempts \ + WHERE principal = ? AND success = 0 AND attempted_at > ?", + ) + .bind(&principal) + .bind(&cutoff) + .fetch_one(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("count_principal_failures: {e}")))?; + Ok(row.0) + }) + } + + fn count_ip_failures( + &self, + source_ip: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let source_ip = source_ip.to_owned(); + Box::pin(async move { + let cutoff = format_timestamp( + OffsetDateTime::now_utc() - time::Duration::seconds(window_seconds), + ); + let row: (i64,) = db::query_as( + "SELECT COUNT(*) FROM login_attempts \ + WHERE source_ip = ? AND success = 0 AND attempted_at > ?", + ) + .bind(&source_ip) + .bind(&cutoff) + .fetch_one(&self.pool) + .await + .map_err(|e| OpError::Internal(format!("count_ip_failures: {e}")))?; + Ok(row.0) + }) + } + + fn record_failed_login(&self, principal: &str, source_ip: Option<&str>) -> BoxFuture<'_, ()> { + let principal = principal.to_owned(); + let source_ip = source_ip.map(str::to_owned); + Box::pin(async move { + let now = format_timestamp(OffsetDateTime::now_utc()); + let result = db::query( + "INSERT INTO login_attempts (principal, attempted_at, success, source_ip) \ + VALUES (?, ?, 0, ?)", + ) + .bind(&principal) + .bind(&now) + .bind(source_ip.as_deref()) + .execute(&self.pool) + .await; + if let Err(e) = result { + tracing::error!("record_failed_login: {e}"); + } + }) + } + + fn cleanup_old_attempts(&self, max_age_seconds: i64) -> BoxFuture<'_, ()> { + Box::pin(async move { + let cutoff = format_timestamp( + OffsetDateTime::now_utc() - time::Duration::seconds(max_age_seconds), + ); + if let Err(e) = db::query("DELETE FROM login_attempts WHERE attempted_at < ?") + .bind(&cutoff) + .execute(&self.pool) + .await + { + tracing::error!("cleanup_old_attempts: {e}"); + } + }) + } +} + +// ── DiagnosticsStore ─────────────────────────────────────────────────── + +impl DiagnosticsStore for DuckDbCatalogStore { + fn count_tables(&self) -> BoxFuture<'_, DiagResult> { + Box::pin(async move { + let (count,): (i64,) = db::query_as("SELECT COUNT(*) FROM tables") + .fetch_one(&self.pool) + .await + .map_err(|e| DiagError::QueryFailed(e.to_string()))?; + Ok(count) + }) + } + + fn count_indexes(&self) -> BoxFuture<'_, DiagResult> { + Box::pin(async move { + let (count,): (i64,) = db::query_as("SELECT COUNT(*) FROM indexes") + .fetch_one(&self.pool) + .await + .map_err(|e| DiagError::QueryFailed(e.to_string()))?; + Ok(count) + }) + } + + fn test_data_database_connection(&self) -> BoxFuture<'_, DiagResult> { + Box::pin(async move { + // Catalog and data share one DuckDB file; report its recorded name. + let row: Option<(String,)> = + db::query_as("SELECT value FROM settings WHERE key = 'data_database_name'") + .fetch_optional(&self.pool) + .await + .map_err(|e| DiagError::QueryFailed(e.to_string()))?; + Ok(row.map_or_else(|| "duckdb (embedded)".to_owned(), |(n,)| n)) + }) + } +} + +// ── CatalogStore supertrait ──────────────────────────────────────────── + +impl CatalogStore for DuckDbCatalogStore { + fn cached_encryption_key(&self) -> Option { + self.encryption_key.as_ref().map(|k| k.to_string()) + } +} diff --git a/crates/storage-duckdb/src/config.rs b/crates/storage-duckdb/src/config.rs new file mode 100644 index 00000000..985690b7 --- /dev/null +++ b/crates/storage-duckdb/src/config.rs @@ -0,0 +1,144 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Storage configuration for the DuckDB backend (`[storage.duckdb]`). + +use std::any::Any; + +use extenddb_storage::config::StorageConfig; +use serde::Deserialize; + +/// DuckDB backend configuration. +/// +/// `path` is the database file location; `:memory:` selects an ephemeral +/// in-memory database. `pool_size` bounds the read connection pool (writes are +/// serialized by the engine regardless). +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DuckDbConfig { + #[serde(default = "default_path")] + pub path: String, + #[serde(default = "default_pool_size")] + pub pool_size: u32, +} + +impl Default for DuckDbConfig { + fn default() -> Self { + Self { + path: default_path(), + pool_size: default_pool_size(), + } + } +} + +/// The compiled-in default database location. +/// +/// With the `memory` feature the default is `:memory:`, producing an ephemeral, +/// bootstrap-on-serve deployment with no file on disk. Otherwise the default is +/// a file in the working directory. +fn default_path() -> String { + if cfg!(feature = "memory") { + ":memory:".to_owned() + } else { + "extenddb.duckdb".to_owned() + } +} + +fn default_pool_size() -> u32 { + 10 +} + +impl StorageConfig for DuckDbConfig { + fn connection_config(&self) -> &str { + &self.path + } + + fn max_connections(&self) -> u32 { + self.pool_size + } + + fn max_catalog_connections(&self) -> u32 { + // Single DuckDB file — catalog and data share one connection pool. + self.pool_size + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +/// Deserialize a `[storage.duckdb]` TOML table into a boxed `StorageConfig`. +pub fn deserialize_config(table: &toml::Table) -> Result, String> { + let mut config: DuckDbConfig = table + .clone() + .try_into() + .map_err(|e: toml::de::Error| format!("Failed to parse duckdb config: {e}"))?; + config.path = resolve_path(config.path); + Ok(Box::new(config)) +} + +/// Resolve a relative database file path to absolute, against the current +/// working directory. +/// +/// This runs at config load — before `serve` daemonizes. Daemonization changes +/// the process working directory, so a relative `path` would otherwise resolve +/// differently (or fail to open) in the daemonized child. In-memory and URI +/// paths are left untouched. +fn resolve_path(path: String) -> String { + if path.contains(":memory:") || path.starts_with("file:") || path.starts_with("duckdb:") { + return path; + } + let p = std::path::Path::new(&path); + if p.is_absolute() { + return path; + } + match std::env::current_dir() { + Ok(cwd) => cwd.join(p).to_string_lossy().into_owned(), + Err(_) => path, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults() { + let c = DuckDbConfig::default(); + #[cfg(not(feature = "memory"))] + assert_eq!(c.path, "extenddb.duckdb"); + #[cfg(feature = "memory")] + assert_eq!(c.path, ":memory:"); + assert_eq!(c.pool_size, 10); + assert_eq!(c.max_connections(), 10); + assert_eq!(c.max_catalog_connections(), 10); + } + + #[cfg(feature = "memory")] + #[test] + fn memory_feature_defaults_to_in_memory() { + assert_eq!(DuckDbConfig::default().path, ":memory:"); + assert_eq!(default_path(), ":memory:"); + } + + #[test] + fn deserialize_full() { + let mut t = toml::Table::new(); + t.insert("path".into(), toml::Value::String(":memory:".into())); + t.insert("pool_size".into(), toml::Value::Integer(4)); + let boxed = deserialize_config(&t).expect("parse"); + assert_eq!(boxed.connection_config(), ":memory:"); + assert_eq!(boxed.max_connections(), 4); + } + + #[test] + fn deserialize_rejects_unknown_field() { + let mut t = toml::Table::new(); + t.insert("bogus".into(), toml::Value::Integer(1)); + assert!(deserialize_config(&t).is_err()); + } +} diff --git a/crates/storage-duckdb/src/create_table.rs b/crates/storage-duckdb/src/create_table.rs new file mode 100644 index 00000000..6b962c0b --- /dev/null +++ b/crates/storage-duckdb/src/create_table.rs @@ -0,0 +1,518 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `create_table` implementation for `DuckDbEngine`. +//! +//! Catalog metadata (table row, indexes, tags, stream shards) is written in one +//! transaction; the per-table/index data tables are created in a second +//! transaction afterward, with catalog cleanup if the data DDL fails. The +//! control-plane delay (`control_plane_delay_seconds`) decides whether the +//! table starts ACTIVE (delay 0) or CREATING with a scheduled transition. + +use crate::db; +use extenddb_core::types::{ + BillingMode, BillingModeSummary, CreateTableInput, GsiDescription, LsiDescription, + ProvisionedThroughputDescription, SseDescription, SseType, TableDescription, TableStatus, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{index_arn, stream_arn, table_arn}; + +use crate::duckdb_util::{format_timestamp, is_unique_violation}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + pub(crate) async fn create_table_impl( + &self, + account_id: &str, + input: CreateTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + let table_id = uuid::Uuid::new_v4().to_string(); + let table_arn = table_arn(&self.region, account_id, &input.table_name); + let billing_mode = input.billing_mode.unwrap_or(BillingMode::Provisioned); + let billing_str = match billing_mode { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + }; + + let to_str = |v: &serde_json::Value| -> Result { + serde_json::to_string(v).map_err(|e| StorageError::Internal(e.to_string())) + }; + let key_schema_json = serde_json::to_string(&input.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs_json = serde_json::to_string(&input.attribute_definitions) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let pt_json = input + .provisioned_throughput + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let stream_json = input + .stream_specification + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let sse_json = input.sse_specification.as_ref().map(to_str).transpose()?; + let on_demand_json = input + .on_demand_throughput + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let deletion_protection = input.deletion_protection_enabled.unwrap_or(false); + + // Control-plane delay decides initial status and scheduled transition. + let delay_secs: f64 = db::query_scalar::( + "SELECT value FROM settings WHERE key = 'control_plane_delay_seconds'", + ) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .and_then(|v| v.parse::().ok()) + .unwrap_or(0.25); + + let now = time::OffsetDateTime::now_utc(); + let creation_ts = format_timestamp(now); + #[allow(clippy::cast_precision_loss)] + let creation_epoch = now.unix_timestamp() as f64; + let (initial_status, status_transition_at) = if delay_secs <= 0.0 { + ("ACTIVE", None) + } else { + ( + "CREATING", + Some(format_timestamp( + now + time::Duration::seconds_f64(delay_secs), + )), + ) + }; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + db::query( + "INSERT INTO tables \ + (account_id, table_name, key_schema, attribute_definitions, billing_mode, \ + provisioned_throughput, stream_specification, table_status, creation_date_time, \ + table_arn, table_id, deletion_protection_enabled, status_transition_at, \ + table_class, sse_specification, on_demand_throughput) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(account_id) + .bind(&input.table_name) + .bind(&key_schema_json) + .bind(&attr_defs_json) + .bind(billing_str) + .bind(&pt_json) + .bind(&stream_json) + .bind(initial_status) + .bind(&creation_ts) + .bind(&table_arn) + .bind(&table_id) + .bind(deletion_protection) + .bind(&status_transition_at) + .bind(&input.table_class) + .bind(&sse_json) + .bind(&on_demand_json) + .execute(&mut *tx) + .await + .map_err(|e| { + if is_unique_violation(&e) { + StorageError::TableAlreadyExists(input.table_name.clone()) + } else { + StorageError::Internal(e.to_string()) + } + })?; + + // GSI / LSI metadata. + let mut gsi_ids: Vec = Vec::new(); + if let Some(gsis) = &input.global_secondary_indexes { + for gsi in gsis { + let ks = serde_json::to_string(&gsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj = serde_json::to_string(&gsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let pt = gsi + .provisioned_throughput + .as_ref() + .map(|pt| { + serde_json::to_string(&ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }) + }) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let index_id = uuid::Uuid::new_v4().to_string(); + db::query( + "INSERT INTO indexes \ + (table_id, index_name, index_id, index_type, key_schema, projection, \ + index_status, provisioned_throughput) \ + VALUES (?, ?, ?, 'GSI', ?, ?, 'ACTIVE', ?)", + ) + .bind(&table_id) + .bind(&gsi.index_name) + .bind(&index_id) + .bind(&ks) + .bind(&proj) + .bind(&pt) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + gsi_ids.push(index_id); + } + } + let mut lsi_ids: Vec = Vec::new(); + if let Some(lsis) = &input.local_secondary_indexes { + for lsi in lsis { + let ks = serde_json::to_string(&lsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj = serde_json::to_string(&lsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let index_id = uuid::Uuid::new_v4().to_string(); + db::query( + "INSERT INTO indexes \ + (table_id, index_name, index_id, index_type, key_schema, projection, \ + index_status, provisioned_throughput) \ + VALUES (?, ?, ?, 'LSI', ?, ?, 'ACTIVE', NULL)", + ) + .bind(&table_id) + .bind(&lsi.index_name) + .bind(&index_id) + .bind(&ks) + .bind(&proj) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + lsi_ids.push(index_id); + } + } + + // Vector indexes. A CreateTable's table is empty, so there is nothing to + // backfill and no `backfilling` member is ever reported on this path. + // The index's status tracks the TABLE's: measured against the service + // (2026-08-21, eu-west-2, three runs polling at 250ms), an index created + // with its table reports CREATING while the table is CREATING and + // reaches ACTIVE in the same DescribeTable poll as the table, with no + // observable gap in either direction. The control-plane worker flips + // both in one pass; see `process_control_plane_transitions`. + let mut vector_ids: Vec = Vec::new(); + if let Some(vis) = &input.vector_indexes { + for vi in vis { + let vec_attr = serde_json::to_string(&vi.vector_attribute) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let search_schema = vi + .search_schema + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj = vi + .projection + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + // Core validation requires Projection, so reaching here + // means the request bypassed validation rather than that + // the caller omitted it. + StorageError::Internal( + "vector index reached storage without a projection".to_owned(), + ) + })?; + let distance = serde_json::to_string(&vi.distance_function) + .map_err(|e| StorageError::Internal(e.to_string()))? + .trim_matches('"') + .to_owned(); + let index_id = uuid::Uuid::new_v4().to_string(); + db::query( + "INSERT INTO vector_indexes \ + (table_id, index_name, index_id, dimensions, distance_function, \ + vector_attribute, search_schema, projection, index_status, backfilling) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)", + ) + .bind(&table_id) + .bind(&vi.index_name) + .bind(&index_id) + .bind(i64::from(vi.dimensions)) + .bind(&distance) + .bind(&vec_attr) + .bind(&search_schema) + .bind(&proj) + .bind(initial_status) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + vector_ids.push(index_id); + } + } + + // Tags. + if let Some(tags) = &input.tags { + for tag in tags { + db::query("INSERT INTO tags (resource_arn, tag_key, tag_value) VALUES (?, ?, ?)") + .bind(&table_arn) + .bind(&tag.key) + .bind(&tag.value) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + + // Stream shards (same transaction — single file). + let stream_label = if input + .stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled) + { + Some(Self::init_stream_shards(&mut tx, account_id, &input.table_name, &table_id).await?) + } else { + None + }; + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Data tables in a second transaction; clean up catalog on failure. + let data_result = async { + let mut data_tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Self::create_data_table( + &mut data_tx, + &table_id, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + if let Some(gsis) = &input.global_secondary_indexes { + for (i, gsi) in gsis.iter().enumerate() { + Self::create_index_data_table( + &mut data_tx, + &gsi_ids[i], + &gsi.key_schema, + &input.attribute_definitions, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } + if let Some(lsis) = &input.local_secondary_indexes { + for (i, lsi) in lsis.iter().enumerate() { + Self::create_index_data_table( + &mut data_tx, + &lsi_ids[i], + &lsi.key_schema, + &input.attribute_definitions, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } + if input.vector_indexes.is_some() { + for index_id in &vector_ids { + Self::create_vector_data_table( + &mut data_tx, + &table_id, + index_id, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok::<(), StorageError>(()) + } + .await; + + if let Err(e) = data_result { + tracing::error!( + "Failed to create data tables for '{}', cleaning up catalog: {e}", + input.table_name + ); + if let Ok(mut cleanup) = self.pool.begin().await { + let _ = crate::referential::delete_table_children(&mut cleanup, &table_id).await; + let _ = db::query("DELETE FROM tables WHERE account_id = ? AND table_name = ?") + .bind(account_id) + .bind(&input.table_name) + .execute(&mut *cleanup) + .await; + let _ = cleanup.commit().await; + } + return Err(e); + } + + self.control_plane_notify.notify_one(); + + // Build the response from in-scope data (avoids a post-commit read race). + let (rcu, wcu) = input.provisioned_throughput.as_ref().map_or((0, 0), |pt| { + (pt.read_capacity_units, pt.write_capacity_units) + }); + + let gsis = input.global_secondary_indexes.as_ref().map(|gs| { + gs.iter() + .map(|g| GsiDescription { + index_name: g.index_name.clone(), + key_schema: g.key_schema.clone(), + projection: g.projection.clone(), + index_status: "ACTIVE".to_owned(), + provisioned_throughput: Some(ProvisionedThroughputDescription { + read_capacity_units: g + .provisioned_throughput + .as_ref() + .map_or(0, |pt| pt.read_capacity_units), + write_capacity_units: g + .provisioned_throughput + .as_ref() + .map_or(0, |pt| pt.write_capacity_units), + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn( + &self.region, + account_id, + &input.table_name, + &g.index_name, + ), + }) + .collect() + }); + let lsis = input.local_secondary_indexes.as_ref().map(|ls| { + ls.iter() + .map(|l| LsiDescription { + index_name: l.index_name.clone(), + key_schema: l.key_schema.clone(), + projection: l.projection.clone(), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn( + &self.region, + account_id, + &input.table_name, + &l.index_name, + ), + }) + .collect() + }); + + let billing_mode_summary = (billing_mode == BillingMode::PayPerRequest).then_some({ + BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: Some(creation_epoch), + } + }); + let latest_stream_arn = stream_label + .as_ref() + .map(|label| stream_arn(&self.region, account_id, &input.table_name, label)); + let response_status = if initial_status == "ACTIVE" { + TableStatus::Active + } else { + TableStatus::Creating + }; + let sse_description = input.sse_specification.as_ref().and_then(|spec| { + let enabled = spec + .get("Enabled") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + enabled.then(|| SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(SseType::KMS), + kms_master_key_arn: Some(format!( + "arn:aws:kms:{}:{}:key/default", + self.region, account_id + )), + }) + }); + + // Echo the vector indexes we just created. Built from the request plus the + // ids assigned above rather than re-read from the catalog, which would add + // a round trip to say something already known. A CreateTable's table is + // empty, so each index is ACTIVE with no `backfilling` member. + let vector_index_descs: Option> = input + .vector_indexes + .as_ref() + .map(|vis| { + vis.iter() + .map(|vi| extenddb_core::types::VectorIndexDescription { + index_name: vi.index_name.clone(), + vector_attribute: vi.vector_attribute.clone(), + dimensions: vi.dimensions, + search_schema: vi.search_schema.clone(), + distance_function: vi.distance_function, + index_status: extenddb_core::types::IndexStatus::Active, + backfilling: None, + index_size_bytes: 0, + item_count: 0, + index_arn: extenddb_storage::util::index_arn( + &self.region, + account_id, + &input.table_name, + &vi.index_name, + ), + projection: vi.projection.clone(), + }) + .collect() + }) + .filter(|v: &Vec<_>| !v.is_empty()); + + Ok(TableDescription { + restore_summary: None, + table_name: input.table_name, + key_schema: input.key_schema, + attribute_definitions: input.attribute_definitions, + table_status: response_status, + creation_date_time: creation_epoch, + table_size_bytes: 0, + item_count: 0, + table_arn, + table_id, + provisioned_throughput: ProvisionedThroughputDescription { + read_capacity_units: rcu, + write_capacity_units: wcu, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + billing_mode_summary, + global_secondary_indexes: gsis, + local_secondary_indexes: lsis, + stream_specification: input.stream_specification, + latest_stream_arn, + latest_stream_label: stream_label, + deletion_protection_enabled: deletion_protection, + sse_description, + table_class_summary: input + .table_class + .as_ref() + .map(|tc| serde_json::json!({ "TableClass": tc })), + on_demand_throughput: input.on_demand_throughput, + // Every field is populated deliberately, with no `..Default::default()` + // spread. This response is the complete description of what was just + // created, so a new core field should break this site and force a + // decision about whether create must report it, rather than silently + // defaulting. Sites that legitimately opt out still use the spread. + vector_indexes: vector_index_descs, + }) + } +} diff --git a/crates/storage-duckdb/src/credential_store.rs b/crates/storage-duckdb/src/credential_store.rs new file mode 100644 index 00000000..8e9c7def --- /dev/null +++ b/crates/storage-duckdb/src/credential_store.rs @@ -0,0 +1,173 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! DuckDB-backed `CredentialStore` for SigV4 verification. +//! +//! `AKIA…` keys resolve to long-lived IAM user access keys; `ASIA…` keys +//! resolve to temporary `AssumeRole` sessions (with expiry enforcement). +//! Secrets are decrypted with AES-256-GCM; the `access_key_id` is the AAD, with +//! a no-AAD fallback for any legacy ciphertext. + +use crate::db; +use aes_gcm::aead::{Aead, Payload}; +use aes_gcm::{Aes256Gcm, Key, KeyInit, Nonce}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use extenddb_auth::{CredentialStore, StoredCredential}; +use extenddb_core::error::DynamoDbError; + +use zeroize::{Zeroize, ZeroizeOnDrop}; + +use crate::duckdb_util::parse_timestamp; + +/// Decrypt a `nonce(12) || ciphertext` secret with AES-256-GCM. +fn decrypt_secret(encrypted: &[u8], key_b64: &str, aad: &str) -> Result { + if encrypted.len() < 28 { + return Err("ciphertext too short (need 12-byte nonce + 16-byte tag)".to_owned()); + } + let key_bytes = BASE64 + .decode(key_b64) + .map_err(|e| format!("decode encryption key: {e}"))?; + if key_bytes.len() != 32 { + return Err("encryption key must be 32 bytes".to_owned()); + } + let cipher = Aes256Gcm::new(Key::::from_slice(&key_bytes)); + let nonce = Nonce::from_slice(&encrypted[..12]); + + if let Ok(plaintext) = cipher.decrypt( + nonce, + Payload { + msg: &encrypted[12..], + aad: aad.as_bytes(), + }, + ) { + return String::from_utf8(plaintext).map_err(|e| format!("secret not UTF-8: {e}")); + } + // Fallback: ciphertext written without AAD. + let plaintext = cipher + .decrypt(nonce, &encrypted[12..]) + .map_err(|e| format!("decrypt: {e}"))?; + String::from_utf8(plaintext).map_err(|e| format!("secret not UTF-8: {e}")) +} + +/// Credential store over the catalog DuckDB pool. +#[derive(Zeroize, ZeroizeOnDrop)] +pub struct DuckDbCredentialStore { + #[zeroize(skip)] + pool: db::Pool, + encryption_key: String, +} + +impl DuckDbCredentialStore { + pub fn new(pool: db::Pool, encryption_key: String) -> Self { + Self { + pool, + encryption_key, + } + } + + async fn lookup_user( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let row: Option<(Vec, String, String, bool)> = db::query_as( + "SELECT secret_key_encrypted, account_id, user_name, is_active \ + FROM access_keys WHERE access_key_id = ?", + ) + .bind(access_key_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + tracing::error!("credential lookup failed for {access_key_id}: {e}"); + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + + let Some((encrypted, account_id, user_name, is_active)) = row else { + return Ok(None); + }; + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!("secret decryption failed for {access_key_id}: {e}"); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: user_name, + session_name: None, + is_session: false, + session_token: None, + is_active, + expires_at: None, + })) + } + + async fn lookup_session( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let row: Option<(Vec, String, String, String, String, String)> = db::query_as( + "SELECT secret_key_encrypted, account_id, role_name, session_name, \ + session_token, expires_at \ + FROM iam_sessions WHERE access_key_id = ?", + ) + .bind(access_key_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + tracing::error!("session lookup failed for {access_key_id}: {e}"); + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + + let Some((encrypted, account_id, role_name, session_name, session_token, expires_at)) = row + else { + return Ok(None); + }; + + let expires = parse_timestamp(&expires_at).map_err(|e| { + tracing::error!("session expiry parse error: {e}"); + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + if expires < time::OffsetDateTime::now_utc() { + return Err(DynamoDbError::ExpiredTokenException( + "The security token included in the request is expired".to_owned(), + )); + } + + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!("session secret decryption failed for {access_key_id}: {e}"); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: role_name, + session_name: Some(session_name), + is_session: true, + session_token: Some(session_token), + is_active: true, + expires_at: Some(expires), + })) + } +} + +#[async_trait::async_trait] +impl CredentialStore for DuckDbCredentialStore { + async fn lookup_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + if access_key_id.starts_with("ASIA") { + self.lookup_session(access_key_id).await + } else if access_key_id.starts_with("AKIA") { + self.lookup_user(access_key_id).await + } else { + Ok(None) + } + } +} diff --git a/crates/storage-duckdb/src/data/data_engine.rs b/crates/storage-duckdb/src/data/data_engine.rs new file mode 100644 index 00000000..1679d94f --- /dev/null +++ b/crates/storage-duckdb/src/data/data_engine.rs @@ -0,0 +1,398 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `DataEngine` trait implementation for `DuckDbEngine`. +//! +//! Each method clones its borrowed arguments into owned values and delegates to +//! the corresponding `*_impl` method inside a boxed future. The transact-write +//! path rebuilds borrowed `TransactWriteOp`s from owned components inside the +//! future, since the trait's borrowed ops cannot outlive the call. + +use extenddb_core::expression::{Expr, ExpressionMaps, KeyCondition, UpdateAction}; +use extenddb_core::types::{Item, ReturnValuesOnConditionCheckFailure, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, IdempotencyKey, StreamCapture, TransactGetOp, TransactWriteOp}; +use futures::future::BoxFuture; + +use crate::store::DuckDbEngine; + +impl DataEngine for DuckDbEngine { + /// Declares vector support by handing over the implementation. `Some(self)` + /// only compiles because `DuckDbEngine` implements `VectorSearchEngine`, so + /// this cannot claim a capability the backend does not have. + fn as_vector_search(&self) -> Option<&dyn extenddb_storage::VectorSearchEngine> { + Some(self) + } + + fn put_item( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.put_item_impl( + &key_info, + item, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn get_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + Box::pin(async move { self.get_item_impl(&key_info, &key).await }) + } + + fn delete_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.delete_item_impl( + &key_info, + &key, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn update_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result<(Option, Option), StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + let actions = actions.to_vec(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.update_item_impl( + &key_info, + &key, + &actions, + return_old, + return_new, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn query( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> BoxFuture<'_, Result<(Vec, Option), StorageError>> { + let key_info = key_info.clone(); + let key_condition = key_condition.clone(); + let maps = maps.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(str::to_owned); + Box::pin(async move { + self.query_impl( + &key_info, + &key_condition, + &maps, + forward, + limit, + exclusive_start_key.as_ref(), + index_name.as_deref(), + ) + .await + }) + } + + fn scan( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> BoxFuture<'_, Result<(Vec, Option), StorageError>> { + let key_info = key_info.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(str::to_owned); + Box::pin(async move { + self.scan_impl( + &key_info, + limit, + exclusive_start_key.as_ref(), + segment, + total_segments, + index_name.as_deref(), + ) + .await + }) + } + + fn transact_get_items( + &self, + ops: &[TransactGetOp<'_>], + ) -> BoxFuture<'_, Result>, StorageError>> { + let owned: Vec<(TableKeyInfo, Item)> = ops + .iter() + .map(|op| (op.key_info.clone(), op.key.clone())) + .collect(); + Box::pin(async move { + let borrowed: Vec = owned + .iter() + .map(|(key_info, key)| TransactGetOp { key_info, key }) + .collect(); + self.transact_get_items_impl(&borrowed).await + }) + } + + fn transact_write_items( + &self, + ops: &[TransactWriteOp<'_>], + idempotency: Option>, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let owned: Vec = ops.iter().map(OwnedWriteOp::from_op).collect(); + let idempotency = idempotency.map(|k| { + ( + k.account_id.to_owned(), + k.token.to_owned(), + k.fingerprint.to_owned(), + ) + }); + Box::pin(async move { + let borrowed: Vec = owned.iter().map(OwnedWriteOp::as_op).collect(); + self.transact_write_items_impl( + &borrowed, + idempotency.as_ref().map(|(a, t, f)| IdempotencyKey { + account_id: a, + token: t, + fingerprint: f, + }), + ) + .await + }) + } + + fn cleanup_expired_idempotency_tokens( + &self, + max_age_seconds: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + self.cleanup_expired_idempotency_tokens_impl(max_age_seconds) + .await + }) + } +} + +/// Owned form of a `TransactWriteOp`, so the borrowed ops can be reconstructed +/// inside the `'static` future. +enum OwnedWriteOp { + Put { + key_info: TableKeyInfo, + item: Item, + condition: Option, + maps: ExpressionMaps, + rv: ReturnValuesOnConditionCheckFailure, + stream: Option, + }, + Delete { + key_info: TableKeyInfo, + key: Item, + condition: Option, + maps: ExpressionMaps, + rv: ReturnValuesOnConditionCheckFailure, + stream: Option, + }, + Update { + key_info: TableKeyInfo, + key: Item, + actions: Vec, + condition: Option, + maps: ExpressionMaps, + rv: ReturnValuesOnConditionCheckFailure, + stream: Option, + }, + ConditionCheck { + key_info: TableKeyInfo, + key: Item, + condition: Expr, + maps: ExpressionMaps, + rv: ReturnValuesOnConditionCheckFailure, + }, +} + +impl OwnedWriteOp { + fn from_op(op: &TransactWriteOp<'_>) -> Self { + match op { + TransactWriteOp::Put { + key_info, + item, + condition, + maps, + return_values_on_ccf, + stream, + } => Self::Put { + key_info: (*key_info).clone(), + item: (*item).clone(), + condition: condition.cloned(), + maps: (*maps).clone(), + rv: *return_values_on_ccf, + stream: stream.clone(), + }, + TransactWriteOp::Delete { + key_info, + key, + condition, + maps, + return_values_on_ccf, + stream, + } => Self::Delete { + key_info: (*key_info).clone(), + key: (*key).clone(), + condition: condition.cloned(), + maps: (*maps).clone(), + rv: *return_values_on_ccf, + stream: stream.clone(), + }, + TransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + return_values_on_ccf, + stream, + } => Self::Update { + key_info: (*key_info).clone(), + key: (*key).clone(), + actions: actions.to_vec(), + condition: condition.cloned(), + maps: (*maps).clone(), + rv: *return_values_on_ccf, + stream: stream.clone(), + }, + TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf, + } => Self::ConditionCheck { + key_info: (*key_info).clone(), + key: (*key).clone(), + condition: (*condition).clone(), + maps: (*maps).clone(), + rv: *return_values_on_ccf, + }, + } + } + + fn as_op(&self) -> TransactWriteOp<'_> { + match self { + Self::Put { + key_info, + item, + condition, + maps, + rv, + stream, + } => TransactWriteOp::Put { + key_info, + item, + condition: condition.as_ref(), + maps, + return_values_on_ccf: *rv, + stream: stream.clone(), + }, + Self::Delete { + key_info, + key, + condition, + maps, + rv, + stream, + } => TransactWriteOp::Delete { + key_info, + key, + condition: condition.as_ref(), + maps, + return_values_on_ccf: *rv, + stream: stream.clone(), + }, + Self::Update { + key_info, + key, + actions, + condition, + maps, + rv, + stream, + } => TransactWriteOp::Update { + key_info, + key, + actions, + condition: condition.as_ref(), + maps, + return_values_on_ccf: *rv, + stream: stream.clone(), + }, + Self::ConditionCheck { + key_info, + key, + condition, + maps, + rv, + } => TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf: *rv, + }, + } + } +} diff --git a/crates/storage-duckdb/src/data/ddl.rs b/crates/storage-duckdb/src/data/ddl.rs new file mode 100644 index 00000000..921cf809 --- /dev/null +++ b/crates/storage-duckdb/src/data/ddl.rs @@ -0,0 +1,744 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! DDL for per-DynamoDB-table and per-index data tables, plus catalog fetches +//! of `TableKeyInfo` / `IndexInfo`. +//! +//! Sort-key columns follow design decision D2: `sk_s`/`base_sk_s` are `TEXT`, +//! `sk_n`/`base_sk_n` are `TEXT` (order-preserving numeric encoding, never +//! `DOUBLE`), `sk_b`/`base_sk_b` are `BLOB`. + +use crate::db; +use extenddb_core::types::{ + AttributeDefinition, IndexInfo, IndexType, KeySchemaElement, Projection, StreamSpecification, + TableKeyInfo, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::sk_column_n; + +use super::{ + all_sort_key_info, data_table_name, index_table_name, vector_table_glob_pattern, + vector_table_name, +}; +use crate::store::DuckDbEngine; + +/// DuckDB column type for the Nth sort-key position and scalar type (D2). +fn sk_col_defs(index: usize) -> [String; 3] { + if index == 0 { + [ + "sk_s TEXT".to_owned(), + "sk_n TEXT".to_owned(), + "sk_b BLOB".to_owned(), + ] + } else { + let n = index + 1; + [ + format!("sk{n}_s TEXT"), + format!("sk{n}_n TEXT"), + format!("sk{n}_b BLOB"), + ] + } +} + +/// DuckDB column type for the Nth base-table sort-key position (D2). +fn base_sk_col_defs(index: usize) -> [String; 3] { + if index == 0 { + [ + "base_sk_s TEXT".to_owned(), + "base_sk_n TEXT".to_owned(), + "base_sk_b BLOB".to_owned(), + ] + } else { + let n = index + 1; + [ + format!("base_sk{n}_s TEXT"), + format!("base_sk{n}_n TEXT"), + format!("base_sk{n}_b BLOB"), + ] + } +} + +impl DuckDbEngine { + /// Create the per-DynamoDB-table data table. + /// + /// # Safety (SQL injection) + /// `table_id` is a server-generated UUID; column names are constants. No + /// user input is interpolated into the DDL. + pub(crate) async fn create_data_table( + tx: &mut db::Transaction, + table_id: &str, + key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let ddb_table = data_table_name(table_id); + let sk_infos = all_sort_key_info(key_schema, attr_defs); + + let ddl = if sk_infos.is_empty() { + format!( + "CREATE TABLE {ddb_table} (pk TEXT NOT NULL PRIMARY KEY, item_data TEXT NOT NULL)" + ) + } else { + let mut col_defs = vec!["pk TEXT NOT NULL".to_owned()]; + let mut pk_cols = vec!["pk".to_owned()]; + for (i, &(_, sk_type)) in sk_infos.iter().enumerate() { + col_defs.extend(sk_col_defs(i)); + pk_cols.push(sk_column_n(i, sk_type)); + } + col_defs.push("item_data TEXT NOT NULL".to_owned()); + format!( + "CREATE TABLE {ddb_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ) + }; + + db::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Create a vector-index data table: one row per indexed vector. + /// + /// `part` is the search-schema HASH value when one is declared, and a single + /// constant otherwise, so an unscoped index is one partition rather than a + /// separate code path. `nrm` is the vector's precomputed L2 norm, so cosine + /// costs one dot product at query time instead of two passes. + /// + /// # Safety (SQL injection) + /// `index_id` is a server-generated UUID and column names are constants, so + /// no user input reaches the DDL. Vector attribute names are stored as data, + /// never as identifiers. + pub(crate) async fn create_vector_data_table( + tx: &mut db::Transaction, + table_id: &str, + index_id: &str, + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, index_id); + let base_sks = all_sort_key_info(base_key_schema, base_attr_defs); + + let mut col_defs = vec![ + "part TEXT NOT NULL".to_owned(), + "base_pk TEXT NOT NULL".to_owned(), + ]; + for i in 0..base_sks.len() { + col_defs.extend(base_sk_col_defs(i)); + } + col_defs.push("vec BLOB NOT NULL".to_owned()); + col_defs.push("nrm DOUBLE NOT NULL".to_owned()); + col_defs.push("item_data TEXT NOT NULL".to_owned()); + + // Keyed by the base item, not by the partition, so one base item yields + // at most one vector row and a re-put replaces rather than duplicates. + let mut pk_cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + pk_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + + let ddl = format!( + "CREATE TABLE {vec_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ); + db::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // The scan is always partition-scoped, so this index is what keeps a + // search off the full table when a HASH element is declared. + let part_idx = + format!("CREATE INDEX \"_vidx_part_{table_id}_{index_id}\" ON {vec_table} (part)"); + db::query(&part_idx) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Drop every vector-index data table belonging to one DynamoDB table. + /// + /// Discovered from `duckdb_tables()` rather than from the catalog, because the + /// caller runs this after the catalog rows have been cascade-deleted in the + /// same transaction, so `vector_indexes` is already empty for this table. + /// Without this, dropping a table would leave its vector data tables behind + /// forever with nothing left pointing at them. + /// Drop one vector index's data table. + /// + /// The table-drop path sweeps `duckdb_tables()` instead, because by the time it + /// runs the catalog rows have already cascade-deleted and the index ids are + /// unreadable. Here the id is known, so the name is derived directly rather + /// than matched by pattern. + pub(crate) async fn drop_vector_data_table_by_id( + pool: &db::Pool, + table_id: &str, + index_id: &str, + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, index_id); + db::query(&format!("DROP TABLE IF EXISTS {vec_table}")) + .execute(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn drop_all_vector_data_tables( + tx: &mut db::Transaction, + table_id: &str, + ) -> Result<(), StorageError> { + let names: Vec = + db::query_scalar("SELECT table_name FROM duckdb_tables() WHERE table_name GLOB ?") + .bind(vector_table_glob_pattern(table_id)) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + for name in names { + // Names come from duckdb_tables(), not from user input, and are quoted. + db::query(&format!("DROP TABLE IF EXISTS \"{name}\"")) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + } + + /// Drop the per-DynamoDB-table data table. + pub(crate) async fn drop_data_table( + tx: &mut db::Transaction, + table_id: &str, + ) -> Result<(), StorageError> { + let ddb_table = data_table_name(table_id); + db::query(&format!("DROP TABLE IF EXISTS {ddb_table}")) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + // Vector data tables are keyed by index, not by table, so they are not + // reached by dropping the item table or by the catalog cascade. + Self::drop_all_vector_data_tables(tx, table_id).await?; + Ok(()) + } + + /// Create a GSI/LSI data table. GSI keys are not unique, so the primary key + /// is `(pk, base_pk, base_sk*)`: the base-table key is what makes a row + /// unique, keeping one row per base item per index partition. + /// + /// Two secondary indexes are created: + /// - `idx_order_*` on `(pk, sk*, base_pk, base_sk*)`, for sort-key ordering + /// within an index partition. Only when the index declares a sort key. + /// - `idx_base_key_*` on `(base_pk, base_sk*)`, for the reverse lookup that + /// index propagation uses to delete an item's old index row. Always. + pub(crate) async fn create_index_data_table( + tx: &mut db::Transaction, + index_id: &str, + index_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let idx_table = index_table_name(index_id); + let idx_sks = all_sort_key_info(index_key_schema, attr_defs); + let base_sks = all_sort_key_info(base_key_schema, base_attr_defs); + + let mut col_defs = vec!["pk TEXT NOT NULL".to_owned()]; + for i in 0..idx_sks.len() { + col_defs.extend(sk_col_defs(i)); + } + col_defs.push("base_pk TEXT NOT NULL".to_owned()); + for i in 0..base_sks.len() { + col_defs.extend(base_sk_col_defs(i)); + } + col_defs.push("item_data TEXT NOT NULL".to_owned()); + + let mut pk_cols = vec!["pk".to_owned(), "base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + pk_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + + let ddl = format!( + "CREATE TABLE {idx_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ); + db::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if !idx_sks.is_empty() { + let mut order_cols = vec!["pk".to_owned()]; + for (i, &(_, sk_type)) in idx_sks.iter().enumerate() { + order_cols.push(sk_column_n(i, sk_type)); + } + order_cols.push("base_pk".to_owned()); + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + order_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + let order_idx = format!( + "CREATE INDEX \"idx_order_{index_id}\" ON {idx_table} ({})", + order_cols.join(", ") + ); + db::query(&order_idx) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Index on the base-table key columns, for `delete_index_row_multi`. + // + // GSI/LSI propagation deletes the old index row by base-table key: + // `DELETE FROM WHERE base_pk = ? AND base_sk* = ?`, with no `pk` + // predicate. Both the PRIMARY KEY and the ordering index above lead + // with `pk`, so neither can serve that filter and DuckDB plans a full + // `SCAN` of the index table on every propagating write. Unconditional + // rather than gated on `idx_sks`, because the reverse lookup happens + // whether or not the index declares a sort key. + { + let mut base_key_cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + base_key_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + let base_key_idx = format!( + "CREATE INDEX \"idx_base_key_{index_id}\" ON {idx_table} ({})", + base_key_cols.join(", ") + ); + db::query(&base_key_idx) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + Ok(()) + } + + /// Drop a GSI/LSI data table. + pub(crate) async fn drop_index_data_table( + tx: &mut db::Transaction, + index_id: &str, + ) -> Result<(), StorageError> { + let idx_table = index_table_name(index_id); + db::query(&format!("DROP TABLE IF EXISTS {idx_table}")) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Fetch `TableKeyInfo` for an ACTIVE table from the catalog. + pub(crate) async fn fetch_table_key_info( + &self, + account_id: &str, + table_name: &str, + ) -> Result { + let row: Option<(String, String, String, String, Option)> = db::query_as( + "SELECT key_schema, attribute_definitions, table_status, table_id, \ + stream_specification \ + FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (ks_str, ad_str, status, table_id, stream_spec_str) = + row.ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; + + match status.as_str() { + "ACTIVE" => {} + // A table still being created, or being deleted, is not usable for + // data-plane operations; DynamoDB reports it as not found (not + // in-use). UPDATING keeps the not-active classification. + "CREATING" | "DELETING" => { + return Err(StorageError::TableNotFound(table_name.to_owned())); + } + _ => return Err(StorageError::TableNotActive(table_name.to_owned())), + } + + let key_schema: Vec = + serde_json::from_str(&ks_str).map_err(|e| StorageError::Internal(e.to_string()))?; + let attribute_definitions: Vec = + serde_json::from_str(&ad_str).map_err(|e| StorageError::Internal(e.to_string()))?; + let stream_specification: Option = stream_spec_str + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Full secondary-index metadata rides on the (cached) TableKeyInfo so + // per-index consumed-capacity accounting avoids a separate + // describe_table round-trip on every write; it also supplies `has_lsi`. + let (global_secondary_indexes, local_secondary_indexes) = + self.fetch_all_index_info(&table_id).await?; + let has_lsi = !local_secondary_indexes.is_empty(); + // Vector indexes ride on the cached key info too, so the write path can + // decide whether any maintenance is needed without a query, and so the + // engine can validate vector attributes on writes. + let vector_indexes = self.fetch_vector_index_key_info(&table_id).await?; + + let key_info = TableKeyInfo { + table_name: table_name.to_owned(), + account_id: account_id.to_owned(), + table_id, + base_key_schema: key_schema.clone(), + key_schema, + attribute_definitions, + has_lsi, + global_secondary_indexes, + local_secondary_indexes, + stream_specification, + // Every field is populated, with no `..Default::default()` spread: a + // new core field should break this site and force a decision about + // whether the write path needs it, rather than silently defaulting. + vector_indexes, + }; + // Catalog metadata that cannot describe its own sort key would make the + // keyed read paths fall back to a partition-only lookup and return the + // wrong item, so refuse it here rather than serve a wrong answer (#259). + key_info + .validate_sort_key_definitions() + .map_err(StorageError::Internal)?; + Ok(key_info) + } + + /// Fetch the vector indexes of a table in the shape the engine caches. + /// + /// Note what this cannot carry: `VectorIndexKeyInfo` has no distance + /// function, so a search still reads the catalog for it. Widening that + /// type would remove the last per-search catalog read. + async fn fetch_vector_index_key_info( + &self, + table_id: &str, + ) -> Result, StorageError> { + let rows: Vec<(String, i64, String, Option, String)> = db::query_as( + "SELECT index_name, dimensions, vector_attribute, search_schema, projection \ + FROM vector_indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut out = Vec::with_capacity(rows.len()); + for (index_name, dimensions, vector_attribute, search_schema, projection) in rows { + let attr: extenddb_core::types::VectorAttribute = + serde_json::from_str(&vector_attribute) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; + let search_schema = match search_schema.as_deref() { + Some(json) => serde_json::from_str(json) + .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?, + None => Vec::new(), + }; + let projection: extenddb_core::types::Projection = serde_json::from_str(&projection) + .map_err(|e| StorageError::Internal(format!("vector projection: {e}")))?; + out.push(extenddb_core::types::VectorIndexKeyInfo { + index_name, + dimensions: u32::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?, + vector_attribute_name: attr.attribute_name, + search_schema, + projection, + }); + } + Ok(out) + } + + /// Fetch every secondary index defined on a table, split into + /// `(global_secondary_indexes, local_secondary_indexes)`. + async fn fetch_all_index_info( + &self, + table_id: &str, + ) -> Result<(Vec, Vec), StorageError> { + let rows: Vec<(String, String, String, String, String)> = db::query_as( + "SELECT index_name, index_type, index_id, key_schema, projection \ + FROM indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut infos = Vec::new(); + for (index_name, idx_type_str, index_id, ks_json, proj_json) in rows { + let index_type = match idx_type_str.as_str() { + "GSI" => IndexType::Gsi, + "LSI" => IndexType::Lsi, + other => { + return Err(StorageError::Internal(format!( + "unknown index type in database: {other}" + ))); + } + }; + let key_schema: Vec = serde_json::from_str(&ks_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection: Projection = serde_json::from_str(&proj_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let info = IndexInfo { + index_name, + index_id, + index_type, + key_schema, + projection, + }; + infos.push(info); + } + // Grouped by core rather than matched here, so a new IndexType variant + // does not break this backend. The string parse above already rejects + // any kind this backend cannot have created. + let grouped = extenddb_core::types::partition_indexes(infos); + Ok((grouped.gsis, grouped.lsis)) + } + + /// Fetch `IndexInfo` for a secondary index, validating the table is ACTIVE. + pub(crate) async fn fetch_index_info( + &self, + account_id: &str, + table_name: &str, + index_name: &str, + ) -> Result { + let row: Option<(String, String)> = db::query_as( + "SELECT table_id, table_status FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id, status) = + row.ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; + match status.as_str() { + "ACTIVE" => {} + // A table still being created, or being deleted, is not usable for + // data-plane operations; DynamoDB reports it as not found (not + // in-use). UPDATING keeps the not-active classification. + "CREATING" | "DELETING" => { + return Err(StorageError::TableNotFound(table_name.to_owned())); + } + _ => return Err(StorageError::TableNotActive(table_name.to_owned())), + } + self.fetch_index_info_by_table_id(&table_id, index_name) + .await + } + + /// Fetch `IndexInfo` using a known `table_id`. + pub(crate) async fn fetch_index_info_by_table_id( + &self, + table_id: &str, + index_name: &str, + ) -> Result { + let row: Option<(String, String, String, String)> = db::query_as( + "SELECT index_type, index_id, key_schema, projection \ + FROM indexes WHERE table_id = ? AND index_name = ?", + ) + .bind(table_id) + .bind(index_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (idx_type_str, index_id, ks_str, proj_str) = + row.ok_or_else(|| StorageError::IndexNotFound(index_name.to_owned()))?; + + let index_type = match idx_type_str.as_str() { + "GSI" => IndexType::Gsi, + "LSI" => IndexType::Lsi, + other => { + return Err(StorageError::Internal(format!( + "unknown index type in database: {other}" + ))); + } + }; + let key_schema: Vec = + serde_json::from_str(&ks_str).map_err(|e| StorageError::Internal(e.to_string()))?; + let projection: Projection = + serde_json::from_str(&proj_str).map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(IndexInfo { + index_name: index_name.to_owned(), + index_id, + index_type, + key_schema, + projection, + }) + } +} + +#[cfg(test)] +mod index_data_table_tests { + use crate::db; + use crate::store::DuckDbEngine; + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + + fn hash(name: &str) -> KeySchemaElement { + KeySchemaElement { + attribute_name: name.to_owned(), + key_type: KeyType::Hash, + } + } + + fn range(name: &str) -> KeySchemaElement { + KeySchemaElement { + attribute_name: name.to_owned(), + key_type: KeyType::Range, + } + } + + fn attr(name: &str, t: ScalarAttributeType) -> AttributeDefinition { + AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type: t, + } + } + + /// Create an index data table through the real DDL path. + async fn make_index_table( + index_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> db::Pool { + let pool = db::Pool::open(":memory:", 1).await.unwrap(); + let mut tx = pool.begin().await.unwrap(); + DuckDbEngine::create_index_data_table( + &mut tx, + "probe", + index_key_schema, + attr_defs, + base_key_schema, + base_attr_defs, + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + pool + } + + /// Column list of a named index, in definition order, read from DuckDB's + /// catalog. The SQLite backend asserts on `EXPLAIN QUERY PLAN` here; DuckDB's + /// optimizer chooses a sequential scan below a row-count threshold whatever + /// indexes exist, so on a probe table the plan says nothing about the index. + /// The index definition does. + async fn index_columns(pool: &db::Pool, index_name: &str) -> Vec { + let row: Option<(String,)> = db::query_as( + "SELECT CAST(expressions AS VARCHAR) FROM duckdb_indexes() WHERE index_name = ?", + ) + .bind(index_name) + .fetch_optional(pool) + .await + .unwrap(); + let Some((expr,)) = row else { + return Vec::new(); + }; + expr.trim_matches(|c| c == '[' || c == ']') + .split(',') + .map(|c| c.trim().trim_matches('"').to_owned()) + .filter(|c| !c.is_empty()) + .collect() + } + + #[tokio::test] + async fn reverse_lookup_delete_has_an_index_composite_base_key() { + let pool = make_index_table( + &[hash("gsi_pk"), range("gsi_sk")], + &[ + attr("gsi_pk", ScalarAttributeType::S), + attr("gsi_sk", ScalarAttributeType::S), + ], + &[hash("id"), range("ts")], + &[ + attr("id", ScalarAttributeType::S), + attr("ts", ScalarAttributeType::S), + ], + ) + .await; + + assert_eq!( + index_columns(&pool, "idx_base_key_probe").await, + vec!["base_pk".to_owned(), "base_sk_s".to_owned()], + "reverse-lookup DELETE must be served by the base-key index" + ); + } + + #[tokio::test] + async fn reverse_lookup_delete_has_an_index_hash_only_base_key() { + let pool = make_index_table( + &[hash("gsi_pk"), range("gsi_sk")], + &[ + attr("gsi_pk", ScalarAttributeType::S), + attr("gsi_sk", ScalarAttributeType::S), + ], + &[hash("id")], + &[attr("id", ScalarAttributeType::S)], + ) + .await; + + assert_eq!( + index_columns(&pool, "idx_base_key_probe").await, + vec!["base_pk".to_owned()], + "hash-only base key must still index the reverse lookup" + ); + } + + /// The ordering index is only created when the index declares a sort key, so + /// a sort-key-less index is the case where the base-key index is the *only* + /// secondary index. It must still be created, which is why that block is + /// unconditional. + #[tokio::test] + async fn reverse_lookup_delete_has_an_index_when_index_has_no_sort_key() { + let pool = make_index_table( + &[hash("gsi_pk")], + &[attr("gsi_pk", ScalarAttributeType::S)], + &[hash("id"), range("ts")], + &[ + attr("id", ScalarAttributeType::S), + attr("ts", ScalarAttributeType::S), + ], + ) + .await; + + assert_eq!( + index_columns(&pool, "idx_base_key_probe").await, + vec!["base_pk".to_owned(), "base_sk_s".to_owned()], + "an index with no sort key still needs the base-key index" + ); + assert!( + index_columns(&pool, "idx_order_probe").await.is_empty(), + "no ordering index without a sort key" + ); + } + + /// Guards the column order. An index on `(base_sk_s, base_pk)` would also + /// satisfy a composite lookup, but would not serve a `base_pk`-only lookup, + /// so pin that the index leads with `base_pk`. + #[tokio::test] + async fn base_key_index_leads_with_base_pk() { + let pool = make_index_table( + &[hash("gsi_pk"), range("gsi_sk")], + &[ + attr("gsi_pk", ScalarAttributeType::S), + attr("gsi_sk", ScalarAttributeType::S), + ], + &[hash("id"), range("ts")], + &[ + attr("id", ScalarAttributeType::S), + attr("ts", ScalarAttributeType::S), + ], + ) + .await; + + assert_eq!( + index_columns(&pool, "idx_base_key_probe") + .await + .first() + .map(String::as_str), + Some("base_pk"), + "base-key index must lead with base_pk" + ); + } +} diff --git a/crates/storage-duckdb/src/data/delete_item.rs b/crates/storage-duckdb/src/data/delete_item.rs new file mode 100644 index 00000000..22707600 --- /dev/null +++ b/crates/storage-duckdb/src/data/delete_item.rs @@ -0,0 +1,120 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `delete_item` for the DuckDB backend. + +use extenddb_core::expression::{Expr, ExpressionMaps}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; + +use super::index::{enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::query::check_condition; +use super::tx_helpers::{delete_item_in_tx, fetch_item_in_tx, write_stream_record_in_tx}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + pub(crate) async fn delete_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + // Read the propagation delay BEFORE taking the write lock. It is a + // runtime setting, not an invariant of this write, so it does not need + // to be read under the lock, and the lock serialises every write in the + // process: work done inside it is the backend's throughput bottleneck. + let system_delay = self.index_propagation_delay().await; + let _writer = self.write_lock.lock().await; + // Read the index set after acquiring the write lock so a concurrently + // added GSI (UpdateTable holds the same lock) is not missed. + let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old = fetch_item_in_tx(&mut tx, key_info, key).await?; + + if condition.is_some() { + let empty = Item::new(); + let target = old.as_ref().unwrap_or(&empty); + if let Err(e) = check_condition(condition, target, maps) { + return match e { + StorageError::ConditionFailed(_) => Err(StorageError::ConditionFailed(old)), + other => Err(other), + }; + } + } + + // Only mutate when an item actually exists; deleting a missing item is + // a no-op (no row removed, no index change, no stream record). + let mut enqueued = false; + if old.is_some() { + delete_item_in_tx(&mut tx, key_info, key).await?; + + if !indexes.is_empty() { + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old.as_ref(), + None, + system_delay, + ) + .await?; + } + // Vector rows for this base item are removed too. `new_item` is None, + // so this is a pure removal, applied in this transaction when the + // propagation delay is 0 and enqueued otherwise. + if !key_info.vector_indexes.is_empty() + && crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + None, + system_delay, + ) + .await? + > 0 + { + enqueued = true; + } + if enqueue_async_indexes( + &mut tx, + key_info, + &indexes, + old.as_ref(), + None, + system_delay, + ) + .await? + > 0 + { + enqueued = true; + } + + if let Some(capture) = stream { + write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), None).await?; + } + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if enqueued { + self.gsi_notify.notify_waiters(); + } + + Ok(if return_old { old } else { None }) + } +} diff --git a/crates/storage-duckdb/src/data/index.rs b/crates/storage-duckdb/src/data/index.rs new file mode 100644 index 00000000..af8ffe83 --- /dev/null +++ b/crates/storage-duckdb/src/data/index.rs @@ -0,0 +1,813 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! GSI/LSI index maintenance for the DuckDB backend. +//! +//! Synchronous indexes (LSIs and zero-delay GSIs) are reconciled in the base +//! write transaction. Async GSIs are deferred via the `gsi_pending` queue — +//! **one self-describing row per index**: each row snapshots the base key +//! schema, attribute definitions, and its single target index definition, so +//! the worker applies with zero catalog reads. Each row carries its index's own +//! (jittered) propagation delay, and `ready_at` is kept monotonic within the +//! base key's `worker_partition` so jitter cannot reorder updates to one item. +//! Index key columns follow D2 — `N` keys are stored as the order-preserving +//! TEXT encoding, `S` as TEXT, `B` as BLOB — via [`sk_bound`]. + +use crate::db; +use extenddb_core::types::{ + AttributeDefinition, Item, KeySchemaElement, Projection, ProjectionType, ScalarAttributeType, + TableKeyInfo, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{composite_pk_to_text, parse_sk, sk_column, sk_column_n}; +use serde::{Deserialize, Serialize}; + +use super::{BoundValue, all_sort_key_info, index_table_name, sk_bound}; + +/// Number of partitions a base key can hash to. Updates to one base item share +/// a partition; `ready_at` is clamped monotonic within a partition so the +/// single drain worker (which orders by `id`) applies them in order. More +/// partitions reduce false serialization between unrelated keys; there is no +/// concurrency cost because DuckDB has a single writer. +const NUM_PARTITIONS: u64 = 16; + +/// Stable partition for a base-table key (FNV-1a, mapped to a partition). A +/// local hash keeps the mapping fixed for a key across builds (`std`'s hasher +/// is not guaranteed stable). +fn partition_for(base_pk_text: &str) -> i64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; // FNV-1a offset basis + for byte in base_pk_text.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); // FNV prime + } + (hash % NUM_PARTITIONS) as i64 +} + +/// Jittered propagation delay (ms), uniform in `[delay_ms / 2, delay_ms]`. +/// +/// Simulates DynamoDB's eventual-consistency variability (a fixed delay is +/// perfectly predictable) while keeping the configured delay a meaningful lower +/// bound — jittering all the way to ~0 would make the delay almost meaningless. +/// `delay_ms <= 1` is returned unchanged. Callers only enqueue async indexes, +/// so `delay_ms >= 1`. +fn jitter_delay_ms(delay_ms: u64) -> u64 { + if delay_ms <= 1 { + delay_ms + } else { + use rand::Rng; + rand::rng().random_range(delay_ms / 2 + 1..=delay_ms) + } +} + +/// A single target index definition, snapshotted at enqueue time. The +/// propagation delay is not stored — it is already encoded in the row's +/// `ready_at`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct GsiIndexDef { + pub(crate) index_id: String, + pub(crate) key_schema: Vec, + pub(crate) projection: Projection, +} + +/// Self-describing apply context serialized into `gsi_pending.index_context`. +/// One per row → exactly one target index, so the worker needs no catalog read. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct GsiApplyContext { + pub(crate) base_key_schema: Vec, + pub(crate) attribute_definitions: Vec, + pub(crate) index: GsiIndexDef, +} + +/// What one `gsi_pending` row asks the worker to do: maintain a GSI, or maintain a +/// vector index. One queue serves both so that updates to a single base item stay +/// ordered across index kinds, which two queues drained independently could not +/// guarantee. +/// +/// `untagged` is load-bearing, not stylistic. A GSI context serializes to exactly +/// the bytes it did before this enum existed, so rows already on disk from an +/// earlier version still deserialize, and rows written now are still readable by +/// one. That matters more than it looks: an unparseable `index_context` is treated +/// as a poison row and *dropped*, so a tagged representation would silently discard +/// every in-flight GSI update across an upgrade. +/// +/// The variants are unambiguous by shape rather than by tag: a GSI context requires +/// `index` and a vector context requires `vector`, and no writer emits the other's +/// field, so for any context this code produces exactly one variant matches. +/// `pending_context_tests` pins both directions, including a verbatim legacy payload. +/// +/// To be exact rather than reassuring: a hand-corrupted payload carrying BOTH fields +/// would match `Gsi`, because untagged tries variants in declaration order and +/// ignores unknown fields. That is unreachable from any serializer here, and a +/// genuinely malformed context is already dropped as a poison row, so it is a +/// property of the representation worth knowing rather than a case to defend. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum PendingApplyContext { + Gsi(GsiApplyContext), + Vector(super::vector_index::VectorApplyContext), +} + +impl PendingApplyContext { + /// The base table's key schema, which both kinds carry and the queue needs in + /// order to place the row in its base key's partition. + fn base_key_schema(&self) -> &[KeySchemaElement] { + match self { + Self::Gsi(c) => &c.base_key_schema, + Self::Vector(c) => &c.base_key_schema, + } + } +} + +/// Apply one claimed row to whichever index kind it describes. +/// +/// Both arms tolerate a data table that no longer exists, so a base table or index +/// dropped while a row was in flight is applied-as-skip rather than logged and +/// dropped as unprocessable. +pub(crate) async fn apply_pending_context( + tx: &mut db::Transaction, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &PendingApplyContext, +) -> Result<(), StorageError> { + match context { + PendingApplyContext::Gsi(c) => apply_claimed_row(tx, old_item, new_item, c).await, + PendingApplyContext::Vector(c) => { + super::vector_index::apply_vector_context(tx, old_item, new_item, c).await + } + } +} + +/// Metadata for a single index, used on the write path and by the GSI worker. +pub(crate) struct IndexMeta { + pub(super) index_id: String, + pub(super) index_name: String, + pub(super) index_type: String, + pub(super) key_schema: Vec, + pub(super) projection: Projection, + /// Per-GSI propagation delay (ms). `None` = use system default; `Some(0)` = + /// synchronous. + pub(super) propagation_delay_ms: Option, +} + +/// Fetch all index metadata for a table from the catalog. +pub(crate) async fn fetch_indexes_for_table( + table_id: &str, + pool: &db::Pool, +) -> Result, StorageError> { + let rows: Vec<(String, String, String, String, String, Option)> = db::query_as( + "SELECT index_id, index_name, index_type, key_schema, projection, propagation_delay_ms \ + FROM indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(pool) + .await + .map_err(crate::duckdb_util::map_db_err)?; + + rows.into_iter() + .map(|(index_id, index_name, index_type, ks, proj, delay)| { + Ok(IndexMeta { + index_id, + index_name, + index_type, + key_schema: serde_json::from_str(&ks) + .map_err(|e| StorageError::Internal(e.to_string()))?, + projection: serde_json::from_str(&proj) + .map_err(|e| StorageError::Internal(e.to_string()))?, + propagation_delay_ms: delay, + }) + }) + .collect() +} + +/// Effective GSI propagation delay (ms): per-GSI override, else system default. +/// `Some(0)` forces synchronous; negative is treated as "use system default". +pub(crate) fn effective_delay(idx: &IndexMeta, system_default: u64) -> u64 { + match idx.propagation_delay_ms { + Some(0) => 0, + Some(ms) if ms > 0 => ms as u64, + _ => system_default, + } +} + +/// Enqueue async GSI propagation for a write: **one self-describing +/// `gsi_pending` row per async index**, each honoring its own effective delay. +/// +/// Returns the number of rows enqueued so callers can skip waking the worker +/// when nothing was queued. Must run inside the base write transaction so the +/// pending rows commit atomically with the item mutation. +pub(crate) async fn enqueue_async_indexes( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + indexes: &[IndexMeta], + old_item: Option<&Item>, + new_item: Option<&Item>, + system_default_delay: u64, +) -> Result { + let mut enqueued = 0usize; + for idx in indexes { + if idx.index_type == "LSI" { + continue; // LSIs are always synchronous. + } + let delay = effective_delay(idx, system_default_delay); + if delay == 0 { + continue; // Synchronous GSI — handled in-txn by sync_indexes. + } + let context = GsiApplyContext { + base_key_schema: key_info.key_schema.clone(), + attribute_definitions: key_info.attribute_definitions.clone(), + index: GsiIndexDef { + index_id: idx.index_id.clone(), + key_schema: idx.key_schema.clone(), + projection: idx.projection.clone(), + }, + }; + enqueue_pending_row( + tx, + &key_info.table_id, + old_item, + new_item, + delay, + &PendingApplyContext::Gsi(context), + ) + .await?; + enqueued += 1; + } + Ok(enqueued) +} + +/// Insert one self-describing `gsi_pending` row inside the base write transaction +/// (zero crash window). Shared by the GSI and vector write paths, which is what puts +/// both index kinds in one totally ordered queue. +/// +/// `delay_ms` is the effective delay; a jitter in `[delay/2, delay]` is applied. +/// `ready_at` is clamped to `max(now + jitter, MAX(ready_at) in the base key's +/// partition)` so a later write that draws a smaller jitter can never become +/// eligible before an earlier one — preserving per-key FIFO when the worker drains +/// the partition in `id` order. +pub(crate) async fn enqueue_pending_row( + tx: &mut db::Transaction, + table_id: &str, + old_item: Option<&Item>, + new_item: Option<&Item>, + delay_ms: u64, + context: &PendingApplyContext, +) -> Result<(), StorageError> { + let old_json = old_item + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let new_json = new_item + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let context_json = + serde_json::to_string(context).map_err(|e| StorageError::Internal(e.to_string()))?; + + // Route all updates for one base item to a single partition (per-key FIFO). + // The base key is immutable: `new_item` carries it for puts/updates, + // `old_item` for deletes. Both context kinds describe the same base table, so + // a GSI row and a vector row for one item hash to the same partition and stay + // mutually ordered. + let worker_partition = match new_item.or(old_item) { + Some(item) => partition_for(&composite_pk_to_text(item, context.base_key_schema())?), + None => 0, + }; + + // ready_at as RFC 3339 so it compares correctly (lexically) against the + // worker's RFC 3339 `now` cutoff and against other rows' ready_at. + let jittered = jitter_delay_ms(delay_ms); + let candidate = crate::duckdb_util::format_timestamp( + time::OffsetDateTime::now_utc() + + time::Duration::milliseconds(i64::try_from(jittered).unwrap_or(i64::MAX)), + ); + // Monotonic clamp within the partition (RFC 3339 strings sort lexically). + let part_max: Option = + db::query_scalar("SELECT MAX(ready_at) FROM gsi_pending WHERE worker_partition = ?") + .bind(worker_partition) + .fetch_one(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + let ready_at = match part_max { + Some(prev) if prev > candidate => prev, + _ => candidate, + }; + + db::query( + "INSERT INTO gsi_pending \ + (table_id, worker_partition, old_item, new_item, index_context, ready_at) \ + VALUES (?, ?, ?, ?, ?, ?)", + ) + .bind(table_id) + .bind(worker_partition) + .bind(&old_json) + .bind(&new_json) + .bind(&context_json) + .bind(&ready_at) + .execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + Ok(()) +} + +/// Project an item according to an index's projection configuration. +pub(crate) fn project_item_for_index( + item: &Item, + index_ks: &[KeySchemaElement], + base_ks: &[KeySchemaElement], + projection: &Projection, +) -> Item { + match projection.projection_type { + ProjectionType::All => item.clone(), + ProjectionType::KeysOnly | ProjectionType::Include => { + let mut projected = Item::new(); + for ks in base_ks.iter().chain(index_ks.iter()) { + if let Some(v) = item.get(&ks.attribute_name) { + projected.insert(ks.attribute_name.clone(), v.clone()); + } + } + if projection.projection_type == ProjectionType::Include + && let Some(attrs) = &projection.non_key_attributes + { + for attr in attrs { + if let Some(v) = item.get(attr) { + projected.insert(attr.clone(), v.clone()); + } + } + } + projected + } + } +} + +/// Whether an item carries every key attribute an index requires. +pub(crate) fn item_has_index_keys(item: &Item, index_ks: &[KeySchemaElement]) -> bool { + index_ks + .iter() + .all(|ks| item.contains_key(&ks.attribute_name)) +} + +/// Synchronously reconcile index tables for a base-item change — but only for +/// indexes that are synchronous: LSIs (always) and GSIs whose effective delay +/// is 0. GSIs with a non-zero delay are deferred via `gsi_pending` and applied +/// by the worker, so they are skipped here. +pub(crate) async fn sync_indexes( + tx: &mut db::Transaction, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + indexes: &[IndexMeta], + old_item: Option<&Item>, + new_item: Option<&Item>, + system_default_delay: u64, +) -> Result<(), StorageError> { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + for idx in indexes { + if idx.index_type != "LSI" && effective_delay(idx, system_default_delay) != 0 { + continue; // Async GSI — applied later by the propagation worker. + } + let idx_table = index_table_name(&idx.index_id); + let idx_sks = all_sort_key_info(&idx.key_schema, attr_defs); + + if let Some(old) = old_item + && item_has_index_keys(old, &idx.key_schema) + { + delete_index_row_multi(tx, &idx_table, old, base_key_schema, &base_sks).await?; + } + + if let Some(new) = new_item + && item_has_index_keys(new, &idx.key_schema) + { + let projected = + project_item_for_index(new, &idx.key_schema, base_key_schema, &idx.projection); + insert_index_row_multi( + tx, + &idx_table, + new, + &projected, + &idx.key_schema, + base_key_schema, + &idx_sks, + &base_sks, + ) + .await?; + } + } + Ok(()) +} + +/// Delete an index row identified by its base-table key columns. +pub(crate) async fn delete_index_row_multi( + tx: &mut db::Transaction, + idx_table: &str, + item: &Item, + base_ks: &[KeySchemaElement], + base_sks: &[(&str, ScalarAttributeType)], +) -> Result<(), StorageError> { + let base_pk_text = composite_pk_to_text(item, base_ks)?; + + let mut where_parts = vec!["base_pk = ?".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + where_parts.push(format!("base_{} = ?", sk_column_n(i, sk_type))); + } + let sql = format!( + "DELETE FROM {idx_table} WHERE {}", + where_parts.join(" AND ") + ); + + let mut query = db::query(&sql).bind(base_pk_text); + for &(sk_name, sk_type) in base_sks { + // Bind a value for every placeholder, mirroring insert_index_row_multi: + // a missing sort-key attribute binds an empty string rather than + // skipping the bind (which would desynchronise placeholders and binds). + let bound = match item.get(sk_name) { + Some(v) => sk_bound(&parse_sk(v, sk_type)?), + None => BoundValue::Text(String::new()), + }; + query = super::bind_bound!(query, bound); + } + query + .execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + Ok(()) +} + +/// Insert (or replace) an index row for a base item. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn insert_index_row_multi( + tx: &mut db::Transaction, + idx_table: &str, + item: &Item, + projected: &Item, + index_ks: &[KeySchemaElement], + base_ks: &[KeySchemaElement], + idx_sks: &[(&str, ScalarAttributeType)], + base_sks: &[(&str, ScalarAttributeType)], +) -> Result<(), StorageError> { + let idx_pk_text = composite_pk_to_text(item, index_ks)?; + let base_pk_text = composite_pk_to_text(item, base_ks)?; + let item_json = + serde_json::to_string(projected).map_err(|e| StorageError::Internal(e.to_string()))?; + + // Columns and the matching bound values, in column order. + let mut cols = vec!["pk".to_owned()]; + let mut values: Vec = vec![BoundValue::Text(idx_pk_text)]; + + for (i, &(sk_name, sk_type)) in idx_sks.iter().enumerate() { + cols.push(sk_column_n(i, sk_type)); + if let Some(v) = item.get(sk_name) { + values.push(sk_bound(&parse_sk(v, sk_type)?)); + } else { + values.push(BoundValue::Text(String::new())); + } + } + + cols.push("base_pk".to_owned()); + values.push(BoundValue::Text(base_pk_text)); + + for (i, &(sk_name, sk_type)) in base_sks.iter().enumerate() { + cols.push(format!("base_{}", sk_column_n(i, sk_type))); + if let Some(v) = item.get(sk_name) { + values.push(sk_bound(&parse_sk(v, sk_type)?)); + } else { + values.push(BoundValue::Text(String::new())); + } + } + + cols.push("item_data".to_owned()); + values.push(BoundValue::Text(item_json)); + + let placeholders = vec!["?"; cols.len()].join(", "); + let sql = format!( + "INSERT OR REPLACE INTO {idx_table} ({}) VALUES ({placeholders})", + cols.join(", ") + ); + + let mut query = db::query(&sql); + for v in values { + query = super::bind_bound!(query, v); + } + query + .execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + Ok(()) +} + +/// Helper: derive the sort-key column name used by an index data table for the +/// Nth index sort key. Exposed for query routing. +#[allow(dead_code)] +pub(crate) fn index_sk_column(index: usize, sk_type: ScalarAttributeType) -> String { + if index == 0 { + sk_column(sk_type).to_owned() + } else { + sk_column_n(index, sk_type) + } +} + +/// True if a storage error is a "missing table" error (the `_ddb_*` index table +/// or a `_vidx_*` vector table was dropped, e.g. the base table was deleted +/// while a `gsi_pending` row was in flight). Such rows are benignly skipped. +pub(crate) fn is_no_such_table(e: &StorageError) -> bool { + matches!(e, StorageError::Internal(msg) if msg.contains("no such table")) +} + +/// Apply one claimed `gsi_pending` row to its single target index, within the +/// worker's transaction, using the row's self-describing `index_context` — no +/// catalog read. A missing index data table (the base table was deleted while +/// the row was in flight) is skipped benignly. +pub(crate) async fn apply_claimed_row( + tx: &mut db::Transaction, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &GsiApplyContext, +) -> Result<(), StorageError> { + let base_sks = all_sort_key_info(&context.base_key_schema, &context.attribute_definitions); + let idx = &context.index; + let idx_table = index_table_name(&idx.index_id); + let idx_sks = all_sort_key_info(&idx.key_schema, &context.attribute_definitions); + + if let Some(old) = old_item + && item_has_index_keys(old, &idx.key_schema) + { + delete_index_row_multi(tx, &idx_table, old, &context.base_key_schema, &base_sks) + .await + .or_else(|e| if is_no_such_table(&e) { Ok(()) } else { Err(e) })?; + } + if let Some(new) = new_item + && item_has_index_keys(new, &idx.key_schema) + { + let projected = project_item_for_index( + new, + &idx.key_schema, + &context.base_key_schema, + &idx.projection, + ); + insert_index_row_multi( + tx, + &idx_table, + new, + &projected, + &idx.key_schema, + &context.base_key_schema, + &idx_sks, + &base_sks, + ) + .await + .or_else(|e| if is_no_such_table(&e) { Ok(()) } else { Err(e) })?; + } + Ok(()) +} + +#[cfg(test)] +mod pending_context_tests { + use super::{GsiApplyContext, GsiIndexDef, PendingApplyContext}; + use crate::data::vector_index::{VectorApplyContext, VectorIndexMeta}; + use crate::db; + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, Projection, ProjectionType, + ScalarAttributeType, + }; + + /// A context written by a build that predates vector rows, verbatim. It must + /// still deserialize. + /// + /// This is the test that protects an upgrade. A row whose `index_context` fails + /// to parse is treated as poison and DROPPED, so if the representation had + /// changed incompatibly, every GSI update in flight at the moment of the upgrade + /// would have been discarded silently, with the item written and its index never + /// catching up. + const LEGACY_GSI_CONTEXT: &str = r#"{ + "base_key_schema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "attribute_definitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "index": { + "index_id": "idx-1", + "key_schema": [{"AttributeName": "gsipk", "KeyType": "HASH"}], + "projection": {"ProjectionType": "ALL"} + } + }"#; + + fn base_ks() -> Vec { + vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }] + } + + fn base_ad() -> Vec { + vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }] + } + + fn gsi_context() -> GsiApplyContext { + GsiApplyContext { + base_key_schema: base_ks(), + attribute_definitions: base_ad(), + index: GsiIndexDef { + index_id: "idx-1".to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "gsipk".to_owned(), + key_type: KeyType::Hash, + }], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + }, + } + } + + fn vector_context() -> VectorApplyContext { + VectorApplyContext { + base_key_schema: base_ks(), + attribute_definitions: base_ad(), + table_id: "t-1".to_owned(), + vector: VectorIndexMeta { + index_id: "vidx-1".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + hash_attribute_name: Some("tenant".to_owned()), + search_schema_attribute_names: vec!["tenant".to_owned()], + }, + } + } + + #[test] + fn a_legacy_gsi_context_still_deserializes_as_a_gsi_row() { + let parsed: PendingApplyContext = + serde_json::from_str(LEGACY_GSI_CONTEXT).expect("legacy context must still parse"); + match parsed { + PendingApplyContext::Gsi(c) => assert_eq!(c.index.index_id, "idx-1"), + PendingApplyContext::Vector(_) => { + panic!("a legacy GSI context must not be read as a vector row") + } + } + } + + /// The bytes a GSI row writes today are the bytes it wrote before the enum + /// existed, so a row written by this build is still readable by the previous + /// one. `untagged` is what buys this, and this test is what keeps it. + #[test] + fn wrapping_a_gsi_context_does_not_change_its_serialized_form() { + let bare = serde_json::to_string(&gsi_context()).expect("bare"); + let wrapped = + serde_json::to_string(&PendingApplyContext::Gsi(gsi_context())).expect("wrapped"); + assert_eq!( + bare, wrapped, + "the queue's on-disk format must not change for GSI rows" + ); + } + + /// A vector context round-trips and carries no GSI discriminant field. + /// + /// Note what this does NOT guard: it still passes if `untagged` is removed, so it + /// is not the protection for on-disk compatibility. The two tests above are, and + /// both fail without `untagged`. This one pins the shape contract that makes the + /// discrimination possible in the first place. + #[test] + fn a_vector_context_round_trips_and_carries_no_gsi_discriminant() { + let json = serde_json::to_string(&PendingApplyContext::Vector(vector_context())) + .expect("serialize"); + assert!( + !json.contains("\"index\":"), + "a vector context must not carry the GSI discriminant field: {json}" + ); + let parsed: PendingApplyContext = serde_json::from_str(&json).expect("deserialize"); + match parsed { + PendingApplyContext::Vector(c) => { + assert_eq!(c.vector.index_id, "vidx-1"); + assert_eq!(c.table_id, "t-1"); + assert_eq!(c.vector.dimensions, 2); + assert_eq!(c.vector.hash_attribute_name.as_deref(), Some("tenant")); + assert_eq!(c.vector.search_schema_attribute_names, ["tenant"]); + assert_eq!( + c.vector.projection.projection_type, + ProjectionType::KeysOnly + ); + } + PendingApplyContext::Gsi(_) => panic!("a vector context must not be read as a GSI row"), + } + } + + /// A GSI row and a vector row for the SAME base item must land in the same queue + /// partition, which is what makes them mutually ordered. + /// + /// This is the claim that reusing one queue buys ordering ACROSS index kinds, and + /// it holds only because both contexts hash the same base key rather than + /// anything index-specific. Asserted on the partition because the partition is the + /// mechanism: two kinds hashing differently would land in separate partitions, + /// each monotonic on its own, leaving the relative order of a GSI and a vector + /// update to one item unconstrained. + /// + /// Driven through the real `enqueue_pending_row` rather than a test shim, so it is + /// the production path being measured. + #[tokio::test] + async fn a_gsi_row_and_a_vector_row_for_one_item_share_a_partition() { + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let the_item: extenddb_core::types::Item = + serde_json::from_str(r#"{"pk":{"S":"shared-key"},"emb":{"L":[{"N":"1"}]}}"#) + .expect("item"); + let mut tx = engine.pool.begin().await.expect("tx"); + for context in [ + PendingApplyContext::Gsi(gsi_context()), + PendingApplyContext::Vector(vector_context()), + ] { + super::enqueue_pending_row(&mut tx, "t-1", None, Some(&the_item), 60_000, &context) + .await + .expect("enqueue"); + } + tx.commit().await.expect("commit"); + + let partitions: Vec<(i64,)> = + db::query_as("SELECT DISTINCT worker_partition FROM gsi_pending") + .fetch_all(&engine.pool) + .await + .expect("partitions"); + assert_eq!( + partitions.len(), + 1, + "a GSI row and a vector row for one base item must share a partition, or \ + their relative order is unconstrained" + ); + let (depth,): (i64,) = db::query_as("SELECT COUNT(*) FROM gsi_pending") + .fetch_one(&engine.pool) + .await + .expect("depth"); + assert_eq!( + depth, 2, + "both kinds must have enqueued, so one partition is not an artefact of a \ + missing row" + ); + } + + /// A catalog created before the rename must keep honouring its operator's value. + /// + /// This is the compatibility property the whole fallback exists for, and the cost + /// of getting it wrong is not cosmetic: the server refuses to start on a + /// catalog-version mismatch rather than migrating, so nothing ever rewrites the old + /// row. Reading past it would silently reset a configured delay to the default, and + /// since 0 means synchronous, the silent change would be from strict to eventually + /// consistent, which is exactly the direction that turns a passing test suite into + /// a flaky one somewhere else. + #[tokio::test] + async fn a_pre_rename_catalog_still_honours_its_configured_delay() { + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + // Reshape the catalog to look as it did before the rename: the legacy key + // only, carrying a deliberately non-default value. + db::query("DELETE FROM settings WHERE key = 'index_propagation_delay_ms'") + .execute(&engine.pool) + .await + .expect("drop canonical row"); + db::query("INSERT INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '0')") + .execute(&engine.pool) + .await + .expect("seed legacy row"); + + assert_eq!( + engine.index_propagation_delay().await, + 0, + "a value set under the pre-rename key must still be honoured, or an \ + operator's synchronous setting silently becomes asynchronous" + ); + } + + /// With both rows present the canonical one wins, deterministically. + /// + /// Reachable if an operator sets the legacy key on a build that predates the + /// canonicalising write path and then upgrades. Without the explicit ordering the + /// winner would be whichever row DuckDB happened to return first. + #[tokio::test] + async fn the_canonical_key_wins_when_both_are_present() { + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + db::query( + "INSERT OR REPLACE INTO settings (key, value) \ + VALUES ('index_propagation_delay_ms', '7'), ('gsi_propagation_delay_ms', '999')", + ) + .execute(&engine.pool) + .await + .expect("seed both rows"); + + assert_eq!( + engine.index_propagation_delay().await, + 7, + "the canonical key must take precedence over the deprecated alias" + ); + } +} diff --git a/crates/storage-duckdb/src/data/mod.rs b/crates/storage-duckdb/src/data/mod.rs new file mode 100644 index 00000000..5bed28c2 --- /dev/null +++ b/crates/storage-duckdb/src/data/mod.rs @@ -0,0 +1,187 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Per-DynamoDB-table data storage for the DuckDB backend. +//! +//! Each virtual DynamoDB table maps to a DuckDB table named `_ddb_`; +//! each secondary index maps to `_ddb_`. The full item is stored as +//! JSON `TEXT` in `item_data`; key columns exist only for lookup and ordering. +//! +//! # Key column types (design decision D2) +//! +//! - Partition key (`pk`): always `TEXT` via the shared `pk_to_text` (equality +//! only) — identical to the PostgreSQL backend. +//! - Sort key, by attribute type: +//! - `S` → `sk_s TEXT` (DuckDB BINARY collation = DynamoDB UTF-8 byte order). +//! - `N` → `sk_n TEXT` holding [`encode_orderable_number`], so lexicographic +//! order equals numeric order with full 38-digit precision (never `DOUBLE`). +//! - `B` → `sk_b BLOB` (memcmp = DynamoDB unsigned byte order). +//! +//! The exact, full-precision value is always preserved in `item_data` JSON, so +//! reads lose nothing; the typed key columns are used only for +//! lookup/range/ordering. + +use extenddb_core::types::{ + AttributeDefinition, Item, KeySchemaElement, KeyType, ScalarAttributeType, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::SortKeyValue; + +use crate::number_key::encode_orderable_number; + +mod data_engine; +mod ddl; +mod delete_item; +mod index; +mod put_item; +mod query; +mod query_scan; +mod transactions; +mod tx_helpers; +mod update_item; +pub(crate) mod vector_index; + +pub(crate) use index::{ + PendingApplyContext, apply_pending_context, insert_index_row_multi, project_item_for_index, +}; +pub(crate) use tx_helpers::upsert_item_in_tx; + +/// Quoted SQL identifier for a virtual DynamoDB table's data table. +pub(crate) fn data_table_name(table_id: &str) -> String { + format!("\"_ddb_{table_id}\"") +} + +/// Quoted SQL identifier for a GSI/LSI data table. +pub(crate) fn index_table_name(index_id: &str) -> String { + format!("\"_ddb_{index_id}\"") +} + +/// Quoted SQL identifier for a vector-index data table. +/// +/// The name carries the base `table_id` as well as the `index_id` so every vector +/// table belonging to a DynamoDB table is discoverable from the table alone. That +/// is what lets `drop_data_table` clean them up: by the time it runs, the catalog +/// rows have already been cascade-deleted inside the same transaction, so the +/// index ids can no longer be read from `vector_indexes`. +/// +/// One row per vector rather than a packed blob per partition. Measured +/// 2026-08-06: a packed blob reads 2 to 4x faster but makes every write +/// O(partition), since inserting one vector rewrites the whole blob (390 MB for +/// a 100k-vector partition at 1024 dimensions). Vector indexes are maintained on +/// every write to an indexed attribute, so that trade is not available. +pub(crate) fn vector_table_name(table_id: &str, index_id: &str) -> String { + format!("\"_vidx_{table_id}_{index_id}\"") +} + +/// `GLOB` pattern matching every vector data table of one DynamoDB table. +/// +/// `GLOB` rather than `LIKE` because the pattern itself is full of underscores, and +/// in `LIKE` an underscore is a single-character wildcard: `_vidx__%` would match +/// far more than it appears to. It can only ever over-match, since a wildcard also +/// matches a literal underscore, and no other table could share the UUID, so `LIKE` +/// was safe in practice. It was not safe for the stated reason, though, and a comment +/// that justifies the wrong half is worse than none. +/// +/// `GLOB` treats `_` literally and uses `*` for the wildcard, so the pattern means +/// what it reads. `table_id` is a server-generated UUID, which contains no `GLOB` +/// metacharacters either. +pub(crate) fn vector_table_glob_pattern(table_id: &str) -> String { + format!("_vidx_{table_id}_*") +} + +/// All RANGE key attributes in key-schema order, paired with their scalar type. +pub(crate) fn all_sort_key_info<'a>( + key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], +) -> Vec<(&'a str, ScalarAttributeType)> { + key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Range) + .filter_map(|ks| { + attr_defs + .iter() + .find(|ad| ad.attribute_name == ks.attribute_name) + .map(|ad| (ks.attribute_name.as_str(), ad.attribute_type)) + }) + .collect() +} + +/// Deserialize an `item_data` JSON value into an `Item`. +pub(crate) fn json_to_item(v: serde_json::Value) -> Result { + serde_json::from_value(v).map_err(|e| StorageError::Internal(e.to_string())) +} + +/// A value bound to a sort-key column, already mapped to its DuckDB storage +/// representation per D2. +#[derive(Debug, Clone)] +pub(crate) enum BoundValue { + /// `TEXT` — used for `S` (raw string) and `N` (order-preserving encoding). + Text(String), + /// `BLOB` — used for `B`. + Blob(Vec), +} + +/// Map a parsed sort-key value to its DuckDB storage representation (D2): +/// `S` → TEXT, `N` → order-preserving TEXT, `B` → BLOB. +pub(crate) fn sk_bound(sk: &SortKeyValue) -> BoundValue { + match sk { + SortKeyValue::S(s) => BoundValue::Text(s.clone()), + SortKeyValue::N(n) => BoundValue::Text(encode_orderable_number(n)), + SortKeyValue::B(b) => BoundValue::Blob(b.clone()), + } +} + +/// Bind a [`BoundValue`] onto a positional `sqlx` query. +macro_rules! bind_bound { + ($query:expr, $bound:expr) => { + match $bound { + crate::data::BoundValue::Text(s) => $query.bind(s), + crate::data::BoundValue::Blob(b) => $query.bind(b), + } + }; +} + +/// Bind `pk` then a sort-key value, then `fetch_optional` a single JSON column. +macro_rules! bind_sk_fetch_optional { + ($sql:expr, $pk:expr, $sk:expr, $executor:expr) => {{ + let __q = crate::db::query_as::<(serde_json::Value,)>($sql).bind($pk); + let __q = match crate::data::sk_bound($sk) { + crate::data::BoundValue::Text(s) => __q.bind(s), + crate::data::BoundValue::Blob(b) => __q.bind(b), + }; + __q.fetch_optional($executor) + .await + .map_err(|e| extenddb_storage::error::StorageError::Internal(e.to_string())) + }}; +} + +/// Bind `pk`, a sort-key value, then `item_json`, and `execute`. +macro_rules! bind_sk_execute { + ($sql:expr, $pk:expr, $sk:expr, $item_json:expr, $executor:expr) => {{ + let __q = crate::db::query($sql).bind($pk); + let __q = match crate::data::sk_bound($sk) { + crate::data::BoundValue::Text(s) => __q.bind(s), + crate::data::BoundValue::Blob(b) => __q.bind(b), + }; + __q.bind($item_json) + .execute($executor) + .await + .map_err(|e| extenddb_storage::error::StorageError::Internal(e.to_string())) + }}; +} + +/// Bind `pk` then a sort-key value, and `execute` (no item payload). +macro_rules! bind_sk_only_execute { + ($sql:expr, $pk:expr, $sk:expr, $executor:expr) => {{ + let __q = crate::db::query($sql).bind($pk); + let __q = match crate::data::sk_bound($sk) { + crate::data::BoundValue::Text(s) => __q.bind(s), + crate::data::BoundValue::Blob(b) => __q.bind(b), + }; + __q.execute($executor) + .await + .map_err(|e| extenddb_storage::error::StorageError::Internal(e.to_string())) + }}; +} + +pub(crate) use {bind_bound, bind_sk_execute, bind_sk_fetch_optional, bind_sk_only_execute}; diff --git a/crates/storage-duckdb/src/data/put_item.rs b/crates/storage-duckdb/src/data/put_item.rs new file mode 100644 index 00000000..0817ac75 --- /dev/null +++ b/crates/storage-duckdb/src/data/put_item.rs @@ -0,0 +1,190 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `put_item` and `get_item` for the DuckDB backend. +//! +//! Writes acquire the engine write lock (D1) and run in a transaction, so the +//! condition check, the write, index sync, and stream capture are one atomic +//! unit with no competing writer — no `INSERT ... ON CONFLICT DO NOTHING` race +//! dance is needed. + +use crate::db; +use extenddb_core::expression::{Expr, ExpressionMaps}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{pk_to_text, sk_column, sk_info}; + +use super::index::{enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::query::check_condition; +use super::tx_helpers::{fetch_item_in_tx, upsert_item_in_tx, write_stream_record_in_tx}; +use super::{bind_sk_fetch_optional, data_table_name, json_to_item}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + pub(crate) async fn put_item_impl( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + // Read the index set only after acquiring the write lock, so a GSI added + // by a concurrent UpdateTable (which holds the same lock) cannot be missed + // and left unmaintained by this write. + // Read the propagation delay BEFORE taking the write lock. It is a + // runtime setting, not an invariant of this write, so it does not need + // to be read under the lock, and the lock serialises every write in the + // process: work done inside it is the backend's throughput bottleneck. + let system_delay = self.index_propagation_delay().await; + let _writer = self.write_lock.lock().await; + let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + + // Index key attributes present in the item must match their declared + // scalar type and be non-empty, matching real DynamoDB. This is up-front + // input validation (a top-level ValidationException), so it runs before + // any write work. Mirrors the PostgreSQL backend. + if !indexes.is_empty() { + let index_refs: Vec> = indexes + .iter() + .map(|idx| extenddb_core::validation::IndexKeyRef { + index_name: &idx.index_name, + key_schema: &idx.key_schema, + }) + .collect(); + extenddb_core::validation::validate_index_keys( + &item, + &index_refs, + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; + } + + let need_old = condition.is_some() || return_old || !indexes.is_empty() || stream.is_some(); + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old = if need_old { + fetch_item_in_tx(&mut tx, key_info, &item).await? + } else { + None + }; + + if condition.is_some() { + let empty = Item::new(); + let target = old.as_ref().unwrap_or(&empty); + if let Err(e) = check_condition(condition, target, maps) { + return match e { + StorageError::ConditionFailed(_) => Err(StorageError::ConditionFailed(old)), + other => Err(other), + }; + } + } + + upsert_item_in_tx(&mut tx, key_info, &item).await?; + + // Synchronous indexes (LSIs + zero-delay GSIs) are applied in-txn; + // async GSIs are enqueued into gsi_pending within the same txn. + if !indexes.is_empty() { + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old.as_ref(), + Some(&item), + system_delay, + ) + .await?; + } + let enqueued_gsi = enqueue_async_indexes( + &mut tx, + key_info, + &indexes, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0; + // Vector indexes: applied in this transaction when the propagation delay is + // 0, otherwise enqueued alongside the async GSI work. Gated on the cached + // key info so a table without them costs no extra query, and deliberately + // outside the `indexes` guard above: a table may have a vector index and no + // GSI or LSI at all. + let enqueued_vector = if key_info.vector_indexes.is_empty() { + false + } else { + crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0 + }; + let enqueued = enqueued_gsi || enqueued_vector; + + if let Some(capture) = stream { + write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) + .await?; + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if enqueued { + self.gsi_notify.notify_waiters(); + } + + Ok(if return_old { old } else { None }) + } + + pub(crate) async fn get_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> Result, StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + let json_opt = if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = extenddb_storage::util::parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + let row: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&sql, pk_text.as_ref(), &sk, &self.pool)?; + row.map(|(v,)| v) + } else { + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let row: Option<(serde_json::Value,)> = db::query_as(&sql) + .bind(pk_text.as_ref()) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + row.map(|(v,)| v) + }; + + json_opt.map(json_to_item).transpose() + } +} diff --git a/crates/storage-duckdb/src/data/query.rs b/crates/storage-duckdb/src/data/query.rs new file mode 100644 index 00000000..c37941cb --- /dev/null +++ b/crates/storage-duckdb/src/data/query.rs @@ -0,0 +1,233 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared query helpers: condition evaluation, key-condition SQL fragments, +//! and dynamic query execution. +//! +//! Sort-key comparisons bind through [`super::sk_bound`], so `N` keys are +//! compared as their order-preserving TEXT encoding (D2): `>`, `<`, `BETWEEN` +//! all remain numerically correct. `begins_with` applies only to `S` (string +//! prefix via the maximal code point `char(1114111)`) and `B` (byte-range via +//! an incremented upper bound); DynamoDB rejects `begins_with` on `N`. + +use crate::db; +use extenddb_core::expression::{self, CompareOp, Expr, ExpressionMaps, SortKeyCondition}; +use extenddb_core::types::{ + AttributeValue, Item, KeySchemaElement, ScalarAttributeType, extract_key, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{SortKeyValue, parse_sk}; + +use super::BoundValue; + +/// Evaluate an optional condition expression against an item. +/// Returns `ConditionFailed(None)` when the condition evaluates to false. +pub(super) fn check_condition( + condition: Option<&Expr>, + item: &Item, + maps: &ExpressionMaps, +) -> Result<(), StorageError> { + if let Some(cond) = condition { + let passed = expression::evaluate_condition(cond, item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + return Err(StorageError::ConditionFailed(None)); + } + } + Ok(()) +} + +/// Resolve a key-condition placeholder expression to its `AttributeValue`. +pub(super) fn resolve_expr_to_av( + expr: &Expr, + maps: &ExpressionMaps, +) -> Result { + match expr { + Expr::Placeholder(name) => maps + .resolve_value(name) + .cloned() + .map_err(|e| StorageError::Validation(e.to_string())), + _ => Err(StorageError::Internal( + "expected placeholder in key condition".to_owned(), + )), + } +} + +/// SQL `WHERE` fragment for a sort-key condition on `sk_col`, together with the +/// values to bind for it. +/// +/// The fragment and its bind list are returned from one function on purpose. +/// `begins_with` on a binary key sometimes has no upper bound at all (see +/// [`binary_prefix_upper_bound`]), so the placeholder count is not fixed by the +/// condition kind alone. Building the SQL in one place and the binds in another +/// meant the two could disagree about how many placeholders exist, which is a +/// bind-offset corruption rather than a visible error. +pub(super) fn build_sk_sql_and_binds( + sk_cond: &SortKeyCondition, + sk_col: &str, + sk_type: ScalarAttributeType, + maps: &ExpressionMaps, +) -> Result<(String, Vec), StorageError> { + match sk_cond { + SortKeyCondition::Compare { op, value, .. } => { + let sql_op = match op { + CompareOp::Eq => "=", + CompareOp::Ne => "<>", + CompareOp::Lt => "<", + CompareOp::Le => "<=", + CompareOp::Gt => ">", + CompareOp::Ge => ">=", + }; + Ok(( + format!(" AND {sk_col} {sql_op} ?"), + vec![parse_sk(&resolve_expr_to_av(value, maps)?, sk_type)?], + )) + } + SortKeyCondition::Between { low, high, .. } => Ok(( + format!(" AND {sk_col} BETWEEN ? AND ?"), + vec![ + parse_sk(&resolve_expr_to_av(low, maps)?, sk_type)?, + parse_sk(&resolve_expr_to_av(high, maps)?, sk_type)?, + ], + )), + SortKeyCondition::BeginsWith { prefix, .. } => { + let sk = parse_sk(&resolve_expr_to_av(prefix, maps)?, sk_type)?; + match sk { + SortKeyValue::B(b) => match binary_prefix_upper_bound(&b) { + Some(upper) => Ok(( + format!(" AND {sk_col} >= ? AND {sk_col} < ?"), + vec![SortKeyValue::B(b), SortKeyValue::B(upper)], + )), + // Every byte string with this prefix is in range and there is + // no finite value above them all, so the only correct upper + // bound is none. Emitting one anyway is what silently dropped + // rows. + None => Ok((format!(" AND {sk_col} >= ?"), vec![SortKeyValue::B(b)])), + }, + // For strings the SQL upper bound is `? || chr(1114111)`, + // so the same prefix is bound twice. + SortKeyValue::S(s) => Ok(( + format!(" AND {sk_col} >= ? AND {sk_col} < (? || chr(1114111))"), + vec![SortKeyValue::S(s.clone()), SortKeyValue::S(s)], + )), + SortKeyValue::N(_) => Err(StorageError::Validation( + "begins_with is not supported on numeric sort keys".to_owned(), + )), + } + } + } +} + +/// Exclusive upper bound for a binary prefix range, or `None` when the range is +/// unbounded above. +/// +/// The bound is the smallest byte string greater than every string having +/// `prefix` as a prefix. Found by incrementing the last byte below `0xFF` and +/// discarding everything after it: `[1, 2]` yields `[1, 3]`, so `[1, 2, 9]` is +/// still included but `[1, 3]` is not. +/// +/// `None` when every byte is `0xFF`, which includes the empty prefix. No finite +/// bound exists in that case, because a longer all-`0xFF` string always sorts +/// after any candidate: with `[0xFF]` the strings `[0xFF, 0xFF]`, +/// `[0xFF, 0xFF, 0xFF]` and so on continue without end. The previous code +/// returned `vec![0xFF; prefix.len() + 1]` here, which is a value inside the +/// matching set rather than above it, so rows were silently dropped: for an +/// empty prefix it produced `[0xFF]`, which excluded every key sorting at or +/// after it, and `begins_with([])` then returned part of the partition while +/// reporting success. Returning `None` and omitting the predicate is the only +/// correct answer that does not depend on a maximum key length, which is +/// operator-configurable via `max_sort_key_size_bytes`. +fn binary_prefix_upper_bound(prefix: &[u8]) -> Option> { + let mut out = prefix.to_vec(); + while let Some(last) = out.last_mut() { + if *last < 0xFF { + *last += 1; + return Some(out); + } + out.pop(); + } + None +} + +/// Build a `LastEvaluatedKey` from an item's key attributes. +pub(super) fn build_key(item: &Item, key_schema: &[KeySchemaElement]) -> Item { + extract_key(item, key_schema) +} + +/// Execute a dynamically-built query, binding `BoundValue`s positionally. +pub(super) async fn execute_dynamic_query( + sql: &str, + values: Vec, + pool: &db::Pool, +) -> Result, StorageError> { + let mut query = db::query_as::<(serde_json::Value,)>(sql); + for v in values { + query = match v { + BoundValue::Text(s) => query.bind(s), + BoundValue::Blob(b) => query.bind(b), + }; + } + let rows: Vec<(serde_json::Value,)> = query + .fetch_all(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(rows.into_iter().map(|(v,)| v).collect()) +} + +#[cfg(test)] +mod tests { + use super::binary_prefix_upper_bound; + + /// Every byte string having `prefix` as a prefix must sort below the bound, + /// and the bound itself must not. Asserted against explicit witnesses rather + /// than trusting the arithmetic, since the defect this replaces was an + /// off-by-domain error that looked arithmetically reasonable. + #[test] + fn the_bound_excludes_itself_and_includes_every_extension() { + let upper = binary_prefix_upper_bound(&[1, 2]).expect("finite bound exists"); + assert_eq!(upper, vec![1, 3]); + for witness in [ + vec![1, 2], + vec![1, 2, 0], + vec![1, 2, 0xFF], + vec![1, 2, 9, 9], + ] { + assert!(witness < upper, "{witness:?} must be inside the range"); + } + assert!(vec![1, 3] >= upper, "the bound must be exclusive"); + } + + /// A trailing `0xFF` carries: the last byte below `0xFF` is incremented and + /// everything after it is discarded. + #[test] + fn a_trailing_all_ones_byte_carries_into_the_previous_byte() { + assert_eq!(binary_prefix_upper_bound(&[1, 0xFF]), Some(vec![2])); + assert_eq!(binary_prefix_upper_bound(&[1, 0xFF, 0xFF]), Some(vec![2])); + let upper = binary_prefix_upper_bound(&[1, 0xFF]).expect("finite bound exists"); + assert!(vec![1, 0xFF, 0xFF] < upper, "extension must be included"); + } + + /// The empty prefix matches everything, so no upper bound can exist. This is + /// the case that silently dropped rows: the old code returned `[0xFF]`, which + /// excluded every key sorting at or after it, so `begins_with([])` returned + /// part of the partition and reported success. + #[test] + fn an_empty_prefix_has_no_upper_bound() { + assert_eq!(binary_prefix_upper_bound(&[]), None); + } + + /// Same defect one length up, and not only for the empty prefix: an all-`0xFF` + /// prefix of any length has no finite bound, because a longer all-`0xFF` + /// string always sorts after any candidate. The old code returned + /// `[0xFF, 0xFF]` for `[0xFF]`, which wrongly excluded `[0xFF, 0xFF, 0xFF]`. + #[test] + fn an_all_ones_prefix_has_no_upper_bound_at_any_length() { + for prefix in [vec![0xFF], vec![0xFF, 0xFF], vec![0xFF; 8]] { + assert_eq!( + binary_prefix_upper_bound(&prefix), + None, + "no finite bound exists above {prefix:?}" + ); + } + } +} diff --git a/crates/storage-duckdb/src/data/query_scan.rs b/crates/storage-duckdb/src/data/query_scan.rs new file mode 100644 index 00000000..9562fb48 --- /dev/null +++ b/crates/storage-duckdb/src/data/query_scan.rs @@ -0,0 +1,465 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `query` and `scan` for the DuckDB backend. +//! +//! Mirrors the PostgreSQL backend's index routing and pagination. For index +//! operations the engine passes `key_info.key_schema` = index key schema and +//! `key_info.base_key_schema` = base table key schema, so the index `sk_*` +//! columns are addressed by the index sort key and `base_*` columns provide +//! tie-breakers. DuckDB differences: positional `?` placeholders, no `COLLATE` +//! (the default BINARY collation already matches DynamoDB byte order, and `N` +//! keys are stored as the order-preserving TEXT encoding per D2), and +//! `rowid % total_segments` for parallel scan. + +use std::fmt::Write; + +use extenddb_core::expression::{ExpressionMaps, KeyCondition, PathElement}; +use extenddb_core::types::{Item, ScalarAttributeType, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{ + encode_netstring_composite, parse_sk, pk_to_text, sk_column, sk_column_n, sk_info, +}; + +use super::query::{build_key, build_sk_sql_and_binds, execute_dynamic_query, resolve_expr_to_av}; +use super::{ + BoundValue, all_sort_key_info, data_table_name, index_table_name, json_to_item, sk_bound, +}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + #[allow(clippy::too_many_arguments)] + pub(crate) async fn query_impl( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + let (ddb_table, is_lsi) = if let Some(idx_name) = index_name { + let info = self + .fetch_index_info_by_table_id(&key_info.table_id, idx_name) + .await?; + ( + index_table_name(&info.index_id), + info.index_type == extenddb_core::types::IndexType::Lsi, + ) + } else { + (data_table_name(&key_info.table_id), false) + }; + + // Partition key (composite via netstring when multi-HASH). + let pk_text = if key_condition.extra_pk_conditions.is_empty() { + pk_to_text(&resolve_expr_to_av(&key_condition.pk_value, maps)?)?.into_owned() + } else { + let mut parts = + vec![pk_to_text(&resolve_expr_to_av(&key_condition.pk_value, maps)?)?.into_owned()]; + for (_, value) in &key_condition.extra_pk_conditions { + parts.push(pk_to_text(&resolve_expr_to_av(value, maps)?)?.into_owned()); + } + encode_netstring_composite(&parts) + }; + + let sk_info_val = sk_info(&key_info.key_schema, &key_info.attribute_definitions); + let all_sks = all_sort_key_info(&key_info.key_schema, &key_info.attribute_definitions); + let base_sk_info: Option<(String, ScalarAttributeType)> = if index_name.is_some() { + sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) + .map(|(n, t)| (n.to_owned(), t)) + } else { + None + }; + + let mut sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let mut binds: Vec = vec![BoundValue::Text(pk_text)]; + + // Primary sort-key condition. + if let (Some(sk_cond), Some((_, sk_type))) = (&key_condition.sk_condition, sk_info_val) { + let (sk_sql, sk_binds) = + build_sk_sql_and_binds(sk_cond, sk_column(sk_type), sk_type, maps)?; + sql.push_str(&sk_sql); + for v in &sk_binds { + binds.push(sk_bound(v)); + } + } + + // Extra RANGE-key equality conditions (multi-RANGE schemas). + for (path, value) in &key_condition.extra_sk_conditions { + let Some(attr_name) = resolve_attr_name(path, maps) else { + continue; + }; + if let Some(pos) = all_sks.iter().position(|(n, _)| *n == attr_name) + && pos > 0 + { + let (_, sk_type) = all_sks[pos]; + let _ = write!(sql, " AND {} = ?", sk_column_n(pos, sk_type)); + binds.push(sk_bound(&parse_sk( + &resolve_expr_to_av(value, maps)?, + sk_type, + )?)); + } + } + + // Pagination. + if exclusive_start_key.is_some() && sk_info_val.is_none() && index_name.is_none() { + return Ok((Vec::new(), None)); + } + if let Some(start_key) = exclusive_start_key { + append_query_pagination( + &mut sql, + &mut binds, + start_key, + sk_info_val, + base_sk_info.as_ref(), + key_info, + index_name.is_some(), + is_lsi, + forward, + )?; + } + + // ORDER BY. + let dir = if forward { "ASC" } else { "DESC" }; + if let Some((_, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + if let Some((_, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + if is_lsi { + let _ = write!(sql, " ORDER BY {sk_col} {dir}, {base_col} {dir}"); + } else { + // GSI: order by the full base primary key after the index SK + // so the ordering matches the pagination tie-breaker exactly. + // Ordering by base SK alone leaves rows that share an index SK + // and a base SK in an arbitrary order, which no + // ExclusiveStartKey can resume from deterministically. + let _ = write!(sql, " ORDER BY {sk_col} {dir}, base_pk ASC, {base_col} ASC"); + } + } else if index_name.is_some() { + let _ = write!(sql, " ORDER BY {sk_col} {dir}, base_pk ASC"); + } else { + let _ = write!(sql, " ORDER BY {sk_col} {dir}"); + } + } else if index_name.is_some() { + if let Some((_, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + let _ = write!(sql, " ORDER BY base_pk {dir}, {base_col} {dir}"); + } else { + let _ = write!(sql, " ORDER BY base_pk {dir}"); + } + } + + let fetch_limit = limit.map_or(1_000_001, |l| l + 1); + let _ = write!(sql, " LIMIT {fetch_limit}"); + + let rows = execute_dynamic_query(&sql, binds, &self.pool).await?; + finalize(rows, limit, &key_info.key_schema) + } + + pub(crate) async fn scan_impl( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + let ddb_table = if let Some(idx_name) = index_name { + let info = self + .fetch_index_info_by_table_id(&key_info.table_id, idx_name) + .await?; + index_table_name(&info.index_id) + } else { + data_table_name(&key_info.table_id) + }; + + let sk_info_val = sk_info(&key_info.key_schema, &key_info.attribute_definitions); + let base_sk_info: Option<(String, ScalarAttributeType)> = if index_name.is_some() { + sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) + .map(|(n, t)| (n.to_owned(), t)) + } else { + None + }; + + let mut sql = format!("SELECT item_data FROM {ddb_table}"); + let mut conditions: Vec = Vec::new(); + let mut binds: Vec = Vec::new(); + + // Parallel scan: disjoint rowid partitioning. seg/total are validated + // non-negative integers at the engine layer, safe to interpolate. + if let (Some(seg), Some(total)) = (segment, total_segments) { + conditions.push(format!("(rowid % {total}) = {seg}")); + } + + if let Some(start_key) = exclusive_start_key { + let pk_name = &key_info.key_schema[0].attribute_name; + if !start_key.contains_key(pk_name) { + return Err(StorageError::Validation( + "The provided starting key is invalid: The provided key element does not match the schema".to_owned(), + )); + } + let pk_text = pk_to_text(start_key.get(pk_name).unwrap())?.into_owned(); + + if index_name.is_some() { + if let Some((sk_name, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + let sk_bv = start_key + .get(sk_name) + .map(|v| parse_sk(v, sk_type)) + .transpose()? + .map(|s| sk_bound(&s)); + let base_pk_text = base_pk_from_start_key(start_key, key_info)?; + if let Some((base_name, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + conditions.push(format!( + "(pk, {sk_col}, base_pk, {base_col}) > (?, ?, ?, ?)" + )); + binds.push(BoundValue::Text(pk_text)); + binds.push(sk_bv.unwrap_or(BoundValue::Text(String::new()))); + binds.push(BoundValue::Text(base_pk_text)); + if let Some(v) = start_key.get(base_name.as_str()) { + binds.push(sk_bound(&parse_sk(v, *base_type)?)); + } else { + binds.push(BoundValue::Text(String::new())); + } + } else { + conditions.push(format!("(pk, {sk_col}, base_pk) > (?, ?, ?)")); + binds.push(BoundValue::Text(pk_text)); + binds.push(sk_bv.unwrap_or(BoundValue::Text(String::new()))); + binds.push(BoundValue::Text(base_pk_text)); + } + } else { + // Hash-only GSI. Include the base sort key in the + // pagination predicate when the base table has one: the + // index PRIMARY KEY is (pk, base_pk, base_sk*), so (pk, + // base_pk) alone is not a total order and would skip rows + // sharing a (pk, base_pk) across a page boundary. + let base_pk_text = base_pk_from_start_key(start_key, key_info)?; + if let Some((base_name, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + conditions.push(format!("(pk, base_pk, {base_col}) > (?, ?, ?)")); + binds.push(BoundValue::Text(pk_text)); + binds.push(BoundValue::Text(base_pk_text)); + if let Some(v) = start_key.get(base_name.as_str()) { + binds.push(sk_bound(&parse_sk(v, *base_type)?)); + } else { + binds.push(BoundValue::Text(String::new())); + } + } else { + conditions.push("(pk, base_pk) > (?, ?)".to_owned()); + binds.push(BoundValue::Text(pk_text)); + binds.push(BoundValue::Text(base_pk_text)); + } + } + } else if let Some((sk_name, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + conditions.push(format!("(pk, {sk_col}) > (?, ?)")); + binds.push(BoundValue::Text(pk_text)); + if let Some(v) = start_key.get(sk_name) { + binds.push(sk_bound(&parse_sk(v, sk_type)?)); + } else { + binds.push(BoundValue::Text(String::new())); + } + } else { + conditions.push("pk > ?".to_owned()); + binds.push(BoundValue::Text(pk_text)); + } + } + + if !conditions.is_empty() { + sql.push_str(" WHERE "); + sql.push_str(&conditions.join(" AND ")); + } + + // Deterministic ordering for pagination. + if index_name.is_some() { + if let Some((_, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + if let Some((_, base_type)) = &base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + let _ = write!(sql, " ORDER BY pk, {sk_col}, base_pk, {base_col}"); + } else { + let _ = write!(sql, " ORDER BY pk, {sk_col}, base_pk"); + } + } else if let Some((_, base_type)) = &base_sk_info { + // Hash-only GSI on a composite-key base table: order by the full + // index key so it is a total order matching the pagination + // predicate above. + let base_col = format!("base_{}", sk_column(*base_type)); + let _ = write!(sql, " ORDER BY pk, base_pk, {base_col}"); + } else { + let _ = write!(sql, " ORDER BY pk, base_pk"); + } + } else if let Some((_, sk_type)) = sk_info_val { + let _ = write!(sql, " ORDER BY pk, {}", sk_column(sk_type)); + } else { + sql.push_str(" ORDER BY pk"); + } + + let fetch_limit = limit.map_or(1_000_001, |l| l + 1); + let _ = write!(sql, " LIMIT {fetch_limit}"); + + let rows = execute_dynamic_query(&sql, binds, &self.pool).await?; + finalize(rows, limit, &key_info.key_schema) + } +} + +/// Resolve a key-condition path's attribute name, handling `#name` references. +fn resolve_attr_name(path: &[PathElement], maps: &ExpressionMaps) -> Option { + match path.first() { + Some(PathElement::Attribute(name)) => { + if let Some(reference) = name.strip_prefix('#') { + maps.names.get(reference).cloned() + } else { + Some(name.clone()) + } + } + _ => None, + } +} + +/// Extract the base-table partition key text from a (combined) start key. +fn base_pk_from_start_key( + start_key: &Item, + key_info: &TableKeyInfo, +) -> Result { + let base_pk_attr = &key_info.base_key_schema[0].attribute_name; + start_key + .get(base_pk_attr) + .map(pk_to_text) + .transpose()? + .map(|c| c.into_owned()) + .ok_or_else(|| { + StorageError::Validation( + "The provided starting key is invalid: missing base table partition key".to_owned(), + ) + }) +} + +/// Append the query pagination predicate and its binds, mirroring the +/// PostgreSQL `build_pagination_where` cases. +#[allow(clippy::too_many_arguments)] +fn append_query_pagination( + sql: &mut String, + binds: &mut Vec, + start_key: &Item, + sk_info_val: Option<(&str, ScalarAttributeType)>, + base_sk_info: Option<&(String, ScalarAttributeType)>, + key_info: &TableKeyInfo, + is_index: bool, + is_lsi: bool, + forward: bool, +) -> Result<(), StorageError> { + let cmp = if forward { ">" } else { "<" }; + + if let Some((sk_name, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + let sk_bv = start_key + .get(sk_name) + .map(|v| parse_sk(v, sk_type)) + .transpose()? + .map(|s| sk_bound(&s)); + + if let Some((base_name, base_type)) = base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + let sk_bv = sk_bv.unwrap_or(BoundValue::Text(String::new())); + let base_sk_bv = if let Some(v) = start_key.get(base_name.as_str()) { + sk_bound(&parse_sk(v, *base_type)?) + } else { + BoundValue::Text(String::new()) + }; + + if is_lsi { + // LSI: every row shares the queried partition key, so the base + // table's sort key alone identifies a row uniquely, and it is a + // user-visible sort dimension so it follows ScanIndexForward. + let _ = write!( + sql, + " AND ({sk_col} {cmp} ? OR ({sk_col} = ? AND {base_col} {cmp} ?))" + ); + binds.push(sk_bv.clone()); + binds.push(sk_bv); + binds.push(base_sk_bv); + } else { + // GSI: the tie-breaker must be the FULL base primary key. Rows + // in a GSI partition are unique on (index SK, base PK, base SK), + // not on (index SK, base SK): many base partitions can project + // the same index SK and the same base SK. Comparing base SK + // alone made a page-two query return nothing whenever the rows + // sharing an index SK also shared a base SK, so a paginating + // client silently stopped after page one. + let base_pk_bv = BoundValue::Text(base_pk_from_start_key(start_key, key_info)?); + let _ = write!( + sql, + " AND ({sk_col} {cmp} ? OR ({sk_col} = ? AND (base_pk > ? \ + OR (base_pk = ? AND {base_col} > ?))))" + ); + binds.push(sk_bv.clone()); + binds.push(sk_bv); + binds.push(base_pk_bv.clone()); + binds.push(base_pk_bv); + binds.push(base_sk_bv); + } + } else if is_index { + let _ = write!( + sql, + " AND ({sk_col} {cmp} ? OR ({sk_col} = ? AND base_pk > ?))" + ); + let sk_bv = sk_bv.unwrap_or(BoundValue::Text(String::new())); + binds.push(sk_bv.clone()); + binds.push(sk_bv); + binds.push(BoundValue::Text(base_pk_from_start_key( + start_key, key_info, + )?)); + } else { + let _ = write!(sql, " AND {sk_col} {cmp} ?"); + binds.push(sk_bv.unwrap_or(BoundValue::Text(String::new()))); + } + } else if is_index { + let base_pk_text = base_pk_from_start_key(start_key, key_info)?; + if let Some((base_name, base_type)) = base_sk_info { + let base_col = format!("base_{}", sk_column(*base_type)); + let _ = write!( + sql, + " AND (base_pk > ? OR (base_pk = ? AND {base_col} > ?))" + ); + binds.push(BoundValue::Text(base_pk_text.clone())); + binds.push(BoundValue::Text(base_pk_text)); + if let Some(v) = start_key.get(base_name.as_str()) { + binds.push(sk_bound(&parse_sk(v, *base_type)?)); + } else { + binds.push(BoundValue::Text(String::new())); + } + } else { + let _ = write!(sql, " AND base_pk > ?"); + binds.push(BoundValue::Text(base_pk_text)); + } + } + Ok(()) +} + +/// Trim the over-fetched extra row, deserialize items, and derive the +/// `LastEvaluatedKey` (storage-side: the queried table's own key; the engine +/// enriches index LEKs with base-table key attributes). +fn finalize( + rows: Vec, + limit: Option, + key_schema: &[extenddb_core::types::KeySchemaElement], +) -> Result<(Vec, Option), StorageError> { + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let actual_limit = limit.map_or(1_000_000_usize, |l| l.max(0) as usize); + let has_more = rows.len() > actual_limit; + let items: Vec = rows + .into_iter() + .take(actual_limit) + .map(json_to_item) + .collect::, _>>()?; + let last_key = if has_more { + items.last().map(|item| build_key(item, key_schema)) + } else { + None + }; + Ok((items, last_key)) +} diff --git a/crates/storage-duckdb/src/data/transactions.rs b/crates/storage-duckdb/src/data/transactions.rs new file mode 100644 index 00000000..9018398b --- /dev/null +++ b/crates/storage-duckdb/src/data/transactions.rs @@ -0,0 +1,506 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `TransactGetItems` / `TransactWriteItems` and idempotency-token cleanup. +//! +//! Writes run under the engine write lock (D1) in a single transaction, so all +//! operations, the idempotency check, and stream capture commit atomically or +//! roll back together. Reads run in one transaction for a consistent snapshot. + +use crate::db; +use std::collections::HashMap; + +use extenddb_core::expression::{self, ExpressionMaps}; +use extenddb_core::types::{CancellationReason, Item, ReturnValuesOnConditionCheckFailure}; +use extenddb_core::validation; +use extenddb_storage::error::StorageError; +use extenddb_storage::{IdempotencyKey, TransactGetOp, TransactWriteOp}; + +use super::index::{IndexMeta, enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::tx_helpers::{ + check_idempotency_token_in_tx, delete_item_in_tx, fetch_item_for_update, fetch_item_in_tx, + upsert_item_in_tx, write_stream_record_in_tx, +}; +use crate::duckdb_util::format_timestamp; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + pub(crate) async fn transact_get_items_impl( + &self, + ops: &[TransactGetOp<'_>], + ) -> Result>, StorageError> { + // Validate keys first, collecting per-item reasons (all-or-nothing). + let mut reasons = Vec::with_capacity(ops.len()); + let mut any_failed = false; + for op in ops { + match validation::validate_key_only( + op.key, + &op.key_info.key_schema, + &op.key_info.attribute_definitions, + ) { + Ok(()) => reasons.push(CancellationReason::none()), + Err(e) => { + any_failed = true; + reasons.push(CancellationReason::validation_error(e.to_string())); + } + } + } + if any_failed { + return Err(StorageError::TransactionCanceled(reasons)); + } + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let mut results = Vec::with_capacity(ops.len()); + for op in ops { + results.push(fetch_item_in_tx(&mut tx, op.key_info, op.key).await?); + } + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(results) + } + + pub(crate) async fn transact_write_items_impl( + &self, + ops: &[TransactWriteOp<'_>], + idempotency: Option>, + ) -> Result<(), StorageError> { + // Read the propagation delay BEFORE taking the write lock. It is a + // runtime setting, not an invariant of this write, so it does not need + // to be read under the lock, and the lock serialises every write in the + // process: work done inside it is the backend's throughput bottleneck. + let system_delay = self.index_propagation_delay().await; + let _writer = self.write_lock.lock().await; + // Fetch index metadata per distinct table AFTER acquiring the write lock, + // so a GSI added by a concurrent UpdateTable (same lock) is not missed and + // left unmaintained by these writes. + let mut table_indexes: HashMap> = HashMap::new(); + for op in ops { + let name = op_table_name(op); + if !table_indexes.contains_key(name) { + let indexes = fetch_indexes_for_table(op_table_id(op), &self.pool).await?; + table_indexes.insert(name.to_owned(), indexes); + } + } + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some(key) = idempotency { + // Idempotency is per-account: the same ClientRequestToken in two + // accounts must not collide. The engine passes the caller's + // account explicitly. + check_idempotency_token_in_tx(&mut tx, key.account_id, key.token, key.fingerprint) + .await?; + } + + let mut reasons: Vec = Vec::with_capacity(ops.len()); + let mut op_items: Vec<(Option, Option)> = Vec::with_capacity(ops.len()); + let mut any_failed = false; + + for op in ops { + let indexes = &table_indexes[op_table_name(op)]; + match execute_transact_write_op( + &mut tx, + op, + indexes, + self.max_item_size_bytes, + system_delay, + ) + .await + { + Ok(items) => { + op_items.push(items); + reasons.push(CancellationReason::none()); + } + Err(TxnOpError::Cancel(r)) => { + op_items.push((None, None)); + any_failed = true; + reasons.push(r); + } + Err(TxnOpError::Validation(msg)) => { + // Up-front input validation (e.g. empty secondary-index key): + // abort the whole transaction with a top-level + // ValidationException, not a per-item cancellation reason. + return Err(StorageError::Validation(msg)); + } + Err(TxnOpError::Storage(e)) => return Err(e), + } + } + + if any_failed { + // Dropping `tx` without commit rolls back all writes. + return Err(StorageError::TransactionCanceled(reasons)); + } + + // Capture stream records after all writes are staged. + for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { + let capture = match op { + TransactWriteOp::Put { stream, .. } + | TransactWriteOp::Delete { stream, .. } + | TransactWriteOp::Update { stream, .. } => stream.as_ref(), + TransactWriteOp::ConditionCheck { .. } => None, + }; + if let Some(capture) = capture { + write_stream_record_in_tx( + &mut tx, + op_key_info(op), + capture, + old_item.as_ref(), + new_item.as_ref(), + ) + .await?; + } + } + + // Persist index work for each op inside the same transaction: async GSI + // rows, and vector maintenance which is applied here when the propagation + // delay is 0 and enqueued otherwise. + // + // Deliberately after every op is staged rather than inside each op, so + // there is one call site instead of three. Both paths stay atomic with the + // item writes either way: the synchronous apply runs in this transaction, + // and a pending row is inserted into it, so a cancelled transaction leaves + // neither behind. The vector apply reads only the op's own before/after + // items, never the base table, so its position relative to the other ops' + // staged writes cannot change its result. + let mut needs_notify = false; + for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { + let indexes = &table_indexes[op_table_name(op)]; + if old_item.is_some() || new_item.is_some() { + let n = enqueue_async_indexes( + &mut tx, + op_key_info(op), + indexes, + old_item.as_ref(), + new_item.as_ref(), + system_delay, + ) + .await?; + if n > 0 { + needs_notify = true; + } + let key_info = op_key_info(op); + if !key_info.vector_indexes.is_empty() { + let n = crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old_item.as_ref(), + new_item.as_ref(), + system_delay, + ) + .await?; + if n > 0 { + needs_notify = true; + } + } + } + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if needs_notify { + self.gsi_notify.notify_waiters(); + } + Ok(()) + } + + pub(crate) async fn cleanup_expired_idempotency_tokens_impl( + &self, + max_age_seconds: i64, + ) -> Result { + let cutoff = format_timestamp( + time::OffsetDateTime::now_utc() - time::Duration::seconds(max_age_seconds), + ); + let result = db::query("DELETE FROM idempotency_tokens WHERE created_at < ?") + .bind(&cutoff) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(result.rows_affected()) + } +} + +fn op_table_name<'a>(op: &'a TransactWriteOp<'_>) -> &'a str { + &op_key_info(op).table_name +} + +fn op_table_id<'a>(op: &'a TransactWriteOp<'_>) -> &'a str { + &op_key_info(op).table_id +} + +fn op_key_info<'a>(op: &'a TransactWriteOp<'_>) -> &'a extenddb_core::types::TableKeyInfo { + match op { + TransactWriteOp::Put { key_info, .. } + | TransactWriteOp::Delete { key_info, .. } + | TransactWriteOp::Update { key_info, .. } + | TransactWriteOp::ConditionCheck { key_info, .. } => key_info, + } +} + +enum TxnOpError { + Cancel(CancellationReason), + /// Up-front input validation failure — aborts the whole transaction with a + /// top-level `ValidationException` (not a per-item cancellation reason). + Validation(String), + Storage(StorageError), +} + +/// Build [`validation::IndexKeyRef`] views over the table's indexes for +/// secondary-index key validation. +pub(crate) fn index_key_refs(indexes: &[IndexMeta]) -> Vec> { + indexes + .iter() + .map(|idx| validation::IndexKeyRef { + index_name: &idx.index_name, + key_schema: &idx.key_schema, + }) + .collect() +} + +/// Execute one transact-write op, returning `(old, new)` images for stream +/// capture, or a cancellation reason on a failed condition / validation. +async fn execute_transact_write_op( + tx: &mut db::Transaction, + op: &TransactWriteOp<'_>, + indexes: &[IndexMeta], + max_item_size_bytes: usize, + system_delay: u64, +) -> Result<(Option, Option), TxnOpError> { + match op { + TransactWriteOp::Put { + key_info, + item, + condition, + maps, + return_values_on_ccf, + .. + } => { + validation::validate_item_keys( + item, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + // Secondary-index key faults split by kind: a type mismatch is a + // per-item cancellation reason; an empty index key is up-front input + // validation (a top-level ValidationException). + let idx_refs = index_key_refs(indexes); + validation::validate_index_key_types(item, &idx_refs, &key_info.attribute_definitions) + .map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + validation::validate_index_key_not_empty( + item, + &idx_refs, + validation::SecondaryIndexEmptyContext::Item, + ) + .map_err(|e| TxnOpError::Validation(e.to_string()))?; + let existing = fetch_item_for_update(tx, key_info, item) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + *condition, + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + upsert_item_in_tx(tx, key_info, item) + .await + .map_err(TxnOpError::Storage)?; + if !indexes.is_empty() { + sync_indexes( + tx, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + existing.as_ref(), + Some(item), + system_delay, + ) + .await + .map_err(TxnOpError::Storage)?; + } + Ok((existing, Some((*item).clone()))) + } + TransactWriteOp::Delete { + key_info, + key, + condition, + maps, + return_values_on_ccf, + .. + } => { + validation::validate_batch_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + *condition, + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + delete_item_in_tx(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + if !indexes.is_empty() { + sync_indexes( + tx, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + existing.as_ref(), + None, + system_delay, + ) + .await + .map_err(TxnOpError::Storage)?; + } + Ok((existing, None)) + } + TransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + return_values_on_ccf, + .. + } => { + validation::validate_batch_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + *condition, + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + let mut item = existing.clone().unwrap_or_else(|| (*key).clone()); + expression::apply_update_validated( + actions, + &mut item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + validation::validate_item_size(&item, max_item_size_bytes).map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + // Secondary-index key validation on the post-update item: a type + // mismatch is a cancellation reason; setting an index key to an + // empty value is a top-level ValidationException. + let idx_refs = index_key_refs(indexes); + validation::validate_index_key_types(&item, &idx_refs, &key_info.attribute_definitions) + .map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + validation::validate_index_key_not_empty( + &item, + &idx_refs, + validation::SecondaryIndexEmptyContext::UpdateExpression, + ) + .map_err(|e| TxnOpError::Validation(e.to_string()))?; + upsert_item_in_tx(tx, key_info, &item) + .await + .map_err(TxnOpError::Storage)?; + if !indexes.is_empty() { + sync_indexes( + tx, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + existing.as_ref(), + Some(&item), + system_delay, + ) + .await + .map_err(TxnOpError::Storage)?; + } + Ok((existing, Some(item))) + } + TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf, + } => { + validation::validate_batch_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + Some(condition), + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + Ok((None, None)) + } + } +} + +/// Evaluate a transaction op's condition, producing a `CancellationReason` on +/// failure (attaching the old item when `ReturnValuesOnConditionCheckFailure` +/// is `AllOld`). +fn eval_condition( + condition: Option<&expression::Expr>, + item: &Item, + maps: &ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + existing: Option<&Item>, +) -> Result<(), TxnOpError> { + if let Some(cond) = condition { + let passed = expression::evaluate_condition(cond, item, maps) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + if !passed { + let returned = if return_values_on_ccf == ReturnValuesOnConditionCheckFailure::AllOld { + existing.cloned() + } else { + None + }; + return Err(TxnOpError::Cancel( + CancellationReason::condition_check_failed_with_item(returned), + )); + } + } + Ok(()) +} diff --git a/crates/storage-duckdb/src/data/tx_helpers.rs b/crates/storage-duckdb/src/data/tx_helpers.rs new file mode 100644 index 00000000..83798a52 --- /dev/null +++ b/crates/storage-duckdb/src/data/tx_helpers.rs @@ -0,0 +1,379 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Transaction helpers shared by the write path and transactions module: +//! in-transaction item fetch/upsert/delete, monotonic stream sequencing, +//! atomic stream-record capture, and idempotency-token checks. +//! +//! There is no `SELECT ... FOR UPDATE`: the engine's `write_lock` serializes +//! all writers (design decision D1), so an in-transaction read followed by a +//! write is already atomic with respect to other writers. + +use crate::db; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use extenddb_core::types::{ + AttributeValue, Item, StreamEventName, StreamRecord, StreamRecordData, StreamViewType, + TableKeyInfo, item_size_bytes, +}; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{pk_to_text, sk_column, sk_info}; + +use super::{ + bind_sk_execute, bind_sk_fetch_optional, bind_sk_only_execute, data_table_name, json_to_item, +}; +use crate::duckdb_util::format_timestamp; + +/// Fetch a single item within a transaction by primary key. +pub(super) async fn fetch_item_in_tx( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + key: &Item, +) -> Result, StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + let json_opt = if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = extenddb_storage::util::parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + let row: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&sql, pk_text.as_ref(), &sk, &mut **tx)?; + row.map(|(v,)| v) + } else { + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let row: Option<(serde_json::Value,)> = db::query_as(&sql) + .bind(pk_text.as_ref()) + .fetch_optional(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + row.map(|(v,)| v) + }; + + json_opt.map(json_to_item).transpose() +} + +/// Fetch for write. DuckDB has no `FOR UPDATE`; the engine write lock provides +/// the serialization, so this is the same as `fetch_item_in_tx`. +pub(super) async fn fetch_item_for_update( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + key: &Item, +) -> Result, StorageError> { + fetch_item_in_tx(tx, key_info, key).await +} + +/// Insert or replace an item within a transaction. +pub(crate) async fn upsert_item_in_tx( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + item: &Item, +) -> Result<(), StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = item + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + let item_json = + serde_json::to_string(item).map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((sk_name, sk_type)) = sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = item + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = extenddb_storage::util::parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!( + "INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES (?, ?, ?) \ + ON CONFLICT (pk, {sk_col}) DO UPDATE SET item_data = excluded.item_data" + ); + bind_sk_execute!(&sql, pk_text.as_ref(), &sk, &item_json, &mut **tx)?; + } else { + let sql = format!( + "INSERT INTO {ddb_table} (pk, item_data) VALUES (?, ?) \ + ON CONFLICT (pk) DO UPDATE SET item_data = excluded.item_data" + ); + db::query(&sql) + .bind(pk_text.as_ref()) + .bind(&item_json) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) +} + +/// Delete an item by key within a transaction. +pub(super) async fn delete_item_in_tx( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + key: &Item, +) -> Result<(), StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + if let Some((sk_name, sk_type)) = sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = extenddb_storage::util::parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!("DELETE FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + bind_sk_only_execute!(&sql, pk_text.as_ref(), &sk, &mut **tx)?; + } else { + let sql = format!("DELETE FROM {ddb_table} WHERE pk = ?"); + db::query(&sql) + .bind(pk_text.as_ref()) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) +} + +/// Allocate the next monotonic stream sequence number within a transaction. +async fn next_stream_seq(tx: &mut db::Transaction) -> Result { + db::query("UPDATE seq_counters SET value = value + 1 WHERE name = 'stream'") + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let value: i64 = db::query_scalar("SELECT value FROM seq_counters WHERE name = 'stream'") + .fetch_one(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(value) +} + +/// Write a stream record in the same transaction as the data write, so capture +/// is atomic with the mutation (a hard requirement for correct streams). +pub(super) async fn write_stream_record_in_tx( + tx: &mut db::Transaction, + key_info: &TableKeyInfo, + capture: &StreamCapture, + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + let event_name = match (old_item, new_item) { + (None, Some(_)) => StreamEventName::Insert, + (Some(_), Some(_)) => StreamEventName::Modify, + (Some(_), None) => StreamEventName::Remove, + (None, None) => return Ok(()), + }; + let source = new_item.or(old_item).expect("one image present"); + + // Keys are the primary-key attributes from whichever image is present. + let keys: Item = key_info + .key_schema + .iter() + .filter_map(|ks| { + source + .get(&ks.attribute_name) + .map(|v| (ks.attribute_name.clone(), v.clone())) + }) + .collect(); + + let new_image = matches!( + capture.view_type, + StreamViewType::NewImage | StreamViewType::NewAndOldImages + ) + .then(|| new_item.cloned()) + .flatten(); + let old_image = matches!( + capture.view_type, + StreamViewType::OldImage | StreamViewType::NewAndOldImages + ) + .then(|| old_item.cloned()) + .flatten(); + + let size_bytes = i64::try_from(item_size_bytes(source)).unwrap_or(i64::MAX); + + // Hash the partition key to a shard. + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_str = source + .get(pk_name) + .map(|v| match v { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => BASE64.encode(b), + _ => String::new(), + }) + .unwrap_or_default(); + + let shards: Vec<(String,)> = + db::query_as("SELECT shard_id FROM stream_shards WHERE table_id = ? ORDER BY shard_id") + .bind(&key_info.table_id) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if shards.is_empty() { + return Ok(()); // Streams not enabled for this table. + } + let idx = (crc32fast::hash(pk_str.as_bytes()) as usize) % shards.len(); + let shard_id = shards[idx].0.clone(); + + let seq = format!("{:021}", next_stream_seq(tx).await?); + let approximate_creation_date_time = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + ) + .unwrap_or(i64::MAX); + + let record = StreamRecord { + event_id: uuid::Uuid::new_v4().to_string(), + event_name, + event_version: "1.1".to_owned(), + event_source: "aws:dynamodb".to_owned(), + aws_region: capture.region.to_string(), + dynamodb: StreamRecordData { + approximate_creation_date_time, + keys, + new_image, + old_image, + sequence_number: seq.clone(), + size_bytes, + stream_view_type: capture.view_type, + }, + user_identity: capture.user_identity.clone(), + }; + let record_json = + serde_json::to_string(&record).map_err(|e| StorageError::Internal(e.to_string()))?; + + db::query( + "INSERT INTO stream_records (shard_id, sequence_number, table_id, event_name, record_data) \ + VALUES (?, ?, ?, ?, ?)", + ) + .bind(&shard_id) + .bind(&seq) + .bind(&key_info.table_id) + .bind(format!("{:?}", record.event_name)) + .bind(&record_json) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +/// Check (and record) an idempotency token within a transaction. Tokens are +/// valid for 10 minutes; the cutoff is computed in Rust as RFC 3339 so it +/// compares correctly against the stored RFC 3339 `created_at`. +pub(super) async fn check_idempotency_token_in_tx( + tx: &mut db::Transaction, + account_id: &str, + token: &str, + fingerprint: &str, +) -> Result<(), StorageError> { + let cutoff = format_timestamp(time::OffsetDateTime::now_utc() - time::Duration::minutes(10)); + let existing: Option<(String,)> = db::query_as( + "SELECT fingerprint FROM idempotency_tokens \ + WHERE account_id = ? AND token = ? AND created_at > ?", + ) + .bind(account_id) + .bind(token) + .bind(&cutoff) + .fetch_optional(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((stored_fp,)) = existing { + return if stored_fp == fingerprint { + Err(StorageError::IdempotentReplay) + } else { + Err(StorageError::IdempotentMismatch) + }; + } + + let now = format_timestamp(time::OffsetDateTime::now_utc()); + db::query( + "INSERT INTO idempotency_tokens (account_id, token, fingerprint, created_at) \ + VALUES (?, ?, ?, ?) \ + ON CONFLICT (account_id, token) DO UPDATE SET fingerprint = excluded.fingerprint, \ + created_at = excluded.created_at", + ) + .bind(account_id) + .bind(token) + .bind(fingerprint) + .bind(&now) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +#[cfg(test)] +mod idempotency_tests { + use super::check_idempotency_token_in_tx; + use crate::db; + use extenddb_storage::error::StorageError; + + async fn pool_with_schema() -> db::Pool { + let pool = db::Pool::open(":memory:", 1).await.unwrap(); + db::query( + "CREATE TABLE idempotency_tokens (\ + account_id TEXT NOT NULL,\ + token TEXT NOT NULL,\ + fingerprint TEXT NOT NULL,\ + created_at TEXT NOT NULL DEFAULT (strftime(now()::TIMESTAMP, '%Y-%m-%dT%H:%M:%S.%gZ')),\ + PRIMARY KEY (account_id, token)\ + )", + ) + .execute(&pool) + .await + .unwrap(); + pool + } + + async fn check( + pool: &db::Pool, + account: &str, + token: &str, + fp: &str, + ) -> Result<(), StorageError> { + let mut tx = pool.begin().await.unwrap(); + let r = check_idempotency_token_in_tx(&mut tx, account, token, fp).await; + tx.commit().await.unwrap(); + r + } + + #[tokio::test] + async fn idempotency_is_scoped_per_account() { + let pool = pool_with_schema().await; + + // First use of (acctA, tok) records it. + check(&pool, "acctA", "tok", "fpX").await.unwrap(); + + // Same account + token + fingerprint = idempotent replay. + assert!(matches!( + check(&pool, "acctA", "tok", "fpX").await, + Err(StorageError::IdempotentReplay) + )); + + // Same account + token, different fingerprint = mismatch. + assert!(matches!( + check(&pool, "acctA", "tok", "fpY").await, + Err(StorageError::IdempotentMismatch) + )); + + // Regression: a DIFFERENT account reusing the same token value must be + // independent — not a replay and not a mismatch. + check(&pool, "acctB", "tok", "fpX").await.unwrap(); + } +} diff --git a/crates/storage-duckdb/src/data/update_item.rs b/crates/storage-duckdb/src/data/update_item.rs new file mode 100644 index 00000000..8018957f --- /dev/null +++ b/crates/storage-duckdb/src/data/update_item.rs @@ -0,0 +1,155 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `update_item` for the DuckDB backend. +//! +//! UpdateItem is an upsert: when the item is absent a new one is created from +//! the key plus the applied actions. The condition is evaluated against the +//! pre-update image (or an empty item), then update actions are applied and the +//! result is validated and written — all under the engine write lock (D1). + +use extenddb_core::expression::{self, Expr, ExpressionMaps, UpdateAction}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_core::validation; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; + +use super::index::{enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::query::check_condition; +use super::transactions::index_key_refs; +use super::tx_helpers::{fetch_item_in_tx, upsert_item_in_tx, write_stream_record_in_tx}; +use crate::store::DuckDbEngine; + +impl DuckDbEngine { + #[allow(clippy::too_many_arguments)] + pub(crate) async fn update_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result<(Option, Option), StorageError> { + // Read the propagation delay BEFORE taking the write lock. It is a + // runtime setting, not an invariant of this write, so it does not need + // to be read under the lock, and the lock serialises every write in the + // process: work done inside it is the backend's throughput bottleneck. + let system_delay = self.index_propagation_delay().await; + let _writer = self.write_lock.lock().await; + // Read the index set after acquiring the write lock so a concurrently + // added GSI (UpdateTable holds the same lock) is not missed. + let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old = fetch_item_in_tx(&mut tx, key_info, key).await?; + + if condition.is_some() { + let empty = Item::new(); + let target = old.as_ref().unwrap_or(&empty); + if let Err(e) = check_condition(condition, target, maps) { + return match e { + StorageError::ConditionFailed(_) => Err(StorageError::ConditionFailed(old)), + other => Err(other), + }; + } + } + + // Start from the existing image, or from the key for a fresh upsert. + let mut item = old.clone().unwrap_or_else(|| key.clone()); + expression::apply_update_validated( + actions, + &mut item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; + validation::validate_item_size(&item, self.max_item_size_bytes) + .map_err(|e| StorageError::Validation(e.to_string()))?; + // Secondary-index key validation on the post-update item, matching the + // TransactWriteItems update path: a wrong-typed index key attribute or + // an index key set to an empty value is a ValidationException up front, + // rather than silently producing a malformed / unmatchable index row. + if !indexes.is_empty() { + let idx_refs = index_key_refs(&indexes); + validation::validate_index_key_types(&item, &idx_refs, &key_info.attribute_definitions) + .map_err(|e| StorageError::Validation(e.to_string()))?; + validation::validate_index_key_not_empty( + &item, + &idx_refs, + validation::SecondaryIndexEmptyContext::UpdateExpression, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; + } + + upsert_item_in_tx(&mut tx, key_info, &item).await?; + + if !indexes.is_empty() { + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old.as_ref(), + Some(&item), + system_delay, + ) + .await?; + } + let enqueued_gsi = enqueue_async_indexes( + &mut tx, + key_info, + &indexes, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0; + // Vector indexes: applied in this transaction when the propagation delay is + // 0, otherwise enqueued alongside the async GSI work. Gated on the cached + // key info so a table without them costs no extra query, and kept outside + // the `indexes` guard because a table may have a vector index and no GSI. + let enqueued_vector = if key_info.vector_indexes.is_empty() { + false + } else { + crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0 + }; + let enqueued = enqueued_gsi || enqueued_vector; + + if let Some(capture) = stream { + write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) + .await?; + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if enqueued { + self.gsi_notify.notify_waiters(); + } + + let old_ret = if return_old { old } else { None }; + let new_ret = if return_new { Some(item) } else { None }; + Ok((old_ret, new_ret)) + } +} diff --git a/crates/storage-duckdb/src/data/vector_index.rs b/crates/storage-duckdb/src/data/vector_index.rs new file mode 100644 index 00000000..1b3f5971 --- /dev/null +++ b/crates/storage-duckdb/src/data/vector_index.rs @@ -0,0 +1,1078 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Vector index maintenance on the write path. +//! +//! Vector indexes are **eventually consistent**, the same model the service gives +//! them and the same model a GSI has here, so maintenance runs on the existing +//! `gsi_pending` queue rather than in the base write transaction. [`maintain_vector_indexes`] +//! is the single entry point and owns that choice: a propagation delay of 0 keeps +//! the work in the caller's transaction, any other delay enqueues it. +//! +//! Reusing the GSI queue is what makes the asynchronous path correct rather than +//! merely deferred, and none of these properties would come free from a queue of +//! its own: +//! +//! * **Crash safety.** The pending row is inserted in the base write transaction, +//! so there is no window in which the item is committed and the index work is +//! not yet durable. The worker claims and applies in one transaction, so a crash +//! mid-apply rolls back and the row is retried. At-least-once delivery is safe +//! because applying a row is idempotent: it deletes the base key's row and +//! reinserts it from the snapshotted item. +//! * **Per-key ordering, across index kinds.** The row's partition is a hash of the +//! *base* key, so a vector row and a GSI row for the same item share a partition, +//! `ready_at` is clamped monotonic within it, and the worker drains in `id` order. +//! Two writes to one item therefore reach both index kinds in write order even +//! though the delay is jittered. +//! * **Snapshot semantics.** The row carries its own [`VectorApplyContext`], so the +//! worker needs no catalog read and an index dropped, or redefined, between +//! enqueue and apply cannot make a queued write unapplicable or retroactively +//! change how it was indexed. +//! +//! A write to a table whose items do not carry the vector still enqueues, because +//! the removal is the point: an item that loses its vector attribute must leave the +//! index, and skipping the enqueue would leave the stale row in place forever. + +use crate::db; +use serde::{Deserialize, Serialize}; + +use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement, SearchSchemaElementType}; +use extenddb_core::validation::{vector_components, vector_norm}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::pk_to_text; + +use super::{BoundValue, all_sort_key_info, sk_bound, vector_table_name}; +use crate::vector_search::partition_value; + +/// A vector index as the write path needs it. +/// +/// Serializable because the asynchronous path snapshots it verbatim into the +/// pending row's [`VectorApplyContext`]. The write path and the worker therefore +/// apply from the *same* description of the index, which is the property that stops +/// a queued write from being reinterpreted under a later definition. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct VectorIndexMeta { + pub index_id: String, + pub dimensions: usize, + pub vector_attribute_name: String, + /// The index's projection, applied to the stored row exactly as the GSI path + /// applies its own. Not applying it was an unexplained divergence from the + /// sibling, and it made a search return attributes the index does not project. + pub projection: extenddb_core::types::Projection, + /// The single HASH element's attribute name, when the index declares one. + /// `None` means the index is unscoped and every row shares one partition. + pub hash_attribute_name: Option, + /// Every attribute named by the SearchSchema, HASH and INLINE_FILTER alike. + /// + /// These are projected regardless of `ProjectionType`, which is the documented + /// rule for a vector index and is NOT GSI `KEYS_ONLY` semantics: `KEYS_ONLY` + /// on a vector index projects the base primary key, the vector attribute and + /// the inline filter attributes. Withholding them is not merely a reporting + /// difference, it breaks search: the filter is evaluated against the stored + /// payload, so a missing filter attribute makes every row fail the predicate + /// and a filtered search match nothing. + pub search_schema_attribute_names: Vec, +} + +/// Load the vector indexes of a table. +/// +/// Read inside the write transaction rather than taken from the cached +/// `TableKeyInfo`, because the cache carries the search schema but not the index +/// id, and the id is what names the data table. +pub(crate) async fn fetch_vector_indexes_for_table( + tx: &mut db::Transaction, + table_id: &str, +) -> Result, StorageError> { + let rows: Vec<(String, i64, String, Option, String)> = db::query_as( + "SELECT index_id, dimensions, vector_attribute, search_schema, projection \ + FROM vector_indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + + let mut out = Vec::with_capacity(rows.len()); + for (index_id, dimensions, vector_attribute, search_schema, projection) in rows { + let attr: extenddb_core::types::VectorAttribute = + serde_json::from_str(&vector_attribute) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; + let (hash_attribute_name, search_schema_attribute_names) = match search_schema.as_deref() { + Some(json) => { + let elements: Vec = + serde_json::from_str(json) + .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?; + let hash = elements + .iter() + .find(|e| e.element_type == SearchSchemaElementType::Hash) + .map(|e| e.attribute_name.clone()); + let all = elements + .into_iter() + .map(|e| e.attribute_name) + .collect::>(); + (hash, all) + } + None => (None, Vec::new()), + }; + let projection: extenddb_core::types::Projection = serde_json::from_str(&projection) + .map_err(|e| StorageError::Internal(format!("vector projection: {e}")))?; + out.push(VectorIndexMeta { + index_id, + dimensions: usize::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?, + vector_attribute_name: attr.attribute_name, + hash_attribute_name, + search_schema_attribute_names, + projection, + }); + } + Ok(out) +} + +/// Whether an item belongs in a vector index. +/// +/// It must carry the vector attribute, and the HASH attribute when the index +/// declares one: without the latter the row could not be placed in a partition, +/// and putting it in the unscoped partition would make it visible to searches of +/// every other partition. Not an error, exactly as a GSI silently omits an item +/// missing its index key. +fn item_is_indexable(item: &Item, meta: &VectorIndexMeta) -> bool { + if !item.contains_key(&meta.vector_attribute_name) { + return false; + } + match &meta.hash_attribute_name { + Some(name) => item.contains_key(name), + None => true, + } +} + +/// The partition column value for an item under one index. +fn item_partition(item: &Item, meta: &VectorIndexMeta) -> Result { + match &meta.hash_attribute_name { + Some(name) => { + let value = item.get(name).ok_or_else(|| { + StorageError::Internal( + "indexable check passed but the hash attribute is absent".to_owned(), + ) + })?; + partition_value(Some((name.as_str(), value))) + } + None => partition_value(None), + } +} + +/// Base-key bind values for a row, in key-schema order. +fn base_key_binds( + item: &Item, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Result, StorageError> { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let pk_attr = &base_key_schema[0].attribute_name; + let pk = item.get(pk_attr).ok_or_else(|| { + StorageError::Internal("item written without its partition key".to_owned()) + })?; + let mut binds = vec![BoundValue::Text(pk_to_text(pk)?.into_owned())]; + for &(name, sk_type) in &base_sks { + // Sort keys use the same storage representation as the GSI/LSI tables: + // order-preserving text for numbers and a BLOB for binary. Encoding them + // any other way would still be self-consistent here but would diverge + // from every other index table for the same item. + match item.get(name) { + Some(value) => binds.push(sk_bound(&extenddb_storage::util::parse_sk(value, sk_type)?)), + None => binds.push(BoundValue::Text(String::new())), + } + } + Ok(binds) +} + +/// Column names for the base key, in key-schema order. +fn base_key_columns( + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Vec { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let mut cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + cols.push(format!( + "base_{}", + extenddb_storage::util::sk_column_n(i, sk_type) + )); + } + cols +} + +/// Everything the propagation worker needs to apply one vector index update, +/// serialized into `gsi_pending.index_context`. +/// +/// `table_id` is carried here even though the queue row has a `table_id` column of +/// its own, because a vector data table is named from the table id *and* the index +/// id. Reading it from the context preserves the invariant that the context alone +/// is sufficient, rather than splitting one apply's inputs across a column and a +/// JSON blob. Both are written from the same variable in the same statement, so +/// they cannot disagree. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct VectorApplyContext { + pub(crate) base_key_schema: Vec, + pub(crate) attribute_definitions: Vec, + pub(crate) table_id: String, + /// Deliberately named `vector` rather than `index`: it is the field whose + /// presence lets the untagged `PendingApplyContext` tell a vector row from a + /// GSI row by shape alone. See that type for why the discriminant is a shape + /// and not a tag. + pub(crate) vector: VectorIndexMeta, +} + +/// Maintain every vector index on a table for one item write. +/// +/// The single entry point for the write path, and the one place that decides +/// between synchronous and asynchronous. `delay_ms` of 0 applies in the caller's +/// transaction; anything else enqueues one `gsi_pending` row per index. Returns the +/// number of rows enqueued, so the caller knows whether to wake the worker, and +/// returns 0 for the synchronous path because there is nothing to wake. +/// +/// Keeping the branch here rather than at each call site matters: there are seven +/// write paths, and a single one that enqueued while also applying inline would +/// double-apply, while one that did neither would silently stop indexing. +/// +/// `old_item` and `new_item` follow the same convention as `sync_indexes`: a put +/// supplies both when replacing, a delete supplies only the old. +pub(crate) async fn maintain_vector_indexes( + tx: &mut db::Transaction, + table_id: &str, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + old_item: Option<&Item>, + new_item: Option<&Item>, + delay_ms: u64, +) -> Result { + // Read inside the transaction rather than from the cached `TableKeyInfo`: the + // cache carries the search schema but not the index id, and the id is what + // names the data table. + let metas = fetch_vector_indexes_for_table(tx, table_id).await?; + if metas.is_empty() { + return Ok(0); + } + let key_cols = base_key_columns(base_key_schema, attr_defs); + + if delay_ms == 0 { + for meta in &metas { + apply_vector_index( + tx, + table_id, + meta, + base_key_schema, + attr_defs, + &key_cols, + old_item, + new_item, + ) + .await?; + } + return Ok(0); + } + + let mut enqueued = 0usize; + for meta in metas { + // Enqueued even when the new item carries no vector: the removal is the + // work in that case, and skipping it would leave a stale row indexed. + let context = super::index::PendingApplyContext::Vector(VectorApplyContext { + base_key_schema: base_key_schema.to_vec(), + attribute_definitions: attr_defs.to_vec(), + table_id: table_id.to_owned(), + vector: meta, + }); + super::index::enqueue_pending_row(tx, table_id, old_item, new_item, delay_ms, &context) + .await?; + enqueued += 1; + } + Ok(enqueued) +} + +/// Apply one claimed vector pending row, from its self-describing context. +/// +/// A missing data table is skipped rather than treated as a failure: the base table +/// or the index itself can be dropped while a row is in flight, which is a routine +/// race and not a defect. +/// +/// This is log hygiene rather than data safety, and worth being exact about. The +/// batch already guards every row with a savepoint, so without this the row would be +/// rolled back and dropped, reaching the same end state by a noisier route. What the +/// tolerance changes is that an expected race stops emitting an ERROR line, which +/// otherwise trains operators to ignore the one signal that says a row was thrown +/// away. Matches the GSI sibling, so both arms of the dispatcher behave alike. +pub(crate) async fn apply_vector_context( + tx: &mut db::Transaction, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &VectorApplyContext, +) -> Result<(), StorageError> { + let key_cols = base_key_columns(&context.base_key_schema, &context.attribute_definitions); + apply_vector_index( + tx, + &context.table_id, + &context.vector, + &context.base_key_schema, + &context.attribute_definitions, + &key_cols, + old_item, + new_item, + ) + .await + .or_else(|e| { + if super::index::is_no_such_table(&e) { + Ok(()) + } else { + Err(e) + } + }) +} + +/// Apply an item write to a single vector index. +/// +/// The delete-then-insert shape matters. An item can move between partitions when +/// its HASH attribute changes, and the row is keyed by the base item rather than by +/// the partition, so an insert alone would leave the old partition's row in place +/// and the item would be findable in two partitions at once. +/// +/// The delete keys off `old_item.or(new_item)` because the base key is immutable, so +/// either carries it. That is what lets a put whose caller had no reason to read the +/// old item still displace the row it replaces. +#[allow(clippy::too_many_arguments)] +async fn apply_vector_index( + tx: &mut db::Transaction, + table_id: &str, + meta: &VectorIndexMeta, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + key_cols: &[String], + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, &meta.index_id); + let where_clause = key_cols + .iter() + .map(|c| format!("{c} = ?")) + .collect::>() + .join(" AND "); + + // Remove any existing row for this base item first, whatever partition it + // was in. + if let Some(source) = old_item.or(new_item) { + let binds = base_key_binds(source, base_key_schema, attr_defs)?; + let sql = format!("DELETE FROM {vec_table} WHERE {where_clause}"); + let mut q = db::query(&sql); + for b in binds { + q = super::bind_bound!(q, b); + } + q.execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + } + + let Some(new_item) = new_item else { + return Ok(()); // A delete: removal above is the whole of the work. + }; + insert_vector_row( + tx, + table_id, + meta, + new_item, + base_key_schema, + attr_defs, + key_cols, + VectorRowConflict::Fail, + ) + .await +} + +/// Write one item's row into one vector index. +/// +/// Shared by the write path and by backfill deliberately. These are the only two +/// producers of a vector row, and a second copy of this logic would be free to +/// drift: a backfilled row shaped differently from a live-written one would search +/// correctly right up until the difference mattered, with nothing to catch it. +/// +/// A non-indexable item is a no-op rather than an error, which is what makes a +/// backfill over a table where only some items carry the vector work. +/// How [`insert_vector_row`] treats a primary-key conflict. +/// +/// `Fail` is the write path's contract: every apply reaches the insert through +/// `apply_vector_index`, which deletes the base key's row first, so a conflict +/// there means a broken invariant and must be loud. `KeepExisting` is the +/// BACKFILL's contract: in synchronous-visibility mode +/// (`index_propagation_delay_ms == 0`) a base write landing mid-backfill is +/// applied to the CREATING index inline rather than queued, and each backfill +/// batch reads the base table LIVE under the write lock, so a row already +/// present was written from the same or a newer base image than the one the +/// backfill just read; keeping it is correct and inserting over it would only +/// fail the whole build. Without this, that collision wedged the index in +/// CREATING permanently (proven by test below). +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum VectorRowConflict { + Fail, + KeepExisting, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn insert_vector_row( + tx: &mut db::Transaction, + table_id: &str, + meta: &VectorIndexMeta, + item: &Item, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + key_cols: &[String], + on_conflict: VectorRowConflict, +) -> Result<(), StorageError> { + if !item_is_indexable(item, meta) { + return Ok(()); + } + let vec_table = vector_table_name(table_id, &meta.index_id); + + let value = item.get(&meta.vector_attribute_name).ok_or_else(|| { + StorageError::Internal("indexable check passed but the vector is absent".to_owned()) + })?; + let components = vector_components(value).ok_or_else(|| { + // Core validates the write before it reaches storage, so a malformed + // vector here means validation was bypassed rather than that a caller + // sent bad input. + StorageError::Internal( + "vector attribute reached storage without passing validation".to_owned(), + ) + })?; + if components.len() != meta.dimensions { + return Err(StorageError::Internal(format!( + "vector has {} components, index declares {}", + components.len(), + meta.dimensions + ))); + } + + let mut blob = Vec::with_capacity(components.len() * 4); + for x in &components { + blob.extend_from_slice(&x.to_le_bytes()); + } + let norm = vector_norm(&components); + let part = item_partition(item, meta)?; + // Projected exactly as the GSI sibling projects, so a search returns what + // the index declares and no more. + let mut projected = + super::index::project_item_for_index(item, &[], base_key_schema, &meta.projection); + // The SearchSchema attributes are always projected, whatever the + // ProjectionType. See `search_schema_attribute_names` for why: the inline + // filter is evaluated against this payload, so dropping the attribute would + // silently turn every filtered search into a zero-result search. + for name in &meta.search_schema_attribute_names { + if !projected.contains_key(name) + && let Some(v) = item.get(name) + { + projected.insert(name.clone(), v.clone()); + } + } + // The vector itself is not kept in the payload: it is already in the `vec` + // column as `f32`, which is the width the service validates against, and the + // search path rebuilds the attribute from those bits. Keeping a verbatim + // decimal copy here duplicated 10 to 15 KB per row at 1024 dimensions and + // would have returned the client's original precision where the service + // returns the narrowed value. + projected.remove(&meta.vector_attribute_name); + let item_json = serde_json::to_string(&projected) + .map_err(|e| StorageError::Internal(format!("serialize item: {e}")))?; + + // On the write path this stays a plain INSERT, deliberately, where the GSI + // sibling uses INSERT OR REPLACE: every apply reaches here through + // `apply_vector_index`, which unconditionally deletes the base key's row + // first, so a conflict means a broken invariant and must fail loudly. The + // backfill passes `KeepExisting` instead; see [`VectorRowConflict`] for + // why an existing row is the same-or-newer generation there and must win. + let cols = std::iter::once("part".to_owned()) + .chain(key_cols.iter().cloned()) + .chain(["vec".to_owned(), "nrm".to_owned(), "item_data".to_owned()]) + .collect::>(); + let placeholders = vec!["?"; cols.len()].join(", "); + let verb = match on_conflict { + VectorRowConflict::Fail => "INSERT", + VectorRowConflict::KeepExisting => "INSERT OR IGNORE", + }; + let sql = format!( + "{verb} INTO {vec_table} ({}) VALUES ({placeholders})", + cols.join(", ") + ); + let key_binds = base_key_binds(item, base_key_schema, attr_defs)?; + let mut q = db::query(&sql).bind(part); + for b in key_binds { + q = super::bind_bound!(q, b); + } + q.bind(blob) + .bind(f64::from(norm)) + .bind(item_json) + .execute(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + Ok(()) +} + +/// Populate a newly created vector index from the base table. +/// +/// Batched by offset exactly as `backfill_gsi` is, and for the same reason: a table +/// large enough to be worth indexing is too large to hold in memory. Returns the +/// number of rows written, which is what distinguishes "backfilled nothing because +/// no item carries the vector" from "backfilled nothing because the scan is broken". +/// Everything a backfill needs that does not change between batches. +/// +/// Bundled because the alternative was an eight-argument function threaded through two +/// drivers, where the only per-batch values are the page size and the cursor. +struct BackfillPlan<'a> { + table_id: &'a str, + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], + key_cols: Vec, +} + +impl<'a> BackfillPlan<'a> { + fn new( + table_id: &'a str, + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], + ) -> Self { + Self { + table_id, + meta, + base_key_schema, + attr_defs, + key_cols: base_key_columns(base_key_schema, attr_defs), + } + } +} + +/// Backfill one batch of existing rows into the vector index. +/// +/// Returns `(written, fetched, last_rowid)`. `fetched` distinguishes a short read +/// (the end) from a full one, and `last_rowid` is the cursor to resume from. +/// +/// Pagination is by KEY, not by `OFFSET`. Offset anchors on a position, so removing any +/// already-scanned row shifts every later position by one and the next batch skips a +/// row entirely. That row is then missing from the index permanently, and no queue +/// entry can repair it, because the skipped row was never written to: only the removed +/// one was. Reproduced before this change: one removal during a backfill left the row +/// at the batch boundary absent from the index. +/// +/// The cursor is `rowid`, not `pk`, and the difference is a correctness matter rather +/// than a preference. Composite-key base tables have `PRIMARY KEY (pk, sk*)`, so `pk` +/// alone is not unique; a `pk > last_pk` cursor whose batch boundary fell inside one +/// partition's sort-key group excluded every remaining row sharing that `pk`, +/// permanently, on both the UpdateTable path and startup reconciliation (which share +/// this function). Reproduced: a five-row partition scanned with a batch of three +/// indexed four rows and skipped two. `rowid` is unique regardless of the key layout, +/// so one query shape serves both. It is a valid cursor because the base tables are +/// ordinary rowid tables and every write is `INSERT ... ON CONFLICT DO UPDATE` +/// (`tx_helpers.rs`), which updates in place and never reassigns a rowid; rows +/// inserted after a batch passed their rowid position are the concurrent writes the +/// queue hold already captures and replays after ACTIVE. +/// +/// It was unreachable while the whole backfill ran in one transaction, and became +/// reachable the moment batches started committing independently. +/// One batch's outcome: rows indexed, rows skipped as poison, rows fetched +/// (for termination), and the cursor for the next batch. +struct BatchOutcome { + written: usize, + skipped: usize, + fetched: i64, + last_rowid: i64, +} + +async fn backfill_vector_batch( + tx: &mut db::Transaction, + plan: &BackfillPlan<'_>, + limit: i64, + after_rowid: i64, +) -> Result { + let base_table = super::data_table_name(plan.table_id); + // `rowid > ?` with -1 as the initial cursor: DuckDB rowids start at 0, so + // the first batch needs no separate query shape. + let sql = + format!("SELECT rowid, item_data FROM {base_table} WHERE rowid > ? ORDER BY rowid LIMIT ?"); + let rows: Vec<(i64, String)> = db::query_as(&sql) + .bind(after_rowid) + .bind(limit) + .fetch_all(&mut **tx) + .await + .map_err(crate::duckdb_util::map_db_err)?; + let fetched = i64::try_from(rows.len()).unwrap_or(limit); + let last_rowid = rows.last().map_or(after_rowid, |(rid, _)| *rid); + let mut written = 0usize; + let mut skipped = 0usize; + for (rowid, item_json) in rows { + // Poison classification. The live write path treats a malformed vector + // as an invariant violation and errors loudly, because core validation + // ran before storage was reached. That reasoning is FALSE here: rows + // written before the index existed never passed vector validation, so + // a malformed or wrong-dimension vector in the base table is expected + // input for a backfill, not a bug. Propagating it wedged the build in + // an infinite recovery loop: the error left the index CREATING, the + // watchdog re-ran the rebuild, and the same row failed again, forever, + // while the CREATING hold also froze every queued index write for the + // table. A row whose stored bytes cannot enter the index is skipped + // and counted instead, exactly as a GSI omits an item whose key + // attribute has the wrong type. Transient failures (the INSERT itself + // erroring) still propagate: those are retryable and must not drop + // rows. + let Ok(item) = serde_json::from_str::(&item_json) else { + tracing::warn!( + rowid, + index = %plan.meta.index_id, + "backfill: stored item is unparseable; skipping row" + ); + skipped += 1; + continue; + }; + if !item_is_indexable(&item, plan.meta) { + continue; + } + let vector_ok = item + .get(&plan.meta.vector_attribute_name) + .and_then(vector_components) + .is_some_and(|c| c.len() == plan.meta.dimensions); + if !vector_ok { + tracing::warn!( + rowid, + index = %plan.meta.index_id, + "backfill: vector attribute malformed or wrong dimension; skipping row" + ); + skipped += 1; + continue; + } + insert_vector_row( + tx, + plan.table_id, + plan.meta, + &item, + plan.base_key_schema, + plan.attr_defs, + &plan.key_cols, + VectorRowConflict::KeepExisting, + ) + .await?; + written += 1; + } + Ok(BatchOutcome { + written, + skipped, + fetched, + last_rowid, + }) +} + +/// A completed backfill: rows indexed and rows skipped as poison. `skipped` +/// is recorded on the catalog row so an ACTIVE index that deliberately omits +/// rows says so, rather than the omission being indistinguishable from a bug. +pub(crate) struct BackfillOutcome { + pub(crate) written: usize, + pub(crate) skipped: usize, +} + +/// Backfill the index in independently committed batches, releasing DuckDB's write +/// lock between them. +/// +/// This is what lets the base table stay writable while an index builds, which is how +/// the service behaves: the table remains ACTIVE and accepts writes throughout, and +/// only the index reports CREATING. Holding one transaction for the whole backfill +/// would block every write until it finished. +/// +/// Releasing the lock is also what creates the ordering hazard this design has to +/// answer. A write landing mid-backfill is enqueued, and if it were applied before the +/// backfill wrote its older snapshot of the same item, the index would converge on the +/// stale generation. The queue worker therefore refuses to claim any row for a table +/// whose vector index is still CREATING, so those writes accumulate and are applied +/// only after this returns and the index flips to ACTIVE. +/// +/// A crash part-way leaves the index in CREATING with some rows written, which +/// `reconcile_incomplete_vector_indexes` repairs at startup by rebuilding it. +pub(crate) async fn backfill_vector_index_in_batches( + pool: &db::Pool, + write_lock: &tokio::sync::Mutex<()>, + table_id: &str, + meta: &VectorIndexMeta, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + batch_delay: std::time::Duration, +) -> Result { + const BATCH: i64 = 500; + let plan = BackfillPlan::new(table_id, meta, base_key_schema, attr_defs); + let mut cursor: i64 = -1; + let mut written = 0usize; + let mut skipped = 0usize; + loop { + let outcome = { + let _writer = write_lock.lock().await; + let mut tx = pool.begin().await.map_err(crate::duckdb_util::map_db_err)?; + let result = backfill_vector_batch(&mut tx, &plan, BATCH, cursor).await?; + tx.commit().await.map_err(crate::duckdb_util::map_db_err)?; + result + }; + written += outcome.written; + skipped += outcome.skipped; + if outcome.fetched < BATCH { + break; + } + cursor = outcome.last_rowid; + // Outside the lock, so a write can actually proceed during the pause. Zero in + // production; a test sets it so a write is guaranteed to land mid-backfill. + if !batch_delay.is_zero() { + tokio::time::sleep(batch_delay).await; + } + } + Ok(BackfillOutcome { written, skipped }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db; + + /// A composite-key table whose partition spans a batch boundary must still + /// backfill every row. + /// + /// This is the regression test for a silent data-loss defect: the cursor was + /// `pk > last_pk`, and on a table with `PRIMARY KEY (pk, sk_s)` a batch ending + /// inside one partition's sort-key group excluded every remaining row sharing + /// that `pk`. Five rows in one partition scanned with a batch of three indexed + /// four and skipped two, permanently, on both the UpdateTable path and startup + /// reconciliation, which share `backfill_vector_batch`. The rowid cursor cannot + /// lose rows because rowid is unique whatever the key layout. + /// + /// Driven through `backfill_vector_batch` directly with a batch of 3 rather + /// than through the drivers, because they hardcode a 500-row batch and seeding + /// 501 rows would test the same lines slower. + #[tokio::test] + async fn a_composite_key_partition_straddling_a_batch_boundary_is_fully_backfilled() { + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let ks = vec![ + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }, + ]; + let ad = vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ]; + + let table_id = "t-composite"; + let mut tx = engine.pool.begin().await.expect("tx"); + crate::DuckDbEngine::create_data_table(&mut tx, table_id, &ks, &ad) + .await + .expect("base table"); + crate::DuckDbEngine::create_vector_data_table(&mut tx, table_id, "vidx-1", &ks, &ad) + .await + .expect("vector table"); + + // One partition with five sort keys plus a second partition, so the batch + // of three ends INSIDE partition "a": exactly the boundary that lost rows. + let base_table = super::super::data_table_name(table_id); + for (pk, sk) in [ + ("a", "1"), + ("a", "2"), + ("a", "3"), + ("a", "4"), + ("a", "5"), + ("b", "1"), + ] { + db::query(&format!( + "INSERT INTO {base_table} (pk, sk_s, item_data) VALUES (?, ?, ?)" + )) + .bind(pk) + .bind(sk) + .bind(format!( + r#"{{"pk":{{"S":"{pk}"}},"sk":{{"S":"{sk}"}},"emb":{{"L":[{{"N":"1"}},{{"N":"0"}}]}}}}"# + )) + .execute(&mut *tx) + .await + .expect("seed"); + } + + let meta = VectorIndexMeta { + index_id: "vidx-1".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: extenddb_core::types::Projection { + projection_type: extenddb_core::types::ProjectionType::All, + non_key_attributes: None, + }, + hash_attribute_name: None, + search_schema_attribute_names: Vec::new(), + }; + let plan = BackfillPlan::new(table_id, &meta, &ks, &ad); + + let mut cursor: i64 = -1; + let mut written = 0usize; + loop { + let outcome = backfill_vector_batch(&mut tx, &plan, 3, cursor) + .await + .expect("batch"); + written += outcome.written; + if outcome.fetched < 3 { + break; + } + cursor = outcome.last_rowid; + } + tx.commit().await.expect("commit"); + + assert_eq!( + written, 6, + "every row must be indexed; the pk-only cursor wrote 4 and skipped 2" + ); + let vec_table = super::super::vector_table_name(table_id, "vidx-1"); + let (rows,): (i64,) = db::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 6, "the index must hold all six rows"); + } + + /// A row whose stored bytes cannot enter the index is skipped and counted, + /// not propagated. Propagating it wedged the build permanently: the error + /// left the index CREATING, the watchdog re-ran the rebuild, the same row + /// failed again, and the CREATING hold froze every queued index write for + /// the table. This is the review finding on this file: "non-conformant + /// items keep failing and the index creation will be stuck in recovery + /// loop forever". + /// + /// Discriminating by construction: the poison rows (a wrong-dimension + /// vector, a non-list vector, and unparseable item bytes) sit BETWEEN good + /// rows, so the pre-fix behaviour (error on first poison row, nothing + /// after it indexed) cannot produce these counts. + #[tokio::test] + async fn poison_rows_are_skipped_and_counted_rather_than_wedging_the_backfill() { + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let ad = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + + let table_id = "t-poison"; + let mut tx = engine.pool.begin().await.expect("tx"); + crate::DuckDbEngine::create_data_table(&mut tx, table_id, &ks, &ad) + .await + .expect("base table"); + crate::DuckDbEngine::create_vector_data_table(&mut tx, table_id, "vidx-p", &ks, &ad) + .await + .expect("vector table"); + + let base_table = super::super::data_table_name(table_id); + // good, wrong-dimension, good, non-list vector, unparseable, good. + let rows: [(&str, String); 6] = [ + ( + "g1", + r#"{"pk":{"S":"g1"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#.to_owned(), + ), + ( + "p1", + r#"{"pk":{"S":"p1"},"emb":{"L":[{"N":"1"}]}}"#.to_owned(), + ), + ( + "g2", + r#"{"pk":{"S":"g2"},"emb":{"L":[{"N":"0"},{"N":"1"}]}}"#.to_owned(), + ), + ( + "p2", + r#"{"pk":{"S":"p2"},"emb":{"S":"not-a-vector"}}"#.to_owned(), + ), + ("p3", "{not json".to_owned()), + ( + "g3", + r#"{"pk":{"S":"g3"},"emb":{"L":[{"N":"1"},{"N":"1"}]}}"#.to_owned(), + ), + ]; + for (pk, item) in &rows { + db::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES (?, ?)" + )) + .bind(pk) + .bind(item) + .execute(&mut *tx) + .await + .expect("seed"); + } + + let meta = VectorIndexMeta { + index_id: "vidx-p".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: extenddb_core::types::Projection { + projection_type: extenddb_core::types::ProjectionType::All, + non_key_attributes: None, + }, + hash_attribute_name: None, + search_schema_attribute_names: Vec::new(), + }; + let plan = BackfillPlan::new(table_id, &meta, &ks, &ad); + + let outcome = backfill_vector_batch(&mut tx, &plan, 100, 0) + .await + .expect("a batch containing poison rows must still complete"); + tx.commit().await.expect("commit"); + + assert_eq!(outcome.written, 3, "the three good rows are indexed"); + assert_eq!(outcome.skipped, 3, "the three poison rows are counted"); + + // The good rows AFTER the poison rows made it in, which is the part the + // pre-fix behaviour cannot do: it stopped at p1. + let vec_table = super::super::vector_table_name(table_id, "vidx-p"); + let (rows,): (i64,) = db::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 3); + } + /// A synchronous-visibility write (`index_propagation_delay_ms == 0`) + /// landing mid-backfill is applied to the CREATING index inline, bypassing + /// the queue's CREATING-hold. When the backfill later reaches that base + /// key it must keep the row the inline write produced rather than fail the + /// whole build on the primary-key conflict: each batch reads the base + /// table live under the write lock, so an existing row was written from + /// the same or a newer base image than the one the batch just read. Before + /// `VectorRowConflict::KeepExisting`, this collision errored the backfill + /// and wedged the index in CREATING permanently. + #[tokio::test] + async fn a_synchronous_write_landing_mid_backfill_does_not_wedge_the_build() { + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + let engine = crate::DuckDbEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let ks = vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }]; + let ad = vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + + let table_id = "t-inline-collision"; + let mut tx = engine.pool.begin().await.expect("tx"); + crate::DuckDbEngine::create_data_table(&mut tx, table_id, &ks, &ad) + .await + .expect("base table"); + crate::DuckDbEngine::create_vector_data_table(&mut tx, table_id, "vidx-c", &ks, &ad) + .await + .expect("vector table"); + // Catalog rows: a table and its index still CREATING with a build in + // flight, which is the state a mid-backfill write observes. + db::query("INSERT INTO accounts (account_id, account_name) VALUES ('a', 'a')") + .execute(&mut *tx) + .await + .expect("account row"); + db::query( + "INSERT INTO tables (account_id, table_name, key_schema, attribute_definitions, \ + billing_mode, table_status, creation_date_time, table_arn, table_id) \ + VALUES ('a', 't', '[]', '[]', 'PAY_PER_REQUEST', 'ACTIVE', 'now', 'arn', ?)", + ) + .bind(table_id) + .execute(&mut *tx) + .await + .expect("table row"); + db::query( + "INSERT INTO vector_indexes (table_id, index_name, index_id, dimensions, \ + distance_function, vector_attribute, search_schema, projection, index_status, \ + backfilling) \ + VALUES (?, 'vidx-c', 'vidx-c', 2, 'COSINE', ?, NULL, ?, 'CREATING', 1)", + ) + .bind(table_id) + .bind(r#"{"AttributeName":"emb"}"#) + .bind(r#"{"ProjectionType":"ALL"}"#) + .execute(&mut *tx) + .await + .expect("index row"); + let base_table = super::super::data_table_name(table_id); + db::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES ('z', ?)" + )) + .bind(r#"{"pk":{"S":"z"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#) + .execute(&mut *tx) + .await + .expect("seed base row"); + + // The inline write: at delay 0 maintenance applies to the CREATING + // index directly rather than enqueueing. + let item: extenddb_core::types::Item = + serde_json::from_str(r#"{"pk":{"S":"z"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#) + .expect("item"); + let enqueued = maintain_vector_indexes(&mut tx, table_id, &ks, &ad, None, Some(&item), 0) + .await + .expect("inline maintenance"); + assert_eq!( + enqueued, 0, + "delay 0 must take the inline arm, not the queue" + ); + tx.commit().await.expect("commit"); + + // The backfill now reaches the same base key and must complete rather + // than error on the conflict, leaving exactly one row for the key. + let write_lock = tokio::sync::Mutex::new(()); + let meta = + fetch_vector_indexes_for_table(&mut engine.pool.begin().await.expect("tx"), table_id) + .await + .expect("metas") + .into_iter() + .find(|m| m.index_id == "vidx-c") + .expect("meta"); + let outcome = backfill_vector_index_in_batches( + &engine.pool, + &write_lock, + table_id, + &meta, + &ks, + &ad, + std::time::Duration::ZERO, + ) + .await + .expect("backfill must not wedge on the inline write's row"); + assert_eq!(outcome.skipped, 0); + let vec_table = super::super::vector_table_name(table_id, "vidx-c"); + let (rows,): (i64,) = db::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!( + rows, 1, + "one base key must yield one index row, not a duplicate" + ); + } +} diff --git a/crates/storage-duckdb/src/db.rs b/crates/storage-duckdb/src/db.rs new file mode 100644 index 00000000..f02eddb8 --- /dev/null +++ b/crates/storage-duckdb/src/db.rs @@ -0,0 +1,1074 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A thin asynchronous facade over the synchronous `duckdb` crate. +//! +//! DuckDB has no `sqlx` driver, so this module supplies the small `sqlx`-shaped +//! surface the rest of the crate uses: a connection [`Pool`], positional +//! [`query`] / [`query_as`] / [`query_scalar`] builders with `.bind()`, and a +//! [`Transaction`] that commits explicitly and rolls back on drop. Everything +//! else in the crate is written against this module rather than against +//! `duckdb` directly, which keeps the storage code free of blocking calls. +//! +//! # Execution model +//! +//! `duckdb::Connection` is `Send` but not `Sync`, and every call on it blocks. +//! Each pooled connection therefore lives in a slot (`tokio::sync::Mutex