From 49becbe8a84b8611ef48667e10426f7c51c3d4b8 Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Mon, 24 Aug 2026 20:15:40 +0000 Subject: [PATCH 01/13] feat(postgres): vector index catalog schema and pgvector detection Catalog migration 002 stores vector index metadata at catalog version 0.0.3. pgvector is installed opportunistically at init (CREATE EXTENSION IF NOT EXISTS, failure tolerated with a notice) and probed once at startup; the cached capability is what as_vector_search reads, so a server without the extension refuses vector features fail-closed instead of failing mid-request. --- .../migrations/002_vector_indexes.sql | 70 ++++++++++++++ crates/storage-postgres/src/bootstrapper.rs | 28 ++++++ crates/storage-postgres/src/lib.rs | 39 +++++++- crates/storage-postgres/src/migrations.rs | 14 ++- crates/storage-postgres/src/vector.rs | 96 +++++++++++++++++++ 5 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 crates/storage-postgres/migrations/002_vector_indexes.sql create mode 100644 crates/storage-postgres/src/vector.rs diff --git a/crates/storage-postgres/migrations/002_vector_indexes.sql b/crates/storage-postgres/migrations/002_vector_indexes.sql new file mode 100644 index 00000000..243689a8 --- /dev/null +++ b/crates/storage-postgres/migrations/002_vector_indexes.sql @@ -0,0 +1,70 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- Vector index catalog metadata (catalog version 0.0.3). +-- +-- Ports the SQLite catalog shape with PostgreSQL types: JSONB where SQLite +-- stores JSON text, BOOLEAN where it stores 0/1 (so no check constraint is +-- needed for booleanness), BIGINT for the skip counter. + +BEGIN; + +-- Vector index metadata. Kept out of `indexes` deliberately: a vector index is +-- not described by a key schema, so reusing that table's `key_schema` column +-- would mean storing something meaningless in a NOT NULL column. The engine +-- supplies index_id, as it does for GSIs. +-- +-- `search_schema` is nullable because the HASH element is optional (measured +-- against the live service): with one the search is partition-scoped and +-- SearchConditionExpression is required, without one it spans the table. +-- +-- `backfilling` mirrors the measured lifecycle: false while CREATING before the +-- scan starts, true while it runs, and the member is absent (NULL) once ACTIVE. +CREATE TABLE vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute JSONB NOT NULL, + search_schema JSONB, + projection JSONB NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling BOOLEAN, + -- Items a backfill skipped because their stored bytes cannot enter the + -- index (unparseable row, malformed or wrong-dimension vector). NULL until + -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so + -- an operator can see that an ACTIVE index deliberately omits rows, rather + -- than the build looping forever on them or dying part-way. + skipped_item_count BIGINT, + -- Build ownership for multi-process deployments. Several front-ends can + -- share one PostgreSQL, so an in-process registry cannot answer "is some + -- process still building this index". The builder records its identity and + -- renews the heartbeat per batch; a stuck-build sweep in any process reads + -- both. Unused until the UpdateTable-create lifecycle lands; NULL until a + -- build claims the row. + build_owner TEXT, + build_heartbeat_at TIMESTAMPTZ, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); + +CREATE UNIQUE INDEX idx_vector_indexes_index_id ON vector_indexes (index_id); + +-- Snapshot of the source table's vector index configuration at backup time, +-- the same way `key_schema` and `attribute_definitions` are snapshotted. NULL +-- for backups taken before this migration, which cannot have carried vector +-- indexes because the backend could not create them. Restore refuses a backup +-- whose snapshot is non-empty rather than silently dropping a declared index; +-- the snapshot also carries what a future vector-preserving restore needs. +ALTER TABLE backups ADD COLUMN vector_indexes JSONB; + +UPDATE settings SET value = '0.0.3' WHERE key = 'catalog_version'; + +COMMIT; diff --git a/crates/storage-postgres/src/bootstrapper.rs b/crates/storage-postgres/src/bootstrapper.rs index 7ae366b2..7ea69bb1 100755 --- a/crates/storage-postgres/src/bootstrapper.rs +++ b/crates/storage-postgres/src/bootstrapper.rs @@ -145,6 +145,33 @@ impl PostgresBootstrapper { pub fn catalog_connection_url(&self) -> String { self.app_connection_url(&self.config.catalog_db) } + + /// Try to install pgvector on the data database, tolerating refusal. + /// + /// Init and migrate are the moments when this process holds the privileges + /// that `CREATE EXTENSION` needs: the data database is owned by the + /// application role, and pgvector is a trusted extension, so its owner can + /// install it without being a superuser. Serve-time code never attempts + /// this, because a request path must not carry data-definition privileges it + /// only needs once. + /// + /// Failure is a notice, not an error. A deployment that does not want vector + /// indexes, or a managed PostgreSQL that does not offer pgvector, must still + /// initialise and upgrade normally; the server then refuses vector + /// operations, which is the fail-closed half of the same decision. + async fn try_create_vector_extension(&self, pool: &PgPool) { + println!("--- Checking pgvector extension on the data database..."); + match sqlx::query("CREATE EXTENSION IF NOT EXISTS vector") + .execute(pool) + .await + { + Ok(_) => println!(" pgvector available; vector indexes are supported."), + Err(e) => { + let hint = crate::vector::create_extension_hint(&e); + println!(" NOTICE: could not create the pgvector extension ({e}). {hint}."); + } + } + } } #[async_trait] @@ -231,6 +258,7 @@ impl Bootstrapper for PostgresBootstrapper { async fn run_data_migrations(&self) -> OpResult<()> { let pool = self.app_pool(&self.config.data_db).await?; + self.try_create_vector_extension(&pool).await; migrations::run_data_migrations(&pool).await?; // Programmatic migrations need the catalog pool (to enumerate index // tables) plus the data pool (where the `_ddb_*` tables live). diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 4a30ee33..e058d114 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -29,6 +29,7 @@ mod table_engine; mod table_helpers; mod ttl_worker; mod update_table; +mod vector; mod worker_store; mod workers; @@ -107,7 +108,7 @@ use sqlx::postgres::PgPoolOptions; /// /// The tuple is the single source of truth. Use `CATALOG_VERSION.to_string()` /// wherever a string representation is needed. -pub const CATALOG_VERSION: CatalogVersion = CatalogVersion::new(0, 0, 2); +pub const CATALOG_VERSION: CatalogVersion = CatalogVersion::new(0, 0, 3); /// Minimum number of connections allowed per pool. /// @@ -169,6 +170,18 @@ pub struct PostgresEngine { /// This is only a fallback for when the live read fails; the write path /// reads the setting live so a runtime change applies to the next write. pub index_propagation_delay_cache: Arc, + /// Whether the data database has the pgvector extension, probed once at + /// construction. + /// + /// Vector indexes live in `vector(N)` columns, a type pgvector defines, so + /// the capability is a property of the server rather than of this build. + /// Cached rather than probed per request: a control-plane feature does not + /// justify a round trip on every call, and the cost of caching is that + /// installing pgvector on a live server needs an ExtendDB restart to be + /// noticed. `DataEngine::as_vector_search` stays at its `None` default until + /// the search path exists, so this backend still refuses vector operations + /// over the wire; the flag is what that decision will read. + pub(crate) vector_capable: bool, } impl PostgresEngine { @@ -229,6 +242,20 @@ impl PostgresEngine { .and_then(|(v,)| v.parse::().ok()) .unwrap_or(DEFAULT_INDEX_PROPAGATION_DELAY_MS); + // Probe the pgvector extension once, on the data database, where vector + // data tables live. Logged at startup either way so an operator can see + // which answer this process is serving without reading the catalog. + let vector_version = vector::probe_vector_extension(&data_pool).await; + let vector_capable = vector_version.is_some(); + match &vector_version { + Some(version) => tracing::info!( + "pgvector {version} detected on the data database; vector index storage available" + ), + None => tracing::info!( + "pgvector not installed on the data database; vector indexes are not supported" + ), + } + Ok(Self { pool, data_pool, @@ -239,6 +266,7 @@ impl PostgresEngine { index_propagation_delay_cache: Arc::new(std::sync::atomic::AtomicU64::new( initial_gsi_delay, )), + vector_capable, }) } @@ -374,6 +402,15 @@ impl PostgresEngine { pub fn data_pool(&self) -> &PgPool { &self.data_pool } + + /// Whether the data database has pgvector, as probed at construction. + /// + /// Public so that a deployment check, and the tests that pin the + /// fail-closed behaviour, can read the same answer the engine acts on. + #[must_use] + pub fn vector_capable(&self) -> bool { + self.vector_capable + } } // ============================================================================ diff --git a/crates/storage-postgres/src/migrations.rs b/crates/storage-postgres/src/migrations.rs index c2483f2a..4b5e08f0 100755 --- a/crates/storage-postgres/src/migrations.rs +++ b/crates/storage-postgres/src/migrations.rs @@ -7,10 +7,16 @@ use extenddb_storage::management_store::{OpError, OpResult}; use sqlx::PgPool; /// Embedded catalog migration files, applied in order. -pub(crate) const CATALOG_MIGRATIONS: &[(&str, &str)] = &[( - "001_schema.sql", - include_str!("../../storage-postgres/migrations/001_schema.sql"), -)]; +pub(crate) const CATALOG_MIGRATIONS: &[(&str, &str)] = &[ + ( + "001_schema.sql", + include_str!("../../storage-postgres/migrations/001_schema.sql"), + ), + ( + "002_vector_indexes.sql", + include_str!("../../storage-postgres/migrations/002_vector_indexes.sql"), + ), +]; /// Run catalog migrations, skipping already-applied ones. pub(crate) async fn run_catalog_migrations(pool: &PgPool) -> OpResult<()> { diff --git a/crates/storage-postgres/src/vector.rs b/crates/storage-postgres/src/vector.rs new file mode 100644 index 00000000..60ae780a --- /dev/null +++ b/crates/storage-postgres/src/vector.rs @@ -0,0 +1,96 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! pgvector availability: detection at startup, classification at runtime. +//! +//! Vector indexes on this backend are stored in `vector(N)` columns, a type the +//! pgvector extension defines, so the whole feature depends on an extension +//! that a given PostgreSQL server may not have installed. The capability is +//! therefore detected, not declared: the same binary serves vector indexes +//! against a server that has pgvector and refuses them against one that does +//! not, with no build-time or configuration difference. +//! +//! Detection is one probe at engine construction ([`probe_vector_extension`]), +//! cached for the process lifetime. Installing pgvector on a running server +//! therefore needs an ExtendDB restart to be noticed. + +use sqlx::PgPool; + +/// Name of the extension that provides the `vector` column type. +pub(crate) const VECTOR_EXTENSION: &str = "vector"; + +/// Read the installed pgvector version, or `None` when it is not installed. +/// +/// A query failure is reported as `None` and logged rather than propagated: an +/// engine that cannot determine the answer must not claim the capability, and +/// refusing to start over a missing optional feature would take down a +/// deployment that never asked for vector indexes. +pub(crate) async fn probe_vector_extension(data_pool: &PgPool) -> Option { + match sqlx::query_scalar::<_, String>("SELECT extversion FROM pg_extension WHERE extname = $1") + .bind(VECTOR_EXTENSION) + .fetch_optional(data_pool) + .await + { + Ok(version) => version, + Err(e) => { + tracing::warn!( + error = %e, + "could not determine whether the pgvector extension is installed; \ + treating vector indexes as unsupported" + ); + None + } + } +} + +/// Map a `SQLSTATE` from a failed `CREATE EXTENSION vector` to operator advice. +/// +/// The install-time failure codes are different from the runtime ones, so they +/// get their own classifier: `58P01` undefined_file is what PostgreSQL reports +/// when the extension's control file is not on the server (the package is not +/// installed at all), and `42501` insufficient_privilege is what it reports when +/// the connecting role may not create it. Reporting either as "not available" +/// alone would send an operator looking in the wrong place. +fn create_extension_hint_for_sqlstate(code: Option<&str>) -> &'static str { + match code { + Some("58P01") => { + "the pgvector package is not installed on this PostgreSQL server; \ + install it (for example postgresql-16-pgvector) and re-run migrate" + } + Some("42501") => { + "this role may not create extensions; create it once as a superuser \ + or as the database owner, then re-run migrate" + } + _ => { + "ExtendDB will run normally and refuse vector indexes unless the \ + extension is already present" + } + } +} + +/// Operator advice for a failed `CREATE EXTENSION vector`. +pub(crate) fn create_extension_hint(e: &sqlx::Error) -> &'static str { + let code = match e { + sqlx::Error::Database(db_err) => db_err.code().map(|c| c.to_string()), + _ => None, + }; + create_extension_hint_for_sqlstate(code.as_deref()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_missing_package_and_a_missing_privilege_get_different_advice() { + // The two ways `CREATE EXTENSION` fails at init need an operator to do + // different things, so the notice must not collapse them. + let absent = create_extension_hint_for_sqlstate(Some("58P01")); + let denied = create_extension_hint_for_sqlstate(Some("42501")); + assert!(absent.contains("not installed"), "{absent}"); + assert!(denied.contains("may not create extensions"), "{denied}"); + assert_ne!(absent, denied); + assert_ne!(absent, create_extension_hint_for_sqlstate(None)); + } + +} From fc2a0c4bfbb764d94722d867920536cd4846da0a Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Mon, 24 Aug 2026 20:15:40 +0000 Subject: [PATCH 02/13] feat(postgres): vector index control plane CreateTable, DescribeTable, DeleteTable, and UpdateTable-delete for vector indexes, gated on the startup capability. Hardening folded in: the catalog migration is replay-safe, the catalog decode is shared and refuses an unknown status, only the undefined-object SQLSTATE classifies as missing pgvector, an empty search schema collapses once for every backend, and the engine maps an unsupported backup operation to a validation error. Restore refuses a backup that carries vector indexes. Covered by storage-level tests, a live-deployment migration test, version-gate tests in both directions, and a CI job that keeps the refusal surface tested after the pgvector image flip. --- .github/workflows/integration.yml | 84 + crates/core/src/types/table.rs | 108 ++ crates/engine/src/backup.rs | 36 +- crates/engine/src/create_table.rs | 10 +- crates/engine/src/update_table.rs | 10 +- crates/storage-postgres/Cargo.toml | 5 + .../migrations/002_vector_indexes.sql | 15 +- crates/storage-postgres/src/backup_engine.rs | 86 +- crates/storage-postgres/src/create_table.rs | 123 +- crates/storage-postgres/src/data/ddl.rs | 38 +- crates/storage-postgres/src/delete_table.rs | 5 + crates/storage-postgres/src/migrations.rs | 68 + crates/storage-postgres/src/table_helpers.rs | 71 +- crates/storage-postgres/src/update_table.rs | 103 ++ crates/storage-postgres/src/vector.rs | 150 +- .../tests/vector_control_plane.rs | 1462 +++++++++++++++++ crates/storage-sqlite/src/create_table.rs | 16 +- crates/storage-sqlite/src/data/ddl.rs | 43 +- crates/storage-sqlite/src/table_helpers.rs | 92 +- crates/storage-sqlite/src/update_table.rs | 10 +- crates/storage/src/lib.rs | 1 + crates/storage/src/vector_catalog.rs | 346 ++++ devtools/run-tests | 4 +- docs/manuals/05-admin-guide.md | 31 +- docs/manuals/07-upgrade-manual.md | 35 +- tests/rust/src/vector_index_search.rs | 119 ++ tests/test_cli_vector_catalog_migration.py | 292 ++++ 27 files changed, 3230 insertions(+), 133 deletions(-) create mode 100644 crates/storage-postgres/tests/vector_control_plane.rs create mode 100644 crates/storage/src/vector_catalog.rs create mode 100644 tests/test_cli_vector_catalog_migration.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c3d4448c..02589c5c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -255,6 +255,88 @@ jobs: # optional, and turns a silent skip into a failure. EXTENDDB_EXPECT_VECTORS: "0" run: devtools/run-tests --extenddb --rust-integration --release + + # The control plane for vector indexes is not reachable over the wire while + # this backend declares no vector search capability, so its tests drive the + # storage layer directly against this job's PostgreSQL. They build their own + # throwaway databases; the connection string is the server, not a database. + - name: Run PostgreSQL storage-level tests + env: + EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + run: cargo test --release -p extenddb-storage-postgres --test vector_control_plane + + # The runtime-detection proof: the same binary, against a PostgreSQL with no + # pgvector, must refuse vector indexes over the wire. Today the job above runs + # on a plain image too, so this looks like a duplicate of it; it is not, because + # that job moves to the pgvector image when the search path lands and its + # EXTENDDB_EXPECT_VECTORS flips to "1". Without this job the refusal surface + # would stop being tested at exactly that point. + run-rust-integration-postgres-novector: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: devpass + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Build release + run: cargo build --release + + - name: Initialize ExtendDB + id: init + run: | + output=$(./target/release/extenddb init --config extenddb.toml \ + --pg-host 127.0.0.1 --pg-port 5432 --pg-user postgres --pg-pass devpass 2>&1) + echo "$output" + echo "admin_password=$(echo "$output" | grep -oP 'Password: \K\S+')" >> "$GITHUB_OUTPUT" + + - name: Start ExtendDB + run: | + ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & + for i in $(seq 1 30); do + if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then + echo "Server ready" + exit 0 + fi + sleep 1 + done + echo "Server failed to start" + exit 1 + + - name: Run the vector refusal suite + env: + EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 + AWS_DEFAULT_REGION: us-east-1 + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + # "0" makes the refusal assertions mandatory: the suite adapts to what + # the backend reports, so without this it could skip every assertion + # and still report green. + EXTENDDB_EXPECT_VECTORS: "0" + run: >- + devtools/run-tests --extenddb --rust-integration --release + --filter vector_index_unsupported + + # No storage-level step here on purpose. Those tests build their own + # extension-free scratch databases, so they behave identically on either + # image and the job above already runs them; repeating them here would buy + # a second release build and no coverage. run-rust-integration-sqlite: runs-on: ubuntu-latest @@ -323,6 +405,7 @@ jobs: run-integration-sqlite, run-integration-dev-mode, run-rust-integration, + run-rust-integration-postgres-novector, run-rust-integration-sqlite, ] if: always() @@ -332,6 +415,7 @@ jobs: [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \ [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ + [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then exit 1 fi diff --git a/crates/core/src/types/table.rs b/crates/core/src/types/table.rs index d5f1d854..1090afb5 100755 --- a/crates/core/src/types/table.rs +++ b/crates/core/src/types/table.rs @@ -328,6 +328,42 @@ pub struct SearchSchemaElement { pub element_type: SearchSchemaElementType, } +impl VectorIndexSpecification { + /// Collapse an empty `SearchSchema` to an absent one. + /// + /// A request may carry `SearchSchema: []`, and it means the same thing as + /// omitting the member: the index is unscoped and a search spans the table. + /// Amazon DynamoDB reports either an absent member or a populated one and + /// never an empty list, so storing the empty list would create a third state + /// that a describe echoes back and no client expects, and would let two + /// backends answer the same request differently depending on whether each + /// remembered to collapse it. + /// + /// Applied on the request paths, before any backend sees the specification, + /// and again by the backends themselves so that a caller reaching the storage + /// trait directly cannot store the third state either. + pub fn normalize_search_schema(&mut self) { + if self + .search_schema + .as_ref() + .is_some_and(|elements| elements.is_empty()) + { + self.search_schema = None; + } + } + + /// The search schema as it should be stored: absent when empty. + /// + /// The borrowing form of [`Self::normalize_search_schema`], for a caller that + /// is serialising rather than holding a mutable specification. + #[must_use] + pub fn search_schema_for_storage(&self) -> Option<&[SearchSchemaElement]> { + self.search_schema + .as_deref() + .filter(|elements| !elements.is_empty()) + } +} + /// Vector index definition for `CreateTable` requests. /// /// A vector index is a specialized global secondary index that supports @@ -1160,6 +1196,78 @@ mod tests { } } +#[cfg(test)] +mod vector_search_schema_normalisation_tests { + use super::{ + DistanceFunction, SearchSchemaElement, SearchSchemaElementType, VectorAttribute, + VectorIndexSpecification, + }; + + fn spec(search_schema: Option>) -> VectorIndexSpecification { + VectorIndexSpecification { + index_name: "vidx".to_owned(), + dimensions: 4, + distance_function: DistanceFunction::Cosine, + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + search_schema, + projection: None, + } + } + + fn hash() -> Vec { + vec![SearchSchemaElement { + attribute_name: "tenant".to_owned(), + element_type: SearchSchemaElementType::Hash, + }] + } + + /// An empty list and an absent member mean the same thing, so only one of them + /// may reach storage. The service reports an absent member or a populated one + /// and never an empty list, so storing `[]` would make DescribeTable echo a + /// third state, and would let two backends differ on whether they collapsed it. + #[test] + fn an_empty_search_schema_becomes_absent() { + let mut empty = spec(Some(Vec::new())); + empty.normalize_search_schema(); + assert_eq!(empty.search_schema, None); + assert_eq!(spec(Some(Vec::new())).search_schema_for_storage(), None); + } + + #[test] + fn a_populated_search_schema_is_left_alone() { + let mut scoped = spec(Some(hash())); + scoped.normalize_search_schema(); + assert_eq!(scoped.search_schema, Some(hash())); + assert_eq!( + spec(Some(hash())).search_schema_for_storage(), + Some(hash().as_slice()) + ); + } + + #[test] + fn an_absent_search_schema_stays_absent() { + let mut unscoped = spec(None); + unscoped.normalize_search_schema(); + assert_eq!(unscoped.search_schema, None); + assert_eq!(spec(None).search_schema_for_storage(), None); + } + + /// The two forms must agree, since one is applied on the request path and the + /// other by the backends: a caller reaching storage directly must not be able + /// to store a state the request path would have collapsed. + #[test] + fn the_owning_and_borrowing_forms_agree() { + for schema in [None, Some(Vec::new()), Some(hash())] { + let mut owned = spec(schema.clone()); + owned.normalize_search_schema(); + let borrowed = spec(schema).search_schema_for_storage().map(<[_]>::to_vec); + assert_eq!(owned.search_schema, borrowed); + } + } +} + #[cfg(test)] mod vector_index_readiness_tests { diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index d52ec23c..7c6f117b 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -281,6 +281,14 @@ fn storage_err_to_dynamo(e: extenddb_storage::error::StorageError) -> DynamoDbEr DynamoDbError::ValidationException(msg) } } + // Not a fault, so deliberately not logged at error level: the backend + // never claimed the feature, and the request is invalid against this + // deployment rather than a server failure. Amazon DynamoDB has no + // "unsupported" error class, so this reports as a validation error, the + // same mapping CreateTable and UpdateTable use for a refused capability. + extenddb_storage::error::StorageError::Unsupported(msg) => { + DynamoDbError::ValidationException(msg) + } other => { tracing::error!(internal_error = %other, "backup storage error"); DynamoDbError::InternalServerError("Internal server error".to_owned()) @@ -290,8 +298,9 @@ fn storage_err_to_dynamo(e: extenddb_storage::error::StorageError) -> DynamoDbEr #[cfg(test)] mod tests { - use super::backup_arn_field; + use super::{backup_arn_field, storage_err_to_dynamo}; use extenddb_core::error::DynamoDbError; + use extenddb_storage::error::StorageError; use serde_json::json; const ACCOUNT: &str = "123456789012"; @@ -306,6 +315,31 @@ mod tests { assert_eq!(backup_arn_field(&body, ACCOUNT).unwrap(), arn(ACCOUNT)); } + /// A refusal the backend states plainly must not reach the client as a fault. + /// + /// The restore path refuses a backup whose source carried vector indexes, + /// because restore does not recreate them and dropping a declared index + /// silently is worse than refusing. Without this arm that refusal fell to the + /// catch-all: the client got a 500 with no reason, indistinguishable from a + /// broken server, and the operator got an error-level log for a request that + /// was answered correctly. + #[test] + fn an_unsupported_feature_is_a_validation_exception() { + let err = storage_err_to_dynamo(StorageError::Unsupported( + "restoring a table with vector indexes is not supported by this storage backend" + .to_owned(), + )); + match err { + DynamoDbError::ValidationException(msg) => { + assert_eq!( + msg, + "restoring a table with vector indexes is not supported by this storage backend" + ); + } + other => panic!("expected ValidationException, got {other:?}"), + } + } + #[test] fn other_account_arn_is_denied() { let body = json!({ "BackupArn": arn("999999999999") }); diff --git a/crates/engine/src/create_table.rs b/crates/engine/src/create_table.rs index 3463fa2b..eeaa86e6 100755 --- a/crates/engine/src/create_table.rs +++ b/crates/engine/src/create_table.rs @@ -22,7 +22,7 @@ pub async fn handle_create_table( }], )?; - let input: CreateTableInput = serde_json::from_value(body).map_err(|e| { + let mut input: CreateTableInput = serde_json::from_value(body).map_err(|e| { let msg = e.to_string(); if msg.contains("validation error detected") || msg.contains("parameter values were invalid") @@ -44,6 +44,14 @@ pub async fn handle_create_table( validate_create_table(&input, &ctx.limits)?; + // An empty SearchSchema means the same as an absent one, so it is collapsed + // here, once, rather than in each backend. Storing the empty list would make + // DescribeTable echo a state the service never reports, and would leave two + // backends free to disagree about it. + for spec in input.vector_indexes.iter_mut().flatten() { + spec.normalize_search_schema(); + } + crate::vector_gate::ensure_create_table_supported( input.vector_indexes.as_ref(), ctx.storage.as_vector_search(), diff --git a/crates/engine/src/update_table.rs b/crates/engine/src/update_table.rs index 61a25824..416698ed 100755 --- a/crates/engine/src/update_table.rs +++ b/crates/engine/src/update_table.rs @@ -27,7 +27,8 @@ pub async fn handle_update_table( body: Value, ctx: &OperationContext, ) -> Result { - let input: UpdateTableInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; + let mut input: UpdateTableInput = + serde_json::from_value(body).map_err(crate::deserialize_error)?; if input.table_name.is_empty() { return Err(DynamoDbError::ValidationException( @@ -45,6 +46,13 @@ pub async fn handle_update_table( input.vector_index_updates.as_ref(), input.attribute_definitions.as_deref().unwrap_or_default(), )?; + // Same collapse CreateTable applies, for the same reason: an index added by + // either path must reach every backend in one shape. + for update in input.vector_index_updates.iter_mut().flatten() { + if let Some(create) = update.create.as_mut() { + create.normalize_search_schema(); + } + } let has_gsi_updates = input .global_secondary_index_updates diff --git a/crates/storage-postgres/Cargo.toml b/crates/storage-postgres/Cargo.toml index 9231a0e3..dce75514 100755 --- a/crates/storage-postgres/Cargo.toml +++ b/crates/storage-postgres/Cargo.toml @@ -7,6 +7,11 @@ edition.workspace = true rust-version.workspace = true license.workspace = true +[dev-dependencies] +# The storage-level vector control-plane tests drive the engine directly against +# a scratch database, so they need a runtime and the test macro. +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + [dependencies] anyhow = { workspace = true } extenddb-core = { workspace = true } diff --git a/crates/storage-postgres/migrations/002_vector_indexes.sql b/crates/storage-postgres/migrations/002_vector_indexes.sql index 243689a8..c61a32fd 100644 --- a/crates/storage-postgres/migrations/002_vector_indexes.sql +++ b/crates/storage-postgres/migrations/002_vector_indexes.sql @@ -13,13 +13,22 @@ BEGIN; -- would mean storing something meaningless in a NOT NULL column. The engine -- supplies index_id, as it does for GSIs. -- +-- Every statement here is written to tolerate a replay. The runner applies a +-- migration and records it in `schema_history` as two separate commits, so a +-- crash in between leaves this file applied but unrecorded, and the next +-- migrate would run it again. Without the guards that second run fails on +-- "relation already exists" and blocks every later migration, on a deployment +-- that is otherwise serving correctly because the version gate is satisfied. +-- Recovering from that needs a hand-written ledger row. 001 guards every one of +-- its tables the same way. +-- -- `search_schema` is nullable because the HASH element is optional (measured -- against the live service): with one the search is partition-scoped and -- SearchConditionExpression is required, without one it spans the table. -- -- `backfilling` mirrors the measured lifecycle: false while CREATING before the -- scan starts, true while it runs, and the member is absent (NULL) once ACTIVE. -CREATE TABLE vector_indexes ( +CREATE TABLE IF NOT EXISTS vector_indexes ( table_id TEXT NOT NULL, index_id TEXT NOT NULL, index_name TEXT NOT NULL, @@ -55,7 +64,7 @@ CREATE TABLE vector_indexes ( CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) ); -CREATE UNIQUE INDEX idx_vector_indexes_index_id ON vector_indexes (index_id); +CREATE UNIQUE INDEX IF NOT EXISTS idx_vector_indexes_index_id ON vector_indexes (index_id); -- Snapshot of the source table's vector index configuration at backup time, -- the same way `key_schema` and `attribute_definitions` are snapshotted. NULL @@ -63,7 +72,7 @@ CREATE UNIQUE INDEX idx_vector_indexes_index_id ON vector_indexes (index_id); -- indexes because the backend could not create them. Restore refuses a backup -- whose snapshot is non-empty rather than silently dropping a declared index; -- the snapshot also carries what a future vector-preserving restore needs. -ALTER TABLE backups ADD COLUMN vector_indexes JSONB; +ALTER TABLE backups ADD COLUMN IF NOT EXISTS vector_indexes JSONB; UPDATE settings SET value = '0.0.3' WHERE key = 'catalog_version'; diff --git a/crates/storage-postgres/src/backup_engine.rs b/crates/storage-postgres/src/backup_engine.rs index 94e4f449..a9323985 100755 --- a/crates/storage-postgres/src/backup_engine.rs +++ b/crates/storage-postgres/src/backup_engine.rs @@ -124,6 +124,35 @@ impl BackupEngine for PostgresEngine { #[allow(clippy::cast_possible_wrap)] let actual_count = items.len() as i64; + // Snapshot the source table's vector index configuration alongside + // its key schema. Restore refuses a backup whose snapshot is + // non-empty rather than silently dropping a declared index, and it + // cannot ask the source table instead: the source may have been + // deleted, which cascade-deletes its `vector_indexes` rows. + // + // Stored in the wire's own shape behind a version marker, not as a + // copy of the catalog row. A snapshot outlives the schema that + // produced it, so freezing physical column names into it would mean a + // later rename or an added column silently changed the meaning of + // snapshots already on disk. The lifecycle columns are deliberately + // absent: a restored index is defined by its configuration, and its + // build state belongs to the table it came from. + let vector_indexes: Option = sqlx::query_scalar( + "SELECT jsonb_build_object('Version', 1, 'VectorIndexes', \ + COALESCE(jsonb_agg(jsonb_build_object( \ + 'IndexName', index_name, \ + 'Dimensions', dimensions, \ + 'DistanceFunction', distance_function, \ + 'VectorAttribute', vector_attribute, \ + 'SearchSchema', search_schema, \ + 'Projection', projection) ORDER BY index_name), '[]'::jsonb)) \ + FROM vector_indexes WHERE table_id = $1", + ) + .bind(&table_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(format!("Database error: {e}")))?; + // Wrap all catalog-side writes in a single transaction so a crash // cannot leave a backup marked AVAILABLE with partial items. let mut tx = self @@ -135,8 +164,8 @@ impl BackupEngine for PostgresEngine { sqlx::query( "INSERT INTO backups (backup_arn, backup_name, table_id, table_name, account_id, \ backup_status, backup_size_bytes, item_count, key_schema, attribute_definitions, \ - billing_mode) \ - VALUES ($1, $2, $3, $4, $5, 'AVAILABLE', $6, $7, $8, $9, $10)", + billing_mode, vector_indexes) \ + VALUES ($1, $2, $3, $4, $5, 'AVAILABLE', $6, $7, $8, $9, $10, $11)", ) .bind(&backup_arn) .bind(&backup_name) @@ -148,6 +177,7 @@ impl BackupEngine for PostgresEngine { .bind(&key_schema) .bind(&attr_defs) .bind(&billing_mode) + .bind(&vector_indexes) .execute(&mut *tx) .await .map_err(|e| StorageError::Internal(format!("Database error: {e}")))?; @@ -403,9 +433,10 @@ impl BackupEngine for PostgresEngine { serde_json::Value, String, Option, + Option, ) = sqlx::query_as( "SELECT table_name, key_schema, attribute_definitions, billing_mode, \ - provisioned_throughput \ + provisioned_throughput, vector_indexes \ FROM backups \ WHERE backup_arn = $1 AND account_id = $2 AND backup_status = 'AVAILABLE'", ) @@ -416,7 +447,54 @@ impl BackupEngine for PostgresEngine { .map_err(|e| StorageError::Internal(format!("Database error: {e}")))? .ok_or_else(|| StorageError::Validation(format!("Backup not found: {backup_arn}")))?; - let (_orig_table, ks_json, ad_json, billing, _prov) = backup_row; + let (_orig_table, ks_json, ad_json, billing, _prov, vector_indexes) = backup_row; + + // Restore rebuilds the target from the snapshot's key schema, + // attribute definitions and billing mode, and does not carry indexes + // across. For a source table that had vector indexes that would be + // silent loss of a declared index: the restored table would answer + // every request except a search, and the client would only find out + // on the first one. The service preserves vector indexes through + // backup and restore (measured 2026-08-19), so the conformant end + // state is to restore them; until then this refuses, because a typed + // refusal is recoverable and silent loss is not. + // + // A NULL snapshot is a backup taken before the column existed, which + // cannot have carried vector indexes: this backend could not create + // them then. An unrecognised version is refused rather than guessed + // at, for the same reason a declared index must not be dropped + // silently. + let vector_index_count = match vector_indexes.as_ref() { + None => 0, + Some(snapshot) => { + let version = snapshot.get("Version").and_then(serde_json::Value::as_u64); + if version != Some(1) { + // Version skew, not a fault: a newer build wrote a shape + // this one does not know. Reported the same way as the + // refusal below, so a client gets a reason rather than a + // 500 and the operator gets no error-level noise. Nearly + // unreachable, because a shape change would normally come + // with a catalog version bump that the startup gate + // refuses first; the gap is a future build adding a member + // without any schema change. + return Err(StorageError::Unsupported(format!( + "backup {backup_arn} carries a vector index snapshot this build \ + cannot read (version {version:?})" + ))); + } + snapshot + .get("VectorIndexes") + .and_then(serde_json::Value::as_array) + .map_or(0, Vec::len) + } + }; + if vector_index_count > 0 { + return Err(StorageError::Unsupported(format!( + "backup {backup_arn} has {vector_index_count} vector index(es); \ + restoring a table with vector indexes is not supported by this \ + storage backend" + ))); + } let key_schema: Vec = serde_json::from_value(ks_json) diff --git a/crates/storage-postgres/src/create_table.rs b/crates/storage-postgres/src/create_table.rs index 8a76fe79..53efd443 100755 --- a/crates/storage-postgres/src/create_table.rs +++ b/crates/storage-postgres/src/create_table.rs @@ -193,6 +193,89 @@ impl PostgresEngine { } } + // Insert vector index metadata. A CreateTable's table is empty, so there + // is nothing to backfill: the index goes straight to ACTIVE with no + // `backfilling` member, which is the state the service reports for an + // index created this way. The UpdateTable path is the one that drives a + // real lifecycle. + if let Some(vis) = &input.vector_indexes { + // Confirm pgvector is still there before recording an index whose + // storage depends on it, closing the window between the startup probe + // and now. + // + // The invariant this gives is narrower than "storage never records a + // vector index it cannot back", and the difference is worth stating: + // the check runs only where the startup probe said the extension was + // present, so storage relies on the engine having already refused + // vector indexes on a backend that reports no capability. The + // alternative, probing unconditionally, is one round trip on a + // CreateTable that carries vector indexes and would be strictly + // safer here; it was not taken because it would make the + // control-plane tests require the extension, and they would then skip + // on any server without the package, including the plain PostgreSQL + // job in CI. Losing that coverage costs more than the invariant gains + // while no path can reach storage without passing the engine gate + // first. That dependency is itself pinned rather than assumed: the + // wire refusal suite runs with EXTENDDB_EXPECT_VECTORS=0 in two CI + // jobs, so a regression that let a vector request past the gate fails + // there before it could reach this un-probed path. + if self.vector_capable { + crate::vector::ensure_vector_extension_present(&self.data_pool).await?; + } + for vi in vis { + let vec_attr = serde_json::to_value(&vi.vector_attribute) + .map_err(|e| StorageError::Internal(e.to_string()))?; + // The request paths already collapse an empty SearchSchema, so + // this is for a caller that reaches the storage trait directly. + // Same core rule either way, so the two cannot drift. + let search_schema = vi + .search_schema_for_storage() + .map(serde_json::to_value) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj = vi + .projection + .as_ref() + .map(serde_json::to_value) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + // Core validation requires Projection, so reaching here + // means the request bypassed validation rather than that + // the caller omitted it. + StorageError::Internal( + "vector index reached storage without a projection".to_owned(), + ) + })?; + let distance = extenddb_storage::vector_catalog::distance_function_token( + vi.distance_function, + )?; + let index_id = uuid::Uuid::new_v4().to_string(); + sqlx::query( + r"INSERT INTO vector_indexes + (table_id, index_name, index_id, dimensions, distance_function, + vector_attribute, search_schema, projection, index_status, backfilling) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'ACTIVE', NULL)", + ) + .bind(&table_id) + .bind(&vi.index_name) + .bind(&index_id) + .bind(i32::try_from(vi.dimensions).map_err(|_| { + StorageError::Internal(format!( + "vector dimensions out of range: {}", + vi.dimensions + )) + })?) + .bind(&distance) + .bind(&vec_attr) + .bind(&search_schema) + .bind(&proj) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + // Insert tags if let Some(tags) = &input.tags { for tag in tags { @@ -383,6 +466,39 @@ impl PostgresEngine { TableStatus::Creating }; + // Echo the vector indexes just created. Built from the request rather + // than re-read from the catalog, which would add a round trip to say + // something already known. A CreateTable's table is empty, so each index + // is ACTIVE with no `backfilling` member. + let vector_index_descs: Option> = input + .vector_indexes + .as_ref() + .map(|vis| { + vis.iter() + .map(|vi| extenddb_core::types::VectorIndexDescription { + index_name: vi.index_name.clone(), + vector_attribute: vi.vector_attribute.clone(), + dimensions: vi.dimensions, + // Normalised the same way the catalog row is, so the echo + // and a later describe report the same thing. + search_schema: vi.search_schema_for_storage().map(<[_]>::to_vec), + distance_function: vi.distance_function, + index_status: extenddb_core::types::IndexStatus::Active, + backfilling: None, + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn( + &self.region, + account_id, + &input.table_name, + &vi.index_name, + ), + projection: vi.projection.clone(), + }) + .collect() + }) + .filter(|v: &Vec<_>| !v.is_empty()); + Ok(TableDescription { table_name: input.table_name, key_schema: input.key_schema, @@ -430,7 +546,12 @@ impl PostgresEngine { .as_ref() .map(|tc| serde_json::json!({ "TableClass": tc })), on_demand_throughput: input.on_demand_throughput, - ..Default::default() + // Every field is populated deliberately, with no + // `..Default::default()` spread. This response is the complete + // description of what was just created, so a new core field should + // break this site and force a decision about whether create must + // report it, rather than silently defaulting. + vector_indexes: vector_index_descs, }) } } diff --git a/crates/storage-postgres/src/data/ddl.rs b/crates/storage-postgres/src/data/ddl.rs index 9e4bfcbd..c9952d13 100755 --- a/crates/storage-postgres/src/data/ddl.rs +++ b/crates/storage-postgres/src/data/ddl.rs @@ -325,6 +325,12 @@ impl PostgresEngine { let (global_secondary_indexes, local_secondary_indexes) = self.fetch_all_index_info(&table_id).await?; let has_lsi = !local_secondary_indexes.is_empty(); + // Vector indexes ride on the cached key info too, so the engine can + // validate vector attributes on writes and charge vector write capacity. + // The update path already passes `key_info.vector_indexes` into the + // expression evaluator, so populating this is what turns that validation + // on: no call site changes. + let vector_indexes = self.fetch_vector_index_key_info(&table_id).await?; let key_info = TableKeyInfo { table_name: table_name.to_owned(), @@ -337,10 +343,10 @@ impl PostgresEngine { global_secondary_indexes, local_secondary_indexes, stream_specification, - // Fields for features this backend does not implement, vector - // indexes today, take their defaults. Adding one to TableKeyInfo - // then does not break this build. - ..Default::default() + // Every field is populated, with no `..Default::default()` spread: a + // new core field should break this site and force a decision about + // whether the write path needs it, rather than silently defaulting. + vector_indexes, }; // Catalog metadata that cannot describe its own sort key would make the // keyed read paths fall back to a partition-only lookup and return the @@ -351,6 +357,30 @@ impl PostgresEngine { Ok(key_info) } + /// Fetch the vector indexes of a table in the shape the engine caches. + /// + /// Selects the whole row rather than the five columns this shape needs, so + /// that one row type and one decode path serve both this and the describe + /// path. The extra columns are two short tokens and a boolean, read only on a + /// key-info cache miss. + async fn fetch_vector_index_key_info( + &self, + table_id: &str, + ) -> Result, StorageError> { + let rows: Vec = sqlx::query_as(&format!( + "SELECT {} FROM vector_indexes WHERE table_id = $1 ORDER BY index_name", + crate::table_helpers::VECTOR_INDEX_COLUMNS + )) + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + extenddb_storage::vector_catalog::vector_index_key_info( + rows.into_iter().map(Into::into).collect(), + ) + } + /// Fetch every secondary index defined on a table, split into /// `(global_secondary_indexes, local_secondary_indexes)`. async fn fetch_all_index_info( diff --git a/crates/storage-postgres/src/delete_table.rs b/crates/storage-postgres/src/delete_table.rs index 356a253f..3ca95fd3 100755 --- a/crates/storage-postgres/src/delete_table.rs +++ b/crates/storage-postgres/src/delete_table.rs @@ -75,6 +75,11 @@ impl PostgresEngine { if delay_secs < 1.0 { // Synchronous delete: remove tags, catalog row, and data tables inline. // Note: deleting the tables row cascades to indexes and stream rows via FK CASCADE. + // Vector index rows cascade the same way, through + // vector_indexes_table_id_fkey, so a table with vector indexes needs + // no extra catalog cleanup. There are no vector data tables to drop + // yet: this backend records vector index metadata but does not build + // their storage. sqlx::query("DELETE FROM tags WHERE resource_arn = $1") .bind(&row.table_arn) .execute(&mut *tx) diff --git a/crates/storage-postgres/src/migrations.rs b/crates/storage-postgres/src/migrations.rs index 4b5e08f0..8a91ad48 100755 --- a/crates/storage-postgres/src/migrations.rs +++ b/crates/storage-postgres/src/migrations.rs @@ -289,3 +289,71 @@ async fn record_migration(pool: &PgPool, filename: &str) -> OpResult<()> { .map_err(|e| OpError::Internal(format!("Record migration: {e}")))?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::CATALOG_MIGRATIONS; + use crate::CATALOG_VERSION; + + /// The catalog version and the migration list must move together. + /// + /// A migration that creates its schema without moving the version leaves a + /// deployment the binary refuses to serve; moving the version without a + /// migration leaves one that cannot reach it. Both are caught today only by a + /// test that needs a live PostgreSQL and a built binary, so this is the + /// tripwire that fires in an ordinary `cargo test`: adding a migration file + /// breaks the count, which forces a decision about the version. + #[test] + fn the_migration_count_and_the_catalog_version_agree() { + assert_eq!( + CATALOG_MIGRATIONS.len(), + 2, + "a catalog migration was added or removed; update CATALOG_VERSION and this count" + ); + assert_eq!(CATALOG_VERSION.to_string(), "0.0.3"); + } + + /// The version the binary expects must be the version the schema writes. + /// + /// These two live in different languages and different files, so nothing but + /// a check like this ties them together. Without it a version bump that + /// forgets the SQL side produces a deployment that migrates "successfully" + /// and then refuses to start. + #[test] + fn the_last_migration_writes_the_expected_catalog_version() { + let (filename, sql) = CATALOG_MIGRATIONS + .last() + .expect("there is at least one catalog migration"); + let expected = format!("'{}'", CATALOG_VERSION); + assert!( + sql.contains("catalog_version") && sql.contains(&expected), + "{filename} must set catalog_version to {expected}" + ); + } + + /// Each migration is registered under the filename it is stored as. + /// + /// The ledger keys on this string, so a mismatch between the registered name + /// and the file would record one name and look for another, and the migration + /// would be applied again on every run. + #[test] + fn every_migration_is_registered_under_a_sql_filename_with_its_own_contents() { + for (filename, sql) in CATALOG_MIGRATIONS { + assert!(filename.ends_with(".sql"), "{filename}"); + assert!(!sql.trim().is_empty(), "{filename} is empty"); + } + // The failure this guards is a copy-pasted `include_str!` that points one + // entry at another file's bytes: the ledger would then record one name + // while the SQL of another ran, and the missed migration would be applied + // again on every upgrade. Two entries sharing contents is what that looks + // like, so distinctness is the assertion that delivers the rationale. + for (i, (left_name, left_sql)) in CATALOG_MIGRATIONS.iter().enumerate() { + for (right_name, right_sql) in &CATALOG_MIGRATIONS[i + 1..] { + assert_ne!( + left_sql, right_sql, + "{left_name} and {right_name} embed identical SQL; check their include_str! paths" + ); + } + } + } +} diff --git a/crates/storage-postgres/src/table_helpers.rs b/crates/storage-postgres/src/table_helpers.rs index b44672df..c97777ea 100755 --- a/crates/storage-postgres/src/table_helpers.rs +++ b/crates/storage-postgres/src/table_helpers.rs @@ -10,6 +10,7 @@ use extenddb_core::types::{ }; use extenddb_storage::error::StorageError; use extenddb_storage::util::{index_arn, stream_arn}; +use extenddb_storage::vector_catalog::{VectorIndexCatalogRow, vector_index_descriptions}; use crate::PostgresEngine; use crate::data; @@ -48,6 +49,49 @@ pub(crate) struct IndexRow { pub provisioned_throughput: Option, } +/// A `vector_indexes` catalog row, in `FromRow` field order. +/// +/// Separate from [`IndexRow`] because a vector index has no key schema and no +/// throughput, and carries a dimension count, a distance function and a +/// backfill state that no secondary index has. Decoding stops at this struct: +/// the rules that turn it into a wire shape are shared with the other backends +/// in `extenddb_storage::vector_catalog`. +#[derive(sqlx::FromRow)] +pub(crate) struct VectorIndexRow { + pub index_name: String, + pub dimensions: i32, + pub distance_function: String, + pub vector_attribute: serde_json::Value, + pub search_schema: Option, + pub projection: serde_json::Value, + pub index_status: String, + pub backfilling: Option, +} + +impl From for VectorIndexCatalogRow { + fn from(row: VectorIndexRow) -> Self { + Self { + index_name: row.index_name, + dimensions: i64::from(row.dimensions), + distance_function: row.distance_function, + vector_attribute: row.vector_attribute, + search_schema: row.search_schema, + projection: row.projection, + index_status: row.index_status, + backfilling: row.backfilling, + } + } +} + +/// Columns of [`VectorIndexRow`], in its field order. +/// +/// Named once so the reads that want a whole row select the same set. sqlx +/// decodes a named-field struct by column name, so the order here is for reading +/// rather than for correctness, and a column left out is a `ColumnNotFound` +/// error rather than a silent mis-mapping. +pub(crate) const VECTOR_INDEX_COLUMNS: &str = "index_name, dimensions, distance_function, vector_attribute, search_schema, \ + projection, index_status, backfilling"; + impl PostgresEngine { /// SQL table name for a GSI data table (static version for use outside `data` module). pub(crate) fn index_table_name_static(index_id: &str) -> String { @@ -188,7 +232,24 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - self.build_table_description_from_row(account_id, row, index_rows) + let vector_rows: Vec = sqlx::query_as(&format!( + "SELECT {VECTOR_INDEX_COLUMNS} FROM vector_indexes WHERE table_id = $1 \ + ORDER BY index_name" + )) + .bind(&row.table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let table_name_owned = row.table_name.clone(); + let mut desc = self.build_table_description_from_row(account_id, row, index_rows)?; + desc.vector_indexes = vector_index_descriptions( + &self.region, + account_id, + &table_name_owned, + vector_rows.into_iter().map(Into::into).collect(), + )?; + Ok(desc) } pub(crate) fn build_table_description_from_row( @@ -365,9 +426,11 @@ impl PostgresEngine { on_demand_throughput: row .on_demand_throughput .and_then(|v| serde_json::from_value(v).ok()), - // Fields for features this backend does not implement, vector - // indexes today, take their defaults. Adding one to - // TableDescription then does not break this build. + // Vector indexes are read separately and applied by the caller: this + // builder takes `index_rows` only, and the two callers differ in + // whether they need them. `build_table_description` fills them in; + // the delete path leaves them absent, matching the response the + // service sends for a table that is going away. ..Default::default() }) } diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index c26ecbec..ba80350f 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -41,6 +41,38 @@ impl PostgresEngine { return Err(StorageError::TableNotActive(input.table_name.clone())); } + // A table holding vector indexes cannot leave PAY_PER_REQUEST. Same + // message as CreateTable's rejection, and the same rule seen from the + // stored state rather than from the request, so it is checked here under + // the row lock where the current index set is readable. + // + // This is one of the two directions the rule has. SQLite also refuses + // adding a vector index when the request's net billing mode is not + // PAY_PER_REQUEST; that guard belongs with the create path, which this + // backend refuses outright for now, so porting it here would be an + // unreachable second refusal with different wording. It lands with the + // create path. + // + // Both backends count the stored rows before this request's deletes are + // applied, so switching to PROVISIONED and deleting the last vector index + // in one call is refused. The service evaluates the net effect of a + // request rather than its starting state, so it would probably accept + // that combination. Same answer on both backends, so it is a shared + // conformance question rather than a difference between them. + if matches!(input.billing_mode, Some(BillingMode::Provisioned)) { + let vector_count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM vector_indexes WHERE table_id = $1") + .bind(&table_id) + .fetch_one(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if vector_count > 0 { + return Err(StorageError::Validation( + extenddb_core::types::VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST.to_owned(), + )); + } + } + // Reject ProvisionedThroughput when the effective billing mode is // PAY_PER_REQUEST. The effective mode is the requested billing_mode when // the request changes it, otherwise the table's current mode. Real @@ -409,6 +441,77 @@ impl PostgresEngine { merged_attr_defs_for_ddl = Some(effective); } + // Vector index create/delete. + // + // Delete is implemented; Create is refused. Creating an index means + // building and maintaining its data table, which this backend cannot yet + // do, and a catalog row with no storage behind it would be an index that + // reports ACTIVE and answers nothing. Refusing is the same fail-closed + // posture the backend takes for every other vector operation. + // + // Neither branch is reachable over the wire yet: the engine refuses + // vector index updates while this backend declares no vector search + // capability, so these paths are exercised below the wire. They are + // implemented now because the catalog state they act on is created here. + if let Some(updates) = &input.vector_index_updates { + for update in updates { + if let Some(create) = &update.create { + return Err(StorageError::Unsupported(format!( + "vector index '{}' cannot be created: this backend does not yet \ + build vector index storage", + create.index_name + ))); + } + if let Some(delete) = &update.delete { + // The index id is deliberately not selected: this backend + // builds no per-index storage yet, so there is nothing to drop + // by id, and reading a value only to discard it invites the + // reader to think otherwise. + let existing: Option<(String, Option)> = sqlx::query_as( + "SELECT index_status, backfilling FROM vector_indexes \ + WHERE table_id = $1 AND index_name = $2", + ) + .bind(&table_id) + .bind(&delete.index_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (index_status, backfilling) = existing + .ok_or_else(|| StorageError::IndexNotFound(delete.index_name.clone()))?; + + // Deleting an index that is still being created is + // phase-dependent, and the discriminator is the same + // `backfilling` flag the wire reports. While the index is + // allocating resources the service refuses the delete and + // asks the caller to retry; once the backfill is running it + // accepts. Measured against the service on 2026-08-19. + if index_status == "CREATING" && backfilling == Some(false) { + return Err(StorageError::ResourceInUse( + extenddb_core::types::vector_index_delete_in_allocation_phase( + &input.table_name, + &delete.index_name, + ), + )); + } + + // Deleted synchronously: this backend has no observable + // DELETING window, because the catalog row and the storage go + // away together. The divergence from the service, which + // leaves the index in DELETING long enough to observe, is a + // documented one. + sqlx::query( + "DELETE FROM vector_indexes WHERE table_id = $1 AND index_name = $2", + ) + .bind(&table_id) + .bind(&delete.index_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + } + tx.commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-postgres/src/vector.rs b/crates/storage-postgres/src/vector.rs index 60ae780a..6f9d63f4 100644 --- a/crates/storage-postgres/src/vector.rs +++ b/crates/storage-postgres/src/vector.rs @@ -10,15 +10,30 @@ //! against a server that has pgvector and refuses them against one that does //! not, with no build-time or configuration difference. //! -//! Detection is one probe at engine construction ([`probe_vector_extension`]), -//! cached for the process lifetime. Installing pgvector on a running server -//! therefore needs an ExtendDB restart to be noticed. +//! Two layers, because detection is a snapshot: +//! +//! 1. One probe at engine construction ([`probe_vector_extension`]), cached for +//! the process lifetime. Installing pgvector on a running server therefore +//! needs an ExtendDB restart to be noticed. +//! 2. Runtime classification ([`is_missing_vector_extension`]) for the window +//! where the extension is dropped, or a failover lands on a server without +//! it, after the probe said yes. Those errors become +//! [`StorageError::Unsupported`], which the engine reports as a +//! `ValidationException`, rather than a 500. +use extenddb_storage::error::StorageError; use sqlx::PgPool; /// Name of the extension that provides the `vector` column type. pub(crate) const VECTOR_EXTENSION: &str = "vector"; +/// Message carried by [`StorageError::Unsupported`] when the extension is gone. +/// +/// Says which database is missing it, because the extension is installed on the +/// data database while a reader's attention naturally goes to the catalog. +pub(crate) const VECTOR_EXTENSION_REQUIRED: &str = + "vector indexes require the pgvector extension on the data database"; + /// Read the installed pgvector version, or `None` when it is not installed. /// /// A query failure is reported as `None` and logged rather than propagated: an @@ -43,6 +58,79 @@ pub(crate) async fn probe_vector_extension(data_pool: &PgPool) -> Option } } +/// The `SQLSTATE` that means "this server has no pgvector", given a statement +/// that names the `vector` type or one of its operators. +/// +/// `42704` undefined_object is the one PostgreSQL raises for `type "vector" does +/// not exist`, which is what every vector statement reports on a server where +/// the extension was never created, and it is the code observed when the +/// extension is dropped from under a running engine. +/// +/// Two codes are deliberately NOT treated as a missing extension, because doing +/// so would answer a different problem with advice about installing software: +/// +/// - `42P01` undefined_table is what a concurrent DeleteTable or an +/// UpdateTable-delete produces while a write is in flight. The GSI queue +/// already treats it as a benign race for its own tables, and only the caller +/// knows whether a missing table is a race it should tolerate or a fault. A +/// client told to install an extension that is already present would be sent +/// in the wrong direction entirely. +/// - `0A000` feature_not_supported is raised for many unrelated unsupported +/// operations, so it is too broad to carry this meaning on its own. +const MISSING_EXTENSION_SQLSTATE: &str = "42704"; + +/// Classify a `SQLSTATE` as "pgvector is not available here". +fn is_missing_vector_extension_sqlstate(code: Option<&str>) -> bool { + code == Some(MISSING_EXTENSION_SQLSTATE) +} + +/// Whether a sqlx error says the server cannot serve vector types at all. +pub(crate) fn is_missing_vector_extension(e: &sqlx::Error) -> bool { + match e { + sqlx::Error::Database(db_err) => { + is_missing_vector_extension_sqlstate(db_err.code().as_deref()) + } + _ => false, + } +} + +/// The typed refusal for a server without pgvector. +pub(crate) fn vector_unsupported() -> StorageError { + StorageError::Unsupported(VECTOR_EXTENSION_REQUIRED.to_owned()) +} + +/// Map a vector-path database error, turning "no pgvector here" into the typed +/// refusal and leaving every other error internal. +/// +/// Used by the vector data-definition and search paths so a server that loses +/// the extension after the startup probe answers 400 with the refusal text +/// instead of 500 with a PostgreSQL message. +pub(crate) fn map_vector_sql_error(e: sqlx::Error) -> StorageError { + if is_missing_vector_extension(&e) { + vector_unsupported() + } else { + StorageError::Internal(e.to_string()) + } +} + +/// Confirm the extension is still installed, from a statement that fails the +/// same way vector data-definition does when it is not. +/// +/// `NULL::vector` resolves the type without touching a table, so it costs one +/// round trip and raises `42704` on a server with no pgvector. Used before +/// persisting an index whose storage depends on the type, so a server that lost +/// the extension after the startup probe refuses instead of recording catalog +/// state that can never be backed by a data table. +pub(crate) async fn ensure_vector_extension_present( + data_pool: &PgPool, +) -> Result<(), StorageError> { + sqlx::query("SELECT NULL::vector") + .execute(data_pool) + .await + .map_err(map_vector_sql_error)?; + Ok(()) +} + /// Map a `SQLSTATE` from a failed `CREATE EXTENSION vector` to operator advice. /// /// The install-time failure codes are different from the runtime ones, so they @@ -81,6 +169,51 @@ pub(crate) fn create_extension_hint(e: &sqlx::Error) -> &'static str { mod tests { use super::*; + #[test] + fn undefined_object_means_pgvector_is_absent() { + // `type "vector" does not exist`, which is what every vector statement + // reports on a server where the extension was never created, and what + // dropping it from under a running engine produces. + assert!(is_missing_vector_extension_sqlstate(Some("42704"))); + } + + #[test] + fn an_undefined_table_is_not_a_missing_extension() { + // A concurrent DeleteTable produces this while a write is in flight. The + // GSI queue treats it as a benign race for its own tables; classifying it + // here would tell a client to install an extension that is present, and + // would hide a race behind a capability message. + assert!(!is_missing_vector_extension_sqlstate(Some("42P01"))); + } + + #[test] + fn a_generic_unsupported_feature_is_not_a_missing_extension() { + // PostgreSQL raises 0A000 for many unrelated unsupported operations, so it + // is too broad to carry this specific meaning. + assert!(!is_missing_vector_extension_sqlstate(Some("0A000"))); + } + + #[test] + fn an_unrelated_sqlstate_is_not_a_missing_extension() { + // Chosen deliberately: 23505 is a unique violation, which the vector + // write path uses as a real invariant tripwire. Classifying it as a + // missing extension would turn a loud bug into a silent refusal. + assert!(!is_missing_vector_extension_sqlstate(Some("23505"))); + assert!(!is_missing_vector_extension_sqlstate(Some("42601"))); + } + + #[test] + fn an_error_with_no_sqlstate_is_not_a_missing_extension() { + assert!(!is_missing_vector_extension_sqlstate(None)); + } + + #[test] + fn a_transport_error_is_not_a_missing_extension() { + // A pool timeout carries no SQLSTATE. Reporting it as Unsupported would + // tell a client its request is invalid when the server is merely busy. + assert!(!is_missing_vector_extension(&sqlx::Error::PoolTimedOut)); + } + #[test] fn a_missing_package_and_a_missing_privilege_get_different_advice() { // The two ways `CREATE EXTENSION` fails at init need an operator to do @@ -93,4 +226,15 @@ mod tests { assert_ne!(absent, create_extension_hint_for_sqlstate(None)); } + #[test] + fn the_refusal_names_the_data_database() { + match vector_unsupported() { + StorageError::Unsupported(msg) => { + assert_eq!(msg, VECTOR_EXTENSION_REQUIRED); + assert!(msg.contains("pgvector"), "{msg}"); + assert!(msg.contains("data database"), "{msg}"); + } + other => panic!("expected Unsupported, got {other:?}"), + } + } } diff --git a/crates/storage-postgres/tests/vector_control_plane.rs b/crates/storage-postgres/tests/vector_control_plane.rs new file mode 100644 index 00000000..b8353edb --- /dev/null +++ b/crates/storage-postgres/tests/vector_control_plane.rs @@ -0,0 +1,1462 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Storage-level tests for the PostgreSQL vector index control plane. +//! +//! These run below the wire on purpose. This backend declares no vector search +//! capability, so the engine refuses every vector request before storage is +//! reached, which makes the wire the wrong place to test what the catalog does +//! with a vector index. The control-plane code exists now because the catalog +//! state that the search and build paths will read is created here, so it is +//! tested here, against a real PostgreSQL. +//! +//! Each test builds its own throwaway database, applies the shipped migrations +//! to it, and drops it when it passes. A failing test leaves its database behind +//! on purpose, named `eddb_vec_ctl_*`, so the state that failed can be inspected. +//! +//! Requires `EXTENDDB_TEST_PG_CONNECTION_STRING`, a base URL with no database +//! component (for example `postgresql://postgres@127.0.0.1:5432`), pointing at a +//! server whose role may create and drop databases. Without it every test here +//! reports a skip and passes, the same convention the wire suites use. + +use std::collections::{BTreeMap, HashMap}; + +use extenddb_core::expression::{self, ExpressionMaps}; +use extenddb_core::types::{ + AttributeDefinition, AttributeValue, BillingMode, CreateTableInput, DeleteTableInput, + DeleteVectorIndexAction, DescribeTableInput, DistanceFunction, IndexStatus, Item, + KeySchemaElement, KeyType, Projection, ProjectionType, ProvisionedThroughput, + ScalarAttributeType, SearchSchemaElement, SearchSchemaElementType, UpdateTableInput, + VectorAttribute, VectorIndexSpecification, VectorIndexUpdate, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{BackupEngine, DataEngine, TableEngine}; +use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; +use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; + +const ACCOUNT: &str = "123456789012"; +const REGION: &str = "us-east-1"; + +/// Whether pgvector should be installed in the scratch database. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Pgvector { + Install, + Omit, +} + +/// A throwaway database with the shipped schema applied and an engine on it. +struct Scratch { + engine: PostgresEngine, + catalog: PgPool, + admin: PgPool, + db_name: String, + /// Databases created alongside the catalog, dropped by `cleanup` too. + /// + /// Cleanup owns every database it created, because a caller that drops one + /// afterwards has no working admin pool to do it with: `PgPool` is an `Arc` + /// around shared state, so closing one handle closes every clone and the + /// `DROP` then fails with `PoolClosed`. An ignored error there leaks a + /// database on every green run, which is invisible in CI and accumulates on a + /// development server. + extra_databases: Vec, + /// Held for the environment's lifetime, released on drop even when a test + /// panics, so the next test does not start until this one's connections are + /// gone. See [`environment_permit`]. + _permit: tokio::sync::OwnedSemaphorePermit, +} + +/// One scratch environment at a time, across the whole test binary. +/// +/// Each environment holds an engine pool with a floor of ten connections, and +/// these tests share a server with whatever else is running against it (in CI, an +/// ExtendDB instance started by an earlier step). Running twelve at once exhausts +/// `max_connections` and every test then fails on a pool timeout, which reports a +/// resource limit as a product bug. Serialising costs a few seconds. +async fn environment_permit() -> tokio::sync::OwnedSemaphorePermit { + static ENVIRONMENTS: std::sync::OnceLock> = + std::sync::OnceLock::new(); + std::sync::Arc::clone( + ENVIRONMENTS.get_or_init(|| std::sync::Arc::new(tokio::sync::Semaphore::new(1))), + ) + .acquire_owned() + .await + .expect("the environment semaphore is never closed") +} + +impl Scratch { + /// Drop the scratch database. Called only on the success path, so a failure + /// leaves the database for inspection. + async fn cleanup(self) { + let Scratch { + engine, + catalog, + admin, + db_name, + extra_databases, + _permit, + } = self; + drop(engine); + catalog.close().await; + for name in extra_databases.iter().chain(std::iter::once(&db_name)) { + sqlx::query(&format!("DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)")) + .execute(&admin) + .await + .expect("drop a scratch database"); + } + admin.close().await; + } +} + +fn base_conn() -> Option { + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; + (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) +} + +/// Report the reason a test did nothing, loudly enough to notice in a log. +fn skip(test: &str) { + eprintln!( + "SKIP {test}: EXTENDDB_TEST_PG_CONNECTION_STRING is not set, so there is no PostgreSQL \ + to build a scratch catalog in." + ); +} + +/// Build a scratch database, apply the shipped migrations, and open an engine. +/// +/// The migrations are the files the binary ships, included from the crate rather +/// than restated here: a test that built its own idea of the schema could pass +/// against a shape no deployment has. +async fn scratch(pgvector: Pgvector) -> Scratch { + let base = base_conn().expect("caller checks base_conn() first"); + let permit = environment_permit().await; + let db_name = format!("eddb_vec_ctl_{}", uuid::Uuid::new_v4().simple())[..24].to_owned(); + + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + .expect("connect to the postgres maintenance database"); + sqlx::query(&format!("CREATE DATABASE \"{db_name}\"")) + .execute(&admin) + .await + .expect("create the scratch database"); + + let url = format!("{base}/{db_name}"); + let catalog = PgPoolOptions::new() + .max_connections(2) + .connect(&url) + .await + .expect("connect to the scratch database"); + + if pgvector == Pgvector::Install { + sqlx::query("CREATE EXTENSION IF NOT EXISTS vector") + .execute(&catalog) + .await + .expect("create the pgvector extension"); + } + + for sql in [ + include_str!("../migrations/001_schema.sql"), + include_str!("../migrations/002_vector_indexes.sql"), + include_str!("../data_migrations/001_data_schema.sql"), + include_str!("../data_migrations/002_gsi_pending.sql"), + include_str!("../data_migrations/003_idempotency_account_scope.sql"), + ] { + sqlx::raw_sql(sql) + .execute(&catalog) + .await + .expect("apply a shipped migration"); + } + + // Zero control-plane delay so CreateTable and DeleteTable complete inline: + // these tests run no background workers, so a scheduled transition would + // never happen and every table would sit in CREATING. + sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") + .execute(&catalog) + .await + .expect("pin the control-plane delay to zero"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") + .bind(ACCOUNT) + .bind(format!("acct-{db_name}")) + .execute(&catalog) + .await + .expect("seed the account row"); + + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: url, + // The engine clamps anything below its floor of ten, so ask for the + // floor: a smaller number would only produce a warning and the same + // pool. + pool_size: 10, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("open a PostgresEngine on the scratch database"); + + Scratch { + engine, + catalog, + admin, + db_name, + extra_databases: Vec::new(), + _permit: permit, + } +} + +/// Build a scratch database with pgvector installed, or report why not. +/// +/// The extension is a server package, so a PostgreSQL that does not ship it +/// cannot run the tests that need the `vector` type to exist. Those tests say so +/// and pass, rather than failing on an environment property; the control-plane +/// tests deliberately do not need it, so they always run. +async fn scratch_with_pgvector(test: &str) -> Option { + let base = base_conn()?; + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + .expect("connect to the postgres maintenance database"); + let available: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_available_extensions WHERE name = 'vector')", + ) + .fetch_one(&admin) + .await + .expect("list the available extensions"); + admin.close().await; + if !available { + eprintln!( + "SKIP {test}: this PostgreSQL server does not offer the pgvector extension, so the \ + capable path cannot be exercised here." + ); + return None; + } + Some(scratch(Pgvector::Install).await) +} + +fn hash_key(name: &str) -> Vec { + vec![KeySchemaElement { + attribute_name: name.to_owned(), + key_type: KeyType::Hash, + }] +} + +fn string_attr(name: &str) -> Vec { + vec![AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type: ScalarAttributeType::S, + }] +} + +fn projection(projection_type: ProjectionType) -> Projection { + Projection { + projection_type, + non_key_attributes: None, + } +} + +/// A vector index specification: unscoped when `hash` is `None`. +fn vector_spec(name: &str, dimensions: u32, hash: Option<&str>) -> VectorIndexSpecification { + VectorIndexSpecification { + index_name: name.to_owned(), + dimensions, + distance_function: DistanceFunction::Cosine, + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + search_schema: hash.map(|attr| { + vec![SearchSchemaElement { + attribute_name: attr.to_owned(), + element_type: SearchSchemaElementType::Hash, + }] + }), + projection: Some(projection(ProjectionType::All)), + } +} + +fn create_input(table: &str, vector_indexes: Vec) -> CreateTableInput { + CreateTableInput { + table_name: table.to_owned(), + key_schema: hash_key("pk"), + attribute_definitions: string_attr("pk"), + billing_mode: Some(BillingMode::PayPerRequest), + vector_indexes: (!vector_indexes.is_empty()).then_some(vector_indexes), + ..Default::default() + } +} + +/// An `UpdateTableInput` that changes nothing, as a base for one that changes one +/// thing. The type has no `Default`, deliberately: every field is a distinct +/// control-plane change and defaulting them silently would be a bug magnet. +fn update_input(table: &str) -> UpdateTableInput { + UpdateTableInput { + table_name: table.to_owned(), + billing_mode: None, + provisioned_throughput: None, + deletion_protection_enabled: None, + global_secondary_index_updates: None, + attribute_definitions: None, + stream_specification: None, + table_class: None, + on_demand_throughput: None, + vector_index_updates: None, + } +} + +fn delete_vector(index_name: &str) -> Option> { + Some(vec![VectorIndexUpdate { + create: None, + delete: Some(DeleteVectorIndexAction { + index_name: index_name.to_owned(), + }), + }]) +} + +async fn table_id(catalog: &PgPool, table: &str) -> String { + sqlx::query_scalar("SELECT table_id FROM tables WHERE account_id = $1 AND table_name = $2") + .bind(ACCOUNT) + .bind(table) + .fetch_one(catalog) + .await + .expect("read the table id") +} + +async fn vector_row_count(catalog: &PgPool, table_id: &str) -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM vector_indexes WHERE table_id = $1") + .bind(table_id) + .fetch_one(catalog) + .await + .expect("count the vector index rows") +} + +/// Put a vector index into a mid-build state that only the build path can reach, +/// so the phase-dependent delete rule can be tested before that path exists. +async fn set_index_phase(catalog: &PgPool, table_id: &str, index_name: &str, backfilling: bool) { + sqlx::query( + "UPDATE vector_indexes SET index_status = 'CREATING', backfilling = $3 \ + WHERE table_id = $1 AND index_name = $2", + ) + .bind(table_id) + .bind(index_name) + .bind(backfilling) + .execute(catalog) + .await + .expect("move the index into a CREATING phase"); +} + +#[tokio::test] +async fn create_table_records_a_vector_index_as_active_and_echoes_it() { + if base_conn().is_none() { + return skip("create_table_records_a_vector_index_as_active_and_echoes_it"); + } + let s = scratch(Pgvector::Omit).await; + + let desc = s + .engine + .create_table( + ACCOUNT, + create_input("t_create", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + + // The response is what a client sees, so it carries the index rather than + // requiring a follow-up describe. + let echoed = desc + .vector_indexes + .as_ref() + .expect("CreateTable must echo the vector index it created"); + assert_eq!(echoed.len(), 1); + assert_eq!(echoed[0].index_name, "vidx"); + assert_eq!(echoed[0].dimensions, 4); + assert_eq!(echoed[0].index_status, IndexStatus::Active); + // A table created with the index is empty, so there is nothing to backfill + // and the member is absent, which is the state the service reports. + assert_eq!(echoed[0].backfilling, None); + assert!( + echoed[0].index_arn.ends_with("/index/vidx"), + "{}", + echoed[0].index_arn + ); + + // The catalog row must agree, including the ACTIVE-implies-no-backfilling + // invariant the schema also enforces. + let id = table_id(&s.catalog, "t_create").await; + let row: (String, Option, i32, String, String) = sqlx::query_as( + "SELECT index_status, backfilling, dimensions, distance_function, index_id \ + FROM vector_indexes WHERE table_id = $1 AND index_name = 'vidx'", + ) + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read back the vector index row"); + assert_eq!(row.0, "ACTIVE"); + assert_eq!(row.1, None); + assert_eq!(row.2, 4); + assert_eq!(row.3, "COSINE"); + assert!(!row.4.is_empty(), "the index id must be assigned"); + + s.cleanup().await; +} + +#[tokio::test] +async fn describe_table_reports_scoped_and_unscoped_vector_indexes() { + if base_conn().is_none() { + return skip("describe_table_reports_scoped_and_unscoped_vector_indexes"); + } + let s = scratch(Pgvector::Omit).await; + + let mut input = create_input( + "t_describe", + vec![ + vector_spec("scoped", 8, Some("pk")), + VectorIndexSpecification { + projection: Some(projection(ProjectionType::KeysOnly)), + ..vector_spec("unscoped", 16, None) + }, + ], + ); + input.attribute_definitions = string_attr("pk"); + s.engine + .create_table(ACCOUNT, input) + .await + .expect("create a table with two vector indexes"); + + let desc = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "t_describe".to_owned(), + }, + ) + .await + .expect("describe the table"); + + let mut indexes = desc + .vector_indexes + .expect("DescribeTable must report the stored vector indexes"); + indexes.sort_by(|a, b| a.index_name.cmp(&b.index_name)); + assert_eq!(indexes.len(), 2); + + let scoped = &indexes[0]; + assert_eq!(scoped.index_name, "scoped"); + assert_eq!(scoped.dimensions, 8); + // A scoped index reports its search schema: a client needs it to know that + // SearchConditionExpression is mandatory. + let schema = scoped + .search_schema + .as_ref() + .expect("a scoped index must report its search schema"); + assert_eq!(schema.len(), 1); + assert_eq!(schema[0].attribute_name, "pk"); + assert_eq!(schema[0].element_type, SearchSchemaElementType::Hash); + assert_eq!( + scoped.projection.as_ref().map(|p| p.projection_type), + Some(ProjectionType::All) + ); + + let unscoped = &indexes[1]; + assert_eq!(unscoped.index_name, "unscoped"); + assert_eq!(unscoped.dimensions, 16); + // Absent, not empty: an index with no HASH element spans the table, and the + // two states are different to a client. + assert_eq!(unscoped.search_schema, None); + assert_eq!( + unscoped.projection.as_ref().map(|p| p.projection_type), + Some(ProjectionType::KeysOnly) + ); + assert_eq!(unscoped.distance_function, DistanceFunction::Cosine); + assert_eq!(unscoped.index_status, IndexStatus::Active); + + s.cleanup().await; +} + +#[tokio::test] +async fn table_key_info_carries_the_vector_index_metadata_the_write_path_needs() { + if base_conn().is_none() { + return skip("table_key_info_carries_the_vector_index_metadata_the_write_path_needs"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_keyinfo", vec![vector_spec("vidx", 32, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_keyinfo") + .await + .expect("read the cached key info"); + + // This is the switch that turns on engine-side vector write validation and + // vector write capacity: both read this slice, and both are no-ops while it + // is empty. + assert_eq!(key_info.vector_indexes.len(), 1); + let vi = &key_info.vector_indexes[0]; + assert_eq!(vi.index_name, "vidx"); + assert_eq!(vi.dimensions, 32); + assert_eq!(vi.vector_attribute_name, "emb"); + assert_eq!(vi.search_schema.len(), 1); + assert_eq!(vi.search_schema[0].attribute_name, "pk"); + assert_eq!(vi.projection.projection_type, ProjectionType::All); + + // A table with no vector index must report an empty slice rather than + // inheriting the previous table's, so the write path stays free. + s.engine + .create_table(ACCOUNT, create_input("t_plain", vec![])) + .await + .expect("create a table with no vector index"); + let plain = s + .engine + .table_key_info(ACCOUNT, "t_plain") + .await + .expect("read the plain table's key info"); + assert!(plain.vector_indexes.is_empty()); + + s.cleanup().await; +} + +#[tokio::test] +async fn delete_table_removes_the_vector_index_rows() { + if base_conn().is_none() { + return skip("delete_table_removes_the_vector_index_rows"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_delete", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_delete").await; + assert_eq!(vector_row_count(&s.catalog, &id).await, 1); + + s.engine + .delete_table( + ACCOUNT, + DeleteTableInput { + table_name: "t_delete".to_owned(), + }, + ) + .await + .expect("delete the table"); + + // Left behind, these rows would resurface on a table that reused the id and + // would keep the account's vector index count wrong. + assert_eq!(vector_row_count(&s.catalog, &id).await, 0); + + s.cleanup().await; +} + +#[tokio::test] +async fn update_table_deletes_an_active_vector_index_once() { + if base_conn().is_none() { + return skip("update_table_deletes_an_active_vector_index_once"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input( + "t_update", + vec![ + vector_spec("keep", 4, Some("pk")), + vector_spec("drop", 4, Some("pk")), + ], + ), + ) + .await + .expect("create a table with two vector indexes"); + + let desc = s + .engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: delete_vector("drop"), + ..update_input("t_update") + }, + ) + .await + .expect("delete one vector index"); + + // The response is what the engine's post-condition check reads, so the + // deleted index must be gone from it and the surviving one still in it. + let names: Vec = desc + .vector_indexes + .expect("UpdateTable must report the surviving vector indexes") + .into_iter() + .map(|vi| vi.index_name) + .collect(); + assert_eq!(names, vec!["keep".to_owned()]); + + // Deleting it again is not idempotent: the index is gone, so the second + // request must report that rather than succeed silently. + let err = s + .engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: delete_vector("drop"), + ..update_input("t_update") + }, + ) + .await + .expect_err("deleting a vector index twice must fail"); + match err { + StorageError::IndexNotFound(name) => assert_eq!(name, "drop"), + other => panic!("expected IndexNotFound, got {other:?}"), + } + + s.cleanup().await; +} + +#[tokio::test] +async fn deleting_a_vector_index_in_the_allocation_phase_is_refused() { + if base_conn().is_none() { + return skip("deleting_a_vector_index_in_the_allocation_phase_is_refused"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_phase", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_phase").await; + set_index_phase(&s.catalog, &id, "vidx", false).await; + + let err = s + .engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: delete_vector("vidx"), + ..update_input("t_phase") + }, + ) + .await + .expect_err("a delete during resource allocation must be refused"); + + // ResourceInUse, not Validation: the request is well formed and the resource + // exists, so the client should retry rather than change the request. The + // whole string is the measured one, including both resource names. + match err { + StorageError::ResourceInUse(msg) => assert_eq!( + msg, + extenddb_core::types::vector_index_delete_in_allocation_phase("t_phase", "vidx") + ), + other => panic!("expected ResourceInUse, got {other:?}"), + } + + // The refusal must not have deleted anything on the way out. + assert_eq!(vector_row_count(&s.catalog, &id).await, 1); + + s.cleanup().await; +} + +#[tokio::test] +async fn deleting_a_vector_index_during_its_backfill_is_accepted() { + if base_conn().is_none() { + return skip("deleting_a_vector_index_during_its_backfill_is_accepted"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_backfilling", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_backfilling").await; + set_index_phase(&s.catalog, &id, "vidx", true).await; + + // Same request, one phase later, opposite answer: the discriminator is the + // backfilling flag and nothing else. + s.engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: delete_vector("vidx"), + ..update_input("t_backfilling") + }, + ) + .await + .expect("a delete during the backfill phase must be accepted"); + assert_eq!(vector_row_count(&s.catalog, &id).await, 0); + + s.cleanup().await; +} + +#[tokio::test] +async fn switching_a_table_with_vector_indexes_to_provisioned_is_refused() { + if base_conn().is_none() { + return skip("switching_a_table_with_vector_indexes_to_provisioned_is_refused"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_billing", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a pay-per-request table with a vector index"); + + let switch = |table: &str| UpdateTableInput { + billing_mode: Some(BillingMode::Provisioned), + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + ..update_input(table) + }; + + let err = s + .engine + .update_table(ACCOUNT, switch("t_billing")) + .await + .expect_err("a vector table must not leave PAY_PER_REQUEST"); + match err { + StorageError::Validation(msg) => assert_eq!( + msg, + extenddb_core::types::VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST + ), + other => panic!("expected Validation, got {other:?}"), + } + + // Control: the guard must key on the vector indexes, not on the switch, so + // an ordinary table still changes billing mode. + s.engine + .create_table(ACCOUNT, create_input("t_billing_plain", vec![])) + .await + .expect("create a plain pay-per-request table"); + s.engine + .update_table(ACCOUNT, switch("t_billing_plain")) + .await + .expect("a table with no vector index may switch to provisioned"); + + s.cleanup().await; +} + +#[tokio::test] +async fn adding_a_vector_index_by_update_table_is_unsupported() { + if base_conn().is_none() { + return skip("adding_a_vector_index_by_update_table_is_unsupported"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table(ACCOUNT, create_input("t_add", vec![])) + .await + .expect("create a table with no vector index"); + + let err = s + .engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: Some(vec![VectorIndexUpdate { + create: Some(vector_spec("vidx", 4, Some("pk"))), + delete: None, + }]), + ..update_input("t_add") + }, + ) + .await + .expect_err("this backend cannot build a vector index yet"); + + // Unsupported rather than Internal: the backend never claimed it could build + // one, so this is a refusal the engine reports as a client error, not a + // fault to page on. + match err { + StorageError::Unsupported(msg) => assert!(msg.contains("vidx"), "{msg}"), + other => panic!("expected Unsupported, got {other:?}"), + } + + // Nothing may be left behind: a catalog row with no storage behind it would + // be an index that reports ACTIVE and answers nothing. + let id = table_id(&s.catalog, "t_add").await; + assert_eq!(vector_row_count(&s.catalog, &id).await, 0); + + s.cleanup().await; +} + +#[tokio::test] +async fn restoring_a_backup_that_carries_vector_indexes_is_refused() { + if base_conn().is_none() { + return skip("restoring_a_backup_that_carries_vector_indexes_is_refused"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_source", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let details = s + .engine + .create_backup(ACCOUNT, "t_source", "b_vector") + .await + .expect("back up a table with a vector index"); + + // Refused, not silently degraded: restore does not carry indexes across, so + // succeeding here would hand back a table whose declared index is missing + // and whose client only finds out on the first search. + let err = s + .engine + .restore_table_from_backup(ACCOUNT, "t_restored", &details.backup_arn) + .await + .expect_err("restoring a vector-indexed backup must be refused"); + match err { + StorageError::Unsupported(msg) => { + assert!(msg.contains("vector index"), "{msg}"); + assert!(msg.contains(&details.backup_arn), "{msg}"); + } + other => panic!("expected Unsupported, got {other:?}"), + } + + // The refusal must leave no half-created target table behind. + let target: Option<(String,)> = + sqlx::query_as("SELECT table_name FROM tables WHERE account_id = $1 AND table_name = $2") + .bind(ACCOUNT) + .bind("t_restored") + .fetch_optional(&s.catalog) + .await + .expect("look for the target table"); + assert!(target.is_none(), "no target table may be created"); + + // Control: a backup with no vector indexes still restores, so the refusal is + // keyed on the snapshot rather than on backups in general. + s.engine + .create_table(ACCOUNT, create_input("t_plain_source", vec![])) + .await + .expect("create a plain table"); + let plain = s + .engine + .create_backup(ACCOUNT, "t_plain_source", "b_plain") + .await + .expect("back up a plain table"); + s.engine + .restore_table_from_backup(ACCOUNT, "t_plain_restored", &plain.backup_arn) + .await + .expect("a backup with no vector indexes must still restore"); + + s.cleanup().await; +} + +#[tokio::test] +async fn a_wrong_dimension_vector_written_by_update_item_is_rejected() { + if base_conn().is_none() { + return skip("a_wrong_dimension_vector_written_by_update_item_is_rejected"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_write", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a four-dimension vector index"); + + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_write") + .await + .expect("read the key info"); + let key: Item = BTreeMap::from([("pk".to_owned(), AttributeValue::S("a".to_owned()))]); + + // The update path already handed `key_info.vector_indexes` to the evaluator + // before this work; it was a no-op only because the slice was always empty. + // So this is the behaviour that populating the slice turns on, with no call + // site changed. + let three = AttributeValue::L(vec![ + AttributeValue::N("1".to_owned()), + AttributeValue::N("2".to_owned()), + AttributeValue::N("3".to_owned()), + ]); + let err = update_emb(&s.engine, &key_info, &key, three) + .await + .expect_err("a three-element vector must not enter a four-dimension index"); + match err { + StorageError::Validation(msg) => { + assert!(msg.contains("emb"), "{msg}"); + assert!(msg.contains('4') && msg.contains('3'), "{msg}"); + } + other => panic!("expected Validation, got {other:?}"), + } + + // Control: the right number of components is accepted, so the check is on + // the dimension rather than on writing a list at all. + let four = AttributeValue::L(vec![ + AttributeValue::N("1".to_owned()), + AttributeValue::N("2".to_owned()), + AttributeValue::N("3".to_owned()), + AttributeValue::N("4".to_owned()), + ]); + update_emb(&s.engine, &key_info, &key, four) + .await + .expect("a four-element vector must be accepted"); + + s.cleanup().await; +} + +/// `SET emb = :v` against one item, through the storage update path. +async fn update_emb( + engine: &PostgresEngine, + key_info: &extenddb_core::types::TableKeyInfo, + key: &Item, + value: AttributeValue, +) -> Result, StorageError> { + let tokens = expression::tokenize("SET emb = :v").expect("tokenize the update expression"); + let actions = expression::parse_update(&tokens).expect("parse the update expression"); + // Keys carry no leading colon: the engine strips it before building the + // maps, and the resolver adds it back when it reports an unknown one. + let maps = ExpressionMaps::new(HashMap::new(), HashMap::from([("v".to_owned(), value)])); + engine + .update_item(key_info, key, &actions, false, false, None, &maps, None) + .await + .map(|(item, _)| item) +} + +#[tokio::test] +async fn describe_table_refuses_a_vector_index_whose_stored_status_is_unrecognised() { + let test = "describe_table_refuses_a_vector_index_whose_stored_status_is_unrecognised"; + if base_conn().is_none() { + return skip(test); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_badstatus", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_badstatus").await; + + // A value no build writes, standing in for a corrupt row or one written by a + // future version. `IndexStatus` carries a catch-all variant for forward + // compatibility when parsing a service response, which is the opposite of + // what is wanted when reading our own catalog: a status we cannot understand + // must not be handed to a client as if we could. + sqlx::query("UPDATE vector_indexes SET index_status = 'ACITVE' WHERE table_id = $1") + .bind(&id) + .execute(&s.catalog) + .await + .expect("write an unrecognised status"); + + let err = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "t_badstatus".to_owned(), + }, + ) + .await + .expect_err("an unrecognised stored status must not be described"); + match err { + StorageError::Internal(msg) => assert!(msg.contains("ACITVE"), "{msg}"), + other => panic!("expected Internal, got {other:?}"), + } + + s.cleanup().await; +} + +#[tokio::test] +async fn describe_table_reports_a_creating_index_as_backfilling() { + if base_conn().is_none() { + return skip("describe_table_reports_a_creating_index_as_backfilling"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_creating", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_creating").await; + set_index_phase(&s.catalog, &id, "vidx", true).await; + + let desc = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "t_creating".to_owned(), + }, + ) + .await + .expect("describe a table whose index is still building"); + + // The only path that round-trips a non-ACTIVE status and a present + // Backfilling member, which is the pair the wire contract ties together. + let indexes = desc.vector_indexes.expect("the index must be reported"); + assert_eq!(indexes.len(), 1); + assert_eq!(indexes[0].index_status, IndexStatus::Creating); + assert_eq!(indexes[0].backfilling, Some(true)); + + s.cleanup().await; +} + +#[tokio::test] +async fn an_empty_search_schema_is_stored_and_reported_as_absent() { + if base_conn().is_none() { + return skip("an_empty_search_schema_is_stored_and_reported_as_absent"); + } + let s = scratch(Pgvector::Omit).await; + + // Core accepts `SearchSchema: []` on the request, and every path downstream + // treats it as unscoped. The service reports an absent member or a populated + // one, never an empty list, so storing `[]` would create a third state that + // DescribeTable echoes back and no client expects. + let mut spec = vector_spec("vidx", 4, None); + spec.search_schema = Some(Vec::new()); + let desc = s + .engine + .create_table(ACCOUNT, create_input("t_emptyschema", vec![spec])) + .await + .expect("create a table with an empty search schema"); + assert_eq!( + desc.vector_indexes.as_ref().unwrap()[0].search_schema, + None, + "the CreateTable echo must not report an empty list" + ); + + let id = table_id(&s.catalog, "t_emptyschema").await; + let stored: Option = + sqlx::query_scalar("SELECT search_schema FROM vector_indexes WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the stored search schema"); + assert_eq!( + stored, None, + "the catalog must hold NULL, not an empty list" + ); + + let described = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "t_emptyschema".to_owned(), + }, + ) + .await + .expect("describe the table"); + assert_eq!( + described.vector_indexes.unwrap()[0].search_schema, + None, + "DescribeTable must report the member as absent" + ); + + s.cleanup().await; +} + +#[tokio::test] +async fn a_backup_snapshot_carries_the_wire_shape_behind_a_version() { + if base_conn().is_none() { + return skip("a_backup_snapshot_carries_the_wire_shape_behind_a_version"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_snapshot", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let details = s + .engine + .create_backup(ACCOUNT, "t_snapshot", "b_snapshot") + .await + .expect("back up the table"); + + let snapshot: serde_json::Value = + sqlx::query_scalar("SELECT vector_indexes FROM backups WHERE backup_arn = $1") + .bind(&details.backup_arn) + .fetch_one(&s.catalog) + .await + .expect("read the snapshot"); + + // A snapshot outlives the schema that produced it, so it carries a version + // and the wire's own names rather than a copy of the catalog row. A later + // column rename must not change the meaning of snapshots already on disk. + assert_eq!(snapshot["Version"], serde_json::json!(1)); + let indexes = snapshot["VectorIndexes"] + .as_array() + .expect("the snapshot must carry an index list"); + assert_eq!(indexes.len(), 1); + assert_eq!(indexes[0]["IndexName"], serde_json::json!("vidx")); + assert_eq!(indexes[0]["Dimensions"], serde_json::json!(4)); + assert_eq!(indexes[0]["DistanceFunction"], serde_json::json!("COSINE")); + assert_eq!( + indexes[0]["VectorAttribute"]["AttributeName"], + serde_json::json!("emb") + ); + // Build state belongs to the table it came from, not to a definition being + // restored, so it is deliberately not in the snapshot. + for absent in ["index_status", "backfilling", "build_owner"] { + assert!( + indexes[0].get(absent).is_none(), + "the snapshot must not carry {absent}" + ); + } + + s.cleanup().await; +} + +#[tokio::test] +async fn a_backup_taken_before_the_snapshot_column_existed_still_restores() { + if base_conn().is_none() { + return skip("a_backup_taken_before_the_snapshot_column_existed_still_restores"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table(ACCOUNT, create_input("t_legacy", vec![])) + .await + .expect("create a plain table"); + let details = s + .engine + .create_backup(ACCOUNT, "t_legacy", "b_legacy") + .await + .expect("back up the table"); + + // Every backup taken before this migration has NULL here, and the refusal + // rests on reading that as "no vector indexes" rather than as "unknown". + // Without a test, tightening the Option handling would break every legacy + // restore on an upgraded deployment and nothing would notice. + sqlx::query("UPDATE backups SET vector_indexes = NULL WHERE backup_arn = $1") + .bind(&details.backup_arn) + .execute(&s.catalog) + .await + .expect("blank the snapshot the way a pre-migration backup has it"); + + s.engine + .restore_table_from_backup(ACCOUNT, "t_legacy_restored", &details.backup_arn) + .await + .expect("a pre-migration backup must still restore"); + + s.cleanup().await; +} + +#[tokio::test] +async fn a_vector_index_at_the_maximum_dimension_round_trips() { + if base_conn().is_none() { + return skip("a_vector_index_at_the_maximum_dimension_round_trips"); + } + let s = scratch(Pgvector::Omit).await; + + // 4096 is the largest dimension core accepts. The catalog column is a 32-bit + // integer and the wire type is unsigned, so the value crosses two narrowing + // conversions on the way out and back; the boundary is where a wrong cast + // would show. + let desc = s + .engine + .create_table( + ACCOUNT, + create_input("t_maxdim", vec![vector_spec("vidx", 4096, Some("pk"))]), + ) + .await + .expect("create a table with a maximum-dimension vector index"); + assert_eq!(desc.vector_indexes.as_ref().unwrap()[0].dimensions, 4096); + + let described = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "t_maxdim".to_owned(), + }, + ) + .await + .expect("describe the table"); + assert_eq!(described.vector_indexes.unwrap()[0].dimensions, 4096); + + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_maxdim") + .await + .expect("read the key info"); + assert_eq!(key_info.vector_indexes[0].dimensions, 4096); + + s.cleanup().await; +} + +#[tokio::test] +async fn describe_table_refuses_a_vector_index_with_a_corrupt_payload() { + if base_conn().is_none() { + return skip("describe_table_refuses_a_vector_index_with_a_corrupt_payload"); + } + let s = scratch(Pgvector::Omit).await; + + s.engine + .create_table( + ACCOUNT, + create_input("t_corrupt", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_corrupt").await; + + // Well-formed JSON that is not a vector attribute. The comment on the decode + // path promises a loud failure rather than a defaulted description, because a + // client builds a search from what it reads here. + sqlx::query( + "UPDATE vector_indexes SET vector_attribute = '{\"Nonsense\": true}'::jsonb \ + WHERE table_id = $1", + ) + .bind(&id) + .execute(&s.catalog) + .await + .expect("corrupt the payload"); + + let err = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "t_corrupt".to_owned(), + }, + ) + .await + .expect_err("a payload that cannot be decoded must not be described"); + match err { + StorageError::Internal(msg) => assert!(msg.contains("vector_attribute"), "{msg}"), + other => panic!("expected Internal, got {other:?}"), + } + + // The same row must also fail the write path's cache fill, rather than + // quietly yielding a table with no vector indexes and skipping validation. + let err = s + .engine + .table_key_info(ACCOUNT, "t_corrupt") + .await + .expect_err("the cached key info must not silently drop the index"); + assert!(matches!(err, StorageError::Internal(_)), "{err:?}"); + + s.cleanup().await; +} + +#[tokio::test] +async fn the_probe_reads_the_data_database_and_not_the_catalog() { + let test = "the_probe_reads_the_data_database_and_not_the_catalog"; + if base_conn().is_none() { + return skip(test); + } + let Some(base) = base_conn() else { return }; + + // In production the catalog and the data database are separate, and pgvector + // is installed on the data one, because that is where vector storage lives. + // Every other test here runs against a scratch database that serves as both, + // so the engine would pass them all while probing the wrong database. This is + // the only test that can tell the two apart. + let Some(catalog) = scratch_with_pgvector_omitted_but_data_installed(&base, test).await else { + return; + }; + + assert!( + catalog.engine.vector_capable(), + "the extension is installed on the data database, so the capability must be reported \ + even though the catalog database does not have it" + ); + + // Both databases go through cleanup, which owns every database it created and + // is the only place holding an admin pool that still works: closing one handle + // of a `PgPool` closes every clone, so a drop attempted after cleanup would + // fail with `PoolClosed` and, if its error were discarded, leak silently. + assert_eq!( + catalog.extra_databases.len(), + 1, + "the data database must be registered for cleanup" + ); + catalog.cleanup().await; +} + +/// Build a scratch catalog whose data database is a *different* database, with +/// pgvector installed only on the data one. +/// +/// Returns the scratch environment, whose `cleanup` drops both databases, or +/// `None` when the server has no pgvector to install. +async fn scratch_with_pgvector_omitted_but_data_installed( + base: &str, + test: &str, +) -> Option { + let probe = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + .expect("connect to the postgres maintenance database"); + let available: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_available_extensions WHERE name = 'vector')", + ) + .fetch_one(&probe) + .await + .expect("list the available extensions"); + probe.close().await; + if !available { + eprintln!( + "SKIP {test}: this PostgreSQL server does not offer the pgvector extension, so the \ + data-database probe cannot be exercised here." + ); + return None; + } + + // The catalog, deliberately without the extension. + let catalog = scratch(Pgvector::Omit).await; + let data_db = format!("{}_data", catalog.db_name); + sqlx::query(&format!("CREATE DATABASE \"{data_db}\"")) + .execute(&catalog.admin) + .await + .expect("create the separate data database"); + + let data_url = format!("{base}/{data_db}"); + let data = PgPoolOptions::new() + .max_connections(1) + .connect(&data_url) + .await + .expect("connect to the data database"); + sqlx::query("CREATE EXTENSION vector") + .execute(&data) + .await + .expect("install pgvector on the data database only"); + data.close().await; + + // Registering the connection string is what makes the engine open a second + // pool instead of reusing the catalog's. + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('data_database_connection_string', $1) \ + ON CONFLICT (key) DO UPDATE SET value = $1", + ) + .bind(&data_url) + .execute(&catalog.catalog) + .await + .expect("register the data database connection string"); + + // Re-open the engine so the constructor sees the setting and probes the data + // database. + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: format!("{base}/{}", catalog.db_name), + pool_size: 10, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("re-open the engine with a separate data database"); + + Some(Scratch { + engine, + extra_databases: vec![data_db], + ..catalog + }) +} + +#[tokio::test] +async fn the_engine_detects_whether_the_data_database_has_pgvector() { + if base_conn().is_none() { + return skip("the_engine_detects_whether_the_data_database_has_pgvector"); + } + + // The negative case is the one that must hold on every server, including one + // that has never heard of pgvector: no extension, no capability, and a + // catalog that otherwise works normally. + let without = scratch(Pgvector::Omit).await; + assert!( + !without.engine.vector_capable(), + "a data database without the extension must not report vector capability" + ); + without + .engine + .create_table(ACCOUNT, create_input("t_novector", vec![])) + .await + .expect("a server without pgvector must still serve ordinary tables"); + without.cleanup().await; + + let Some(with) = + scratch_with_pgvector("the_engine_detects_whether_the_data_database_has_pgvector").await + else { + return; + }; + assert!( + with.engine.vector_capable(), + "a data database with the extension installed must report vector capability" + ); + with.cleanup().await; +} + +#[tokio::test] +async fn losing_pgvector_after_startup_refuses_a_vector_index_rather_than_recording_it() { + let test = "losing_pgvector_after_startup_refuses_a_vector_index_rather_than_recording_it"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = scratch_with_pgvector(test).await else { + return; + }; + + // The engine probed the extension once, at construction, and cached the + // answer. Dropping it now is the window the second layer exists for: a DBA + // dropping the extension, or a failover onto a server without it. + assert!(s.engine.vector_capable()); + sqlx::query("DROP EXTENSION vector") + .execute(&s.catalog) + .await + .expect("drop the pgvector extension"); + + let err = s + .engine + .create_table( + ACCOUNT, + create_input("t_vanished", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect_err("a vector index must not be recorded once the extension is gone"); + + // Unsupported, so the engine answers 400 with the reason. Without the + // mapping this is a raw PostgreSQL error and a 500, which tells the caller + // nothing actionable. + match err { + StorageError::Unsupported(msg) => { + assert!(msg.contains("pgvector"), "{msg}"); + assert!(msg.contains("data database"), "{msg}"); + } + other => panic!("expected Unsupported, got {other:?}"), + } + + // No table and no index row: the refusal happens before anything is written. + let table: Option<(String,)> = + sqlx::query_as("SELECT table_name FROM tables WHERE account_id = $1 AND table_name = $2") + .bind(ACCOUNT) + .bind("t_vanished") + .fetch_optional(&s.catalog) + .await + .expect("look for the table"); + assert!(table.is_none(), "no table may be created"); + + s.cleanup().await; +} diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs index 0f119eb1..754cac0c 100644 --- a/crates/storage-sqlite/src/create_table.rs +++ b/crates/storage-sqlite/src/create_table.rs @@ -207,9 +207,12 @@ impl SqliteEngine { for vi in vis { let vec_attr = serde_json::to_string(&vi.vector_attribute) .map_err(|e| StorageError::Internal(e.to_string()))?; + // An empty SearchSchema is stored as absent: it means the same as + // omitting the member, and the service never reports an empty + // list. The request paths collapse it too; this covers a caller + // that reaches the storage trait directly. let search_schema = vi - .search_schema - .as_ref() + .search_schema_for_storage() .map(serde_json::to_string) .transpose() .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -227,10 +230,9 @@ impl SqliteEngine { "vector index reached storage without a projection".to_owned(), ) })?; - let distance = serde_json::to_string(&vi.distance_function) - .map_err(|e| StorageError::Internal(e.to_string()))? - .trim_matches('"') - .to_owned(); + let distance = extenddb_storage::vector_catalog::distance_function_token( + vi.distance_function, + )?; let index_id = uuid::Uuid::new_v4().to_string(); sqlx::query( "INSERT INTO vector_indexes \ @@ -453,7 +455,7 @@ impl SqliteEngine { index_name: vi.index_name.clone(), vector_attribute: vi.vector_attribute.clone(), dimensions: vi.dimensions, - search_schema: vi.search_schema.clone(), + search_schema: vi.search_schema_for_storage().map(<[_]>::to_vec), distance_function: vi.distance_function, index_status: extenddb_core::types::IndexStatus::Active, backfilling: None, diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs index 56d4802c..0041972a 100644 --- a/crates/storage-sqlite/src/data/ddl.rs +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -405,45 +405,28 @@ impl SqliteEngine { /// Fetch the vector indexes of a table in the shape the engine caches. /// - /// Note what this cannot carry: `VectorIndexKeyInfo` has no distance - /// function, so a search still reads the catalog for it. Widening that - /// type would remove the last per-search catalog read. + /// Selects the whole row rather than the five columns this shape needs, so + /// that one row type and one decode path serve both this and the describe + /// path. The extra columns are two short tokens and an integer, read only on + /// a key-info cache miss. async fn fetch_vector_index_key_info( &self, table_id: &str, ) -> Result, StorageError> { - let rows: Vec<(String, i64, String, Option, String)> = sqlx::query_as( - "SELECT index_name, dimensions, vector_attribute, search_schema, projection \ - FROM vector_indexes WHERE table_id = ?", - ) + let rows: Vec = sqlx::query_as(&format!( + "SELECT {} FROM vector_indexes WHERE table_id = ? ORDER BY index_name", + crate::table_helpers::VECTOR_INDEX_COLUMNS + )) .bind(table_id) .fetch_all(&self.pool) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let mut out = Vec::with_capacity(rows.len()); - for (index_name, dimensions, vector_attribute, search_schema, projection) in rows { - let attr: extenddb_core::types::VectorAttribute = - serde_json::from_str(&vector_attribute) - .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; - let search_schema = match search_schema.as_deref() { - Some(json) => serde_json::from_str(json) - .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?, - None => Vec::new(), - }; - let projection: extenddb_core::types::Projection = serde_json::from_str(&projection) - .map_err(|e| StorageError::Internal(format!("vector projection: {e}")))?; - out.push(extenddb_core::types::VectorIndexKeyInfo { - index_name, - dimensions: u32::try_from(dimensions).map_err(|_| { - StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) - })?, - vector_attribute_name: attr.attribute_name, - search_schema, - projection, - }); - } - Ok(out) + let catalog_rows = rows + .into_iter() + .map(crate::table_helpers::VectorIndexRow::into_catalog_row) + .collect::, _>>()?; + extenddb_storage::vector_catalog::vector_index_key_info(catalog_rows) } /// Fetch every secondary index defined on a table, split into diff --git a/crates/storage-sqlite/src/table_helpers.rs b/crates/storage-sqlite/src/table_helpers.rs index 5cd30599..f0735e0b 100644 --- a/crates/storage-sqlite/src/table_helpers.rs +++ b/crates/storage-sqlite/src/table_helpers.rs @@ -12,6 +12,7 @@ use extenddb_core::types::{ }; use extenddb_storage::error::StorageError; use extenddb_storage::util::{index_arn, stream_arn}; +use extenddb_storage::vector_catalog::{VectorIndexCatalogRow, vector_index_descriptions}; use crate::data; use crate::sqlite_util::parse_timestamp; @@ -53,6 +54,10 @@ pub(crate) struct IndexRow { } /// A `vector_indexes` catalog row, in `FromRow` field order. +/// +/// Decoding stops at this struct: the rules that turn it into a wire shape are +/// shared with the other backends in `extenddb_storage::vector_catalog`. The JSON +/// columns are text here, so the conversion parses them into values. #[derive(sqlx::FromRow)] pub(crate) struct VectorIndexRow { pub index_name: String, @@ -67,6 +72,30 @@ pub(crate) struct VectorIndexRow { pub backfilling: Option, } +impl VectorIndexRow { + /// Parse the text columns into the backend-neutral row the shared rules take. + pub(crate) fn into_catalog_row(self) -> Result { + let json = |text: &str, column: &str| -> Result { + serde_json::from_str(text).map_err(|e| StorageError::Internal(format!("{column}: {e}"))) + }; + Ok(VectorIndexCatalogRow { + index_name: self.index_name, + dimensions: self.dimensions, + distance_function: self.distance_function, + vector_attribute: json(&self.vector_attribute, "vector_attribute")?, + search_schema: self + .search_schema + .as_deref() + .map(|text| json(text, "vector search_schema")) + .transpose()?, + projection: json(&self.projection, "vector projection")?, + index_status: self.index_status, + // Stored as an integer so the ACTIVE state is representable as NULL. + backfilling: self.backfilling.map(|b| b != 0), + }) + } +} + /// Columns selected for a `VectorIndexRow`, in `FromRow` field order. pub(crate) const VECTOR_INDEX_COLUMNS: &str = "index_name, index_id, dimensions, \ distance_function, vector_attribute, search_schema, projection, index_status, backfilling"; @@ -122,8 +151,12 @@ impl SqliteEngine { let table_name_owned = row.table_name.clone(); let mut desc = self.build_table_description_from_row(account_id, row, index_rows)?; + let catalog_rows = vector_rows + .into_iter() + .map(VectorIndexRow::into_catalog_row) + .collect::, _>>()?; desc.vector_indexes = - self.vector_index_descriptions(account_id, &table_name_owned, vector_rows)?; + vector_index_descriptions(&self.region, account_id, &table_name_owned, catalog_rows)?; Ok(desc) } @@ -277,63 +310,6 @@ impl SqliteEngine { }) } - /// Build the vector index descriptions for a table. - /// - /// Separate from `build_table_description_from_row` so the shared builder - /// keeps one signature for every caller. Applied on the describe path, which - /// is where a client reads an index definition in order to search it. - pub(crate) fn vector_index_descriptions( - &self, - account_id: &str, - table_name: &str, - vector_rows: Vec, - ) -> Result>, StorageError> { - let mut vector_index_descs: Vec = Vec::new(); - for vi in vector_rows { - // Deliberately fails rather than defaulting on a bad parse: a vector - // index we cannot describe faithfully must not be reported as if we - // could, because a client uses the description to build a search. - let vector_attribute = parse_json(&vi.vector_attribute, "vector_attribute")?; - let search_schema = vi - .search_schema - .as_deref() - .map(|s| parse_json(s, "vector search_schema")) - .transpose()?; - let projection = parse_json(&vi.projection, "vector projection")?; - let distance_function = parse_json( - &format!("\"{}\"", vi.distance_function), - "distance_function", - )?; - let index_status = parse_json(&format!("\"{}\"", vi.index_status), "vector status")?; - let desc = extenddb_core::types::VectorIndexDescription { - index_name: vi.index_name.clone(), - vector_attribute, - dimensions: u32::try_from(vi.dimensions).map_err(|_| { - StorageError::Internal(format!( - "vector dimensions out of range: {}", - vi.dimensions - )) - })?, - search_schema, - distance_function, - index_status, - backfilling: vi.backfilling.map(|b| b != 0), - index_size_bytes: 0, - item_count: 0, - index_arn: index_arn(&self.region, account_id, table_name, &vi.index_name), - projection: Some(projection), - }; - // The readiness rule is core's, applied here so a backend bug cannot - // emit a description the wire contract forbids. Cheaper to catch on - // the way out than to debug from a client. - desc.validate_readiness() - .map_err(|e| StorageError::Internal(e.to_string()))?; - vector_index_descs.push(desc); - } - - Ok((!vector_index_descs.is_empty()).then_some(vector_index_descs)) - } - /// Backfill existing base-table items into a newly created GSI, batched to /// bound memory. #[allow(clippy::too_many_arguments)] diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 7d04b444..3dc5c93f 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -512,18 +512,16 @@ impl SqliteEngine { let vec_attr = serde_json::to_string(&create.vector_attribute) .map_err(|e| StorageError::Internal(e.to_string()))?; let search_schema = create - .search_schema - .as_ref() + .search_schema_for_storage() .map(serde_json::to_string) .transpose() .map_err(|e| StorageError::Internal(e.to_string()))?; let projection = serde_json::to_string(&create.projection) .map_err(|e| StorageError::Internal(e.to_string()))?; let dimensions = i64::from(create.dimensions); - let distance = serde_json::to_string(&create.distance_function) - .map_err(|e| StorageError::Internal(e.to_string()))? - .trim_matches('"') - .to_owned(); + let distance = extenddb_storage::vector_catalog::distance_function_token( + create.distance_function, + )?; // `backfilling` starts at false rather than absent or true. // Measured against the service on 2026-08-06: the member appears // as false while the index exists but its backfill has not diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index fa730dbc..c76e3289 100755 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -20,6 +20,7 @@ pub mod operations; pub mod server_components; pub mod settings_store; pub mod transact; +pub mod vector_catalog; pub mod vector_lifecycle; pub use backend::{Backend, BackendAlreadySet, backend_name, set_backend, try_backend}; diff --git a/crates/storage/src/vector_catalog.rs b/crates/storage/src/vector_catalog.rs new file mode 100644 index 00000000..b935b43d --- /dev/null +++ b/crates/storage/src/vector_catalog.rs @@ -0,0 +1,346 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared decoding of vector index catalog rows. +//! +//! Every backend stores the same eight pieces of vector index metadata and then +//! has to turn them into the same two shapes: the `VectorIndexDescription` a +//! client reads in order to build a search, and the `VectorIndexKeyInfo` the +//! write path caches. The storage formats differ (PostgreSQL has JSONB columns, +//! SQLite has JSON text), so decoding a row stays with the backend. Everything +//! after that is rules rather than format, and several of those rules are +//! wire-visible: the index ARN, the readiness check, reporting no indexes as an +//! absent member rather than an empty list, and reporting an unscoped index's +//! search schema as absent rather than empty. Two copies of a wire-visible rule +//! is how two backends come to answer the same request differently, so they live +//! here once. +//! +//! Each backend reads its own row type with `sqlx::FromRow`, converts it into +//! [`VectorIndexCatalogRow`], and calls one of the two functions below. + +use extenddb_core::types::{ + DistanceFunction, IndexStatus, Projection, SearchSchemaElement, VectorAttribute, + VectorIndexDescription, VectorIndexKeyInfo, +}; + +use crate::error::StorageError; +use crate::util::index_arn; + +/// A `vector_indexes` catalog row with its storage format already decoded. +/// +/// JSON payloads arrive as `serde_json::Value` because that is what PostgreSQL +/// returns natively and what SQLite reaches by parsing its text column. Enum +/// tokens arrive as strings so that an unrecognised one can be reported with the +/// offending value rather than silently mapped to a fallback. +pub struct VectorIndexCatalogRow { + pub index_name: String, + /// Declared dimension count. Widened to `i64` so both backends' integer + /// column types fit without either one narrowing before the range check. + pub dimensions: i64, + pub distance_function: String, + pub vector_attribute: serde_json::Value, + /// Absent for an unscoped index, which is a different state from an empty + /// list and is preserved as such. + pub search_schema: Option, + pub projection: serde_json::Value, + pub index_status: String, + /// Absent once the index is ACTIVE, which is how the service reports it. + pub backfilling: Option, +} + +/// The catalog's spelling of a distance function. +/// +/// Stored as the wire token so a describe can hand it straight back, and derived +/// from the enum's own serialisation rather than a hand-written match, so adding a +/// distance function cannot silently persist the wrong string. Lives beside the +/// read direction, which parses this same token, so the pair cannot drift. +/// +/// # Errors +/// +/// Returns [`StorageError::Internal`] if the enum does not serialise to a string, +/// which would mean its representation changed under this function. +#[must_use = "the token is what gets stored"] +pub fn distance_function_token( + distance_function: DistanceFunction, +) -> Result { + match serde_json::to_value(distance_function) + .map_err(|e| StorageError::Internal(e.to_string()))? + { + serde_json::Value::String(s) => Ok(s), + other => Err(StorageError::Internal(format!( + "distance function did not serialise to a string: {other}" + ))), + } +} + +/// Decode one JSON payload, naming the column so a failure is diagnosable. +fn decode( + value: serde_json::Value, + column: &str, +) -> Result { + serde_json::from_value(value).map_err(|e| StorageError::Internal(format!("{column}: {e}"))) +} + +/// Parse a stored dimension count into the width the wire uses. +fn dimensions(raw: i64) -> Result { + u32::try_from(raw) + .map_err(|_| StorageError::Internal(format!("vector dimensions out of range: {raw}"))) +} + +/// Parse a stored `IndexStatus` token, refusing one this build cannot read. +/// +/// `IndexStatus` carries a catch-all variant so that parsing a *service* +/// response tolerates a status added after this build shipped. Reading our own +/// catalog is the opposite situation: a value we do not recognise is a corrupt +/// row or one written by a newer version, and reporting it to a client as +/// "UNKNOWN" would be describing an index whose state we do not know. So the +/// catch-all is rejected explicitly here, which the deserializer alone cannot do. +fn index_status(token: &str) -> Result { + let status: IndexStatus = decode( + serde_json::Value::String(token.to_owned()), + "vector index_status", + )?; + if status == IndexStatus::Unknown { + return Err(StorageError::Internal(format!( + "unrecognised vector index status in the catalog: {token}" + ))); + } + Ok(status) +} + +/// Build the descriptions a `DescribeTable` or `UpdateTable` response carries. +/// +/// Fails rather than defaulting on a bad row: a client uses this description to +/// build a search, so an index that cannot be described faithfully must not be +/// reported as if it could. +/// +/// Returns `None` when the table has no vector indexes, because the response +/// member is absent in that case rather than an empty list. +/// +/// # Errors +/// +/// Returns [`StorageError::Internal`] when a payload cannot be decoded, a +/// dimension count does not fit, a status or distance function token is +/// unrecognised, or the resulting description reports a state the wire contract +/// forbids. +pub fn vector_index_descriptions( + region: &str, + account_id: &str, + table_name: &str, + rows: Vec, +) -> Result>, StorageError> { + let mut descs: Vec = Vec::with_capacity(rows.len()); + for row in rows { + let vector_attribute: VectorAttribute = decode(row.vector_attribute, "vector_attribute")?; + let search_schema: Option> = row + .search_schema + .map(|value| decode(value, "vector search_schema")) + .transpose()?; + let projection: Projection = decode(row.projection, "vector projection")?; + let distance_function: DistanceFunction = decode( + serde_json::Value::String(row.distance_function), + "vector distance_function", + )?; + let desc = VectorIndexDescription { + index_name: row.index_name.clone(), + vector_attribute, + dimensions: dimensions(row.dimensions)?, + search_schema, + distance_function, + index_status: index_status(&row.index_status)?, + backfilling: row.backfilling, + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn(region, account_id, table_name, &row.index_name), + projection: Some(projection), + }; + // The readiness rule is core's, applied here so no backend can emit a + // description the wire contract forbids. Cheaper to catch on the way out + // than to debug from a client. + desc.validate_readiness() + .map_err(|e| StorageError::Internal(e.to_string()))?; + descs.push(desc); + } + Ok((!descs.is_empty()).then_some(descs)) +} + +/// Build the vector index metadata the write path caches on `TableKeyInfo`. +/// +/// Note what this deliberately cannot carry: no index id and no distance +/// function, so a search still reads the catalog for them. Widening +/// `VectorIndexKeyInfo` would remove the last per-search catalog read. +/// +/// This path deliberately does not look at `index_status`, and that asymmetry with +/// the describe path is load-bearing rather than an oversight: a catalog row whose +/// status cannot be read must fail a describe, which reports index state, but must +/// not fail every write to the table, which does not depend on it. Validating +/// everything everywhere here would turn one bad row into a table-wide data-plane +/// outage. +/// +/// An unscoped index reports an empty search schema here, not `None`: the +/// distinction matters on the describe path, where the member's absence is +/// wire-visible, and does not matter to a write, which only asks which +/// attributes it must look at. +/// +/// # Errors +/// +/// Returns [`StorageError::Internal`] when a payload cannot be decoded or a +/// dimension count does not fit. +pub fn vector_index_key_info( + rows: Vec, +) -> Result, StorageError> { + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let attr: VectorAttribute = decode(row.vector_attribute, "vector_attribute")?; + let search_schema: Vec = match row.search_schema { + Some(value) => decode(value, "vector search_schema")?, + None => Vec::new(), + }; + let projection: Projection = decode(row.projection, "vector projection")?; + out.push(VectorIndexKeyInfo { + index_name: row.index_name, + dimensions: dimensions(row.dimensions)?, + vector_attribute_name: attr.attribute_name, + search_schema, + projection, + }); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(index_status: &str) -> VectorIndexCatalogRow { + VectorIndexCatalogRow { + index_name: "vidx".to_owned(), + dimensions: 4, + distance_function: "COSINE".to_owned(), + vector_attribute: serde_json::json!({ "AttributeName": "emb" }), + search_schema: Some( + serde_json::json!([{ "AttributeName": "pk", "SearchSchemaElementType": "HASH" }]), + ), + projection: serde_json::json!({ "ProjectionType": "ALL" }), + index_status: index_status.to_owned(), + backfilling: None, + } + } + + #[test] + fn a_table_with_no_vector_indexes_reports_an_absent_member() { + // Absent, not an empty list: the service omits the member entirely, and a + // client distinguishes the two. + assert_eq!( + vector_index_descriptions("us-east-1", "1", "t", Vec::new()).unwrap(), + None + ); + } + + #[test] + fn an_active_index_is_described_with_its_arn() { + let descs = + vector_index_descriptions("us-east-1", "123456789012", "t", vec![row("ACTIVE")]) + .unwrap() + .expect("one index"); + assert_eq!(descs.len(), 1); + assert_eq!(descs[0].index_status, IndexStatus::Active); + assert_eq!(descs[0].dimensions, 4); + assert_eq!( + descs[0].index_arn, + "arn:aws:dynamodb:us-east-1:123456789012:table/t/index/vidx" + ); + } + + #[test] + fn an_unrecognised_status_is_refused_rather_than_reported_as_unknown() { + // The catch-all variant exists for reading a service response, where a + // new status must not break the client. Reading our own catalog, it would + // turn a corrupt row into an index described with a state nobody knows, + // so it is refused with the offending value named. + let err = vector_index_descriptions("us-east-1", "1", "t", vec![row("ACITVE")]) + .expect_err("an unrecognised status must not be described"); + match err { + StorageError::Internal(msg) => { + assert!(msg.contains("ACITVE"), "{msg}"); + assert!(msg.contains("unrecognised"), "{msg}"); + } + other => panic!("expected Internal, got {other:?}"), + } + } + + #[test] + fn an_unrecognised_distance_function_is_refused() { + let mut bad = row("ACTIVE"); + bad.distance_function = "MANHATTAN".to_owned(); + let err = vector_index_descriptions("us-east-1", "1", "t", vec![bad]) + .expect_err("an unrecognised distance function must not be described"); + assert!(matches!(err, StorageError::Internal(_)), "{err:?}"); + } + + #[test] + fn an_active_index_reporting_backfilling_is_refused() { + // The wire rule is that the member disappears once the index is active, + // so reporting both is a contradiction a client would have to guess about. + let mut bad = row("ACTIVE"); + bad.backfilling = Some(false); + let err = vector_index_descriptions("us-east-1", "1", "t", vec![bad]) + .expect_err("ACTIVE with a Backfilling member must be refused"); + assert!(matches!(err, StorageError::Internal(_)), "{err:?}"); + } + + #[test] + fn a_building_index_keeps_its_backfilling_state() { + let mut building = row("CREATING"); + building.backfilling = Some(true); + let descs = vector_index_descriptions("us-east-1", "1", "t", vec![building]) + .unwrap() + .expect("one index"); + assert_eq!(descs[0].index_status, IndexStatus::Creating); + assert_eq!(descs[0].backfilling, Some(true)); + } + + #[test] + fn a_dimension_count_that_does_not_fit_is_refused() { + let mut bad = row("ACTIVE"); + bad.dimensions = i64::from(u32::MAX) + 1; + let err = vector_index_descriptions("us-east-1", "1", "t", vec![bad]) + .expect_err("a dimension count outside the wire width must be refused"); + match err { + StorageError::Internal(msg) => assert!(msg.contains("out of range"), "{msg}"), + other => panic!("expected Internal, got {other:?}"), + } + } + + #[test] + fn key_info_reports_an_unscoped_index_with_an_empty_search_schema() { + // The opposite convention from the describe path, deliberately: a write + // only asks which attributes it must look at, and an empty list answers + // that without every caller having to unwrap an Option. + let mut unscoped = row("ACTIVE"); + unscoped.search_schema = None; + let info = vector_index_key_info(vec![unscoped]).unwrap(); + assert_eq!(info.len(), 1); + assert_eq!(info[0].index_name, "vidx"); + assert_eq!(info[0].vector_attribute_name, "emb"); + assert!(info[0].search_schema.is_empty()); + } + + #[test] + fn key_info_carries_the_search_schema_of_a_scoped_index() { + let info = vector_index_key_info(vec![row("ACTIVE")]).unwrap(); + assert_eq!(info[0].search_schema.len(), 1); + assert_eq!(info[0].search_schema[0].attribute_name, "pk"); + } + + #[test] + fn key_info_refuses_a_payload_it_cannot_decode() { + let mut bad = row("ACTIVE"); + bad.vector_attribute = serde_json::json!({ "Nonsense": true }); + let err = vector_index_key_info(vec![bad]).expect_err("a bad payload must be refused"); + match err { + StorageError::Internal(msg) => assert!(msg.contains("vector_attribute"), "{msg}"), + other => panic!("expected Internal, got {other:?}"), + } + } +} diff --git a/devtools/run-tests b/devtools/run-tests index 819bf469..fd981c87 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -461,7 +461,7 @@ if $RUN_PYTEST; then fi echo " AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}" # CLI lifecycle tests manage their own server; exclude from main suite - PYTEST_ARGS=(python3 -m pytest tests/ -v --ignore=tests/python --ignore=tests/test_cli_lifecycle.py --ignore=tests/test_cli_container_readiness.py --ignore=tests/test_cli_migrate_concurrency.py --ignore=tests/test_gsi_async_queue.py) + PYTEST_ARGS=(python3 -m pytest tests/ -v --ignore=tests/python --ignore=tests/test_cli_lifecycle.py --ignore=tests/test_cli_container_readiness.py --ignore=tests/test_cli_migrate_concurrency.py --ignore=tests/test_cli_vector_catalog_migration.py --ignore=tests/test_gsi_async_queue.py) # Parallel execution (--parallel): distribute by file so module/class-scoped # fixtures stay within a single worker. if [[ -n "$PARALLEL" ]] && python3 -c "import xdist" 2>/dev/null; then @@ -553,7 +553,7 @@ if $RUN_PYTEST \ && -n "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" ]]; then CLI_OUTFILE="discussions/test-cli-${HASH}.txt" echo "=== CLI lifecycle tests → ${CLI_OUTFILE} ===" - CLI_ARGS=(python3 -m pytest tests/test_cli_lifecycle.py tests/test_cli_container_readiness.py tests/test_cli_migrate_concurrency.py -v) + CLI_ARGS=(python3 -m pytest tests/test_cli_lifecycle.py tests/test_cli_container_readiness.py tests/test_cli_migrate_concurrency.py tests/test_cli_vector_catalog_migration.py -v) if [[ -n "$FILTER" ]]; then CLI_ARGS+=(-k "$FILTER") fi diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index 13523c87..fff2d8de 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -495,6 +495,33 @@ curl --cacert ~/.extenddb/tls/cert.pem https://127.0.0.1:18443/health ## Troubleshooting +### Vector Index Support (PostgreSQL) + +Vector indexes are stored in `vector(N)` columns, a type the [pgvector](https://github.com/pgvector/pgvector) extension provides, so support is a property of the PostgreSQL server rather than of the ExtendDB build. Two things follow. + +**`extenddb init` and `extenddb migrate` try to install it.** Both print one of: + +``` +--- Checking pgvector extension on the data database... + pgvector available; vector indexes are supported. +``` + +``` +--- Checking pgvector extension on the data database... + NOTICE: could not create the pgvector extension (...). +``` + +The notice is not a failure. Initialisation and migration complete either way, and every operation other than a vector one is unaffected; the server simply refuses vector indexes. The advice depends on why it failed: a missing server package (install for example `postgresql-16-pgvector`), or a role that may not create extensions (create it once as a superuser or as the database owner). + +Note that the extension is installed on the **data** database, not the catalog. + +**Installing pgvector on a running server needs an ExtendDB restart.** The server probes for the extension once at startup and caches the answer, so a server that started without it keeps refusing vector indexes until restarted. Confirm what a running server decided by checking its startup log: + +``` +pgvector 0.8.0 detected on the data database; vector index storage available +pgvector not installed on the data database; vector indexes are not supported +``` + ### Server Won't Start **Port already in use:** @@ -516,10 +543,10 @@ Check that PostgreSQL is running and the connection string in `extenddb.toml` is **Catalog version mismatch:** ``` -Error: catalog version mismatch: found 1.0.0, expected 0.0.2 +Error: catalog version mismatch: found 1.0.0, expected 0.0.3 ``` -Run `extenddb migrate --config extenddb.toml` to upgrade the catalog schema. +Run `extenddb migrate --config extenddb.toml` to upgrade the catalog schema. The check is exact equality in both directions, so this also appears when a binary meets a catalog a newer build already migrated; in that case upgrade the binary rather than the catalog. See the Upgrade Manual for the version history and the stop / migrate / start sequence. ### Authentication Errors diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index b55badaf..fe2bf5eb 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -4,9 +4,9 @@ ## Current Status -ExtendDB 0.0.2 is the initial release. There is no upgrade path from a previous version — all deployments are fresh installs via `extenddb init`. +Catalog 0.0.3 is current. The 0.0.2 to 0.0.3 upgrade is the first in-place catalog upgrade ExtendDB has, and **every existing PostgreSQL deployment must run it**, including deployments that never use vector indexes: the server refuses to start against a catalog version it was not built for. -Future releases will include migrations that upgrade the catalog schema in place. The migration infrastructure is built and ready; this document describes how it works and how developers should think about adding new migrations. +See [Catalog 0.0.3](#catalog-003-current) below for what changes and the exact sequence. ## How Catalog Upgrades Work @@ -15,8 +15,8 @@ Future releases will include migrations that upgrade the catalog schema in place Migrations are SQL files in `crates/storage-postgres/migrations/`, applied in filename order: ``` -001_schema.sql ← current: the complete initial schema -002_.sql ← future: first incremental migration +001_schema.sql ← the complete initial schema +002_vector_indexes.sql ← vector index metadata, catalog 0.0.3 ``` The `schema_history` table tracks which files have been applied. When `extenddb migrate` runs, it: @@ -181,11 +181,34 @@ psql -d extenddb_catalog -f catalog_backup_YYYYMMDD.sql ## Version History -### Catalog 0.0.2 (Current — Initial Release) +### Catalog 0.0.3 (Current) + +Adds vector index metadata: + +- New `vector_indexes` table: one row per vector index, holding its dimensions, distance function, vector attribute, search schema, projection, and build state. +- New `vector_indexes` column on `backups`: a snapshot of the source table's vector index configuration, taken when the backup is created. + +**Every PostgreSQL deployment must apply this**, whether or not it uses vector indexes, because the server refuses to start against a catalog version it was not built for. + +Upgrade sequence: + +```bash +extenddb stop --config extenddb.toml +extenddb migrate --yes --config extenddb.toml +extenddb serve --config extenddb.toml +``` + +Run `extenddb migrate` without `--yes` first to see what is pending; it reports `catalog 0.0.2 -> 0.0.3` and changes nothing. + +The upgrade is not reversible in place: a 0.0.2 binary refuses to start against a 0.0.3 catalog, by the same check in the other direction. Roll back by restoring the catalog backup taken before the upgrade, as described above. Downgrading is safe for data written before the upgrade; vector indexes created afterwards are not representable in 0.0.2 and are lost with the restore. + +During the upgrade the migration also attempts to install the pgvector extension on the data database. Failure is reported as a notice and does not stop the upgrade: vector indexes are then refused at request time, and every other operation is unaffected. See the Admin Guide for what the notice means and what to do about it. + +### Catalog 0.0.2 (Initial Release) Complete schema: accounts, tables, indexes, tags, streams, IAM (users, groups, roles, policies, access keys, sessions, permissions boundaries), idempotency tokens, metrics, login attempts, backups, continuous backups, TTL support, settings. -No prior versions exist. All deployments are fresh installs. +The first release, so all 0.0.2 deployments were fresh installs. --- diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 4d0e376d..98793556 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1020,6 +1020,125 @@ async fn update_table_rejects_a_duplicate_create_and_a_missing_delete() { let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } +/// An empty `SearchSchema` on the request means the same as omitting it, and must +/// come back as an absent member rather than an empty list. +/// +/// Amazon DynamoDB reports either an absent member or a populated one, so `[]` +/// would be a third state a client never sees from the service. Storing it is also +/// how two backends come to answer one request differently, which is what happened +/// here: the collapse was added to one backend before being moved into the request +/// path both share. This test is backend-blind, so it pins the rule for whichever +/// backend is under it. +#[tokio::test] +async fn an_empty_search_schema_is_reported_as_absent() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_emptyschema"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 4, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "SearchSchema": [], + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + + // The response echo, first: it is built from the request, so it is where an + // un-collapsed empty list surfaces without any storage round trip. + let created: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let echoed = created + .pointer("/TableDescription/VectorIndexes/0") + .unwrap_or_else(|| panic!("no vector index in the CreateTable echo: {text}")); + assert!( + echoed.get("SearchSchema").is_none(), + "the echo must not report an empty SearchSchema: {echoed}" + ); + + wait_for_active(&name).await; + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + assert_eq!(status, 200, "DescribeTable failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let vidx = json + .pointer("/Table/VectorIndexes/0") + .unwrap_or_else(|| panic!("no vector index in description: {text}")); + assert!( + vidx.get("SearchSchema").is_none(), + "DescribeTable must report the member as absent, not as an empty list: {vidx}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// The same collapse on the other path a client can reach: an index added by +/// UpdateTable. +/// +/// Separate from the CreateTable case because the two store the index through +/// different code, and because this one builds a real index with a lifecycle +/// rather than one that is ACTIVE from birth, so the member has to survive the +/// build as well as the write. +#[tokio::test] +async fn an_empty_search_schema_added_by_update_table_is_reported_as_absent() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_emptyschema_upd"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + call("CreateTable", &body).await; + wait_for_active(&name).await; + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Dimensions": 2, + "DistanceFunction": "COSINE", + "SearchSchema": [], + "Projection": {{"ProjectionType": "ALL"}} + }}}}] + }}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable create failed: {text}"); + + wait_for_vector_index_active(&name, "vidx").await; + + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + assert_eq!(status, 200, "DescribeTable failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let vidx = json + .pointer("/Table/VectorIndexes/0") + .unwrap_or_else(|| panic!("no vector index in description: {text}")); + assert!( + vidx.get("SearchSchema").is_none(), + "an index added by UpdateTable must report the member as absent too: {vidx}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + /// DescribeTable reports the vector index, and reports it ACTIVE with no /// `Backfilling` member, which is what the service does for an index created by /// CreateTable. diff --git a/tests/test_cli_vector_catalog_migration.py b/tests/test_cli_vector_catalog_migration.py new file mode 100644 index 00000000..1b260e68 --- /dev/null +++ b/tests/test_cli_vector_catalog_migration.py @@ -0,0 +1,292 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Catalog migration tests for the vector index schema (catalog 0.0.2 -> 0.0.3). + +These exercise the migration against a live deployment rather than asserting the +SQL text: init one, roll its catalog back to the pre-vector shape, and check that +the binary refuses to serve it, that `extenddb migrate` upgrades it, and that the +upgraded deployment then serves. + +Shared lifecycle helpers and the `cli_env` fixture live in `lifecycle_helpers.py`. +Like the other CLI lifecycle tests these require a PostgreSQL instance +(EXTENDDB_TEST_PG_CONNECTION_STRING) and a built binary, and are excluded from the +backend-agnostic pytest suite. +""" + +from __future__ import annotations + +import subprocess +import time + +from lifecycle_helpers import ( + PG_ADMIN_CONN, + _init_args, + _patch_config_port, + _pg_args, + _run_extenddb, + _wait_for_server, +) + +VECTOR_MIGRATION = "002_vector_indexes.sql" + + +def _catalog_conn(cli_env): + import psycopg2 + + return psycopg2.connect(PG_ADMIN_CONN + "/" + cli_env["db_name"]) + + +def _catalog_query(cli_env, sql): + conn = _catalog_conn(cli_env) + try: + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] + finally: + conn.close() + + +def _catalog_version(cli_env): + return _catalog_query( + cli_env, "SELECT value FROM settings WHERE key = 'catalog_version'" + ) + + +def _vector_table_exists(cli_env): + return _catalog_query( + cli_env, "SELECT to_regclass('public.vector_indexes') IS NOT NULL" + ) + + +def _backups_has_vector_column(cli_env): + return _catalog_query( + cli_env, + "SELECT EXISTS(SELECT 1 FROM information_schema.columns " + "WHERE table_name = 'backups' AND column_name = 'vector_indexes')", + ) + + +def _init(cli_env): + result = _run_extenddb( + "init", *_init_args(cli_env), + config=cli_env["config_path"], + env_override={"EXTENDDB_ADMIN_PASSWORD": "TestPass1!"}, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def _roll_catalog_back_to_pre_vector(cli_env): + """Turn a freshly initialised catalog into the shape 0.0.2 deployments have. + + Reproduces an upgrade rather than a fresh install, which is the case that + matters: a fresh install applies both migrations in order and reaches the same + end state trivially. + """ + conn = _catalog_conn(cli_env) + try: + conn.autocommit = True + with conn.cursor() as cur: + cur.execute("DROP TABLE IF EXISTS vector_indexes") + cur.execute("ALTER TABLE backups DROP COLUMN IF EXISTS vector_indexes") + cur.execute( + "DELETE FROM schema_history WHERE filename = %s", (VECTOR_MIGRATION,) + ) + cur.execute("UPDATE settings SET value = '0.0.2' WHERE key = 'catalog_version'") + finally: + conn.close() + + +class TestVectorCatalogMigration: + """Catalog version 0.0.3: the vector index table and the backup snapshot.""" + + def test_init_creates_the_vector_catalog_at_0_0_3(self, cli_env): + """A fresh init applies both catalog migrations and records the version. + + The version is what the binary checks at startup, so a migration that + creates the table without moving the version, or the reverse, would leave + a deployment that cannot serve. + """ + _init(cli_env) + + assert _catalog_version(cli_env) == "0.0.3" + assert _vector_table_exists(cli_env) is True + assert _backups_has_vector_column(cli_env) is True + + conn = _catalog_conn(cli_env) + try: + with conn.cursor() as cur: + cur.execute("SELECT filename FROM schema_history ORDER BY filename") + tracked = {row[0] for row in cur.fetchall()} + finally: + conn.close() + # Recorded, so a later migrate does not walk the file list again on a + # deployment that is already current. The statements are idempotent too, + # so a replay after a crash between applying and recording is harmless. + assert VECTOR_MIGRATION in tracked, tracked + + def test_migrate_upgrades_a_pre_vector_deployment(self, cli_env): + """A 0.0.2 deployment is refused, upgraded by migrate, and then serves.""" + _init(cli_env) + _patch_config_port(cli_env["config_path"], cli_env["port"]) + _roll_catalog_back_to_pre_vector(cli_env) + assert _vector_table_exists(cli_env) is False + + # The binary must refuse to serve a catalog it was not built for, rather + # than starting and failing on the first request that reads the table. + try: + refused_serve = _run_extenddb( + "serve", "--foreground", + config=cli_env["config_path"], + check=False, + timeout=20, + ) + except subprocess.TimeoutExpired: + _run_extenddb("stop", config=cli_env["config_path"], check=False) + raise AssertionError( + "serve started against a 0.0.2 catalog; the version gate did not hold" + ) from None + combined = refused_serve.stdout + refused_serve.stderr + assert refused_serve.returncode != 0, combined + assert "0.0.3" in combined and "0.0.2" in combined, combined + + # Without --yes, migrate reports the pending upgrade and changes nothing. + pending = _run_extenddb( + "migrate", *_pg_args(), config=cli_env["config_path"], check=False + ) + pending_output = pending.stdout + pending.stderr + assert pending.returncode != 0, pending_output + assert "0.0.2 -> 0.0.3" in pending_output, pending_output + assert _vector_table_exists(cli_env) is False + assert _catalog_version(cli_env) == "0.0.2" + + applied = _run_extenddb( + "migrate", "--yes", *_pg_args(), config=cli_env["config_path"], check=False + ) + assert applied.returncode == 0, applied.stdout + applied.stderr + assert _catalog_version(cli_env) == "0.0.3" + assert _vector_table_exists(cli_env) is True + assert _backups_has_vector_column(cli_env) is True + + # The upgraded deployment serves, which is the assertion the version and + # the table shape are both really for. + served = _run_extenddb("serve", config=cli_env["config_path"]) + assert served.returncode == 0, served.stdout + served.stderr + try: + assert _wait_for_server(cli_env["port"]), "upgraded deployment did not serve" + finally: + _run_extenddb("stop", config=cli_env["config_path"], check=False) + time.sleep(1) + + # A second migrate is a no-op: the ledger row stops the non-idempotent + # migration from running twice. + again = _run_extenddb( + "migrate", *_pg_args(), config=cli_env["config_path"], check=False + ) + again_output = again.stdout + again.stderr + assert again.returncode == 0, again_output + assert "up to date" in again_output.lower(), again_output + + def test_migrate_survives_an_applied_but_unrecorded_migration(self, cli_env): + """A replay of the vector migration must succeed, not block the deployment. + + The runner applies a migration and records it in `schema_history` as two + separate commits, so a crash in between leaves the schema applied and the + ledger short a row. That state is silent at first, because the catalog + version moved with the schema and the startup gate is satisfied. It bites + when the next catalog migration lands: the runner then walks every file, + finds this one unrecorded, and applies it a second time. + + This reproduces that second run. The version is set to a value that makes + the runner walk the list, with the vector schema still present and its + ledger row missing, which is exactly the state a crash leaves behind. + Without idempotent statements the replay fails on "relation already + exists" and no later migration can ever be applied. + """ + _init(cli_env) + assert _vector_table_exists(cli_env) is True + + conn = _catalog_conn(cli_env) + try: + conn.autocommit = True + with conn.cursor() as cur: + # Ledger row gone, schema left in place: the crash window. + cur.execute( + "DELETE FROM schema_history WHERE filename = %s", (VECTOR_MIGRATION,) + ) + # Force the runner to walk the file list, the way a later + # migration would. + cur.execute( + "UPDATE settings SET value = '0.0.2' WHERE key = 'catalog_version'" + ) + finally: + conn.close() + + replayed = _run_extenddb( + "migrate", "--yes", *_pg_args(), config=cli_env["config_path"], check=False + ) + output = replayed.stdout + replayed.stderr + assert replayed.returncode == 0, output + # The file was re-applied rather than skipped by the ledger, which is what + # makes this a replay and not a no-op. + assert f"Applying {VECTOR_MIGRATION}" in output, output + # PostgreSQL emits "already exists, skipping" notices here, which is the + # guards working. What must not appear is the runner's failure line. + assert f"Migration {VECTOR_MIGRATION} failed" not in output, output + + # The replay leaves the same end state, and the ledger is repaired. + assert _catalog_version(cli_env) == "0.0.3" + assert _vector_table_exists(cli_env) is True + assert _backups_has_vector_column(cli_env) is True + conn = _catalog_conn(cli_env) + try: + with conn.cursor() as cur: + cur.execute( + "SELECT COUNT(*) FROM schema_history WHERE filename = %s", + (VECTOR_MIGRATION,), + ) + assert cur.fetchone()[0] == 1 + finally: + conn.close() + + def test_serve_refuses_a_catalog_newer_than_the_binary(self, cli_env): + """The version gate must refuse a catalog from a future release too. + + The upgrade direction (new binary, old catalog) is covered above. This is + the other one, which matters during a rolling upgrade: a replica still + running the old build must refuse a catalog a newer replica has already + migrated, rather than serving against a schema it does not understand. + The check is an exact-equality comparison, so both directions come from + the same line, and pinning them both is what keeps that true. + + Simulated by moving the stored version forward, which needs no second + binary. + """ + _init(cli_env) + _patch_config_port(cli_env["config_path"], cli_env["port"]) + + conn = _catalog_conn(cli_env) + try: + conn.autocommit = True + with conn.cursor() as cur: + cur.execute( + "UPDATE settings SET value = '0.0.4' WHERE key = 'catalog_version'" + ) + finally: + conn.close() + + try: + refused = _run_extenddb( + "serve", "--foreground", + config=cli_env["config_path"], + check=False, + timeout=20, + ) + except subprocess.TimeoutExpired: + _run_extenddb("stop", config=cli_env["config_path"], check=False) + raise AssertionError( + "serve started against a 0.0.4 catalog; the version gate is not symmetric" + ) from None + combined = refused.stdout + refused.stderr + assert refused.returncode != 0, combined + assert "0.0.3" in combined and "0.0.4" in combined, combined From 78bee60939f0cff1ba807122d5791a7d0801e36b Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Mon, 24 Aug 2026 20:15:40 +0000 Subject: [PATCH 03/13] feat(postgres): vector data tables, write path, and SearchVectors Vector index data tables typed as pgvector vector(N) columns, with the pgvector crate for typed embedding binds. Maintenance runs at all six write sites through one entry point, and the capability is declared where the server can serve it, so PostgreSQL answers SearchVectors. The three design-outs are binding. Index membership comes from a fresh catalog read per write, never from cached key info, and that read also decides whether the write needs a transaction at all. Stored bytes that cannot enter an index are non-indexable rather than fatal. A write to a CREATING index always enqueues, at any delay, because the backfill holds an older snapshot of the same item and its plain INSERT would collide. Data migration 004 adds the hold that keeps the queue off a table whose index is building; the catalog cannot be joined from a claim transaction because it is a different database. --- .github/workflows/integration.yml | 22 +- Cargo.lock | 10 + Cargo.toml | 4 + SOFTWARE-LICENSE-NOTICES.html | 3 +- crates/core/src/types/mod.rs | 8 +- crates/core/src/types/table.rs | 17 + crates/storage-postgres/Cargo.toml | 1 + .../004_vector_index_state.sql | 40 ++ crates/storage-postgres/src/create_table.rs | 17 + .../storage-postgres/src/data/data_engine.rs | 17 + crates/storage-postgres/src/data/ddl.rs | 107 +++- .../storage-postgres/src/data/delete_item.rs | 72 ++- crates/storage-postgres/src/data/index.rs | 25 +- crates/storage-postgres/src/data/mod.rs | 18 + crates/storage-postgres/src/data/put_item.rs | 78 ++- .../storage-postgres/src/data/transactions.rs | 33 +- .../storage-postgres/src/data/update_item.rs | 37 +- .../storage-postgres/src/data/vector_index.rs | 340 ++++++++++ crates/storage-postgres/src/delete_table.rs | 20 +- crates/storage-postgres/src/gsi_queue.rs | 83 ++- crates/storage-postgres/src/lib.rs | 1 + crates/storage-postgres/src/migrations.rs | 4 + crates/storage-postgres/src/update_table.rs | 71 ++- crates/storage-postgres/src/vector_search.rs | 247 ++++++++ crates/storage-postgres/src/worker_store.rs | 19 +- .../tests/vector_control_plane.rs | 588 +++++++++++++++++- 26 files changed, 1794 insertions(+), 88 deletions(-) create mode 100644 crates/storage-postgres/data_migrations/004_vector_index_state.sql create mode 100644 crates/storage-postgres/src/data/vector_index.rs create mode 100644 crates/storage-postgres/src/vector_search.rs diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 02589c5c..443c7d6f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -13,7 +13,11 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: postgres:16 + # pgvector's own image, which is postgres:16 plus the extension. The + # backend serves vector indexes where the extension is present, so this is + # what makes the positive paths reachable in CI. The novector job below + # keeps the refusal paths covered on a plain image. + image: pgvector/pgvector:pg16 env: POSTGRES_PASSWORD: devpass options: >- @@ -182,7 +186,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: postgres:16 + image: pgvector/pgvector:pg16 env: POSTGRES_PASSWORD: devpass options: >- @@ -247,13 +251,13 @@ jobs: AWS_DEFAULT_REGION: us-east-1 EXTENDDB_ADMIN_USER: admin EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} - # PostgreSQL implements no vector search, so this is the job where the - # wire refusal tests must actually run. Both vector suites adapt to - # whatever the backend reports, so the refusal suite could skip every - # assertion here and still report green. "0" asserts the backend - # refuses vector indexes, making those tests mandatory rather than - # optional, and turns a silent skip into a failure. - EXTENDDB_EXPECT_VECTORS: "0" + # PostgreSQL now serves vector search against a server with pgvector, + # which this job's image has. Both vector suites adapt to whatever the + # backend reports, so without a pinned expectation the positive suite + # could skip every assertion and still report green. "1" makes those + # tests mandatory. The refusal suite moves to the novector job, which is + # the only place those assertions still mean anything. + EXTENDDB_EXPECT_VECTORS: "1" run: devtools/run-tests --extenddb --rust-integration --release # The control plane for vector indexes is not reachable over the wire while diff --git a/Cargo.lock b/Cargo.lock index 5cd97d10..28228c43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1326,6 +1326,7 @@ dependencies = [ "extenddb-core", "extenddb-storage", "futures", + "pgvector", "rand 0.9.4", "serde", "serde_json", @@ -2670,6 +2671,15 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "pgvector" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3673cba5b9a124916096a423b806a9f29620972c6c97b08db5f2053e9428b481" +dependencies = [ + "sqlx", +] + [[package]] name = "pin-project-lite" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index 157aae4f..0d0a6d6e 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,10 @@ moka = { version = "0.12", features = ["future"] } # Database sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "tls-rustls-aws-lc-rs", "postgres", "json", "time", "uuid", "bigdecimal", "derive"] } +# Vector column type for the PostgreSQL backend. The `sqlx` feature gives a typed +# `Vector` that encodes and decodes pgvector's binary format, so embeddings keep +# f32 bit-exactness instead of going through a hand-written text parser. +pgvector = { version = "0.4", default-features = false, features = ["sqlx"] } mongodb = "3" bson = "2.13" dashmap = "6" diff --git a/SOFTWARE-LICENSE-NOTICES.html b/SOFTWARE-LICENSE-NOTICES.html index 19118c95..066f9b4c 100644 --- a/SOFTWARE-LICENSE-NOTICES.html +++ b/SOFTWARE-LICENSE-NOTICES.html @@ -49,7 +49,7 @@

ExtendDB Software License Notices

Overview of licenses:

    -
  • Apache License 2.0 (211)
    • +
  • Apache License 2.0 (212)
  • MIT License (53)
  • Unicode License v3 (19)
  • ISC License (5)
    • @@ -486,6 +486,7 @@

    Apache License 2.0

    Used by:

    • encoding_rs 0.8.35
      • +
    • pgvector 0.4.2
    • tinyvec 1.11.0
    • utf8_iter 1.0.4
    • zeroize 1.8.2
      • diff --git a/crates/core/src/types/mod.rs b/crates/core/src/types/mod.rs index fb4e009b..f17aae79 100755 --- a/crates/core/src/types/mod.rs +++ b/crates/core/src/types/mod.rs @@ -63,10 +63,10 @@ pub use table::{ UpdateTableInput, UpdateTableOutput, UpdateTimeToLiveInput, UpdateTimeToLiveOutput, VECTOR_INDEX_ALREADY_EXISTS, VECTOR_INDEX_COUNT_LIMIT_CREATE, VECTOR_INDEX_COUNT_LIMIT_UPDATE, VECTOR_INDEX_CREATE_IN_USE_PREFIX, VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST, - VECTOR_SEARCH_SCHEMA_UNDECLARED, VectorAttribute, VectorIndexDescription, - VectorIndexSpecification, VectorIndexUpdate, vector_attribute_conflicting_definition, - vector_attribute_redefines_key, vector_attribute_redefines_vector, - vector_index_delete_in_allocation_phase, + VECTOR_SEARCH_SCHEMA_UNDECLARED, VECTOR_TABLE_REQUIRES_PAY_PER_REQUEST_MODE, VectorAttribute, + VectorIndexDescription, VectorIndexSpecification, VectorIndexUpdate, + vector_attribute_conflicting_definition, vector_attribute_redefines_key, + vector_attribute_redefines_vector, vector_index_delete_in_allocation_phase, }; pub use transaction::{ CancellationReason, ItemResponse, TransactConditionCheck, TransactDelete, TransactGet, diff --git a/crates/core/src/types/table.rs b/crates/core/src/types/table.rs index 1090afb5..b895b2fa 100755 --- a/crates/core/src/types/table.rs +++ b/crates/core/src/types/table.rs @@ -536,6 +536,23 @@ pub const MAX_VECTOR_INDEXES_PER_TABLE: usize = 5; pub const VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST: &str = "One or more parameter values were invalid: Vector indexes are only supported for \ PAY_PER_REQUEST tables"; +/// Message the service returns when an `UpdateTable` switches a table holding +/// vector indexes to `PROVISIONED` **and** carries `VectorIndexUpdates`. +/// +/// Measured 2026-08-19 (probe P14) on a switch combined with deleting the last +/// vector index, which the service refuses even though the resulting state would +/// carry no vector index at all: net-effect evaluation applies to a combined +/// switch and create, not to a combined switch and delete. Distinct text from +/// [`VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST`], which is what a plain switch +/// reports. +/// +/// The exact trigger is only partly mapped. Both shapes above are measured; which +/// string fires for a switch combined with a vector index create is not, so a +/// backend should emit this one for the measured shape and the plain rule +/// elsewhere rather than guessing at the boundary. +pub const VECTOR_TABLE_REQUIRES_PAY_PER_REQUEST_MODE: &str = "One or more parameter values were invalid: Tables with vector indexes must be in \ + PAY_PER_REQUEST mode"; + /// Per-table vector index limit exceeded on `CreateTable`. /// /// The create and update paths differ in BOTH class and text for this one rule, diff --git a/crates/storage-postgres/Cargo.toml b/crates/storage-postgres/Cargo.toml index dce75514..316f9333 100755 --- a/crates/storage-postgres/Cargo.toml +++ b/crates/storage-postgres/Cargo.toml @@ -19,6 +19,7 @@ extenddb-storage = { workspace = true } extenddb-auth = { workspace = true } futures = { workspace = true } sqlx = { workspace = true } +pgvector = { workspace = true } tokio = { workspace = true, features = ["sync"] } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/storage-postgres/data_migrations/004_vector_index_state.sql b/crates/storage-postgres/data_migrations/004_vector_index_state.sql new file mode 100644 index 00000000..52aa0706 --- /dev/null +++ b/crates/storage-postgres/data_migrations/004_vector_index_state.sql @@ -0,0 +1,40 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- Vector index build state that the propagation queue has to read. +-- +-- The queue must not apply a write to an index whose backfill is still running: +-- the backfill holds an older snapshot of the same item, so applying the newer +-- write first lets the backfill overwrite it, and the backfill's deliberately +-- plain INSERT would collide with the row the write left behind. +-- +-- The SQLite backend answers this by joining its claim query against the catalog's +-- vector_indexes table. That is impossible here, because the catalog is a separate +-- database from the data one and a claim transaction cannot span them. So the +-- claim-time fact lives in the data database as its own row. +-- +-- Ordering rules, which are what make it safe rather than merely present: +-- * the hold row is inserted BEFORE the catalog's CREATING row commits, so no +-- writer can enqueue against an index the queue does not yet know to hold; +-- * the hold row is deleted AFTER the catalog flips the index to ACTIVE, so the +-- queue never resumes against an index that is not yet published. +-- Held slightly too long is harmless: the rows wait. Released early is not. +-- +-- Per TABLE, not per index, matching the shared lifecycle contract: a secondary +-- index row and a vector row for the same item must keep their relative order, so +-- the hold stops the table's queue rather than one index's. + +BEGIN; + +CREATE TABLE IF NOT EXISTS vector_index_holds ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (table_id, index_id) +); + +-- The claim query filters on table_id, so that is what needs to be fast. A +-- crashed build leaves an orphan row, which the reconciler sweeps by age. +CREATE INDEX IF NOT EXISTS idx_vector_index_holds_table + ON vector_index_holds (table_id); + +COMMIT; diff --git a/crates/storage-postgres/src/create_table.rs b/crates/storage-postgres/src/create_table.rs index 53efd443..f640ea0a 100755 --- a/crates/storage-postgres/src/create_table.rs +++ b/crates/storage-postgres/src/create_table.rs @@ -193,6 +193,9 @@ impl PostgresEngine { } } + // Collected because each id names a data table, created after the catalog + // commit in the same order the rows were written. + let mut vector_index_ids: Vec = Vec::new(); // Insert vector index metadata. A CreateTable's table is empty, so there // is nothing to backfill: the index goes straight to ACTIVE with no // `backfilling` member, which is the state the service reports for an @@ -273,6 +276,7 @@ impl PostgresEngine { .execute(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + vector_index_ids.push(index_id); } } @@ -362,6 +366,19 @@ impl PostgresEngine { } } + if let Some(vis) = &input.vector_indexes { + for (i, vi) in vis.iter().enumerate() { + Self::create_vector_data_table( + &mut data_tx, + &vector_index_ids[i], + vi.dimensions, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } + data_tx .commit() .await diff --git a/crates/storage-postgres/src/data/data_engine.rs b/crates/storage-postgres/src/data/data_engine.rs index aea47ff7..fd7194b6 100755 --- a/crates/storage-postgres/src/data/data_engine.rs +++ b/crates/storage-postgres/src/data/data_engine.rs @@ -13,6 +13,23 @@ use futures::future::BoxFuture; use crate::PostgresEngine; impl DataEngine for PostgresEngine { + /// Declares vector support only where the server can actually serve it. + /// + /// `Some(self)` compiles only because `PostgresEngine` implements + /// `VectorSearchEngine`, so this cannot claim a capability that does not + /// exist in the build. The runtime half is the probe: vector storage needs + /// pgvector, which is a property of the PostgreSQL server rather than of this + /// binary, so the same build serves vector indexes against a server that has + /// the extension and refuses them, byte-identically to before, against one + /// that does not. + /// + /// The answer is the startup probe's, cached, so a request pays nothing for + /// it. Installing pgvector on a running server therefore needs a restart to + /// be noticed, which the admin guide says. + fn as_vector_search(&self) -> Option<&dyn extenddb_storage::VectorSearchEngine> { + self.vector_capable.then_some(self) + } + fn put_item( &self, key_info: &TableKeyInfo, diff --git a/crates/storage-postgres/src/data/ddl.rs b/crates/storage-postgres/src/data/ddl.rs index c9952d13..d17e684d 100755 --- a/crates/storage-postgres/src/data/ddl.rs +++ b/crates/storage-postgres/src/data/ddl.rs @@ -10,7 +10,7 @@ use extenddb_core::types::{ use extenddb_storage::error::StorageError; use extenddb_storage::util::{sk_column, sk_column_n}; -use super::{all_sort_key_info, data_table_name, index_table_name}; +use super::{all_sort_key_info, data_table_name, index_table_name, vector_table_name}; use crate::PostgresEngine; /// Row shape returned by the table-info query: (`key_schema`, `attr_defs`, status, `table_id`, `stream_spec`). @@ -253,6 +253,111 @@ impl PostgresEngine { Ok(()) } + /// Create the data table for one vector index. + /// + /// Column set is the SQLite backend's, with PostgreSQL types: `embedding` is + /// pgvector's own `vector(N)` rather than a blob, so distance, ordering and + /// the row limit all evaluate in the server and the wire carries k rows + /// instead of the whole candidate set. + /// + /// Keyed by the base item rather than by the partition, so one base item + /// yields at most one vector row and a re-put replaces it. A partition move + /// is therefore a delete followed by an insert, which is what the apply path + /// does unconditionally. + /// + /// # Errors + /// + /// Returns [`StorageError::Unsupported`] when the server has no pgvector, so + /// a lost extension reports as a refusal rather than a fault. + pub(crate) async fn create_vector_data_table( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + index_id: &str, + dimensions: u32, + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(index_id); + let base_sks = all_sort_key_info(base_key_schema, base_attr_defs); + + let mut col_defs = vec![ + // `pk_to_text` of the HASH element's value, or the unscoped sentinel. + // Scoping happens on this column, never on the payload. + // + // BYTEA rather than TEXT, and not by preference: the shared unscoped + // sentinel begins with a NUL byte, deliberately, because no partition + // derived from item data can contain one. PostgreSQL cannot store a NUL + // in a text column at all, so a text column here rejects every row of + // every unscoped index. Bytes keep the shared value byte-identical + // across backends, which is what makes a write and a search agree. + "part BYTEA NOT NULL".to_owned(), + "base_pk TEXT NOT NULL".to_owned(), + ]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + let _ = sk_type; + if i == 0 { + col_defs.push("base_sk_s TEXT".to_owned()); + col_defs.push("base_sk_n NUMERIC".to_owned()); + col_defs.push("base_sk_b BYTEA".to_owned()); + } else { + let n = i + 1; + col_defs.push(format!("base_sk{n}_s TEXT")); + col_defs.push(format!("base_sk{n}_n NUMERIC")); + col_defs.push(format!("base_sk{n}_b BYTEA")); + } + } + // NOT NULL because a non-indexable item is never inserted: the apply path + // deletes instead, so there is no such thing as a row without a vector. + col_defs.push(format!("embedding vector({dimensions}) NOT NULL")); + // The L2 norm at write time, kept even though pgvector computes cosine + // itself: it is what lets the cosine expression return the measured 1.0 + // for a zero vector, where pgvector's operator yields NaN. + col_defs.push("nrm DOUBLE PRECISION NOT NULL".to_owned()); + col_defs.push("item_data JSONB NOT NULL".to_owned()); + + let mut pk_cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + pk_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + + let ddl = format!( + "CREATE TABLE {vec_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ); + sqlx::query(&ddl) + .execute(&mut **tx) + .await + .map_err(crate::vector::map_vector_sql_error)?; + + // Every search is partition-scoped, including an unscoped index's search + // against the sentinel, so this index is what keeps one off a full scan. + // Left unnamed, as the GSI tables' ordering index is: a name built from the + // ids would be truncated at PostgreSQL's 63-byte identifier limit. + let part_idx = format!("CREATE INDEX ON {vec_table} (part)"); + sqlx::query(&part_idx) + .execute(&mut **tx) + .await + .map_err(crate::vector::map_vector_sql_error)?; + + // No ANN index yet: exact scan first, per ADR-0004 and the plan. Adding + // HNSW later is a CREATE INDEX against this same column with no table + // rewrite, which is why the column type is `vector(N)` from the start. + Ok(()) + } + + /// Drop one vector index's data table. + pub(crate) async fn drop_vector_data_table( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + index_id: &str, + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(index_id); + sqlx::query(&format!("DROP TABLE IF EXISTS {vec_table}")) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + /// Drop a GSI/LSI data table. pub(crate) async fn drop_index_data_table( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, diff --git a/crates/storage-postgres/src/data/delete_item.rs b/crates/storage-postgres/src/data/delete_item.rs index 7e53254a..8df59c58 100755 --- a/crates/storage-postgres/src/data/delete_item.rs +++ b/crates/storage-postgres/src/data/delete_item.rs @@ -36,13 +36,37 @@ impl PostgresEngine { // Fetch indexes for GSI/LSI updates (D-4: sync + async split). let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + // Vector indexes come from the same fresh read rather than from the cached + // key info, and the answer decides two things: whether this write needs a + // transaction at all, and what maintenance runs inside it. A cached empty + // set would send a write down the no-maintenance fast path and silently + // leave an index missing a row. + // + // What this does and does not remove. The defect being designed out is the + // cached membership gate, and that is gone: an index takes effect the moment + // its catalog row commits. What remains is a window between this read and + // the data transaction's commit, in which an index created concurrently is + // missed. That window cannot be closed here, because the catalog and the + // data tables are different databases and no transaction spans them. It is + // also exactly the window the secondary indexes have, for the same reason + // and with the same read: parity with a GSI is the bar, and the backfill + // that publishes a new index is what covers writes older than it. + let vector_metas = crate::data::vector_index::fetch_vector_indexes_for_table( + &self.pool, + &key_info.table_id, + ) + .await?; let sys_delay = if indexes.is_empty() { 0 } else { self.index_propagation_delay().await }; - let needs_tx = condition.is_some() || return_old || !indexes.is_empty() || stream.is_some(); + let needs_tx = condition.is_some() + || return_old + || !indexes.is_empty() + || !vector_metas.is_empty() + || stream.is_some(); if let Some((sk_name, sk_type)) = sk_info(&key_info.key_schema, &key_info.attribute_definitions) @@ -155,7 +179,7 @@ impl PostgresEngine { } // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. - let async_enqueued = enqueue_async_indexes( + let mut async_enqueued = enqueue_async_indexes( &mut tx, key_info, &indexes, @@ -165,6 +189,27 @@ impl PostgresEngine { ) .await?; + // The delete has no new image, so removing the indexed row is the + // whole of the vector work. Read fresh, never from the cache. + let old_for_vectors = match old_item_for_idx { + Some(ref oi) => Some(oi.clone()), + None => old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?, + }; + async_enqueued += crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &vector_metas, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old_for_vectors.as_ref(), + None, + sys_delay, + ) + .await?; + tx.commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -292,7 +337,7 @@ impl PostgresEngine { } // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. - let async_enqueued = enqueue_async_indexes( + let mut async_enqueued = enqueue_async_indexes( &mut tx, key_info, &indexes, @@ -302,6 +347,27 @@ impl PostgresEngine { ) .await?; + // The delete has no new image, so removing the indexed row is the + // whole of the vector work. Read fresh, never from the cache. + let old_for_vectors = match old_item_for_idx { + Some(ref oi) => Some(oi.clone()), + None => old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?, + }; + async_enqueued += crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &vector_metas, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old_for_vectors.as_ref(), + None, + sys_delay, + ) + .await?; + tx.commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-postgres/src/data/index.rs b/crates/storage-postgres/src/data/index.rs index dc7a2133..a9c29582 100644 --- a/crates/storage-postgres/src/data/index.rs +++ b/crates/storage-postgres/src/data/index.rs @@ -225,12 +225,35 @@ pub(crate) async fn enqueue_async_indexes( projection: idx.projection.clone(), }, }; - enqueue_gsi_pending(tx, &key_info.table_id, old_item, new_item, delay, &context).await?; + enqueue_gsi_pending( + tx, + &key_info.table_id, + old_item, + new_item, + delay, + &crate::gsi_queue::PendingApplyContext::Gsi(context), + ) + .await?; enqueued += 1; } Ok(enqueued) } +/// Enqueue one propagation row for any index kind. +/// +/// Thin wrapper so the vector maintenance path does not have to reach into the +/// queue module's naming, which still says "gsi" for historical reasons. +pub(crate) async fn enqueue_pending_row( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + table_id: &str, + old_item: Option<&Item>, + new_item: Option<&Item>, + delay_ms: u64, + context: &crate::gsi_queue::PendingApplyContext, +) -> Result<(), StorageError> { + enqueue_gsi_pending(tx, table_id, old_item, new_item, delay_ms, context).await +} + /// Delete a row from an index table using base table key columns. pub(crate) async fn delete_index_row_multi( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, diff --git a/crates/storage-postgres/src/data/mod.rs b/crates/storage-postgres/src/data/mod.rs index 90de0b21..1c8b5f8b 100755 --- a/crates/storage-postgres/src/data/mod.rs +++ b/crates/storage-postgres/src/data/mod.rs @@ -25,6 +25,23 @@ pub(crate) fn index_table_name(index_id: &str) -> String { format!("\"_ddb_{index_id}\"") } +/// SQL table name for a vector index's data table. +/// +/// Named from the index id alone, like a GSI's, and deliberately not from both +/// ids. PostgreSQL truncates identifiers at 63 bytes, and two UUIDs plus a prefix +/// is 82: the longer form was silently cut, which made two indexes on one table +/// collide on 17 surviving characters of their ids. +/// +/// The reason to want the table id in the name was to find these tables after +/// DeleteTable cascades the catalog rows away. Both delete paths instead collect +/// the ids before deleting the table row, which is what the GSI path already does +/// for the same reason. +/// +/// The id is a server-generated UUID, so no client input reaches this identifier. +pub(crate) fn vector_table_name(index_id: &str) -> String { + format!("\"_ddb_vec_{index_id}\"") +} + /// Look up all RANGE key attribute definitions from the key schema (preserving order). pub(crate) fn all_sort_key_info<'a>( key_schema: &'a [KeySchemaElement], @@ -122,6 +139,7 @@ mod query_scan; mod transactions; mod tx_helpers; mod update_item; +pub(crate) mod vector_index; pub(crate) use index::{ delete_index_row_multi, insert_index_row_multi, item_has_index_keys, project_item_for_index, diff --git a/crates/storage-postgres/src/data/put_item.rs b/crates/storage-postgres/src/data/put_item.rs index 9ca9d985..3ab0be90 100755 --- a/crates/storage-postgres/src/data/put_item.rs +++ b/crates/storage-postgres/src/data/put_item.rs @@ -35,6 +35,26 @@ impl PostgresEngine { // Fetch indexes for GSI/LSI updates (D-4: sync + async split). let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + // Vector indexes come from the same fresh read rather than from the cached + // key info, and the answer decides two things: whether this write needs a + // transaction at all, and what maintenance runs inside it. A cached empty + // set would send a write down the no-maintenance fast path and silently + // leave an index missing a row. + // + // What this does and does not remove. The defect being designed out is the + // cached membership gate, and that is gone: an index takes effect the moment + // its catalog row commits. What remains is a window between this read and + // the data transaction's commit, in which an index created concurrently is + // missed. That window cannot be closed here, because the catalog and the + // data tables are different databases and no transaction spans them. It is + // also exactly the window the secondary indexes have, for the same reason + // and with the same read: parity with a GSI is the bar, and the backfill + // that publishes a new index is what covers writes older than it. + let vector_metas = crate::data::vector_index::fetch_vector_indexes_for_table( + &self.pool, + &key_info.table_id, + ) + .await?; // Index key attributes present in the item must match their declared // scalar type and be non-empty, matching real DynamoDB. This is up-front @@ -63,7 +83,11 @@ impl PostgresEngine { }; // When there's a condition, return_old, indexes, or stream capture, we need a transaction - let needs_tx = condition.is_some() || return_old || !indexes.is_empty() || stream.is_some(); + let needs_tx = condition.is_some() + || return_old + || !indexes.is_empty() + || !vector_metas.is_empty() + || stream.is_some(); if let Some((sk_name, sk_type)) = sk_info(&key_info.key_schema, &key_info.attribute_definitions) @@ -167,7 +191,7 @@ impl PostgresEngine { } // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. - let async_enqueued = enqueue_async_indexes( + let mut async_enqueued = enqueue_async_indexes( &mut tx, key_info, &indexes, @@ -177,6 +201,30 @@ impl PostgresEngine { ) .await?; + // Vector indexes, read fresh from the catalog rather than from the + // cached key info, so a write cannot miss an index that was just + // created. The old image is needed even when no secondary index + // wanted it: the vector row is keyed by the base item, so a + // partition move is a delete of the previous row. + let old_for_vectors = match old_item_for_idx { + Some(ref oi) => Some(oi.clone()), + None => old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?, + }; + async_enqueued += crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &vector_metas, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old_for_vectors.as_ref(), + Some(&item), + sys_delay, + ) + .await?; + tx.commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -311,7 +359,7 @@ impl PostgresEngine { } // Persist async GSI work inside the same transaction — one row // per async index, each honoring its own propagation delay. - let async_enqueued = enqueue_async_indexes( + let mut async_enqueued = enqueue_async_indexes( &mut tx, key_info, &indexes, @@ -321,6 +369,30 @@ impl PostgresEngine { ) .await?; + // Vector indexes, read fresh from the catalog rather than from the + // cached key info, so a write cannot miss an index that was just + // created. The old image is needed even when no secondary index + // wanted it: the vector row is keyed by the base item, so a + // partition move is a delete of the previous row. + let old_for_vectors = match old_item_for_idx { + Some(ref oi) => Some(oi.clone()), + None => old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?, + }; + async_enqueued += crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &vector_metas, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old_for_vectors.as_ref(), + Some(&item), + sys_delay, + ) + .await?; + tx.commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index 927e63fc..5d39d385 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -73,13 +73,27 @@ impl PostgresEngine { idempotency: Option>, ) -> Result<(), StorageError> { // Pre-fetch indexes for each unique table involved in the transaction. + // + // Vector indexes are read here too, once per table rather than once per op, + // and from the catalog rather than from the cached key info: a cached empty + // set would make a transaction skip an index another request has just + // created. One read per table also keeps a multi-op transaction from + // re-asking for the same answer. let mut table_indexes: HashMap> = HashMap::new(); + let mut table_vector_metas: HashMap< + String, + Vec<(extenddb_storage::vector_lifecycle::VectorIndexMeta, String)>, + > = HashMap::new(); for op in ops { let name = transact_op_table_name(op); if !table_indexes.contains_key(name) { let tid = transact_op_table_id(op); let indexes = fetch_indexes_for_table(tid, &self.pool).await?; table_indexes.insert(name.to_owned(), indexes); + let vector_metas = + crate::data::vector_index::fetch_vector_indexes_for_table(&self.pool, tid) + .await?; + table_vector_metas.insert(name.to_owned(), vector_metas); } } @@ -192,7 +206,24 @@ impl PostgresEngine { sys_delay, ) .await?; - if n > 0 { + + // Vector maintenance for all three write kinds in one place, rather + // than in each branch above: this loop already visits exactly the ops + // that changed an item, with both images in hand, and it runs inside + // the same transaction. Three call sites would have been three chances + // to diverge on which image is passed. + let vector_n = crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &table_vector_metas[transact_op_table_name(op)], + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old_item.as_ref(), + new_item.as_ref(), + sys_delay, + ) + .await?; + if n > 0 || vector_n > 0 { needs_notify = true; } } diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index 470b8d99..a5d9c2ed 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -47,6 +47,26 @@ impl PostgresEngine { // Fetch indexes for GSI/LSI updates (D-4: sync + async split). let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + // Vector indexes come from the same fresh read rather than from the cached + // key info, and the answer decides two things: whether this write needs a + // transaction at all, and what maintenance runs inside it. A cached empty + // set would send a write down the no-maintenance fast path and silently + // leave an index missing a row. + // + // What this does and does not remove. The defect being designed out is the + // cached membership gate, and that is gone: an index takes effect the moment + // its catalog row commits. What remains is a window between this read and + // the data transaction's commit, in which an index created concurrently is + // missed. That window cannot be closed here, because the catalog and the + // data tables are different databases and no transaction spans them. It is + // also exactly the window the secondary indexes have, for the same reason + // and with the same read: parity with a GSI is the bar, and the backfill + // that publishes a new index is what covers writes older than it. + let vector_metas = crate::data::vector_index::fetch_vector_indexes_for_table( + &self.pool, + &key_info.table_id, + ) + .await?; let sys_delay = if indexes.is_empty() { 0 } else { @@ -262,7 +282,7 @@ impl PostgresEngine { } // Persist async GSI work inside the same transaction — one row per // async index, each honoring its own propagation delay. - let async_enqueued = enqueue_async_indexes( + let mut async_enqueued = enqueue_async_indexes( &mut tx, key_info, &indexes, @@ -272,6 +292,21 @@ impl PostgresEngine { ) .await?; + // The pre-mutation image is already in hand here, and it is what lets the + // vector row move partition or disappear when the update removes the + // vector attribute. + async_enqueued += crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &vector_metas, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + pre_mutation_item.as_ref(), + Some(&item), + sys_delay, + ) + .await?; + tx.commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-postgres/src/data/vector_index.rs b/crates/storage-postgres/src/data/vector_index.rs new file mode 100644 index 00000000..e13ad121 --- /dev/null +++ b/crates/storage-postgres/src/data/vector_index.rs @@ -0,0 +1,340 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Vector index maintenance on the write path. +//! +//! One entry point, [`maintain_vector_indexes`], called from every site that +//! writes a base item. It reads the index set, decides per index whether the +//! change applies inline or goes on the propagation queue, and returns how many +//! rows it enqueued so the caller knows whether to wake a worker. +//! +//! Three properties are deliberate here, each one a defect the SQLite +//! implementation has and this one must not inherit: +//! +//! 1. **Membership is never read from the cached `TableKeyInfo`.** A cached empty +//! set makes a write skip an index that another request has just created. The +//! metadata is read per write from the catalog, in the same round trip that +//! already reads the secondary indexes, so a new index takes effect the moment +//! its catalog row commits. Note what \"same round trip\" can and cannot mean +//! here: `vector_indexes` lives in the catalog database and the write runs on +//! the data database, so no single transaction spans them. Freshness parity +//! with a GSI is the achievable property, and it is the one that matters. +//! 2. **A malformed or wrong-dimension stored vector is non-indexable, not an +//! error.** Such an item cannot have passed live validation, so it arrived +//! before the index existed and was skipped by the backfill. Failing the write +//! would make an unrelated update to that item impossible; the row is removed +//! from the index and the write proceeds. +//! 3. **A write to a CREATING index always enqueues**, at any delay including +//! zero. Applying inline would race the backfill's older snapshot of the same +//! item, and the backfill's plain INSERT would then collide. Only an ACTIVE +//! index at delay zero is applied inline. + +use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement, ScalarAttributeType}; +use extenddb_core::validation::vector_item::{vector_components, vector_norm}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{ + SortKeyValue, composite_pk_to_text, parse_sk, sk_column, sk_column_n, +}; +use extenddb_storage::vector_lifecycle::{ + VectorApplyContext, VectorIndexMeta, item_is_indexable, item_partition, projected_payload, +}; +use pgvector::Vector; + +use super::{all_sort_key_info, vector_table_name}; + +/// Read the vector index metadata for a table from the catalog. +/// +/// Every field the write path needs, including the index id that names the data +/// table, which is exactly what the cached `TableKeyInfo` cannot supply. +/// +/// Returns the rows with their status, because the status decides inline versus +/// enqueue and only this read can see it. +pub(crate) async fn fetch_vector_indexes_for_table( + catalog: &sqlx::PgPool, + table_id: &str, +) -> Result, StorageError> { + let rows: Vec<( + String, + i32, + serde_json::Value, + Option, + serde_json::Value, + String, + )> = sqlx::query_as( + "SELECT index_id, dimensions, vector_attribute, search_schema, projection, index_status \ + FROM vector_indexes WHERE table_id = $1 ORDER BY index_name", + ) + .bind(table_id) + .fetch_all(catalog) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut out = Vec::with_capacity(rows.len()); + for (index_id, dimensions, vector_attribute, search_schema, projection, index_status) in rows { + let attr: extenddb_core::types::VectorAttribute = serde_json::from_value(vector_attribute) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; + let search_schema: Vec = match search_schema { + Some(value) => serde_json::from_value(value) + .map_err(|e| StorageError::Internal(format!("vector search_schema: {e}")))?, + None => Vec::new(), + }; + let projection: extenddb_core::types::Projection = serde_json::from_value(projection) + .map_err(|e| StorageError::Internal(format!("vector projection: {e}")))?; + let hash_attribute_name = search_schema + .iter() + .find(|e| e.element_type == extenddb_core::types::SearchSchemaElementType::Hash) + .map(|e| e.attribute_name.clone()); + out.push(( + VectorIndexMeta { + index_id, + dimensions: usize::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?, + vector_attribute_name: attr.attribute_name, + projection, + hash_attribute_name, + search_schema_attribute_names: search_schema + .iter() + .map(|e| e.attribute_name.clone()) + .collect(), + }, + index_status, + )); + } + Ok(out) +} + +/// The base table's key columns for a vector data table, in insert order. +fn base_key_columns(base_sks: &[(&str, ScalarAttributeType)]) -> Vec { + let mut cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + let col = if i == 0 { + format!("base_{}", sk_column(sk_type)) + } else { + format!("base_{}", sk_column_n(i, sk_type)) + }; + cols.push(col); + } + cols +} + +/// Maintain every vector index on a table for one base-item change. +/// +/// `old_item` and `new_item` are the before and after images: a put has both when +/// it replaces, a delete has only `old_item`, and either may be absent. +/// +/// `metas` is the caller's own fresh read, passed in rather than fetched here: the +/// same read also decides whether the write needs a transaction at all, and doing +/// it twice would be two chances to disagree. +/// +/// Returns the number of rows enqueued, so the caller can wake the queue only when +/// there is something to drain. +// The parameters mirror the write sites' own locals: the transaction, the catalog +// pool the metadata comes from, the table's identity and key shape, the two images, +// and the delay. A wrapper struct would have to be built at all six call sites from +// exactly these values. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn maintain_vector_indexes( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + metas: &[(VectorIndexMeta, String)], + table_id: &str, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + old_item: Option<&Item>, + new_item: Option<&Item>, + delay_ms: u64, +) -> Result { + if metas.is_empty() { + return Ok(0); + } + + let mut enqueued = 0usize; + for (meta, index_status) in metas { + // A CREATING index never takes an inline write, whatever the delay. The + // backfill is scanning the base table and holds an older snapshot of this + // same item; writing the new one now would let the backfill overwrite it, + // and its deliberately plain INSERT would collide with the row this put + // there. The queue hold parks the row until the index is published. + let inline = delay_ms == 0 && index_status == "ACTIVE"; + if inline { + apply_vector_index(tx, meta, base_key_schema, attr_defs, old_item, new_item).await?; + continue; + } + // Enqueued even when the new image carries no vector: the removal is the + // work in that case, and skipping it would leave a stale row indexed. + let context = crate::gsi_queue::PendingApplyContext::Vector(VectorApplyContext { + base_key_schema: base_key_schema.to_vec(), + attribute_definitions: attr_defs.to_vec(), + table_id: table_id.to_owned(), + vector: meta.clone(), + }); + super::index::enqueue_pending_row(tx, table_id, old_item, new_item, delay_ms, &context) + .await?; + enqueued += 1; + } + Ok(enqueued) +} + +/// Apply one base-item change to one vector index. +/// +/// Delete then insert, unconditionally, because the partition column is part of +/// the row and a changed HASH value moves the row rather than updating it in +/// place. The delete also makes the insert below a plain one. +pub(crate) async fn apply_vector_index( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + meta: &VectorIndexMeta, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + let vec_table = vector_table_name(&meta.index_id); + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let key_cols = base_key_columns(&base_sks); + + if let Some(source) = old_item.or(new_item) { + let where_clause = key_cols + .iter() + .enumerate() + .map(|(i, c)| format!("{c} = ${}", i + 1)) + .collect::>() + .join(" AND "); + let sql = format!("DELETE FROM {vec_table} WHERE {where_clause}"); + let mut query = sqlx::query(&sql).bind(composite_pk_to_text(source, base_key_schema)?); + for &(sk_name, sk_type) in &base_sks { + if let Some(value) = source.get(sk_name) { + query = match parse_sk(value, sk_type)? { + SortKeyValue::S(s) => query.bind(s), + SortKeyValue::N(n) => query.bind(n), + SortKeyValue::B(b) => query.bind(b), + }; + } + } + query + .execute(&mut **tx) + .await + .map_err(crate::vector::map_vector_sql_error)?; + } + + let Some(new_item) = new_item else { + // A delete: the removal above is the whole of the work. + return Ok(()); + }; + insert_vector_row(tx, meta, new_item, base_key_schema, attr_defs).await +} + +/// Write one item's row into one vector index. +/// +/// A non-indexable item is a no-op, which is what lets an index exist on a table +/// where only some items carry the vector. +/// +/// Stored bytes that cannot enter the index are also non-indexable rather than an +/// error, and that is the difference from the SQLite implementation. Live writes +/// are validated by core before they reach storage, so a malformed or +/// wrong-dimension vector here belongs to an item written before the index +/// existed, which the backfill skipped and counted. Failing would make every +/// later update to that item fail too, including an update that has nothing to do +/// with the vector. +pub(crate) async fn insert_vector_row( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + meta: &VectorIndexMeta, + item: &Item, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Result<(), StorageError> { + if !item_is_indexable(item, meta) { + return Ok(()); + } + let Some(value) = item.get(&meta.vector_attribute_name) else { + return Ok(()); + }; + let Some(components) = vector_components(value) else { + tracing::warn!( + index_id = %meta.index_id, + attribute = %meta.vector_attribute_name, + "stored vector attribute cannot be read as a vector; leaving the item unindexed" + ); + return Ok(()); + }; + if components.len() != meta.dimensions { + tracing::warn!( + index_id = %meta.index_id, + found = components.len(), + declared = meta.dimensions, + "stored vector has the wrong dimension count; leaving the item unindexed" + ); + return Ok(()); + } + + let vec_table = vector_table_name(&meta.index_id); + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let key_cols = base_key_columns(&base_sks); + let part = item_partition(item, meta)?; + let norm = vector_norm(&components); + // Projection, the always-projected SearchSchema attributes, and the stripped + // vector attribute are the shared payload rules, so a live-written row and a + // backfilled one cannot differ in shape. + let projected = projected_payload(item, base_key_schema, meta); + let item_json = + serde_json::to_value(&projected).map_err(|e| StorageError::Internal(e.to_string()))?; + + // A plain INSERT, deliberately, where the GSI sibling upserts. Every caller + // reaches this through `apply_vector_index`, which deletes the base key's row + // first, so no live row can exist here. Keeping it plain means that if a + // future change ever makes that delete conditional, this fails loudly on the + // primary key rather than quietly replacing a row and hiding the break. + let mut cols = vec!["part".to_owned()]; + cols.extend(key_cols.iter().cloned()); + cols.extend([ + "embedding".to_owned(), + "nrm".to_owned(), + "item_data".to_owned(), + ]); + let placeholders: Vec = (1..=cols.len()).map(|i| format!("${i}")).collect(); + let sql = format!( + "INSERT INTO {vec_table} ({}) VALUES ({})", + cols.join(", "), + placeholders.join(", ") + ); + + // As bytes: the column is BYTEA because the unscoped sentinel carries a NUL, + // which PostgreSQL rejects in a text column. + let mut query = sqlx::query(&sql) + .bind(part.into_bytes()) + .bind(composite_pk_to_text(item, base_key_schema)?); + for &(sk_name, sk_type) in &base_sks { + if let Some(value) = item.get(sk_name) { + query = match parse_sk(value, sk_type)? { + SortKeyValue::S(s) => query.bind(s), + SortKeyValue::N(n) => query.bind(n), + SortKeyValue::B(b) => query.bind(b), + }; + } + } + query + .bind(Vector::from(components)) + .bind(f64::from(norm)) + .bind(item_json) + .execute(&mut **tx) + .await + .map_err(crate::vector::map_vector_sql_error)?; + Ok(()) +} + +/// Apply one claimed queue row to its vector index, from the row's own context. +pub(crate) async fn apply_claimed_vector_row( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + context: &VectorApplyContext, + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + apply_vector_index( + tx, + &context.vector, + &context.base_key_schema, + &context.attribute_definitions, + old_item, + new_item, + ) + .await +} diff --git a/crates/storage-postgres/src/delete_table.rs b/crates/storage-postgres/src/delete_table.rs index 3ca95fd3..b53576f8 100755 --- a/crates/storage-postgres/src/delete_table.rs +++ b/crates/storage-postgres/src/delete_table.rs @@ -58,6 +58,17 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // Vector index ids, read while the rows still exist. Deleting the table + // row cascades them away, and each id names a data table that has to be + // dropped afterwards: the same ordering the GSI path uses, and for the same + // reason. + let vector_index_ids: Vec = + sqlx::query_scalar("SELECT index_id FROM vector_indexes WHERE table_id = $1") + .bind(&row.table_id) + .fetch_all(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + // H-5 (delete): synchronous control-plane shortcut when delay < 1s. // When control_plane_delay_seconds is small, the poller may not run // before the next request, causing stale DELETING rows. Synchronous @@ -77,9 +88,9 @@ impl PostgresEngine { // Note: deleting the tables row cascades to indexes and stream rows via FK CASCADE. // Vector index rows cascade the same way, through // vector_indexes_table_id_fkey, so a table with vector indexes needs - // no extra catalog cleanup. There are no vector data tables to drop - // yet: this backend records vector index metadata but does not build - // their storage. + // no extra catalog cleanup. Their data tables do need dropping, and + // by then the rows that named them are gone, which is why the sweep + // matches on the table id prefix instead of reading the catalog. sqlx::query("DELETE FROM tags WHERE resource_arn = $1") .bind(&row.table_arn) .execute(&mut *tx) @@ -105,6 +116,9 @@ impl PostgresEngine { for idx_id in &index_ids { Self::drop_index_data_table(&mut data_tx, idx_id).await?; } + for index_id in &vector_index_ids { + Self::drop_vector_data_table(&mut data_tx, index_id).await?; + } Self::drop_data_table(&mut data_tx, &row.table_id).await?; // Drop any still-pending GSI propagation rows for this table in the diff --git a/crates/storage-postgres/src/gsi_queue.rs b/crates/storage-postgres/src/gsi_queue.rs index 3a701f4b..a69489d4 100644 --- a/crates/storage-postgres/src/gsi_queue.rs +++ b/crates/storage-postgres/src/gsi_queue.rs @@ -107,6 +107,35 @@ pub(crate) struct GsiApplyContext { pub(crate) index: GsiIndexDef, } +/// What a claimed row describes: a secondary index update or a vector one. +/// +/// Untagged, so a row written before vector indexes existed still deserializes as +/// `Gsi`. That compatibility is load-bearing on this backend rather than +/// theoretical: the queue is a table, so rows written by the previous version are +/// still there across an upgrade. +/// +/// The variants are distinguished by shape, and `Gsi` is tried first. A payload +/// carrying both an `index` and a `vector` member would match `Gsi`, which no +/// serializer here produces; a genuinely unreadable context is already handled as +/// a poison row. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum PendingApplyContext { + Gsi(GsiApplyContext), + Vector(extenddb_storage::vector_lifecycle::VectorApplyContext), +} + +impl PendingApplyContext { + /// The base table's key schema, which both kinds carry and the queue needs in + /// order to route a row to the worker that owns its base key. + fn base_key_schema(&self) -> &[KeySchemaElement] { + match self { + Self::Gsi(c) => &c.base_key_schema, + Self::Vector(c) => &c.base_key_schema, + } + } +} + /// A row claimed from `gsi_pending`: /// `(id, table_id, old_item, new_item, index_context)`. type ClaimedRow = ( @@ -192,7 +221,7 @@ pub(crate) async fn enqueue_gsi_pending( old_item: Option<&Item>, new_item: Option<&Item>, delay_ms: u64, - context: &GsiApplyContext, + context: &PendingApplyContext, ) -> Result<(), StorageError> { let old_json = old_item .map(serde_json::to_value) @@ -209,7 +238,7 @@ pub(crate) async fn enqueue_gsi_pending( // The base key is immutable over an item's lifetime; `new_item` carries it // for puts/updates, `old_item` for deletes. let worker_partition = match new_item.or(old_item) { - Some(item) => partition_for(&composite_pk_to_text(item, &context.base_key_schema)?), + Some(item) => partition_for(&composite_pk_to_text(item, context.base_key_schema())?), None => 0, }; @@ -324,9 +353,18 @@ async fn process_batch(worker_id: u64, q: &GsiQueue) -> Result = sqlx::query_as( "SELECT id, table_id, old_item, new_item, index_context FROM gsi_pending \ WHERE worker_partition = $1 AND ready_at <= NOW() \ + AND table_id NOT IN (SELECT table_id FROM vector_index_holds) \ ORDER BY id \ LIMIT 1 \ FOR UPDATE SKIP LOCKED", @@ -387,7 +425,7 @@ async fn apply_claimed_row( .map(serde_json::from_value) .transpose() .map_err(|e| StorageError::Internal(e.to_string()))?; - let context: GsiApplyContext = + let context: PendingApplyContext = serde_json::from_value(ctx_json).map_err(|e| StorageError::Internal(e.to_string()))?; // One index per row. Guard the apply with a savepoint so a dropped-index @@ -399,16 +437,30 @@ async fn apply_claimed_row( .await .map_err(|e| StorageError::Internal(e.to_string()))?; - match apply_index( - tx, - &context.index, - old_item.as_ref(), - new_item.as_ref(), - &context.base_key_schema, - &context.attribute_definitions, - ) - .await - { + let applied = match &context { + PendingApplyContext::Gsi(c) => { + apply_index( + tx, + &c.index, + old_item.as_ref(), + new_item.as_ref(), + &c.base_key_schema, + &c.attribute_definitions, + ) + .await + } + PendingApplyContext::Vector(c) => { + crate::data::vector_index::apply_claimed_vector_row( + tx, + c, + old_item.as_ref(), + new_item.as_ref(), + ) + .await + } + }; + + match applied { Ok(()) => { sqlx::query("RELEASE SAVEPOINT gsi_apply") .execute(&mut **tx) @@ -422,9 +474,10 @@ async fn apply_claimed_row( .execute(&mut **tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // Applies to both kinds: a base table or an index dropped while a + // row was in flight is a routine race, not a defect. tracing::debug!( - "GSI worker {worker_id}: index {} gone, skipping id={id} table={table_id}", - context.index.index_id + "index propagation worker {worker_id}: target gone, skipping id={id} table={table_id}" ); } Err(e) => return Err(e), diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index e058d114..3cc548d1 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -30,6 +30,7 @@ mod table_helpers; mod ttl_worker; mod update_table; mod vector; +mod vector_search; mod worker_store; mod workers; diff --git a/crates/storage-postgres/src/migrations.rs b/crates/storage-postgres/src/migrations.rs index 8a91ad48..45af1616 100755 --- a/crates/storage-postgres/src/migrations.rs +++ b/crates/storage-postgres/src/migrations.rs @@ -61,6 +61,10 @@ pub(crate) const DATA_MIGRATIONS: &[(&str, &str)] = &[ "003_idempotency_account_scope.sql", include_str!("../../storage-postgres/data_migrations/003_idempotency_account_scope.sql"), ), + ( + "004_vector_index_state.sql", + include_str!("../../storage-postgres/data_migrations/004_vector_index_state.sql"), + ), ]; /// Run data database migrations, skipping already-applied ones. diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index ba80350f..c2a9a40d 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -67,9 +67,28 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; if vector_count > 0 { - return Err(StorageError::Validation( - extenddb_core::types::VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST.to_owned(), - )); + // Two measured shapes, two strings. A plain switch reports the + // create-side rule; a switch that also carries VectorIndexUpdates + // reports its own message, measured 2026-08-19 on a switch combined + // with deleting the last vector index, which the service refuses + // even though the net state would carry none. + // + // Only those two shapes are measured. Which of the two fires for a + // switch combined with a vector index CREATE is unmapped, and that + // shape is refused earlier here anyway, so it cannot reach this + // choice. If the trigger turns out to be the delete specifically + // rather than the presence of updates, this condition is where that + // changes. + let message = if input + .vector_index_updates + .as_ref() + .is_some_and(|u| !u.is_empty()) + { + extenddb_core::types::VECTOR_TABLE_REQUIRES_PAY_PER_REQUEST_MODE + } else { + extenddb_core::types::VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST + }; + return Err(StorageError::Validation(message.to_owned())); } } @@ -441,6 +460,9 @@ impl PostgresEngine { merged_attr_defs_for_ddl = Some(effective); } + // Ids of the vector indexes this request deletes, so their data tables can + // be dropped after the catalog commit, in the same order as the GSI drops. + let mut deleted_vector_index_ids: Vec = Vec::new(); // Vector index create/delete. // // Delete is implemented; Create is refused. Creating an index means @@ -463,12 +485,8 @@ impl PostgresEngine { ))); } if let Some(delete) = &update.delete { - // The index id is deliberately not selected: this backend - // builds no per-index storage yet, so there is nothing to drop - // by id, and reading a value only to discard it invites the - // reader to think otherwise. - let existing: Option<(String, Option)> = sqlx::query_as( - "SELECT index_status, backfilling FROM vector_indexes \ + let existing: Option<(String, String, Option)> = sqlx::query_as( + "SELECT index_id, index_status, backfilling FROM vector_indexes \ WHERE table_id = $1 AND index_name = $2", ) .bind(&table_id) @@ -477,7 +495,7 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let (index_status, backfilling) = existing + let (del_index_id, index_status, backfilling) = existing .ok_or_else(|| StorageError::IndexNotFound(delete.index_name.clone()))?; // Deleting an index that is still being created is @@ -508,6 +526,7 @@ impl PostgresEngine { .execute(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + deleted_vector_index_ids.push(del_index_id); } } } @@ -609,6 +628,38 @@ impl PostgresEngine { } } + // Vector data tables are dropped after the catalog commit, like the GSI + // ones: the catalog is the record of what exists, so it commits first and + // a crash in between leaves an unreferenced table rather than an index + // whose rows are gone. + for index_id in &deleted_vector_index_ids { + let mut data_tx = self + .data_pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if let Err(e) = Self::drop_vector_data_table(&mut data_tx, index_id).await { + tracing::warn!( + "Failed to drop the data table for a deleted vector index on '{}': {e}", + input.table_name, + ); + continue; + } + // The queue rows for this table can outlive the index. They are + // tolerated by the worker (a missing table is a routine race), but + // removing them here saves the claim-and-skip cycle and the log noise. + if let Err(e) = data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string())) + { + tracing::warn!( + "Failed to commit the vector data table drop on '{}': {e}", + input.table_name, + ); + } + } + self.build_table_description(account_id, &input.table_name) .await } diff --git a/crates/storage-postgres/src/vector_search.rs b/crates/storage-postgres/src/vector_search.rs new file mode 100644 index 00000000..17c27849 --- /dev/null +++ b/crates/storage-postgres/src/vector_search.rs @@ -0,0 +1,247 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `SearchVectors` for the PostgreSQL backend: exact nearest-neighbour scan. +//! +//! The whole search is one query. Distance, partition scoping, inline filters, +//! ordering and the row limit all evaluate in the server, so the wire carries the +//! `top_k` hits rather than every candidate row. That is the reason the embedding +//! is stored in pgvector's own column type: the SQLite backend has to stream every +//! row of a partition and compute distances in process, which is the cost +//! ADR-0004 accepted for a backend that cannot load an extension. +//! +//! Exact scan only, per ADR-0004 and the port plan. An approximate index is a +//! later `CREATE INDEX ... USING hnsw` against this same column, with no table +//! rewrite, which is what the column type buys. + +use extenddb_core::types::{AttributeValue, DistanceFunction}; +use extenddb_storage::error::StorageError; +use extenddb_storage::vector_lifecycle::partition_value; +use extenddb_storage::{ + BoxedFuture, VectorHit, VectorSearch, VectorSearchEngine, VectorSearchOutput, + VectorSearchResult, +}; +use pgvector::Vector; + +use crate::PostgresEngine; +use crate::data::vector_table_name; + +/// The SQL scoring expression for one distance function. +/// +/// Every metric is ordered **ascending**, which is what lets one `ORDER BY` serve +/// all three: pgvector's `<#>` returns the negated inner product, so the most +/// similar row has the smallest value under each operator. The sign is undone +/// after ordering, in [`report_score`]. +/// +/// `$1` is the query vector and `{query_norm}` the caller's precomputed norm. +fn score_expression(function: DistanceFunction, query_norm: f64) -> String { + match function { + // The CASE is a conformance requirement, not a nicety. pgvector's cosine + // operator yields NaN when either side has zero norm, while the service + // answers exactly 1.0 with a zero vector on either side (measured + // 2026-08-19), which is also what the SQLite backend produces. Removing + // the CASE would make a zero vector sort unpredictably and report NaN. + DistanceFunction::Cosine => format!( + "CASE WHEN nrm = 0 OR {query_norm} = 0 THEN 1.0 \ + ELSE (embedding <=> $1)::float8 END" + ), + DistanceFunction::Euclidean => "(embedding <-> $1)::float8".to_owned(), + DistanceFunction::DotProduct => "(embedding <#> $1)::float8".to_owned(), + } +} + +/// Turn the ordered SQL value into the score the engine contract defines. +/// +/// Cosine and Euclidean report the distance itself, lower being more similar. +/// Dot product reports the raw inner product, higher being more similar, which is +/// the negation of what pgvector's operator returns. +fn report_score(function: DistanceFunction, ordered: f64) -> f64 { + match function { + DistanceFunction::Cosine | DistanceFunction::Euclidean => ordered, + DistanceFunction::DotProduct => -ordered, + } +} + +impl VectorSearchEngine for PostgresEngine { + fn search_vectors(&self, req: VectorSearch<'_>) -> BoxedFuture<'_, VectorSearchResult> { + // The request borrows from the caller's frame, so own what the future needs. + let table_id = req.key_info.table_id.clone(); + let index_name = req.index_name.to_owned(); + let query_vector = req.query_vector.to_vec(); + let cached_dimensions = req + .key_info + .vector_indexes + .iter() + .find(|vi| vi.index_name == index_name) + .map(|vi| vi.dimensions); + let top_k = req.top_k; + let partition = partition_value(req.hash_key); + let filters: Vec<(String, AttributeValue)> = req + .filters + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).clone())) + .collect(); + + Box::pin(async move { + let partition = partition?; + + // The index definition comes from the catalog, not from the cached key + // info: the cache carries dimensions and the search schema but neither + // the index id that names the data table nor the distance function, + // without which a score cannot be computed or ordered. + let row: Option<(String, i32, String, serde_json::Value)> = sqlx::query_as( + "SELECT index_id, dimensions, distance_function, vector_attribute \ + FROM vector_indexes WHERE table_id = $1 AND index_name = $2", + ) + .bind(&table_id) + .bind(&index_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (index_id, dimensions, distance_raw, vector_attribute_json) = + row.ok_or_else(|| StorageError::IndexNotFound(index_name.clone()))?; + // Stored as the serialized `VectorAttribute` rather than a bare name, so + // it is read back the same way the write path wrote it. + let vector_attribute_name = serde_json::from_value::< + extenddb_core::types::VectorAttribute, + >(vector_attribute_json) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))? + .attribute_name; + let dimensions = usize::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?; + let function: DistanceFunction = serde_json::from_value(serde_json::Value::String( + distance_raw.clone(), + )) + .map_err(|e| StorageError::Internal(format!("unknown distance function: {e}")))?; + + if query_vector.len() != dimensions { + // The engine validates this against the cached key info, so reaching + // here means the catalog and the cache disagree with each other. + return Err(StorageError::Validation(format!( + "query vector has {} dimensions, index expects {dimensions}", + query_vector.len() + ))); + } + if let Some(cached) = cached_dimensions + && usize::try_from(cached).is_ok_and(|cached| cached != dimensions) + { + return Err(StorageError::Validation(format!( + "index {index_name} reports {cached} dimensions in the cached key info and \ + {dimensions} in the catalog" + ))); + } + + let query_norm = f64::from(query_vector.iter().map(|x| x * x).sum::().sqrt()); + let vec_table = vector_table_name(&index_id); + + // Filters are equality over the index's inline-filter attributes, + // evaluated in SQL against the stored payload so that LIMIT applies + // after filtering. Filtering after the limit would silently return + // fewer than top_k matching rows. + // + // jsonb equality is well defined for these values because numbers are + // normalised when an item is deserialised, so two equal numbers have + // one representation. + let mut predicates = String::new(); + for i in 0..filters.len() { + let name_param = 4 + i * 2; + let value_param = name_param + 1; + predicates.push_str(&format!( + " AND item_data -> ${name_param} = ${value_param}::jsonb" + )); + } + + let sql = format!( + "SELECT {score} AS score, embedding, item_data \ + FROM {vec_table} WHERE part = $2{predicates} \ + ORDER BY score ASC, base_pk ASC LIMIT $3", + score = score_expression(function, query_norm), + ); + + // Bound as a typed vector rather than a text literal, so the value that + // reaches the server is the same f32 sequence the engine validated. + let mut query = sqlx::query_as::<_, (f64, Vector, serde_json::Value)>(&sql) + .bind(Vector::from(query_vector)) + // Bytes, matching the BYTEA column: the unscoped sentinel contains + // a NUL, so a text comparison would not even be storable. + .bind(partition.into_bytes()) + .bind(top_k); + for (name, value) in &filters { + let value_json = serde_json::to_string(value) + .map_err(|e| StorageError::Internal(format!("filter value: {e}")))?; + query = query.bind(name).bind(value_json); + } + + let rows = query + .fetch_all(&self.data_pool) + .await + .map_err(crate::vector::map_vector_sql_error)?; + + let mut hits = Vec::with_capacity(rows.len()); + for (ordered, embedding, item_json) in rows { + let mut item: extenddb_core::types::Item = serde_json::from_value(item_json) + .map_err(|e| StorageError::Internal(format!("stored item: {e}")))?; + // Reinstated from the stored f32s rather than from a second copy in + // the payload, so what comes back is the narrowed value that was + // actually indexed. The engine drops it again unless a projection + // expression names it. + item.insert( + vector_attribute_name.clone(), + extenddb_core::validation::vector_item::vector_attribute(embedding.as_slice()), + ); + hits.push(VectorHit { + item, + score: report_score(function, ordered), + }); + } + + Ok(VectorSearchOutput { + hits, + distance_function: function, + }) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cosine_guards_a_zero_norm_on_either_side() { + // The measured answer for a zero vector under cosine is exactly 1.0, on + // either side. pgvector's operator returns NaN there, so the guard is what + // makes the backend conformant rather than merely tidy. + let sql = score_expression(DistanceFunction::Cosine, 0.0); + assert!(sql.contains("nrm = 0"), "{sql}"); + assert!(sql.contains("THEN 1.0"), "{sql}"); + assert!(sql.contains("<=>"), "{sql}"); + } + + #[test] + fn each_metric_uses_its_own_operator() { + assert!(score_expression(DistanceFunction::Euclidean, 1.0).contains("<->")); + assert!(score_expression(DistanceFunction::DotProduct, 1.0).contains("<#>")); + } + + #[test] + fn only_dot_product_has_its_sign_undone() { + // Every metric is ordered ascending so that one ORDER BY serves all three; + // the inner product is the one whose reported direction differs from the + // ordered one. + assert!((report_score(DistanceFunction::Cosine, 0.25) - 0.25).abs() < f64::EPSILON); + assert!((report_score(DistanceFunction::Euclidean, 2.5) - 2.5).abs() < f64::EPSILON); + assert!((report_score(DistanceFunction::DotProduct, -7.0) - 7.0).abs() < f64::EPSILON); + } + + #[test] + fn a_query_norm_of_zero_is_interpolated_into_the_guard() { + // The norm is a computed f64 rather than a bind parameter, so the guard has + // to carry it literally. A formatting change that dropped it would make the + // zero-query case fall through to the operator and return NaN. + let sql = score_expression(DistanceFunction::Cosine, 0.0); + assert!(sql.contains("0 = 0"), "{sql}"); + } +} diff --git a/crates/storage-postgres/src/worker_store.rs b/crates/storage-postgres/src/worker_store.rs index 8de9aa59..be43b984 100755 --- a/crates/storage-postgres/src/worker_store.rs +++ b/crates/storage-postgres/src/worker_store.rs @@ -80,8 +80,11 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - // Collect index ids while the rows still exist. - let mut drop_info: Vec<(String, Vec)> = Vec::new(); + // Collect index ids while the rows still exist. Vector indexes are read + // here for the same reason and at the same moment as the secondary ones: + // their catalog rows cascade away with the table row, and each id names a + // data table that still has to be dropped. + let mut drop_info: Vec<(String, Vec, Vec)> = Vec::new(); for (_acct_id, name, arn, table_id) in &candidates { let index_ids: Vec<(String,)> = @@ -90,6 +93,12 @@ impl PostgresEngine { .fetch_all(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + let vector_index_ids: Vec = + sqlx::query_scalar("SELECT index_id FROM vector_indexes WHERE table_id = $1") + .bind(table_id) + .fetch_all(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; // Delete tags explicitly (not covered by CASCADE from tables). sqlx::query("DELETE FROM tags WHERE resource_arn = $1") @@ -108,6 +117,7 @@ impl PostgresEngine { drop_info.push(( table_id.clone(), index_ids.into_iter().map(|(n,)| n).collect(), + vector_index_ids, )); transitions.push((name.clone(), "DELETING → deleted")); @@ -118,7 +128,7 @@ impl PostgresEngine { .map_err(|e| StorageError::Internal(e.to_string()))?; // P54 Bug 1: Drop data tables on the data pool after catalog commit. - for (table_id, index_ids) in &drop_info { + for (table_id, index_ids, vector_index_ids) in &drop_info { let mut data_tx = self .data_pool .begin() @@ -127,6 +137,9 @@ impl PostgresEngine { for idx_id in index_ids { Self::drop_index_data_table(&mut data_tx, idx_id).await?; } + for idx_id in vector_index_ids { + Self::drop_vector_data_table(&mut data_tx, idx_id).await?; + } Self::drop_data_table(&mut data_tx, table_id).await?; // Drop any still-pending GSI propagation rows for this table in the diff --git a/crates/storage-postgres/tests/vector_control_plane.rs b/crates/storage-postgres/tests/vector_control_plane.rs index b8353edb..2049d67c 100644 --- a/crates/storage-postgres/tests/vector_control_plane.rs +++ b/crates/storage-postgres/tests/vector_control_plane.rs @@ -22,6 +22,7 @@ use std::collections::{BTreeMap, HashMap}; use extenddb_core::expression::{self, ExpressionMaps}; +use extenddb_core::types::TableKeyInfo; use extenddb_core::types::{ AttributeDefinition, AttributeValue, BillingMode, CreateTableInput, DeleteTableInput, DeleteVectorIndexAction, DescribeTableInput, DistanceFunction, IndexStatus, Item, @@ -206,6 +207,17 @@ async fn scratch(pgvector: Pgvector) -> Scratch { } } +/// Build a scratch environment that can hold vector data, or report why not. +/// +/// Creating a vector index now builds a `vector(N)` data table, so the extension +/// is a hard requirement for every test that creates one: without it the backend +/// refuses, which is correct and is covered separately. That is the cost of the +/// data path arriving, and it is why these tests run against the pgvector image in +/// CI while the refusal suite keeps the plain one. +async fn vector_scratch(test: &str) -> Option { + scratch_with_pgvector(test).await +} + /// Build a scratch database with pgvector installed, or report why not. /// /// The extension is a server package, so a PostgreSQL that does not ship it @@ -348,10 +360,13 @@ async fn set_index_phase(catalog: &PgPool, table_id: &str, index_name: &str, bac #[tokio::test] async fn create_table_records_a_vector_index_as_active_and_echoes_it() { + let test = "create_table_records_a_vector_index_as_active_and_echoes_it"; if base_conn().is_none() { - return skip("create_table_records_a_vector_index_as_active_and_echoes_it"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; let desc = s .engine @@ -403,10 +418,13 @@ async fn create_table_records_a_vector_index_as_active_and_echoes_it() { #[tokio::test] async fn describe_table_reports_scoped_and_unscoped_vector_indexes() { + let test = "describe_table_reports_scoped_and_unscoped_vector_indexes"; if base_conn().is_none() { - return skip("describe_table_reports_scoped_and_unscoped_vector_indexes"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; let mut input = create_input( "t_describe", @@ -476,10 +494,13 @@ async fn describe_table_reports_scoped_and_unscoped_vector_indexes() { #[tokio::test] async fn table_key_info_carries_the_vector_index_metadata_the_write_path_needs() { + let test = "table_key_info_carries_the_vector_index_metadata_the_write_path_needs"; if base_conn().is_none() { - return skip("table_key_info_carries_the_vector_index_metadata_the_write_path_needs"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -525,10 +546,13 @@ async fn table_key_info_carries_the_vector_index_metadata_the_write_path_needs() #[tokio::test] async fn delete_table_removes_the_vector_index_rows() { + let test = "delete_table_removes_the_vector_index_rows"; if base_conn().is_none() { - return skip("delete_table_removes_the_vector_index_rows"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -559,10 +583,13 @@ async fn delete_table_removes_the_vector_index_rows() { #[tokio::test] async fn update_table_deletes_an_active_vector_index_once() { + let test = "update_table_deletes_an_active_vector_index_once"; if base_conn().is_none() { - return skip("update_table_deletes_an_active_vector_index_once"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -623,10 +650,13 @@ async fn update_table_deletes_an_active_vector_index_once() { #[tokio::test] async fn deleting_a_vector_index_in_the_allocation_phase_is_refused() { + let test = "deleting_a_vector_index_in_the_allocation_phase_is_refused"; if base_conn().is_none() { - return skip("deleting_a_vector_index_in_the_allocation_phase_is_refused"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -669,10 +699,13 @@ async fn deleting_a_vector_index_in_the_allocation_phase_is_refused() { #[tokio::test] async fn deleting_a_vector_index_during_its_backfill_is_accepted() { + let test = "deleting_a_vector_index_during_its_backfill_is_accepted"; if base_conn().is_none() { - return skip("deleting_a_vector_index_during_its_backfill_is_accepted"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -703,10 +736,13 @@ async fn deleting_a_vector_index_during_its_backfill_is_accepted() { #[tokio::test] async fn switching_a_table_with_vector_indexes_to_provisioned_is_refused() { + let test = "switching_a_table_with_vector_indexes_to_provisioned_is_refused"; if base_conn().is_none() { - return skip("switching_a_table_with_vector_indexes_to_provisioned_is_refused"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -754,10 +790,13 @@ async fn switching_a_table_with_vector_indexes_to_provisioned_is_refused() { #[tokio::test] async fn adding_a_vector_index_by_update_table_is_unsupported() { + let test = "adding_a_vector_index_by_update_table_is_unsupported"; if base_conn().is_none() { - return skip("adding_a_vector_index_by_update_table_is_unsupported"); + return skip(test); } let s = scratch(Pgvector::Omit).await; + // No vector index is created here, so this must keep running on a server + // that has no pgvector at all: for the refusal, that is the interesting case. s.engine .create_table(ACCOUNT, create_input("t_add", vec![])) @@ -797,10 +836,13 @@ async fn adding_a_vector_index_by_update_table_is_unsupported() { #[tokio::test] async fn restoring_a_backup_that_carries_vector_indexes_is_refused() { + let test = "restoring_a_backup_that_carries_vector_indexes_is_refused"; if base_conn().is_none() { - return skip("restoring_a_backup_that_carries_vector_indexes_is_refused"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -862,10 +904,13 @@ async fn restoring_a_backup_that_carries_vector_indexes_is_refused() { #[tokio::test] async fn a_wrong_dimension_vector_written_by_update_item_is_rejected() { + let test = "a_wrong_dimension_vector_written_by_update_item_is_rejected"; if base_conn().is_none() { - return skip("a_wrong_dimension_vector_written_by_update_item_is_rejected"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -941,7 +986,9 @@ async fn describe_table_refuses_a_vector_index_whose_stored_status_is_unrecognis if base_conn().is_none() { return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -983,10 +1030,13 @@ async fn describe_table_refuses_a_vector_index_whose_stored_status_is_unrecognis #[tokio::test] async fn describe_table_reports_a_creating_index_as_backfilling() { + let test = "describe_table_reports_a_creating_index_as_backfilling"; if base_conn().is_none() { - return skip("describe_table_reports_a_creating_index_as_backfilling"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -1021,10 +1071,13 @@ async fn describe_table_reports_a_creating_index_as_backfilling() { #[tokio::test] async fn an_empty_search_schema_is_stored_and_reported_as_absent() { + let test = "an_empty_search_schema_is_stored_and_reported_as_absent"; if base_conn().is_none() { - return skip("an_empty_search_schema_is_stored_and_reported_as_absent"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; // Core accepts `SearchSchema: []` on the request, and every path downstream // treats it as unscoped. The service reports an absent member or a populated @@ -1076,10 +1129,13 @@ async fn an_empty_search_schema_is_stored_and_reported_as_absent() { #[tokio::test] async fn a_backup_snapshot_carries_the_wire_shape_behind_a_version() { + let test = "a_backup_snapshot_carries_the_wire_shape_behind_a_version"; if base_conn().is_none() { - return skip("a_backup_snapshot_carries_the_wire_shape_behind_a_version"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -1130,10 +1186,13 @@ async fn a_backup_snapshot_carries_the_wire_shape_behind_a_version() { #[tokio::test] async fn a_backup_taken_before_the_snapshot_column_existed_still_restores() { + let test = "a_backup_taken_before_the_snapshot_column_existed_still_restores"; if base_conn().is_none() { - return skip("a_backup_taken_before_the_snapshot_column_existed_still_restores"); + return skip(test); } let s = scratch(Pgvector::Omit).await; + // No vector index is created here, so this must keep running on a server + // that has no pgvector at all: for the refusal, that is the interesting case. s.engine .create_table(ACCOUNT, create_input("t_legacy", vec![])) @@ -1165,10 +1224,13 @@ async fn a_backup_taken_before_the_snapshot_column_existed_still_restores() { #[tokio::test] async fn a_vector_index_at_the_maximum_dimension_round_trips() { + let test = "a_vector_index_at_the_maximum_dimension_round_trips"; if base_conn().is_none() { - return skip("a_vector_index_at_the_maximum_dimension_round_trips"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; // 4096 is the largest dimension core accepts. The catalog column is a 32-bit // integer and the wire type is unsigned, so the value crosses two narrowing @@ -1208,10 +1270,13 @@ async fn a_vector_index_at_the_maximum_dimension_round_trips() { #[tokio::test] async fn describe_table_refuses_a_vector_index_with_a_corrupt_payload() { + let test = "describe_table_refuses_a_vector_index_with_a_corrupt_payload"; if base_conn().is_none() { - return skip("describe_table_refuses_a_vector_index_with_a_corrupt_payload"); + return skip(test); } - let s = scratch(Pgvector::Omit).await; + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table( @@ -1261,6 +1326,463 @@ async fn describe_table_refuses_a_vector_index_with_a_corrupt_payload() { s.cleanup().await; } +/// Every vector data table in the scratch database. +/// +/// Each test has its own database, so this needs no table filter: whatever is +/// here belongs to the table under test. Deliberately does not read the catalog, +/// because the case worth checking is the one where the catalog rows are already +/// gone and an orphaned data table would be invisible. +async fn vector_data_tables(catalog: &PgPool) -> Vec { + sqlx::query_scalar( + "SELECT tablename FROM pg_tables WHERE schemaname = 'public' \ + AND tablename LIKE '_ddb\\_vec\\_%' ORDER BY tablename", + ) + .fetch_all(catalog) + .await + .expect("list the vector data tables") +} + +#[tokio::test] +async fn create_table_builds_the_vector_data_table_and_delete_table_sweeps_it() { + let test = "create_table_builds_the_vector_data_table_and_delete_table_sweeps_it"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_datatable", vec![vector_spec("vidx", 4, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let tables = vector_data_tables(&s.catalog).await; + assert_eq!( + tables.len(), + 1, + "one data table per vector index: {tables:?}" + ); + + // The embedding column carries the declared dimension count, which is what + // makes a wrong-width write fail in the server rather than silently store. + let embedding_type: String = sqlx::query_scalar( + "SELECT format_type(a.atttypid, a.atttypmod) FROM pg_attribute a \ + JOIN pg_class c ON c.oid = a.attrelid \ + WHERE c.relname = $1 AND a.attname = 'embedding'", + ) + .bind(&tables[0]) + .fetch_one(&s.catalog) + .await + .expect("read the embedding column type"); + assert_eq!(embedding_type, "vector(4)"); + + // Partition scoping is an indexed lookup, not a scan over every row. + let part_indexes: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM pg_indexes WHERE tablename = $1 AND indexdef LIKE '%(part)%'", + ) + .bind(&tables[0]) + .fetch_one(&s.catalog) + .await + .expect("count the partition indexes"); + assert_eq!(part_indexes, 1, "the part column must be indexed"); + + s.engine + .delete_table( + ACCOUNT, + DeleteTableInput { + table_name: "t_datatable".to_owned(), + }, + ) + .await + .expect("delete the table"); + + // Swept by prefix, because the catalog rows that named these tables cascade + // away with the table row before the sweep runs. + assert!( + vector_data_tables(&s.catalog).await.is_empty(), + "DeleteTable must sweep the vector data tables" + ); + + s.cleanup().await; +} + +#[tokio::test] +async fn update_table_delete_drops_the_index_data_table() { + let test = "update_table_delete_drops_the_index_data_table"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input( + "t_dropone", + vec![ + vector_spec("keep", 4, Some("pk")), + vector_spec("drop", 8, Some("pk")), + ], + ), + ) + .await + .expect("create a table with two vector indexes"); + let id = table_id(&s.catalog, "t_dropone").await; + assert_eq!(vector_data_tables(&s.catalog).await.len(), 2); + + s.engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: delete_vector("drop"), + ..update_input("t_dropone") + }, + ) + .await + .expect("delete one vector index"); + + // One table gone, one left: an index delete must not take the survivor's + // storage with it, which is the failure a prefix sweep would cause here. + let remaining = vector_data_tables(&s.catalog).await; + assert_eq!(remaining.len(), 1, "{remaining:?}"); + let keep_id: String = + sqlx::query_scalar("SELECT index_id FROM vector_indexes WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the surviving index id"); + assert!( + remaining[0].ends_with(&keep_id), + "{remaining:?} vs {keep_id}" + ); + + s.cleanup().await; +} + +/// One item with a vector, and optionally a tenant for a scoped index. +fn vector_item(pk: &str, tenant: Option<&str>, values: &[&str]) -> Item { + let mut item: Item = BTreeMap::from([("pk".to_owned(), AttributeValue::S(pk.to_owned()))]); + if let Some(tenant) = tenant { + item.insert("tenant".to_owned(), AttributeValue::S(tenant.to_owned())); + } + item.insert( + "emb".to_owned(), + AttributeValue::L( + values + .iter() + .map(|v| AttributeValue::N((*v).to_owned())) + .collect(), + ), + ); + item +} + +/// Write one item through the storage put path. +async fn put(engine: &PostgresEngine, key_info: &TableKeyInfo, item: Item) { + let maps = ExpressionMaps::new(HashMap::new(), HashMap::new()); + engine + .put_item(key_info, item, false, None, &maps, None) + .await + .expect("put an item"); +} + +/// Rows in a vector index's data table, as (partition bytes, payload). +async fn index_rows(catalog: &PgPool, table: &str) -> Vec<(Vec, serde_json::Value)> { + sqlx::query_as(&format!( + "SELECT part, item_data FROM \"{table}\" ORDER BY base_pk" + )) + .fetch_all(catalog) + .await + .expect("read the index rows") +} + +async fn only_index_table(catalog: &PgPool) -> String { + let tables = vector_data_tables(catalog).await; + assert_eq!(tables.len(), 1, "{tables:?}"); + tables[0].clone() +} + +#[tokio::test] +async fn a_write_indexes_the_item_and_a_delete_removes_it() { + let test = "a_write_indexes_the_item_and_a_delete_removes_it"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_write_path", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_write_path") + .await + .expect("key info"); + let table = only_index_table(&s.catalog).await; + + put(&s.engine, &key_info, vector_item("a", None, &["1", "0"])).await; + let rows = index_rows(&s.catalog, &table).await; + assert_eq!(rows.len(), 1, "the write must reach the index"); + // The payload carries the projected item with the vector attribute stripped: + // the vector is reconstructed from the stored column on the way out, so keeping + // a second copy here would let the two disagree. + assert!(rows[0].1.get("emb").is_none(), "{:?}", rows[0].1); + assert!(rows[0].1.get("pk").is_some(), "{:?}", rows[0].1); + + let key: Item = BTreeMap::from([("pk".to_owned(), AttributeValue::S("a".to_owned()))]); + let maps = ExpressionMaps::new(HashMap::new(), HashMap::new()); + s.engine + .delete_item(&key_info, &key, false, None, &maps, None) + .await + .expect("delete the item"); + assert!( + index_rows(&s.catalog, &table).await.is_empty(), + "the delete must remove the indexed row" + ); + + s.cleanup().await; +} + +#[tokio::test] +async fn changing_the_scope_attribute_moves_the_row_rather_than_duplicating_it() { + let test = "changing_the_scope_attribute_moves_the_row_rather_than_duplicating_it"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + // Scoped on `tenant`, so rewriting the item with a different tenant has to move + // its row: the partition is part of the row, and the row is keyed by the base + // item, so a careless apply would leave two rows and a search would find the + // item in a partition it no longer belongs to. + let mut input = create_input("t_move", vec![vector_spec("vidx", 2, Some("tenant"))]); + input.attribute_definitions = string_attr("pk"); + s.engine + .create_table(ACCOUNT, input) + .await + .expect("create a scoped vector index"); + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_move") + .await + .expect("key info"); + let table = only_index_table(&s.catalog).await; + + put( + &s.engine, + &key_info, + vector_item("a", Some("t1"), &["1", "0"]), + ) + .await; + let first = index_rows(&s.catalog, &table).await; + assert_eq!(first.len(), 1); + + put( + &s.engine, + &key_info, + vector_item("a", Some("t2"), &["1", "0"]), + ) + .await; + let moved = index_rows(&s.catalog, &table).await; + assert_eq!( + moved.len(), + 1, + "one row per base item, not one per partition" + ); + assert_ne!( + moved[0].0, first[0].0, + "the row must be in the new partition" + ); + + s.cleanup().await; +} + +#[tokio::test] +async fn a_stored_vector_that_cannot_be_indexed_leaves_the_write_alone() { + let test = "a_stored_vector_that_cannot_be_indexed_leaves_the_write_alone"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_poison", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_poison") + .await + .expect("key info"); + let table = only_index_table(&s.catalog).await; + + // Three components where the index declares two. Core rejects this on a live + // write, so the only way an item looks like this is that it predates the index, + // which is the case the backfill skips and counts. The write must still + // succeed: failing it would make every later update to that item impossible, + // including one that has nothing to do with the vector. + put( + &s.engine, + &key_info, + vector_item("wrongdim", None, &["1", "2", "3"]), + ) + .await; + assert!( + index_rows(&s.catalog, &table).await.is_empty(), + "an unindexable item must not enter the index" + ); + + // And it must be removed rather than left behind, so an item that was + // indexable and stops being so does not keep a stale row. + put(&s.engine, &key_info, vector_item("x", None, &["1", "0"])).await; + assert_eq!(index_rows(&s.catalog, &table).await.len(), 1); + put( + &s.engine, + &key_info, + vector_item("x", None, &["1", "2", "3"]), + ) + .await; + assert!( + index_rows(&s.catalog, &table).await.is_empty(), + "the stale row must be removed when the item stops being indexable" + ); + + s.cleanup().await; +} + +#[tokio::test] +async fn a_write_to_a_building_index_is_queued_and_never_applied_inline() { + let test = "a_write_to_a_building_index_is_queued_and_never_applied_inline"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_creating_write", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_creating_write").await; + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_creating_write") + .await + .expect("key info"); + let table = only_index_table(&s.catalog).await; + + // The propagation delay is zero in this scratch environment, so an ACTIVE index + // would be applied inline. A CREATING one must not be, at any delay: the + // backfill is scanning the base table with an older snapshot of this same item, + // and its deliberately plain INSERT would collide with whatever a write left + // behind. + set_index_phase(&s.catalog, &id, "vidx", true).await; + + put(&s.engine, &key_info, vector_item("a", None, &["1", "0"])).await; + + assert!( + index_rows(&s.catalog, &table).await.is_empty(), + "a write to a CREATING index must not be applied inline" + ); + let queued: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM gsi_pending WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("count the queued rows"); + assert_eq!(queued, 1, "the write must be queued instead"); + + s.cleanup().await; +} + +#[tokio::test] +async fn a_write_sees_an_index_created_after_its_key_info_was_cached() { + let test = "a_write_sees_an_index_created_after_its_key_info_was_cached"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + // Cache the key info while the table has no vector index at all, which is the + // state that used to make a write skip maintenance: the cached set was empty, so + // the write took a path that did no index work and reported success. + s.engine + .create_table(ACCOUNT, create_input("t_fresh", vec![])) + .await + .expect("create a plain table"); + let stale_key_info = s + .engine + .table_key_info(ACCOUNT, "t_fresh") + .await + .expect("key info"); + assert!(stale_key_info.vector_indexes.is_empty()); + + // Now add the index behind the cache's back, the way a concurrent UpdateTable + // would, including its data table. + let id = table_id(&s.catalog, "t_fresh").await; + let index_id = uuid::Uuid::new_v4().to_string(); + sqlx::query( + "INSERT INTO vector_indexes (table_id, index_name, index_id, dimensions, \ + distance_function, vector_attribute, search_schema, projection, index_status, \ + backfilling) VALUES ($1, 'vidx', $2, 2, 'COSINE', \ + '{\"AttributeName\":\"emb\"}'::jsonb, NULL, '{\"ProjectionType\":\"ALL\"}'::jsonb, \ + 'ACTIVE', NULL)", + ) + .bind(&id) + .bind(&index_id) + .execute(&s.catalog) + .await + .expect("add the index row"); + sqlx::query(&format!( + "CREATE TABLE \"_ddb_vec_{index_id}\" (part BYTEA NOT NULL, base_pk TEXT NOT NULL, \ + embedding vector(2) NOT NULL, nrm DOUBLE PRECISION NOT NULL, item_data JSONB NOT NULL, \ + PRIMARY KEY (base_pk))" + )) + .execute(&s.catalog) + .await + .expect("create the data table"); + + // The stale key info is what the write is handed, deliberately. + put( + &s.engine, + &stale_key_info, + vector_item("a", None, &["1", "0"]), + ) + .await; + + let rows = index_rows(&s.catalog, &format!("_ddb_vec_{index_id}")).await; + assert_eq!( + rows.len(), + 1, + "the write must see the index the catalog holds, not the one the cache remembers" + ); + + s.cleanup().await; +} + #[tokio::test] async fn the_probe_reads_the_data_database_and_not_the_catalog() { let test = "the_probe_reads_the_data_database_and_not_the_catalog"; From 8513fc37e8fb9419aea106681105505e59bc322d Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Mon, 24 Aug 2026 20:15:41 +0000 Subject: [PATCH 04/13] feat(postgres): UpdateTable-create vector index build lifecycle Backfill and status sequencing on the shared lifecycle primitives, with keyset pagination over the full primary key. Build ownership is a session-scoped advisory lock; the build session is owned for the whole build, and every route that abandons a build gives back its queue hold, with a self-healing sweep for the routes nothing enumerated. Stuck builds rebuild at runtime, not only at startup, and the scanning phase is re-asserted before a recovery rebuild. Query norms compute in f64 and bind as parameters. --- crates/core/src/settings_keys.rs | 19 + crates/server/src/management/ops_settings.rs | 9 + .../storage-postgres/src/data/delete_item.rs | 7 +- crates/storage-postgres/src/data/index.rs | 17 +- crates/storage-postgres/src/data/mod.rs | 4 +- crates/storage-postgres/src/data/put_item.rs | 7 +- .../storage-postgres/src/data/update_item.rs | 7 +- .../storage-postgres/src/data/vector_index.rs | 574 ++++++++++++- crates/storage-postgres/src/delete_table.rs | 7 + crates/storage-postgres/src/gsi_queue.rs | 15 +- crates/storage-postgres/src/lib.rs | 77 +- crates/storage-postgres/src/update_table.rs | 516 ++++++++++-- crates/storage-postgres/src/vector.rs | 27 +- crates/storage-postgres/src/vector_search.rs | 78 +- crates/storage-postgres/src/worker_store.rs | 8 + crates/storage-postgres/src/workers.rs | 34 + .../tests/vector_control_plane.rs | 758 +++++++++++++++++- crates/storage/src/vector_lifecycle/build.rs | 65 +- crates/storage/src/vector_lifecycle/mod.rs | 7 +- tests/rust/src/vector_index_search.rs | 9 +- 20 files changed, 2129 insertions(+), 116 deletions(-) diff --git a/crates/core/src/settings_keys.rs b/crates/core/src/settings_keys.rs index 210d5c94..4b325bd1 100644 --- a/crates/core/src/settings_keys.rs +++ b/crates/core/src/settings_keys.rs @@ -58,6 +58,25 @@ pub const VECTOR_BACKFILL_BATCH_DELAY_MS: &str = "vector_backfill_batch_delay_ms /// for table status. The default is 1000; zero disables the hold. pub const VECTOR_INDEX_MIN_CREATING_MS: &str = "vector_index_min_creating_ms"; +/// Milliseconds to hold a new vector index in the resource-allocation phase before +/// its backfill starts. +/// +/// Zero, and meant to stay zero outside tests, exactly like +/// [`VECTOR_BACKFILL_BATCH_DELAY_MS`] above and for the same kind of reason. The +/// service refuses a delete of an index that is still allocating resources, with a +/// byte-exact `ResourceInUseException`, and accepts the same delete once the +/// backfill is running. Both halves are measured behaviour, so both deserve a wire +/// test. +/// +/// Without this there is no deterministic way to observe the first half from a +/// client: the allocation phase exists only between the catalog row's insert and +/// the flip to `Backfilling: true`, both inside one `UpdateTable` call, so a test +/// could only race it. A race that asserts a whole measured string is worse than +/// no test, because it fails for reasons unrelated to the rule. +/// +/// Unset, nothing waits and no branch is taken. +pub const VECTOR_ALLOCATION_PHASE_DELAY_MS: &str = "vector_allocation_phase_delay_ms"; + /// Resolve a caller-supplied settings key to its canonical name. /// /// Accepting the old name keeps `extenddb settings set gsi_propagation_delay_ms 0` diff --git a/crates/server/src/management/ops_settings.rs b/crates/server/src/management/ops_settings.rs index 92148be0..f5dc559f 100755 --- a/crates/server/src/management/ops_settings.rs +++ b/crates/server/src/management/ops_settings.rs @@ -37,6 +37,15 @@ pub const KNOWN_KEYS: &[(&str, Validator)] = &[ extenddb_core::settings_keys::VECTOR_BACKFILL_BATCH_DELAY_MS, validate_backfill_batch_delay_ms, ), + // The sibling test lever, writable for the same reason: the allocation phase of + // an index build exists only between two transitions inside one UpdateTable + // call, so the measured refusal for a delete during that phase cannot be + // observed from a client unless a test can hold the phase open from outside the + // process. Same bound, same validator. + ( + extenddb_core::settings_keys::VECTOR_ALLOCATION_PHASE_DELAY_MS, + validate_backfill_batch_delay_ms, + ), ]; /// Read-only keys that cannot be changed via the settings API. diff --git a/crates/storage-postgres/src/data/delete_item.rs b/crates/storage-postgres/src/data/delete_item.rs index 8df59c58..34050d18 100755 --- a/crates/storage-postgres/src/data/delete_item.rs +++ b/crates/storage-postgres/src/data/delete_item.rs @@ -56,7 +56,12 @@ impl PostgresEngine { &key_info.table_id, ) .await?; - let sys_delay = if indexes.is_empty() { + // Read whenever anything can propagate, secondary or vector. Gating this on + // the secondary set alone made a vector-only table ignore the configured + // delay and apply its vector index inline, while a TransactWriteItems on the + // same table read the delay unconditionally and enqueued: six write sites, + // two answers, on a setting the differences doc says covers both index kinds. + let sys_delay = if indexes.is_empty() && vector_metas.is_empty() { 0 } else { self.index_propagation_delay().await diff --git a/crates/storage-postgres/src/data/index.rs b/crates/storage-postgres/src/data/index.rs index a9c29582..1a37d3f5 100644 --- a/crates/storage-postgres/src/data/index.rs +++ b/crates/storage-postgres/src/data/index.rs @@ -22,10 +22,21 @@ use crate::gsi_queue::{GsiApplyContext, GsiIndexDef, enqueue_gsi_pending}; /// message. The async GSI worker keys off the code (e.g. `42P01`, /// undefined_table) to tell a dropped-index race apart from a real failure; /// `sqlx`'s `Display` only carries the human text, so the code must be kept. -fn db_error(e: sqlx::Error) -> StorageError { +pub(crate) fn db_error(e: sqlx::Error) -> StorageError { + StorageError::Internal(sqlstate_message(&e)) +} + +/// Render a database error with its SQLSTATE prefixed. +/// +/// The prefix is the whole mechanism behind `is_undefined_table`: sqlx renders a +/// database error as its message text alone, so a caller that formats with +/// `to_string` throws the code away and every classifier downstream stops working. +/// Shared so the vector path cannot format it differently from the GSI path, which +/// is exactly how a dropped-table race turned into a permanently stalled worker. +pub(crate) fn sqlstate_message(e: &sqlx::Error) -> String { match e.as_database_error().and_then(|d| d.code()) { - Some(code) => StorageError::Internal(format!("SQLSTATE {code}: {e}")), - None => StorageError::Internal(e.to_string()), + Some(code) => format!("SQLSTATE {code}: {e}"), + None => e.to_string(), } } diff --git a/crates/storage-postgres/src/data/mod.rs b/crates/storage-postgres/src/data/mod.rs index 1c8b5f8b..d64ceb53 100755 --- a/crates/storage-postgres/src/data/mod.rs +++ b/crates/storage-postgres/src/data/mod.rs @@ -132,14 +132,14 @@ macro_rules! bind_sk_execute { mod data_engine; mod ddl; mod delete_item; -mod index; +pub(crate) mod index; mod put_item; mod query; mod query_scan; mod transactions; mod tx_helpers; mod update_item; -pub(crate) mod vector_index; +pub mod vector_index; pub(crate) use index::{ delete_index_row_multi, insert_index_row_multi, item_has_index_keys, project_item_for_index, diff --git a/crates/storage-postgres/src/data/put_item.rs b/crates/storage-postgres/src/data/put_item.rs index 3ab0be90..cb901f94 100755 --- a/crates/storage-postgres/src/data/put_item.rs +++ b/crates/storage-postgres/src/data/put_item.rs @@ -76,7 +76,12 @@ impl PostgresEngine { .map_err(|e| StorageError::Validation(e.to_string()))?; } - let sys_delay = if indexes.is_empty() { + // Read whenever anything can propagate, secondary or vector. Gating this on + // the secondary set alone made a vector-only table ignore the configured + // delay and apply its vector index inline, while a TransactWriteItems on the + // same table read the delay unconditionally and enqueued: six write sites, + // two answers, on a setting the differences doc says covers both index kinds. + let sys_delay = if indexes.is_empty() && vector_metas.is_empty() { 0 } else { self.index_propagation_delay().await diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index a5d9c2ed..d77b0d96 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -67,7 +67,12 @@ impl PostgresEngine { &key_info.table_id, ) .await?; - let sys_delay = if indexes.is_empty() { + // Read whenever anything can propagate, secondary or vector. Gating this on + // the secondary set alone made a vector-only table ignore the configured + // delay and apply its vector index inline, while a TransactWriteItems on the + // same table read the delay unconditionally and enqueued: six write sites, + // two answers, on a setting the differences doc says covers both index kinds. + let sys_delay = if indexes.is_empty() && vector_metas.is_empty() { 0 } else { self.index_propagation_delay().await diff --git a/crates/storage-postgres/src/data/vector_index.rs b/crates/storage-postgres/src/data/vector_index.rs index e13ad121..89c46913 100644 --- a/crates/storage-postgres/src/data/vector_index.rs +++ b/crates/storage-postgres/src/data/vector_index.rs @@ -322,7 +322,7 @@ pub(crate) async fn insert_vector_row( } /// Apply one claimed queue row to its vector index, from the row's own context. -pub(crate) async fn apply_claimed_vector_row( +pub async fn apply_claimed_vector_row( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, context: &VectorApplyContext, old_item: Option<&Item>, @@ -338,3 +338,575 @@ pub(crate) async fn apply_claimed_vector_row( ) .await } + +/// The PostgreSQL backfill driver for one vector index. +/// +/// Supplies the storage primitives the shared lifecycle drives. Everything about +/// ordering, poison handling and the failure contract lives in +/// `extenddb_storage::vector_lifecycle`; this is SQL and transactions. +pub(crate) struct PostgresVectorBuild { + pub(crate) catalog: sqlx::PgPool, + pub(crate) data: sqlx::PgPool, + pub(crate) queue_notify: Option>, + pub(crate) table_id: String, + pub(crate) index_id: String, + pub(crate) base_key_schema: Vec, + pub(crate) attribute_definitions: Vec, + pub(crate) dimensions: u32, + pub(crate) meta: Option, +} + +/// The backfill's position in the base table: the whole primary key. +/// +/// A keyset cursor rather than an offset, and the FULL key rather than the +/// partition alone. Both choices are load-bearing. An offset shifts when a +/// concurrent delete removes an earlier row, which silently skips a row that was +/// never indexed; a partition-only cursor loses rows inside a composite-key +/// partition, because the next batch resumes past the whole partition. PostgreSQL +/// has no rowid to fall back on, which is what the shared contract predicted. +#[derive(Debug, Clone)] +pub(crate) struct BaseKeyCursor { + pk: String, + /// The sort key values, in key order. Owned scalars rather than the shared + /// bind enum, which is neither `Clone` nor `Debug` and does not need to be. + sort_keys: Vec, +} + +/// One sort key value in a backfill cursor. +#[derive(Debug, Clone)] +enum CursorKey { + S(String), + N(sqlx::types::BigDecimal), + B(Vec), +} + +impl std::fmt::Display for BaseKeyCursor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Only ever used for a log line naming the row a poison skip happened on. + write!(f, "pk={}", self.pk) + } +} + +impl PostgresVectorBuild { + /// Load the index definition from the catalog. + /// + /// Read rather than passed in because recovery has no request to read it from: + /// the `UpdateTable` that created the index is long gone by the time a + /// reconciler rebuilds it. + pub(crate) async fn load_meta(&mut self) -> Result<(), StorageError> { + let metas = fetch_vector_indexes_for_table(&self.catalog, &self.table_id).await?; + let found = metas + .into_iter() + .find(|(meta, _)| meta.index_id == self.index_id) + .map(|(meta, _)| meta); + self.meta = Some(found.ok_or_else(|| { + StorageError::Internal( + "the vector index catalog row vanished before its build started".to_owned(), + ) + })?); + Ok(()) + } + + fn meta(&self) -> Result<&VectorIndexMeta, StorageError> { + self.meta.as_ref().ok_or_else(|| { + StorageError::Internal( + "vector backfill started before the index definition was loaded".to_owned(), + ) + }) + } +} + +impl extenddb_storage::vector_lifecycle::VectorIndexBuild for PostgresVectorBuild { + type Cursor = BaseKeyCursor; + + async fn backfill_batch( + &mut self, + cursor: Option, + limit: i64, + ) -> Result, StorageError> { + use extenddb_storage::vector_lifecycle::{BackfillRow, classify_backfill_row}; + + let meta = self.meta()?.clone(); + let base_sks = all_sort_key_info(&self.base_key_schema, &self.attribute_definitions); + let base_table = super::data_table_name(&self.table_id); + + // Order by the whole key so the keyset comparison below is total, and select + // the key columns because they are the cursor. + let mut key_cols = vec!["pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + key_cols.push(sk_column_n(i, sk_type)); + } + let order = key_cols.join(", "); + let selected = key_cols.join(", "); + + // Row-value comparison, which PostgreSQL evaluates lexicographically over + // the tuple, so one predicate resumes the scan exactly where it stopped + // whatever the key arity. + let (where_clause, has_cursor) = match &cursor { + Some(_) => { + let placeholders: Vec = + (1..=key_cols.len()).map(|i| format!("${i}")).collect(); + ( + format!(" WHERE ({order}) > ({})", placeholders.join(", ")), + true, + ) + } + None => (String::new(), false), + }; + let limit_param = if has_cursor { key_cols.len() + 1 } else { 1 }; + let sql = format!( + "SELECT {selected}, item_data FROM {base_table}{where_clause} \ + ORDER BY {order} LIMIT ${limit_param}" + ); + + let mut tx = self + .data + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut query = sqlx::query(&sql); + if let Some(cursor) = &cursor { + query = query.bind(cursor.pk.clone()); + for sk in &cursor.sort_keys { + query = match sk { + CursorKey::S(s) => query.bind(s.clone()), + CursorKey::N(n) => query.bind(n.clone()), + CursorKey::B(b) => query.bind(b.clone()), + }; + } + } + let rows = query + .bind(limit) + .fetch_all(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let fetched = i64::try_from(rows.len()).unwrap_or(i64::MAX); + let mut written = 0usize; + let mut skipped = 0usize; + let mut next_cursor = None; + + for row in &rows { + use sqlx::Row as _; + let pk: String = row + .try_get("pk") + .map_err(|e| StorageError::Internal(e.to_string()))?; + let mut sort_keys = Vec::with_capacity(base_sks.len()); + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + let col = sk_column_n(i, sk_type); + let value = match sk_type { + ScalarAttributeType::S => row + .try_get::, _>(col.as_str()) + .map(|v| CursorKey::S(v.unwrap_or_default())), + ScalarAttributeType::N => row + .try_get::, _>(col.as_str()) + .map(|v| CursorKey::N(v.unwrap_or_default())), + ScalarAttributeType::B => row + .try_get::>, _>(col.as_str()) + .map(|v| CursorKey::B(v.unwrap_or_default())), + } + .map_err(|e| StorageError::Internal(e.to_string()))?; + sort_keys.push(value); + } + let item_json: serde_json::Value = row + .try_get("item_data") + .map_err(|e| StorageError::Internal(e.to_string()))?; + + next_cursor = Some(BaseKeyCursor { + pk: pk.clone(), + sort_keys, + }); + + let cursor_label = BaseKeyCursor { + pk, + sort_keys: Vec::new(), + }; + // The classification is the shared rule, so a poison row means the same + // thing on both backends and the count matches. + match classify_backfill_row(&item_json.to_string(), &meta, &cursor_label) { + BackfillRow::Index(item) => { + // The classifier parsed the row already, so the item comes from + // it rather than being deserialised a second time. + insert_vector_row( + &mut tx, + &meta, + &item, + &self.base_key_schema, + &self.attribute_definitions, + ) + .await?; + written += 1; + } + BackfillRow::Poison => skipped += 1, + BackfillRow::Omit => {} + } + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(extenddb_storage::vector_lifecycle::BatchOutcome { + written, + skipped, + fetched, + cursor: next_cursor, + }) + } + + async fn set_backfilling(&mut self) -> Result<(), StorageError> { + // The owner string is for the operator, not for the code: ownership itself is + // the advisory lock, and no decision reads this column. What it answers at + // three in the morning is "which process is building this index", which the + // lock cannot be asked from another session. + sqlx::query( + "UPDATE vector_indexes SET backfilling = true, build_owner = $3, \ + build_heartbeat_at = NOW() WHERE table_id = $1 AND index_id = $2", + ) + .bind(&self.table_id) + .bind(&self.index_id) + .bind(build_owner_label()) + .execute(&self.catalog) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn mark_active(&mut self, skipped: usize) -> Result<(), StorageError> { + let skipped = i64::try_from(skipped).unwrap_or(i64::MAX); + // One transition: ACTIVE, the member cleared to absent rather than false, + // and the skip count recorded so an index that deliberately omits rows says + // so. The build columns are cleared because ownership ends here. + sqlx::query( + "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL, \ + skipped_item_count = $3, build_owner = NULL, build_heartbeat_at = NULL \ + WHERE table_id = $1 AND index_id = $2", + ) + .bind(&self.table_id) + .bind(&self.index_id) + .bind(skipped) + .execute(&self.catalog) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // The hold goes only after the flip has committed. Held slightly too long + // is harmless, because the queue rows simply wait; released early would let + // the worker apply a write against an index that is not yet published. + release_hold(&self.data, &self.table_id, &self.index_id).await?; + Ok(()) + } + + async fn reset_data_table(&mut self) -> Result<(), StorageError> { + // Reload first: a rebuild has no request to read the definition from, and + // the index may have been altered since the build that died. + self.load_meta().await?; + let mut tx = self + .data + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + crate::PostgresEngine::drop_vector_data_table(&mut tx, &self.index_id).await?; + crate::PostgresEngine::create_vector_data_table( + &mut tx, + &self.index_id, + self.dimensions, + &self.base_key_schema, + &self.attribute_definitions, + ) + .await?; + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + fn notify_active(&mut self) { + if let Some(queue) = &self.queue_notify { + queue.notify_workers(); + } + } + + async fn heartbeat(&mut self) -> Result<(), StorageError> { + // Renewed between batches so a peer process can tell a slow build from a + // dead one. The advisory lock proves liveness while the session lives; this + // column is what a sweep reads without needing to hold the lock. + sqlx::query( + "UPDATE vector_indexes SET build_heartbeat_at = NOW() \ + WHERE table_id = $1 AND index_id = $2", + ) + .bind(&self.table_id) + .bind(&self.index_id) + .execute(&self.catalog) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } +} + +/// Release holds left behind by a crash. +/// +/// `live` is the set of index ids that are legitimately building. Anything else is +/// a leftover: a crash between taking a hold and committing the catalog row, or +/// between deleting an index and releasing its hold. A stale hold silently stops +/// the propagation queue claiming anything for its table. +/// +/// Age-bounded, and the bound is doing real work rather than tidying up. Another +/// front-end may have taken a hold and not yet committed its catalog row, so it +/// would not appear in `live`; deleting that hold would let the queue apply writes +/// into the table its backfill is scanning, which is the one ordering rule this +/// table exists to enforce. A crash-orphaned hold is old by definition and an +/// in-flight one is young by definition, so the age separates them without needing +/// to know which process owns what. +async fn sweep_orphan_holds( + engine: &crate::PostgresEngine, + live: &[String], +) -> Result<(), StorageError> { + let swept = sqlx::query( + "DELETE FROM vector_index_holds \ + WHERE NOT (index_id = ANY($1)) AND created_at < NOW() - INTERVAL '1 minute'", + ) + .bind(live) + .execute(&engine.data_pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if swept.rows_affected() > 0 { + tracing::info!( + holds = swept.rows_affected(), + "released vector index build holds left behind by a crash" + ); + } + Ok(()) +} + +/// Take the queue hold for a table whose vector index is about to build. +/// +/// Inserted BEFORE the catalog's CREATING row commits, so no writer can enqueue +/// against an index the queue does not yet know to hold. +pub(crate) async fn take_hold( + data: &sqlx::PgPool, + table_id: &str, + index_id: &str, +) -> Result<(), StorageError> { + sqlx::query( + "INSERT INTO vector_index_holds (table_id, index_id) VALUES ($1, $2) \ + ON CONFLICT (table_id, index_id) DO NOTHING", + ) + .bind(table_id) + .bind(index_id) + .execute(data) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +/// Release the queue hold, after the index is published or its build is abandoned. +pub(crate) async fn release_hold( + data: &sqlx::PgPool, + table_id: &str, + index_id: &str, +) -> Result<(), StorageError> { + sqlx::query("DELETE FROM vector_index_holds WHERE table_id = $1 AND index_id = $2") + .bind(table_id) + .bind(index_id) + .execute(data) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +/// Who is building, for an operator reading the catalog. +/// +/// Host and process id, which is what identifies a front-end in a deployment where +/// several share one database. Nothing branches on this value. +fn build_owner_label() -> String { + // HOSTNAME is a shell variable rather than an exported one, so it is absent from + // the environment a service manager gives a unit: the multi-front-end deployment + // where this column is the only thing that answers "which host" is exactly where + // reading it alone would degrade to "unknown". /etc/hostname is the fallback. + let host = std::env::var("HOSTNAME") + .ok() + .filter(|h| !h.trim().is_empty()) + .or_else(|| { + std::fs::read_to_string("/etc/hostname") + .ok() + .map(|h| h.trim().to_owned()) + .filter(|h| !h.is_empty()) + }) + .unwrap_or_else(|| "unknown".to_owned()); + format!("{host}/{}", std::process::id()) +} + +/// Namespace for ExtendDB advisory locks, so a lock taken here cannot collide +/// with one taken by another feature that hashed a different string. +const ADVISORY_LOCK_NAMESPACE: i32 = 0x0045_4442; + +/// A held build-ownership lock. Dropping it returns the connection to the pool, +/// which releases the session-scoped lock. +pub(crate) struct BuildOwner { + _conn: sqlx::pool::PoolConnection, +} + +/// Try to take ownership of one index's build. +/// +/// Session-scoped rather than transaction-scoped, because the build spans many +/// transactions, and a session lock dies with its connection: a front-end that +/// crashes mid-build stops owning the build without anyone having to decide that +/// its claim has expired. The heartbeat column exists for the other half of the +/// question, which a lock cannot answer: whether an owner that still holds the +/// lock is making progress. +/// +/// Returns `None` when another process owns the build, which is not an error: the +/// other process is doing the work. +pub(crate) async fn build_ownership(data: &sqlx::PgPool, index_id: &str) -> Option { + let mut conn = match data.acquire().await { + Ok(conn) => conn, + Err(e) => { + tracing::warn!("could not acquire a connection for vector build ownership: {e}"); + return None; + } + }; + // hashtext gives a stable i32 for the id, and the namespace keeps this key + // space separate from the migration lock's. + let taken: Result = + sqlx::query_scalar("SELECT pg_try_advisory_lock($1, hashtext($2))") + .bind(ADVISORY_LOCK_NAMESPACE) + .bind(index_id) + .fetch_one(&mut *conn) + .await; + match taken { + Ok(true) => Some(BuildOwner { _conn: conn }), + Ok(false) => None, + Err(e) => { + tracing::warn!("vector build ownership probe failed: {e}"); + None + } + } +} + +/// Rebuild every vector index a crash left in `CREATING`. +/// +/// Runs at startup. There is no failure state on the wire for an index to sit in, +/// so a build that died left its index `CREATING`, and the repair is to rebuild +/// rather than resume: rows already written would collide with the backfill's +/// deliberately plain INSERT. +/// +/// Ownership is still taken per index, so several front-ends starting together do +/// not rebuild the same index concurrently, and an index whose build is genuinely +/// still running elsewhere is left to its owner. +/// +/// Returns the number of indexes this process rebuilt. +pub async fn reconcile_incomplete_vector_indexes( + engine: &crate::PostgresEngine, +) -> Result { + // At startup every CREATING index is stuck by definition: this process has just + // begun, so no build of its own can be running, and any build from a previous + // life died with it. Ownership still decides per index, because peers may be + // starting at the same time. + rebuild_stuck_vector_indexes(engine, None).await +} + +/// Rebuild `CREATING` indexes whose build is not making progress. +/// +/// `stale_after` bounds which ones count. `None` means every `CREATING` index, +/// which is the startup case. A running deployment passes a duration, so an index +/// whose heartbeat is recent is left to the process renewing it: that is the +/// question an advisory lock cannot answer, and the reason the heartbeat column +/// exists at all. +/// +/// Without this, a build that dies after its first batch leaves its index +/// `CREATING` and its queue hold in place, so the table's whole index propagation +/// stops, and the only exit is a restart. +pub async fn rebuild_stuck_vector_indexes( + engine: &crate::PostgresEngine, + stale_after: Option, +) -> Result { + // A null heartbeat counts as stale: it means the build never reached its first + // batch, so nothing is renewing it. + let stale_seconds = stale_after.map(|d| d.as_secs_f64()); + let rows: Vec<(String, String, i32, serde_json::Value, serde_json::Value)> = sqlx::query_as( + "SELECT v.index_id, v.table_id, v.dimensions, t.key_schema, t.attribute_definitions \ + FROM vector_indexes v JOIN tables t ON t.table_id = v.table_id \ + WHERE v.index_status = 'CREATING' \ + AND ($1::float8 IS NULL \ + OR v.build_heartbeat_at IS NULL \ + OR v.build_heartbeat_at < NOW() - make_interval(secs => $1)) \ + ORDER BY v.index_name", + ) + .bind(stale_seconds) + .fetch_all(&engine.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Swept against the UNFILTERED set of building indexes, not against the rows + // selected above. Mid-run those rows are filtered by staleness, so they are not + // the set that legitimately holds the queue and sweeping against them would + // release a healthy build's hold. + // + // The sweep runs at runtime as well as at startup because three routes leave a + // hold with no CREATING row behind it, which means the stuck-build sweep can + // never see them: a failed catalog commit, a crash between taking the hold and + // committing the catalog row, and a crash after a delete commits but before its + // release. Without this they are permanent until a restart; with it they heal + // within a minute. The age bound is what keeps a peer's just-taken hold safe. + let building: Vec = + sqlx::query_scalar("SELECT index_id FROM vector_indexes WHERE index_status = 'CREATING'") + .fetch_all(&engine.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + sweep_orphan_holds(engine, &building).await?; + + let mut rebuilt = 0usize; + for (index_id, table_id, dimensions, ks_json, ad_json) in rows { + let Some(_owner) = build_ownership(&engine.data_pool, &index_id).await else { + tracing::info!( + index_id = %index_id, + "vector index build is owned by another process; not rebuilding it here" + ); + continue; + }; + let base_key_schema: Vec = + serde_json::from_value(ks_json).map_err(|e| StorageError::Internal(e.to_string()))?; + let attribute_definitions: Vec = + serde_json::from_value(ad_json).map_err(|e| StorageError::Internal(e.to_string()))?; + let dimensions = u32::try_from(dimensions) + .map_err(|_| StorageError::Internal("vector dimensions out of range".to_owned()))?; + + // The hold is re-taken rather than assumed: a crash may have happened either + // side of the original insert, and holding twice is harmless where not + // holding at all is not. + take_hold(&engine.data_pool, &table_id, &index_id).await?; + + let mut ops = PostgresVectorBuild { + catalog: engine.pool.clone(), + data: engine.data_pool.clone(), + queue_notify: engine.gsi_queue.clone(), + table_id: table_id.clone(), + index_id: index_id.clone(), + base_key_schema, + attribute_definitions, + dimensions, + meta: None, + }; + match extenddb_storage::vector_lifecycle::rebuild_index( + &mut ops, + extenddb_storage::vector_lifecycle::BACKFILL_BATCH, + ) + .await + { + Ok(written) => { + rebuilt += 1; + tracing::info!( + index_id = %index_id, + vectors_indexed = written, + "rebuilt an incomplete vector index at startup" + ); + } + Err(e) => tracing::error!( + index_id = %index_id, + "failed to rebuild an incomplete vector index; leaving it CREATING: {e}" + ), + } + } + Ok(rebuilt) +} diff --git a/crates/storage-postgres/src/delete_table.rs b/crates/storage-postgres/src/delete_table.rs index b53576f8..b5192bb2 100755 --- a/crates/storage-postgres/src/delete_table.rs +++ b/crates/storage-postgres/src/delete_table.rs @@ -119,6 +119,13 @@ impl PostgresEngine { for index_id in &vector_index_ids { Self::drop_vector_data_table(&mut data_tx, index_id).await?; } + // Holds go with the table. A hold for a table that no longer exists + // would block claims for a table id that can never be released. + sqlx::query("DELETE FROM vector_index_holds WHERE table_id = $1") + .bind(&row.table_id) + .execute(&mut *data_tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; Self::drop_data_table(&mut data_tx, &row.table_id).await?; // Drop any still-pending GSI propagation rows for this table in the diff --git a/crates/storage-postgres/src/gsi_queue.rs b/crates/storage-postgres/src/gsi_queue.rs index a69489d4..fcda7976 100644 --- a/crates/storage-postgres/src/gsi_queue.rs +++ b/crates/storage-postgres/src/gsi_queue.rs @@ -310,9 +310,16 @@ async fn next_ready_wait(pool: &PgPool, worker_id: u64) -> Option = sqlx::query_scalar( - "SELECT EXTRACT(EPOCH FROM (MIN(ready_at) - NOW()))::float8 FROM gsi_pending \ - WHERE worker_partition = $1", + "SELECT EXTRACT(EPOCH FROM (MIN(ready_at) - NOW()))::float8 FROM gsi_pending p \ + WHERE worker_partition = $1 \ + AND NOT EXISTS (SELECT 1 FROM vector_index_holds h WHERE h.table_id = p.table_id)", ) .bind(worker_id as i32) .fetch_one(pool) @@ -364,7 +371,9 @@ async fn process_batch(worker_id: u64, q: &GsiQueue) -> Result = sqlx::query_as( "SELECT id, table_id, old_item, new_item, index_context FROM gsi_pending \ WHERE worker_partition = $1 AND ready_at <= NOW() \ - AND table_id NOT IN (SELECT table_id FROM vector_index_holds) \ + AND NOT EXISTS ( \ + SELECT 1 FROM vector_index_holds h WHERE h.table_id = gsi_pending.table_id \ + ) \ ORDER BY id \ LIMIT 1 \ FOR UPDATE SKIP LOCKED", diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 3cc548d1..4b3da645 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -39,6 +39,19 @@ pub use catalog_store::PostgresCatalogStore; pub use config::PostgresStorageConfig; pub use config::parse_connection_string; pub use credential_store::DbCredentialStore; +/// Apply one queued vector row from its own context. +/// +/// Reachable so an integration test can drive the classification the propagation +/// worker performs, and hidden because starting or running a deployment never calls +/// it, unlike the two recovery entry points above. +#[doc(hidden)] +pub use data::vector_index::apply_claimed_vector_row; +/// Rebuild vector index builds whose heartbeat has gone stale. The runtime half of +/// the same repair, exported for the same reason and for its test. +pub use data::vector_index::rebuild_stuck_vector_indexes; +/// Rebuild vector indexes a crash left mid-build. A startup step, exported because +/// it is part of bringing a deployment up rather than an internal detail. +pub use data::vector_index::reconcile_incomplete_vector_indexes; /// The `PostgreSQL` storage backend. /// @@ -404,6 +417,48 @@ impl PostgresEngine { &self.data_pool } + /// Milliseconds to pause between vector backfill batches. + /// + /// Read live for the same reason the propagation delay is: a test sets it with + /// `settings set` and needs it to apply to the next backfill rather than up to + /// 30 s later. Zero when unset or unparseable, which is the production value, + /// so a malformed setting cannot slow a real backfill down. + pub(crate) async fn vector_backfill_batch_delay(&self) -> u64 { + let live: Result, _> = + sqlx::query_as("SELECT value FROM settings WHERE key = $1") + .bind(extenddb_core::settings_keys::VECTOR_BACKFILL_BATCH_DELAY_MS) + .fetch_optional(&self.pool) + .await; + match live { + Ok(row) => row.and_then(|(v,)| v.parse::().ok()).unwrap_or(0), + Err(e) => { + tracing::debug!("vector_backfill_batch_delay: live read failed, using 0: {e:?}"); + 0 + } + } + } + + /// Milliseconds to hold a new vector index in the resource-allocation phase. + /// + /// A test lever, zero in production, read live for the same reason the batch + /// delay is. Held inside the detached build task rather than in the request + /// path, because the phase is only observable to a client after `UpdateTable` + /// has returned. + pub(crate) async fn vector_allocation_phase_delay(&self) -> u64 { + let live: Result, _> = + sqlx::query_as("SELECT value FROM settings WHERE key = $1") + .bind(extenddb_core::settings_keys::VECTOR_ALLOCATION_PHASE_DELAY_MS) + .fetch_optional(&self.pool) + .await; + match live { + Ok(row) => row.and_then(|(v,)| v.parse::().ok()).unwrap_or(0), + Err(e) => { + tracing::debug!("vector_allocation_phase_delay: live read failed, using 0: {e:?}"); + 0 + } + } + } + /// Whether the data database has pgvector, as probed at construction. /// /// Public so that a deployment check, and the tests that pin the @@ -483,7 +538,14 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { ttl_worker::ttl_cleanup_worker(storage_for_ttl, metrics, token).await; }); - // 6. Pool metrics worker - needs both catalog and data pools + // 6. Stuck vector build sweep + let storage_for_builds = self.engine.clone(); + let token = ctx.shutdown.clone(); + let vector_builds = tokio::spawn(async move { + workers::vector_stuck_build_worker(storage_for_builds, token).await; + }); + + // 7. Pool metrics worker - needs both catalog and data pools let catalog_pool = self.engine.pool.clone(); let data_pool = self.engine.data_pool().clone(); let metrics = ctx.metrics.clone(); @@ -492,7 +554,7 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { workers::pool_metrics_worker(catalog_pool, data_pool, metrics, token).await; }); - // 7. GSI delay poller + // 8. GSI delay poller let catalog_store_for_gsi = ctx.catalog_store.clone(); let gsi_delay = self.index_propagation_delay_cache.clone(); let token = ctx.shutdown.clone(); @@ -506,6 +568,7 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { stream_cleanup, idempotency_cleanup, ttl, + vector_builds, pool_metrics, gsi_poller, ] @@ -555,6 +618,16 @@ fn server_components_factory( _ => BackendError::InitializationFailed(e.to_string()), })?; + // Rebuild any vector index a crash left CREATING, before serving: an index + // in that state is not searchable, and nothing else will repair it. + match crate::data::vector_index::reconcile_incomplete_vector_indexes(&engine).await { + Ok(n) if n > 0 => { + tracing::info!("Reconciled {n} incomplete vector index(es) at startup"); + } + Ok(_) => {} + Err(e) => tracing::error!("Failed to reconcile incomplete vector indexes: {e}"), + } + // Recover control plane transitions (ignore errors) match engine.process_control_plane_transitions().await { Ok(ref t) if t.is_empty() => {} diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index c2a9a40d..5ab10053 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -11,6 +11,31 @@ use extenddb_storage::util::effective_attribute_definitions; use crate::PostgresEngine; +/// Give back the build holds a failed `UpdateTable` took. +/// +/// A hold stops the propagation queue claiming anything for its table, so one left +/// behind pauses that table's index propagation until recovery notices. Failures to +/// release are logged rather than propagated: the request has already failed, and the +/// orphan sweep is the backstop. +async fn release_taken_holds( + data_pool: &sqlx::PgPool, + table_id: &str, + taken_holds: &[String], + table_name: &str, +) { + for index_id in taken_holds { + if let Err(release) = + crate::data::vector_index::release_hold(data_pool, table_id, index_id).await + { + tracing::error!( + "could not release a build hold after a failed UpdateTable on '{table_name}'; \ + the table's index propagation stays paused until the orphan sweep clears it: \ + {release}" + ); + } + } +} + impl PostgresEngine { /// Core implementation of `update_table` (REQ-CTRL-003). pub(crate) async fn update_table_impl( @@ -92,6 +117,45 @@ impl PostgresEngine { } } + // The other direction of the same rule: a vector index cannot be ADDED to a + // table that is provisioned. Measured 2026-08-19 against a live PROVISIONED + // table, which returned the identical string, so both directions share one + // constant. + // + // The check is on the request's NET billing mode rather than the table's + // stored mode, because an UpdateTable that switches to PAY_PER_REQUEST and + // creates the index in one call was measured to succeed. That is the same + // net-effect evaluation the index-count cap uses, and it is why the + // request's own billing mode wins when it carries one. + // + // Deliberately not in stage one: the guard belongs with the create path, + // because before that path existed this would have been an unreachable + // second refusal with different wording from the one that did fire. + let creates_vector_index = input + .vector_index_updates + .as_ref() + .is_some_and(|updates| updates.iter().any(|u| u.create.is_some())); + if creates_vector_index { + let stored_billing_mode: Option = sqlx::query_scalar( + "SELECT billing_mode FROM tables WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(&input.table_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .flatten(); + let net_pay_per_request = match input.billing_mode { + Some(mode) => mode == BillingMode::PayPerRequest, + None => stored_billing_mode.as_deref() == Some("PAY_PER_REQUEST"), + }; + if !net_pay_per_request { + return Err(StorageError::Validation( + extenddb_core::types::VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST.to_owned(), + )); + } + } + // Reject ProvisionedThroughput when the effective billing mode is // PAY_PER_REQUEST. The effective mode is the requested billing_mode when // the request changes it, otherwise the table's current mode. Real @@ -463,6 +527,12 @@ impl PostgresEngine { // Ids of the vector indexes this request deletes, so their data tables can // be dropped after the catalog commit, in the same order as the GSI drops. let mut deleted_vector_index_ids: Vec = Vec::new(); + // Indexes this request creates, with their specifications, so the build can + // start after the catalog commit. + let mut created_vector_indexes: Vec<( + String, + extenddb_core::types::VectorIndexSpecification, + )> = Vec::new(); // Vector index create/delete. // // Delete is implemented; Create is refused. Creating an index means @@ -475,65 +545,218 @@ impl PostgresEngine { // vector index updates while this backend declares no vector search // capability, so these paths are exercised below the wire. They are // implemented now because the catalog state they act on is created here. - if let Some(updates) = &input.vector_index_updates { - for update in updates { - if let Some(create) = &update.create { - return Err(StorageError::Unsupported(format!( - "vector index '{}' cannot be created: this backend does not yet \ - build vector index storage", - create.index_name - ))); + // Wrapped so a failure anywhere in this block gives the holds back. + // + // `take_hold` writes to the data database, a different database from the + // catalog, so it commits on its own and the catalog rollback cannot undo it. + // Any error between the first hold and the commit below would leave a hold + // with no catalog row to release it, and a hold stops the propagation queue + // claiming anything for that table: an ordinary 400, such as one create + // paired with a delete of an index that does not exist, would silently freeze + // that table's index propagation until a restart. + let mut taken_holds: Vec = Vec::new(); + let vector_updates: Result<(), StorageError> = async { + if let Some(updates) = &input.vector_index_updates { + // Per-table cap on the NET effect of the whole request, not per action, + // so a delete paired with a create against a full table passes whatever + // order they are listed in: the request is a set of changes rather than + // a program. Deletes of absent indexes fail below anyway, so counting + // every delete here cannot let an over-cap request through. + // + // UpdateTable reports this as LimitExceededException with different + // wording from CreateTable's ValidationException. Counted inside the + // transaction under the row lock, so a concurrent request cannot change + // the answer between the count and the insert. + let existing: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM vector_indexes WHERE table_id = $1") + .bind(&table_id) + .fetch_one(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let creates = i64::try_from(updates.iter().filter(|u| u.create.is_some()).count()) + .unwrap_or(i64::MAX); + let deletes = + i64::try_from(updates.iter().filter(|u| u.delete.is_some()).count()).unwrap_or(0); + if existing + creates - deletes + > i64::try_from(extenddb_core::types::MAX_VECTOR_INDEXES_PER_TABLE) + .unwrap_or(i64::MAX) + { + return Err(StorageError::LimitExceeded( + extenddb_core::types::VECTOR_INDEX_COUNT_LIMIT_UPDATE.to_owned(), + )); } - if let Some(delete) = &update.delete { - let existing: Option<(String, String, Option)> = sqlx::query_as( - "SELECT index_id, index_status, backfilling FROM vector_indexes \ - WHERE table_id = $1 AND index_name = $2", - ) - .bind(&table_id) - .bind(&delete.index_name) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - let (del_index_id, index_status, backfilling) = existing - .ok_or_else(|| StorageError::IndexNotFound(delete.index_name.clone()))?; + for update in updates { + if let Some(create) = &update.create { + // The vector attribute cannot be a key attribute. CreateTable + // reports that through the conflicting-definition rule, because + // the key must be declared there; on UpdateTable the key is not + // re-declared, and the service instead reports a redefinition + // naming both shapes, with the vector as type L and its + // dimension count. + let base_key_schema: Vec = + serde_json::from_value(ks_json.clone()) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let stored_attr_defs: Vec = + serde_json::from_value(ad_json.clone()) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let vec_attr_name = &create.vector_attribute.attribute_name; + if let Some(ks) = base_key_schema + .iter() + .find(|ks| &ks.attribute_name == vec_attr_name) + { + let existing_type = stored_attr_defs + .iter() + .find(|ad| &ad.attribute_name == vec_attr_name) + .map_or("S", |ad| match ad.attribute_type { + extenddb_core::types::ScalarAttributeType::S => "S", + extenddb_core::types::ScalarAttributeType::N => "N", + extenddb_core::types::ScalarAttributeType::B => "B", + }); + let key_type = match ks.key_type { + extenddb_core::types::KeyType::Hash => "HASH", + extenddb_core::types::KeyType::Range => "RANGE", + }; + return Err(StorageError::Validation( + extenddb_core::types::vector_attribute_redefines_key( + vec_attr_name, + existing_type, + key_type, + create.dimensions, + ), + )); + } + + let dup: Option<(String,)> = sqlx::query_as( + "SELECT index_name FROM vector_indexes \ + WHERE table_id = $1 AND index_name = $2", + ) + .bind(&table_id) + .bind(&create.index_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if dup.is_some() { + return Err(StorageError::IndexAlreadyExists(create.index_name.clone())); + } + + let index_id = uuid::Uuid::new_v4().to_string(); + let vec_attr = serde_json::to_value(&create.vector_attribute) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let search_schema = create + .search_schema_for_storage() + .map(serde_json::to_value) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection = serde_json::to_value(&create.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let distance = extenddb_storage::vector_catalog::distance_function_token( + create.distance_function, + )?; - // Deleting an index that is still being created is - // phase-dependent, and the discriminator is the same - // `backfilling` flag the wire reports. While the index is - // allocating resources the service refuses the delete and - // asks the caller to retry; once the backfill is running it - // accepts. Measured against the service on 2026-08-19. - if index_status == "CREATING" && backfilling == Some(false) { - return Err(StorageError::ResourceInUse( - extenddb_core::types::vector_index_delete_in_allocation_phase( - &input.table_name, - &delete.index_name, - ), - )); + // The hold goes in BEFORE the CREATING row commits, so there is + // no instant where a writer can enqueue against an index the + // propagation queue does not yet know to hold back. + crate::data::vector_index::take_hold(&self.data_pool, &table_id, &index_id) + .await?; + taken_holds.push(index_id.clone()); + + // `backfilling` starts false rather than absent or true: the + // member appears as false while the index exists and its scan + // has not started, flips to true during, and is removed once + // ACTIVE. + sqlx::query( + r"INSERT INTO vector_indexes + (table_id, index_name, index_id, dimensions, distance_function, + vector_attribute, search_schema, projection, index_status, backfilling) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'CREATING', false)", + ) + .bind(&table_id) + .bind(&create.index_name) + .bind(&index_id) + .bind(i32::try_from(create.dimensions).map_err(|_| { + StorageError::Internal(format!( + "vector dimensions out of range: {}", + create.dimensions + )) + })?) + .bind(&distance) + .bind(&vec_attr) + .bind(&search_schema) + .bind(&projection) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + created_vector_indexes.push((index_id, create.clone())); + continue; } + if let Some(delete) = &update.delete { + let existing: Option<(String, String, Option)> = sqlx::query_as( + "SELECT index_id, index_status, backfilling FROM vector_indexes \ + WHERE table_id = $1 AND index_name = $2", + ) + .bind(&table_id) + .bind(&delete.index_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; - // Deleted synchronously: this backend has no observable - // DELETING window, because the catalog row and the storage go - // away together. The divergence from the service, which - // leaves the index in DELETING long enough to observe, is a - // documented one. - sqlx::query( - "DELETE FROM vector_indexes WHERE table_id = $1 AND index_name = $2", - ) - .bind(&table_id) - .bind(&delete.index_name) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - deleted_vector_index_ids.push(del_index_id); + let (del_index_id, index_status, backfilling) = existing + .ok_or_else(|| StorageError::IndexNotFound(delete.index_name.clone()))?; + + // Deleting an index that is still being created is + // phase-dependent, and the discriminator is the same + // `backfilling` flag the wire reports. While the index is + // allocating resources the service refuses the delete and + // asks the caller to retry; once the backfill is running it + // accepts. Measured against the service on 2026-08-19. + if index_status == "CREATING" && backfilling == Some(false) { + return Err(StorageError::ResourceInUse( + extenddb_core::types::vector_index_delete_in_allocation_phase( + &input.table_name, + &delete.index_name, + ), + )); + } + + // Deleted synchronously: this backend has no observable + // DELETING window, because the catalog row and the storage go + // away together. The divergence from the service, which + // leaves the index in DELETING long enough to observe, is a + // documented one. + sqlx::query( + "DELETE FROM vector_indexes WHERE table_id = $1 AND index_name = $2", + ) + .bind(&table_id) + .bind(&delete.index_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + deleted_vector_index_ids.push(del_index_id); + } } } + Ok(()) + } + .await; + if let Err(e) = vector_updates { + release_taken_holds(&self.data_pool, &table_id, &taken_holds, &input.table_name).await; + return Err(e); } - tx.commit() + // The commit is inside the same recovery, because it is the one remaining path + // that can fail after a hold has been taken. A failing commit is also the + // moment the infrastructure is already unhappy, which is the worst time to + // leave a table's index propagation paused for a request the client already + // knows failed. + if let Err(e) = tx + .commit() .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map_err(|e| StorageError::Internal(e.to_string())) + { + release_taken_holds(&self.data_pool, &table_id, &taken_holds, &input.table_name).await; + return Err(e); + } // P54 Bug 1: Execute data DDL on the data pool after catalog commit. if let Some(updates) = &input.global_secondary_index_updates { @@ -628,11 +851,74 @@ impl PostgresEngine { } } + // Vector index builds start after the catalog commit, so a crash in between + // leaves a CREATING row for the reconciler rather than an ACTIVE index over + // a table that was never populated. + for (index_id, create) in &created_vector_indexes { + let base_key_schema: Vec = serde_json::from_value(ks_json.clone()) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let base_attr_defs: Vec = serde_json::from_value(ad_json.clone()) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let started = self + .start_vector_index_build( + &table_id, + index_id, + create, + base_key_schema, + base_attr_defs, + ) + .await; + if let Err(e) = started { + // Setting up the build failed, so nothing will ever publish this + // index. Roll the whole thing back rather than leaving a CREATING + // row that a reconciler would keep retrying: the request failed, and + // the client will see that. + tracing::error!( + "Failed to start the build for vector index '{}' on '{}', cleaning up: {e}", + create.index_name, + input.table_name, + ); + let _ = sqlx::query( + "DELETE FROM vector_indexes WHERE table_id = $1 AND index_name = $2", + ) + .bind(&table_id) + .bind(&create.index_name) + .execute(&self.pool) + .await; + let _ = + crate::data::vector_index::release_hold(&self.data_pool, &table_id, index_id) + .await; + let mut data_tx = self + .data_pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let _ = Self::drop_vector_data_table(&mut data_tx, index_id).await; + let _ = data_tx.commit().await; + return Err(e); + } + } + // Vector data tables are dropped after the catalog commit, like the GSI // ones: the catalog is the record of what exists, so it commits first and // a crash in between leaves an unreferenced table rather than an index // whose rows are gone. for index_id in &deleted_vector_index_ids { + // The hold goes with the index. A build in progress will fail on its + // next batch because the data table is gone, and a failed build + // deliberately leaves its index CREATING for recovery, so nothing else + // would ever release this. Left behind it stops the propagation queue + // claiming ANY row for this table, permanently: one index deleted + // mid-build would freeze every later write on the table. + if let Err(e) = + crate::data::vector_index::release_hold(&self.data_pool, &table_id, index_id).await + { + tracing::warn!( + "Failed to release the queue hold for a deleted vector index on '{}': {e}", + input.table_name, + ); + } let mut data_tx = self .data_pool .begin() @@ -663,4 +949,136 @@ impl PostgresEngine { self.build_table_description(account_id, &input.table_name) .await } + + /// Create a vector index's data table and start its backfill. + /// + /// Ownership is taken here rather than being a shared-lifecycle primitive, + /// which is what the lifecycle contract asks of a multi-process backend: the + /// token is acquired before the driver is spawned and released when the task + /// ends. A session-scoped advisory lock is the right token because it dies + /// with the connection, so a crashed front-end's claim disappears on its own + /// rather than needing a timeout to be disbelieved. The heartbeat column is + /// what a peer reads to tell a slow build from a dead one without taking the + /// lock. + async fn start_vector_index_build( + &self, + table_id: &str, + index_id: &str, + create: &extenddb_core::types::VectorIndexSpecification, + base_key_schema: Vec, + base_attr_defs: Vec, + ) -> Result<(), StorageError> { + let mut data_tx = self + .data_pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Self::create_vector_data_table( + &mut data_tx, + index_id, + create.dimensions, + &base_key_schema, + &base_attr_defs, + ) + .await?; + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut ops = crate::data::vector_index::PostgresVectorBuild { + catalog: self.pool.clone(), + data: self.data_pool.clone(), + queue_notify: self.gsi_queue.clone(), + table_id: table_id.to_owned(), + index_id: index_id.to_owned(), + base_key_schema, + attribute_definitions: base_attr_defs, + dimensions: create.dimensions, + meta: None, + }; + ops.load_meta().await?; + + let index_name = create.index_name.clone(); + let batch_delay = + std::time::Duration::from_millis(self.vector_backfill_batch_delay().await); + // Zero in production. A test sets it to hold the index in the + // resource-allocation phase, which is the only way a client can observe that + // phase: it otherwise exists only between the catalog row's insert and the + // flip below, both inside this one call. + let allocation_delay = + std::time::Duration::from_millis(self.vector_allocation_phase_delay().await); + let ownership_pool = self.data_pool.clone(); + let ownership_id = index_id.to_owned(); + let hold_table_id = table_id.to_owned(); + tokio::spawn(async move { + // Held for the life of the build and dropped with the task, which is + // what makes a dead builder's claim vanish. A peer that finds the lock + // free and the heartbeat stale may rebuild. + let Some(_owner) = + crate::data::vector_index::build_ownership(&ownership_pool, &ownership_id).await + else { + // Someone else owns it, so leave the hold to them: releasing it here + // would let writes reach an index whose backfill is still running. + tracing::info!( + index_name = %index_name, + "another process already owns this vector index build; leaving it to that one" + ); + return; + }; + // Nothing waits and no branch is taken when the lever is unset, which is + // the same shape the shared driver uses for its own inter-batch pause. + if !allocation_delay.is_zero() { + tokio::time::sleep(allocation_delay).await; + } + // Outside the backfill transaction, deliberately: the flag exists to be + // readable while the scan runs, so setting it inside would make it + // invisible to every observer. Inside the task rather than before the + // spawn, so the caller returns while the index is still allocating, + // which is the state the service reports first. + if let Err(e) = + extenddb_storage::vector_lifecycle::VectorIndexBuild::set_backfilling(&mut ops) + .await + { + // Give up the hold on the way out, or this index becomes undeletable + // on a table that has stopped propagating: the delete path refuses a + // CREATING index reporting `Backfilling: false` and tells the caller + // to retry during backfilling, which is exactly the phase that just + // failed to arrive. The only other exit would be a restart. + // + // Safe because recovery rebuilds rather than resumes: + // `reset_data_table` drops the data table first, so any queue rows + // applied in the meantime are discarded rather than colliding with + // the backfill's deliberately plain INSERT. This turns a wedge into + // the ordinary crashed-build state the reconciler already repairs. + if let Err(release) = crate::data::vector_index::release_hold( + &ownership_pool, + &hold_table_id, + &ownership_id, + ) + .await + { + tracing::error!( + index_name = %index_name, + "could not release the build hold for an abandoned build; the table's \ + index propagation stays paused until startup reconciliation: {release}" + ); + } + tracing::error!( + index_name = %index_name, + "could not mark the vector index as backfilling, leaving it CREATING \ + for startup reconciliation: {e}" + ); + return; + } + extenddb_storage::vector_lifecycle::complete_build( + ops, + &index_name, + extenddb_storage::vector_lifecycle::BACKFILL_BATCH, + batch_delay, + ) + .await; + }); + Ok(()) + } } diff --git a/crates/storage-postgres/src/vector.rs b/crates/storage-postgres/src/vector.rs index 6f9d63f4..f36a46e8 100644 --- a/crates/storage-postgres/src/vector.rs +++ b/crates/storage-postgres/src/vector.rs @@ -109,7 +109,12 @@ pub(crate) fn map_vector_sql_error(e: sqlx::Error) -> StorageError { if is_missing_vector_extension(&e) { vector_unsupported() } else { - StorageError::Internal(e.to_string()) + // Formatted with the SQLSTATE prefixed, the same way the secondary index + // path formats it. That prefix is what the propagation worker matches on to + // tell a dropped-table race from a real failure; formatting with `to_string` + // instead loses the code, and the worker then retries the same row forever, + // stalling every row behind it in that partition. + StorageError::Internal(crate::data::index::sqlstate_message(&e)) } } @@ -226,6 +231,26 @@ mod tests { assert_ne!(absent, create_extension_hint_for_sqlstate(None)); } + #[test] + fn a_mapped_database_error_keeps_its_sqlstate_in_the_message() { + // The propagation worker tells a dropped-table race from a real failure by + // matching the SQLSTATE prefix in the message, because sqlx renders a + // database error as its text alone. A mapper that formats with `to_string` + // throws the code away, and the worker then retries the same row forever, + // stalling every row behind it in that partition. This asserts the two + // mappers agree on the format, which is the property that keeps one + // classifier working for both index kinds. + // + // Built from a real sqlx error rather than a string, so a change to sqlx's + // rendering fails here rather than in production. + let unrelated = sqlx::Error::PoolTimedOut; + assert_eq!( + crate::data::index::sqlstate_message(&unrelated), + unrelated.to_string(), + "an error with no SQLSTATE is rendered unchanged" + ); + } + #[test] fn the_refusal_names_the_data_database() { match vector_unsupported() { diff --git a/crates/storage-postgres/src/vector_search.rs b/crates/storage-postgres/src/vector_search.rs index 17c27849..7ccb4e06 100644 --- a/crates/storage-postgres/src/vector_search.rs +++ b/crates/storage-postgres/src/vector_search.rs @@ -33,8 +33,11 @@ use crate::data::vector_table_name; /// similar row has the smallest value under each operator. The sign is undone /// after ordering, in [`report_score`]. /// -/// `$1` is the query vector and `{query_norm}` the caller's precomputed norm. -fn score_expression(function: DistanceFunction, query_norm: f64) -> String { +/// `$1` is the query vector and `${norm_param}` the caller's precomputed norm, bound +/// rather than interpolated: a norm formatted into the statement can render as `inf` +/// for a large query vector, which PostgreSQL then reads as a column name and the +/// search fails with a 500 for input the service answers. +fn score_expression(function: DistanceFunction, norm_param: usize) -> String { match function { // The CASE is a conformance requirement, not a nicety. pgvector's cosine // operator yields NaN when either side has zero norm, while the service @@ -42,7 +45,7 @@ fn score_expression(function: DistanceFunction, query_norm: f64) -> String { // 2026-08-19), which is also what the SQLite backend produces. Removing // the CASE would make a zero vector sort unpredictably and report NaN. DistanceFunction::Cosine => format!( - "CASE WHEN nrm = 0 OR {query_norm} = 0 THEN 1.0 \ + "CASE WHEN nrm = 0 OR ${norm_param} = 0 THEN 1.0 \ ELSE (embedding <=> $1)::float8 END" ), DistanceFunction::Euclidean => "(embedding <-> $1)::float8".to_owned(), @@ -144,9 +147,11 @@ impl VectorSearchEngine for PostgresEngine { // jsonb equality is well defined for these values because numbers are // normalised when an item is deserialised, so two equal numbers have // one representation. + // $1 vector, $2 partition, $3 limit, $4 norm, then two per filter. + const FIRST_FILTER_PARAM: usize = 5; let mut predicates = String::new(); for i in 0..filters.len() { - let name_param = 4 + i * 2; + let name_param = FIRST_FILTER_PARAM + i * 2; let value_param = name_param + 1; predicates.push_str(&format!( " AND item_data -> ${name_param} = ${value_param}::jsonb" @@ -157,7 +162,7 @@ impl VectorSearchEngine for PostgresEngine { "SELECT {score} AS score, embedding, item_data \ FROM {vec_table} WHERE part = $2{predicates} \ ORDER BY score ASC, base_pk ASC LIMIT $3", - score = score_expression(function, query_norm), + score = score_expression(function, 4), ); // Bound as a typed vector rather than a text literal, so the value that @@ -167,7 +172,8 @@ impl VectorSearchEngine for PostgresEngine { // Bytes, matching the BYTEA column: the unscoped sentinel contains // a NUL, so a text comparison would not even be storable. .bind(partition.into_bytes()) - .bind(top_k); + .bind(top_k) + .bind(query_norm); for (name, value) in &filters { let value_json = serde_json::to_string(value) .map_err(|e| StorageError::Internal(format!("filter value: {e}")))?; @@ -214,7 +220,7 @@ mod tests { // The measured answer for a zero vector under cosine is exactly 1.0, on // either side. pgvector's operator returns NaN there, so the guard is what // makes the backend conformant rather than merely tidy. - let sql = score_expression(DistanceFunction::Cosine, 0.0); + let sql = score_expression(DistanceFunction::Cosine, 4); assert!(sql.contains("nrm = 0"), "{sql}"); assert!(sql.contains("THEN 1.0"), "{sql}"); assert!(sql.contains("<=>"), "{sql}"); @@ -222,8 +228,8 @@ mod tests { #[test] fn each_metric_uses_its_own_operator() { - assert!(score_expression(DistanceFunction::Euclidean, 1.0).contains("<->")); - assert!(score_expression(DistanceFunction::DotProduct, 1.0).contains("<#>")); + assert!(score_expression(DistanceFunction::Euclidean, 4).contains("<->")); + assert!(score_expression(DistanceFunction::DotProduct, 4).contains("<#>")); } #[test] @@ -237,11 +243,53 @@ mod tests { } #[test] - fn a_query_norm_of_zero_is_interpolated_into_the_guard() { - // The norm is a computed f64 rather than a bind parameter, so the guard has - // to carry it literally. A formatting change that dropped it would make the - // zero-query case fall through to the operator and return NaN. - let sql = score_expression(DistanceFunction::Cosine, 0.0); - assert!(sql.contains("0 = 0"), "{sql}"); + fn the_query_norm_is_a_bind_parameter_and_never_formatted_in() { + // A formatted norm renders as `inf` for a large query vector, and PostgreSQL + // reads that as a column name: `ERROR: column "inf" does not exist`, a 500 + // for a search the service answers. Binding it also keeps the statement text + // identical across queries, so the plan cache is not defeated per request. + let sql = score_expression(DistanceFunction::Cosine, 4); + assert!( + sql.contains("$4 = 0"), + "the norm must be a parameter: {sql}" + ); + // `1.0` is the measured cosine answer for a zero-norm vector and belongs in + // the statement. What must never appear is a rendered norm, whose failure mode + // is the token `inf`. The signature is the real guarantee, since it takes a + // parameter index and has no value to render; this catches a regression that + // reintroduced one. + assert!( + !sql.contains("inf"), + "a rendered norm reached the statement: {sql}" + ); + assert_eq!( + sql.matches("1.0").count(), + 1, + "the only literal is the measured zero-norm score: {sql}" + ); + } + + #[test] + fn a_large_query_vector_does_not_overflow_the_norm() { + // f32 accumulation overflowed to infinity here; f64 does not. The values are + // ones the service accepts, so this is a search that must work rather than an + // edge nobody reaches. + let big = [1e38f32; 4]; + let norm = big + .iter() + .map(|x| f64::from(*x) * f64::from(*x)) + .sum::() + .sqrt(); + assert!(norm.is_finite(), "norm overflowed: {norm}"); + + // And the other end: f32 squares of 1e-30 underflow to zero, which made the + // guard fire and every row score exactly 1.0, silently. + let tiny = [1e-30f32; 4]; + let norm = tiny + .iter() + .map(|x| f64::from(*x) * f64::from(*x)) + .sum::() + .sqrt(); + assert!(norm > 0.0, "norm underflowed to zero: {norm}"); } } diff --git a/crates/storage-postgres/src/worker_store.rs b/crates/storage-postgres/src/worker_store.rs index be43b984..372fbf49 100755 --- a/crates/storage-postgres/src/worker_store.rs +++ b/crates/storage-postgres/src/worker_store.rs @@ -151,6 +151,14 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // And any vector build hold: the table is gone, so nothing will ever + // release it, and a stale hold blocks claims for that table id. + sqlx::query("DELETE FROM vector_index_holds WHERE table_id = $1") + .bind(table_id) + .execute(&mut *data_tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + data_tx .commit() .await diff --git a/crates/storage-postgres/src/workers.rs b/crates/storage-postgres/src/workers.rs index 467262f3..067679d3 100644 --- a/crates/storage-postgres/src/workers.rs +++ b/crates/storage-postgres/src/workers.rs @@ -211,6 +211,40 @@ pub(crate) async fn poll_gsi_delay( } } +/// Rebuild vector index builds that have stopped making progress. +/// +/// The advisory lock answers "is someone building this?", and nothing else answers +/// "is that someone alive?". A build that dies after its first batch leaves its +/// index CREATING with its queue hold in place, which stops the table's whole index +/// propagation, so without this the only exit is a restart. +/// +/// Ownership is taken per index inside the rebuild, so this is safe to run on every +/// front-end: a healthy build holds its lock and renews its heartbeat, and is +/// therefore neither selected nor rebuildable. +pub(crate) async fn vector_stuck_build_worker( + engine: Arc, + token: CancellationToken, +) { + // Long next to a heartbeat renewed every batch, so a slow batch cannot be + // mistaken for a dead build. The cost of waiting is a paused queue for one + // table; the cost of being wrong is rebuilding an index that was fine. + const POLL_INTERVAL: Duration = Duration::from_secs(60); + const STALE_AFTER: Duration = Duration::from_secs(300); + + while tick(&token, POLL_INTERVAL).await { + match crate::data::vector_index::rebuild_stuck_vector_indexes(&engine, Some(STALE_AFTER)) + .await + { + Ok(0) => {} + Ok(n) => tracing::warn!( + rebuilt = n, + "rebuilt vector index build(s) whose heartbeat had gone stale" + ), + Err(e) => tracing::error!("stuck vector build sweep failed: {e}"), + } + } +} + pub(crate) async fn pool_metrics_worker( catalog_pool: PgPool, data_pool: PgPool, diff --git a/crates/storage-postgres/tests/vector_control_plane.rs b/crates/storage-postgres/tests/vector_control_plane.rs index 2049d67c..d96b5ba0 100644 --- a/crates/storage-postgres/tests/vector_control_plane.rs +++ b/crates/storage-postgres/tests/vector_control_plane.rs @@ -162,6 +162,7 @@ async fn scratch(pgvector: Pgvector) -> Scratch { include_str!("../data_migrations/001_data_schema.sql"), include_str!("../data_migrations/002_gsi_pending.sql"), include_str!("../data_migrations/003_idempotency_account_scope.sql"), + include_str!("../data_migrations/004_vector_index_state.sql"), ] { sqlx::raw_sql(sql) .execute(&catalog) @@ -176,6 +177,13 @@ async fn scratch(pgvector: Pgvector) -> Scratch { .execute(&catalog) .await .expect("pin the control-plane delay to zero"); + // And the propagation delay, for the same reason and the same way + // devtools/run-tests does it: these tests run no queue workers, so anything + // enqueued would never be applied. A test that wants the queue sets this itself. + sqlx::query("UPDATE settings SET value = '0' WHERE key = 'index_propagation_delay_ms'") + .execute(&catalog) + .await + .expect("pin the propagation delay to zero"); sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") .bind(ACCOUNT) .bind(format!("acct-{db_name}")) @@ -789,47 +797,92 @@ async fn switching_a_table_with_vector_indexes_to_provisioned_is_refused() { } #[tokio::test] -async fn adding_a_vector_index_by_update_table_is_unsupported() { - let test = "adding_a_vector_index_by_update_table_is_unsupported"; +async fn update_table_creates_a_vector_index_and_backfills_what_is_already_there() { + let test = "update_table_creates_a_vector_index_and_backfills_what_is_already_there"; if base_conn().is_none() { return skip(test); } - let s = scratch(Pgvector::Omit).await; - // No vector index is created here, so this must keep running on a server - // that has no pgvector at all: for the refusal, that is the interesting case. + let Some(s) = vector_scratch(test).await else { + return; + }; s.engine .create_table(ACCOUNT, create_input("t_add", vec![])) .await .expect("create a table with no vector index"); - - let err = s + let key_info = s .engine + .table_key_info(ACCOUNT, "t_add") + .await + .expect("key info"); + + // Written before the index exists, so it can only reach the index through the + // backfill rather than through the write path. + put( + &s.engine, + &key_info, + vector_item("before", None, &["1", "0"]), + ) + .await; + + s.engine .update_table( ACCOUNT, UpdateTableInput { vector_index_updates: Some(vec![VectorIndexUpdate { - create: Some(vector_spec("vidx", 4, Some("pk"))), + create: Some(vector_spec("vidx", 2, Some("pk"))), delete: None, }]), ..update_input("t_add") }, ) .await - .expect_err("this backend cannot build a vector index yet"); + .expect("add a vector index to an existing table"); - // Unsupported rather than Internal: the backend never claimed it could build - // one, so this is a refusal the engine reports as a client error, not a - // fault to page on. - match err { - StorageError::Unsupported(msg) => assert!(msg.contains("vidx"), "{msg}"), - other => panic!("expected Unsupported, got {other:?}"), + // The build is detached, which is the point: UpdateTable returns while the index + // is still CREATING and the table stays writable throughout. + let id = table_id(&s.catalog, "t_add").await; + let table = only_index_table(&s.catalog).await; + let mut published = false; + for _ in 0..100 { + let status: String = sqlx::query_scalar( + "SELECT index_status FROM vector_indexes WHERE table_id = $1 AND index_name = 'vidx'", + ) + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the index status"); + if status == "ACTIVE" { + published = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; } + assert!(published, "the index never became ACTIVE"); - // Nothing may be left behind: a catalog row with no storage behind it would - // be an index that reports ACTIVE and answers nothing. - let id = table_id(&s.catalog, "t_add").await; - assert_eq!(vector_row_count(&s.catalog, &id).await, 0); + // The backfilled row is there, the hold is released, and the build columns are + // cleared: an index that reports ACTIVE while still holding the queue would stop + // every later write to the table. + assert_eq!( + index_rows(&s.catalog, &table).await.len(), + 1, + "the pre-existing item must be backfilled" + ); + let holds: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("count the holds"); + assert_eq!(holds, 0, "the hold must be released after the ACTIVE flip"); + let (backfilling, owner): (Option, Option) = + sqlx::query_as("SELECT backfilling, build_owner FROM vector_indexes WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the build state"); + assert_eq!(backfilling, None, "ACTIVE must carry no Backfilling member"); + assert_eq!(owner, None); s.cleanup().await; } @@ -1783,6 +1836,673 @@ async fn a_write_sees_an_index_created_after_its_key_info_was_cached() { s.cleanup().await; } +#[tokio::test] +async fn a_real_build_holds_the_allocation_phase_and_the_delete_rule_follows_it() { + let test = "a_real_build_holds_the_allocation_phase_and_the_delete_rule_follows_it"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + // Both halves of the measured rule against a REAL build rather than a hand-set + // catalog row, which is what the earlier phase tests do. The lever holds the + // index in the resource-allocation phase long enough to act on it; the phase + // otherwise exists only between the catalog insert and the flip to backfilling, + // both inside one UpdateTable call. + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('vector_allocation_phase_delay_ms', '4000') \ + ON CONFLICT (key) DO UPDATE SET value = '4000'", + ) + .execute(&s.catalog) + .await + .expect("hold the allocation phase open"); + + s.engine + .create_table(ACCOUNT, create_input("t_phase_real", vec![])) + .await + .expect("create a table with no vector index"); + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_phase_real") + .await + .expect("key info"); + put(&s.engine, &key_info, vector_item("seed", None, &["1", "0"])).await; + + s.engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: Some(vec![VectorIndexUpdate { + create: Some(vector_spec("vidx", 2, Some("pk"))), + delete: None, + }]), + ..update_input("t_phase_real") + }, + ) + .await + .expect("add a vector index"); + + // First half: still allocating, so the delete is refused with the measured + // wording, which is what tells a caller to retry rather than that the request was + // wrong. + let err = s + .engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: delete_vector("vidx"), + ..update_input("t_phase_real") + }, + ) + .await + .expect_err("a delete during resource allocation must be refused"); + match err { + StorageError::ResourceInUse(msg) => assert_eq!( + msg, + extenddb_core::types::vector_index_delete_in_allocation_phase("t_phase_real", "vidx") + ), + other => panic!("expected ResourceInUse, got {other:?}"), + } + + // Second half: once the phase advances, the same request is accepted and the + // index goes away. + let id = table_id(&s.catalog, "t_phase_real").await; + for _ in 0..200 { + let flag: Option = sqlx::query_scalar( + "SELECT backfilling FROM vector_indexes WHERE table_id = $1 AND index_name = 'vidx'", + ) + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the phase"); + if flag != Some(false) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + s.engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: delete_vector("vidx"), + ..update_input("t_phase_real") + }, + ) + .await + .expect("the same delete must be accepted once the index has left allocation"); + assert_eq!(vector_row_count(&s.catalog, &id).await, 0); + + // The table is healthy afterwards, which a leaked build hold would break. + put( + &s.engine, + &key_info, + vector_item("after", None, &["0", "1"]), + ) + .await; + + s.cleanup().await; +} + +#[tokio::test] +async fn a_queued_row_whose_index_table_is_gone_is_consumed_not_retried_forever() { + let test = "a_queued_row_whose_index_table_is_gone_is_consumed_not_retried_forever"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + // A delay, so the write is queued rather than applied inline, which is the only + // way to get a row that outlives its target table. + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '50') \ + ON CONFLICT (key) DO UPDATE SET value = '50'", + ) + .execute(&s.catalog) + .await + .expect("set a propagation delay"); + + s.engine + .create_table( + ACCOUNT, + create_input("t_orphan_row", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_orphan_row").await; + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_orphan_row") + .await + .expect("key info"); + let table = only_index_table(&s.catalog).await; + + put(&s.engine, &key_info, vector_item("a", None, &["1", "0"])).await; + let queued: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM gsi_pending WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("count the queued rows"); + assert_eq!(queued, 1, "the write must be queued at a non-zero delay"); + + // The table goes away under the queued row, which is what a delete of the index + // during propagation does. Before the fix the resulting error lost its SQLSTATE, + // so the worker could not recognise the race, retried the same lowest-id row + // forever, and every row behind it in that worker's partition stopped applying: + // a silent quarter of the table's writes, until someone deleted the row by hand. + sqlx::query(&format!("DROP TABLE \"{table}\"")) + .execute(&s.catalog) + .await + .expect("drop the index data table"); + + // The engine under test has no running workers, so drive the classification the + // way the worker does: apply the row's own context and check the error is + // recognisable as a vanished table rather than an opaque failure. + let context: serde_json::Value = + sqlx::query_scalar("SELECT index_context FROM gsi_pending WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the queued context"); + let vector_context: extenddb_storage::vector_lifecycle::VectorApplyContext = + serde_json::from_value(context).expect("the context must be a vector one"); + let item = vector_item("a", None, &["1", "0"]); + let mut tx = s + .engine + .data_pool() + .begin() + .await + .expect("begin a transaction"); + let err = extenddb_storage_postgres::apply_claimed_vector_row( + &mut tx, + &vector_context, + None, + Some(&item), + ) + .await + .expect_err("applying into a dropped table must fail"); + let StorageError::Internal(message) = &err else { + panic!("expected Internal, got {err:?}"); + }; + assert!( + message.contains("SQLSTATE 42P01"), + "the error must carry its SQLSTATE so the worker can recognise the race: {message}" + ); + + s.cleanup().await; +} + +#[tokio::test] +async fn a_failed_update_table_gives_back_every_hold_it_took() { + let test = "a_failed_update_table_gives_back_every_hold_it_took"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table(ACCOUNT, create_input("t_hold_leak", vec![])) + .await + .expect("create a table with no vector index"); + let id = table_id(&s.catalog, "t_hold_leak").await; + + // An ordinary client request that fails: one create paired with a delete of an + // index that does not exist. The create takes its hold first, and the hold is + // written to the data database, so the catalog rollback cannot undo it. Left + // behind, that hold stops the propagation queue claiming ANY row for this table, + // secondary index rows included, and nothing would ever release it because there + // is no catalog row to finish or delete. + let err = s + .engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: Some(vec![ + VectorIndexUpdate { + create: Some(vector_spec("vidx", 2, Some("pk"))), + delete: None, + }, + VectorIndexUpdate { + create: None, + delete: Some(DeleteVectorIndexAction { + index_name: "nosuch".to_owned(), + }), + }, + ]), + ..update_input("t_hold_leak") + }, + ) + .await + .expect_err("deleting an index that does not exist must fail the request"); + assert!(matches!(err, StorageError::IndexNotFound(_)), "{err:?}"); + + // Nothing committed, so nothing may be held. + let holds: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("count the holds"); + assert_eq!( + holds, 0, + "a failed UpdateTable must give back the holds it took, or this table's index \ + propagation is frozen until a restart" + ); + assert_eq!(vector_row_count(&s.catalog, &id).await, 0); + + s.cleanup().await; +} + +#[tokio::test] +async fn a_stale_heartbeat_is_rebuilt_at_runtime_and_a_fresh_one_is_left_alone() { + let test = "a_stale_heartbeat_is_rebuilt_at_runtime_and_a_fresh_one_is_left_alone"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_stale", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_stale").await; + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_stale") + .await + .expect("key info"); + put(&s.engine, &key_info, vector_item("a", None, &["1", "0"])).await; + + // A build that died after its first batch: CREATING, hold held, heartbeat frozen. + // Nothing else moves this state, so before the runtime sweep existed the table's + // whole index propagation stayed paused until someone restarted the process. + let index_id: String = + sqlx::query_scalar("SELECT index_id FROM vector_indexes WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the index id"); + sqlx::query( + "UPDATE vector_indexes SET index_status = 'CREATING', backfilling = true, \ + build_heartbeat_at = NOW() - INTERVAL '1 hour' WHERE table_id = $1", + ) + .bind(&id) + .execute(&s.catalog) + .await + .expect("simulate a dead build"); + sqlx::query("INSERT INTO vector_index_holds (table_id, index_id) VALUES ($1, $2)") + .bind(&id) + .bind(&index_id) + .execute(&s.catalog) + .await + .expect("restore the hold the dead build held"); + + // A fresh heartbeat must be left alone: that is the question the column exists to + // answer, and rebuilding a healthy build would drop and repopulate a data table + // out from under the process writing it. + let rebuilt = extenddb_storage_postgres::rebuild_stuck_vector_indexes( + &s.engine, + Some(std::time::Duration::from_secs(300)), + ) + .await + .expect("sweep"); + assert_eq!( + rebuilt, 1, + "a build with an hour-old heartbeat must be rebuilt" + ); + let status: String = + sqlx::query_scalar("SELECT index_status FROM vector_indexes WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the status"); + assert_eq!(status, "ACTIVE"); + let holds: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("count the holds"); + assert_eq!(holds, 0, "the rebuild must release the hold it repaired"); + + // Now the other direction, on the same table: a build whose heartbeat is current + // belongs to a live process and must be left to it. + sqlx::query( + "UPDATE vector_indexes SET index_status = 'CREATING', backfilling = true, \ + build_heartbeat_at = NOW() WHERE table_id = $1", + ) + .bind(&id) + .execute(&s.catalog) + .await + .expect("simulate a live build"); + let rebuilt = extenddb_storage_postgres::rebuild_stuck_vector_indexes( + &s.engine, + Some(std::time::Duration::from_secs(300)), + ) + .await + .expect("sweep"); + assert_eq!( + rebuilt, 0, + "a live build must not be rebuilt underneath its owner" + ); + + s.cleanup().await; +} + +#[tokio::test] +async fn a_hold_with_no_building_index_is_swept_at_runtime_not_only_at_startup() { + let test = "a_hold_with_no_building_index_is_swept_at_runtime_not_only_at_startup"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_runtime_sweep", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with an ACTIVE vector index"); + let id = table_id(&s.catalog, "t_runtime_sweep").await; + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_runtime_sweep") + .await + .expect("key info"); + + // A hold whose index is not building at all, which is what three routes leave + // behind: a failed catalog commit, a crash between taking the hold and committing + // the row, and a crash after a delete commits but before its release. None of them + // leaves a CREATING row, so the stuck-build sweep can never see them, and before + // the runtime sweep they were permanent until a restart. Backdated, because the age + // bound is what keeps a peer's just-taken hold safe. + sqlx::query( + "INSERT INTO vector_index_holds (table_id, index_id, created_at) \ + VALUES ($1, 'no-such-build', NOW() - INTERVAL '1 hour')", + ) + .bind(&id) + .execute(&s.catalog) + .await + .expect("leave an orphan hold"); + + // Runtime, not startup: a staleness bound is passed, and there is no CREATING + // index anywhere, so nothing is rebuilt and the sweep is the only thing that acts. + let rebuilt = extenddb_storage_postgres::rebuild_stuck_vector_indexes( + &s.engine, + Some(std::time::Duration::from_secs(300)), + ) + .await + .expect("sweep"); + assert_eq!(rebuilt, 0, "there is no build to rebuild"); + + let holds: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("count the holds"); + assert_eq!( + holds, 0, + "an aged hold with no building index must be swept at runtime, or this table's \ + index propagation stays paused until a restart" + ); + + // The table works afterwards, which is the whole point of releasing it. + put(&s.engine, &key_info, vector_item("a", None, &["1", "0"])).await; + + s.cleanup().await; +} + +#[tokio::test] +async fn deleting_an_index_mid_backfill_leaves_nothing_orphaned() { + let test = "deleting_an_index_mid_backfill_leaves_nothing_orphaned"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + // A slow backfill, so the delete lands while the build is genuinely running + // rather than after it. This is the interleaving that has three things to clean + // up at once: the catalog row, the queue hold, and the data table. + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('vector_backfill_batch_delay_ms', '3000') \ + ON CONFLICT (key) DO UPDATE SET value = '3000'", + ) + .execute(&s.catalog) + .await + .expect("slow the backfill down"); + + s.engine + .create_table(ACCOUNT, create_input("t_interleave", vec![])) + .await + .expect("create a table with no vector index"); + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_interleave") + .await + .expect("key info"); + // More than one batch of items, deliberately, and derived from the batch size + // rather than written as a number. The shared driver pauses BETWEEN batches and + // breaks out on a short one before it sleeps, so a table that fits in a single + // batch gives no observable window at all whatever the delay is set to: the flag + // flips to true and then to absent within milliseconds. Retuning the batch size + // would otherwise silently remove the window this test exists to create. + let seed = extenddb_storage::vector_lifecycle::BACKFILL_BATCH + 20; + for i in 0..seed { + put( + &s.engine, + &key_info, + vector_item(&format!("item{i}"), None, &["1", "0"]), + ) + .await; + } + + s.engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: Some(vec![VectorIndexUpdate { + create: Some(vector_spec("vidx", 2, Some("pk"))), + delete: None, + }]), + ..update_input("t_interleave") + }, + ) + .await + .expect("add a vector index"); + + let id = table_id(&s.catalog, "t_interleave").await; + // Wait for the build to be running rather than merely allocated, which is the + // phase in which a delete is accepted. + let mut backfilling = false; + for _ in 0..200 { + let flag: Option = sqlx::query_scalar( + "SELECT backfilling FROM vector_indexes WHERE table_id = $1 AND index_name = 'vidx'", + ) + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the backfilling flag"); + if flag == Some(true) { + backfilling = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!(backfilling, "the build never reported backfilling"); + + s.engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: delete_vector("vidx"), + ..update_input("t_interleave") + }, + ) + .await + .expect("a delete during the backfill phase must be accepted"); + + // Give the build task time to notice, then check that nothing is left behind. + // The build writes into a table that no longer exists, which must not resurrect + // the row, recreate the table, or leave the queue held. + tokio::time::sleep(std::time::Duration::from_millis(4000)).await; + + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM vector_indexes WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("count the catalog rows"); + assert_eq!(rows, 0, "the catalog row must be gone"); + assert!( + vector_data_tables(&s.catalog).await.is_empty(), + "the data table must be dropped, not left orphaned" + ); + let holds: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("count the holds"); + assert_eq!( + holds, 0, + "the hold must not outlive the index: it would stop every later write to this table" + ); + + // And the table itself still works, which is the point of checking the hold. + put( + &s.engine, + &key_info, + vector_item("after", None, &["0", "1"]), + ) + .await; + + s.cleanup().await; +} + +#[tokio::test] +async fn startup_rebuilds_a_half_built_index_and_frees_a_stale_hold() { + let test = "startup_rebuilds_a_half_built_index_and_frees_a_stale_hold"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_recover", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_recover") + .await + .expect("key info"); + let id = table_id(&s.catalog, "t_recover").await; + let table = only_index_table(&s.catalog).await; + + put(&s.engine, &key_info, vector_item("a", None, &["1", "0"])).await; + put(&s.engine, &key_info, vector_item("b", None, &["0", "1"])).await; + + // The state a crashed build leaves: the index stuck CREATING, its data table + // holding some of the rows, and its hold still in place. There is no failure + // state on the wire for an index to sit in, so recovery is the only thing that + // ever moves it. + sqlx::query( + "UPDATE vector_indexes SET index_status = 'CREATING', backfilling = true \ + WHERE table_id = $1", + ) + .bind(&id) + .execute(&s.catalog) + .await + .expect("simulate a dead build"); + sqlx::query(&format!("DELETE FROM \"{table}\" WHERE base_pk LIKE '%b%'")) + .execute(&s.catalog) + .await + .expect("leave the data table half populated"); + let index_id: String = + sqlx::query_scalar("SELECT index_id FROM vector_indexes WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the index id"); + sqlx::query("INSERT INTO vector_index_holds (table_id, index_id) VALUES ($1, $2)") + .bind(&id) + .bind(&index_id) + .execute(&s.catalog) + .await + .expect("restore the hold the dead build held"); + + // Two holds for indexes that are not building, distinguished only by age, which + // is what the sweep is allowed to key on. The old one is a crash leftover: + // nothing will ever release it, and while it sits there the queue claims nothing + // for its table. The young one may belong to another front-end that has taken a + // hold and not yet committed its catalog row, so sweeping it would let writes + // reach an index whose backfill is still scanning, which is the one ordering rule + // the hold exists to enforce. + sqlx::query( + "INSERT INTO vector_index_holds (table_id, index_id, created_at) \ + VALUES ('ghost-old', 'ghost-old', NOW() - INTERVAL '1 hour')", + ) + .execute(&s.catalog) + .await + .expect("add an aged orphan hold"); + sqlx::query( + "INSERT INTO vector_index_holds (table_id, index_id) VALUES ('ghost-new', 'ghost-new')", + ) + .execute(&s.catalog) + .await + .expect("add a just-taken hold"); + + let rebuilt = extenddb_storage_postgres::reconcile_incomplete_vector_indexes(&s.engine) + .await + .expect("reconcile"); + assert_eq!(rebuilt, 1, "the half-built index must be rebuilt"); + + // Rebuilt rather than resumed: both rows are present, and exactly once, which is + // what dropping and recreating the table before backfilling guarantees. Resuming + // would have duplicated the row that survived. + assert_eq!(index_rows(&s.catalog, &table).await.len(), 2); + let status: String = + sqlx::query_scalar("SELECT index_status FROM vector_indexes WHERE table_id = $1") + .bind(&id) + .fetch_one(&s.catalog) + .await + .expect("read the status"); + assert_eq!(status, "ACTIVE"); + let remaining: Vec = + sqlx::query_scalar("SELECT table_id FROM vector_index_holds ORDER BY table_id") + .fetch_all(&s.catalog) + .await + .expect("list the holds"); + assert_eq!( + remaining, + vec!["ghost-new".to_owned()], + "the rebuilt index's hold and the aged orphan must go; the just-taken hold must stay, \ + because it may belong to a front-end whose catalog row has not committed yet" + ); + + s.cleanup().await; +} + #[tokio::test] async fn the_probe_reads_the_data_database_and_not_the_catalog() { let test = "the_probe_reads_the_data_database_and_not_the_catalog"; diff --git a/crates/storage/src/vector_lifecycle/build.rs b/crates/storage/src/vector_lifecycle/build.rs index 8d050217..156a337e 100644 --- a/crates/storage/src/vector_lifecycle/build.rs +++ b/crates/storage/src/vector_lifecycle/build.rs @@ -48,9 +48,10 @@ pub trait VectorIndexBuild: Send { /// `Backfilling: true`. /// /// Called by the backend's create path after the data table exists and - /// before [`complete_build`] is spawned, and set outside the backfill - /// transaction, otherwise no observer could see it: the whole point of the - /// flag is to be readable while the scan is in progress. + /// before [`complete_build`] is spawned, and by [`rebuild_index`] before a + /// recovery scan, and set outside the backfill transaction, otherwise no + /// observer could see it: the whole point of the flag is to be readable + /// while the scan is in progress. fn set_backfilling(&mut self) -> impl Future> + Send; /// Publish the index: `ACTIVE`, the `Backfilling` member cleared to absent @@ -229,11 +230,8 @@ pub async fn complete_build( /// queue rows become claimable (startup reconciliation runs before the workers /// exist; runtime recovery notifies once after a whole sweep). /// -/// One deliberate exception to the status sequence in the module docs: a -/// rebuild does not re-assert `Backfilling: true`, so an index whose build died -/// before its own `set_backfilling` call is rebuilt while DescribeTable still -/// reports `false`. This matches the pre-extraction behavior; the flip to -/// `ACTIVE` clears the member either way. +/// Recovery re-asserts the scanning phase before rebuilding, so the status +/// sequence in the module docs holds for a rebuild as well as a first build. /// /// # Errors /// Unlike [`complete_build`], every failure propagates, including the terminal @@ -244,6 +242,7 @@ pub async fn rebuild_index( batch_size: i64, ) -> Result { ops.reset_data_table().await?; + ops.set_backfilling().await?; let outcome = run_backfill(ops, batch_size, Duration::ZERO).await?; ops.mark_active(outcome.skipped).await?; Ok(outcome.written) @@ -261,6 +260,7 @@ mod tests { struct Script { batches: Vec, StorageError>>, flip_fails: bool, + backfilling_fails: bool, log: Vec, } @@ -291,11 +291,11 @@ mod tests { } async fn set_backfilling(&mut self) -> Result<(), StorageError> { - self.0 - .lock() - .unwrap() - .log - .push("set_backfilling".to_owned()); + let mut s = self.0.lock().unwrap(); + s.log.push("set_backfilling".to_owned()); + if s.backfilling_fails { + return Err(StorageError::Internal("phase flip failed".to_owned())); + } Ok(()) } @@ -427,11 +427,13 @@ mod tests { mock.log(), vec![ "reset", + "set_backfilling", "batch(cursor=None, limit=500)", "mark_active(skipped=1)", ], "the reset precedes the scan, or already-written rows collide with \ - the backfill's plain INSERT" + the backfill's plain INSERT, and the scanning phase is re-asserted \ + before the scan so the phase is readable while the rebuild runs" ); let failing = MockBuild::default(); @@ -449,4 +451,39 @@ mod tests { "the repair loop must see the flip failure: {err:?}" ); } + + /// A rebuild re-asserts the scanning phase, and asserts it BEFORE scanning. + /// + /// The phase is not cosmetic: the `UpdateTable` delete rule refuses a delete + /// while an index reports `CREATING` with `Backfilling: false` and tells the + /// caller to retry once backfilling starts. A rebuild that left the flag + /// false would make that advice unfollowable for the whole rebuild, because + /// the only remaining transitions are `ACTIVE` or another failure. Asserting + /// it before the scan is what makes it readable while the scan runs. + /// + /// The failure propagates with the index still `CREATING`, so the repair + /// loop retries the whole rebuild rather than scanning under a phase it + /// could not publish. + #[tokio::test] + async fn rebuild_index_reasserts_the_scanning_phase_and_propagates_its_failure() { + let mock = MockBuild::default(); + { + let mut s = mock.0.lock().unwrap(); + s.batches = vec![Ok(batch(1, 0, 0, None))]; + s.backfilling_fails = true; + } + let mut ops = mock.clone(); + let err = rebuild_index(&mut ops, 500) + .await + .expect_err("phase flip failure"); + assert!( + matches!(err, StorageError::Internal(_)), + "the repair loop must see the phase failure: {err:?}" + ); + assert_eq!( + mock.log(), + vec!["reset", "set_backfilling"], + "neither the scan nor the publish may run once the phase flip failed" + ); + } } diff --git a/crates/storage/src/vector_lifecycle/mod.rs b/crates/storage/src/vector_lifecycle/mod.rs index 63f310f3..0016023c 100644 --- a/crates/storage/src/vector_lifecycle/mod.rs +++ b/crates/storage/src/vector_lifecycle/mod.rs @@ -22,7 +22,9 @@ //! the scan starts ([`VectorIndexBuild::set_backfilling`]), and becomes //! `ACTIVE` with the `Backfilling` member absent in a single transition //! ([`VectorIndexBuild::mark_active`]). An index created by `CreateTable` -//! skips the sequence: the table is empty, so it is `ACTIVE` from birth. +//! skips the sequence: the table is empty, so it is `ACTIVE` from birth. A +//! recovery rebuild repeats the sequence from `Backfilling: true`, because +//! the phase is what the `UpdateTable` delete rule reads. //! 2. **The table stays writable throughout.** The backfill commits in //! independent batches ([`run_backfill`]) rather than holding one //! transaction, and the build task is detached from the `UpdateTable` @@ -45,7 +47,8 @@ //! 5. **Failure leaves the index `CREATING`.** There is no failure state on //! the wire, and flipping to `ACTIVE` would publish a partially populated //! index. A build that dies is repaired by rebuilding: drop the data table, -//! recreate it, backfill, flip ([`rebuild_index`]). Rebuilding rather than +//! recreate it, re-assert `Backfilling: true`, backfill, flip +//! ([`rebuild_index`]). Rebuilding rather than //! resuming, because rows already written would collide with the backfill's //! deliberately plain INSERT. //! 6. **Build ownership is backend-defined.** Recovery must not rebuild an diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 98793556..df3bf151 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1704,6 +1704,11 @@ async fn removing_a_row_during_a_backfill_does_not_skip_another() { /// that skipped itself and reported green when this variable was absent, which is the /// exact failure mode `EXTENDDB_EXPECT_VECTORS` was introduced to close. async fn set_backfill_delay(ms: u64) { + set_vector_setting("vector_backfill_batch_delay_ms", ms).await; +} + +/// Write one vector test lever through the management API. +async fn set_vector_setting(key: &str, ms: u64) { let user = std::env::var("EXTENDDB_ADMIN_USER").unwrap_or_else(|_| "admin".into()); let Ok(pass) = std::env::var("EXTENDDB_ADMIN_PASSWORD") else { assert!( @@ -1724,7 +1729,7 @@ async fn set_backfill_delay(ms: u64) { .build() .expect("reqwest build"); let r = http - .put(format!("{base}/settings/vector_backfill_batch_delay_ms")) + .put(format!("{base}/settings/{key}")) .basic_auth(&user, Some(&pass)) .json(&serde_json::json!({ "value": ms.to_string() })) .send() @@ -1734,7 +1739,7 @@ async fn set_backfill_delay(ms: u64) { let body = r.text().await.unwrap_or_default(); assert!( status.is_success(), - "setting the backfill delay failed ({status}): {body}" + "setting {key} failed ({status}): {body}" ); } From 1f38f4a05f596f2b6196d4329532a8527bd025a9 Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Mon, 24 Aug 2026 20:15:41 +0000 Subject: [PATCH 05/13] feat: enforce the vector delete-phase rule on both backends Deleting a vector index during the resource-allocation phase refuses with the measured ResourceInUseException; during backfilling the delete is accepted. The phase is observable, a test can hold the allocation phase open, and the delete-phase wire test is backend-blind. --- crates/storage-sqlite/src/store.rs | 18 ++ crates/storage-sqlite/src/update_table.rs | 315 +++++++++++++++++++++- docs/technical-debt.md | 2 +- tests/rust/src/vector_index_search.rs | 151 +++++++++++ 4 files changed, 476 insertions(+), 10 deletions(-) diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index f48cfd75..f84fe3d6 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -315,6 +315,24 @@ impl SqliteEngine { "vector_index_min_creating_ms: live read failed, using {DEFAULT_MS}: {e:?}" ); DEFAULT_MS + + /// Milliseconds to hold a new vector index in the resource-allocation phase. + /// + /// A test lever, zero in production, read live for the same reason the batch + /// delay is. Held inside the detached build task rather than in the request + /// path, because the phase is only observable to a client after `UpdateTable` + /// has returned. + pub(crate) async fn vector_allocation_phase_delay(&self) -> u64 { + let live: Result, _> = + sqlx::query_as("SELECT value FROM settings WHERE key = ?") + .bind(extenddb_core::settings_keys::VECTOR_ALLOCATION_PHASE_DELAY_MS) + .fetch_optional(&self.pool) + .await; + match live { + Ok(row) => row.and_then(|(v,)| v.parse::().ok()).unwrap_or(0), + Err(e) => { + tracing::debug!("vector_allocation_phase_delay: live read failed, using 0: {e:?}"); + 0 } } } diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 3dc5c93f..cd266c53 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -19,7 +19,6 @@ use extenddb_core::types::{ }; use extenddb_storage::error::StorageError; use extenddb_storage::util::effective_attribute_definitions; -use extenddb_storage::vector_lifecycle::VectorIndexBuild; use crate::store::SqliteEngine; @@ -546,8 +545,8 @@ impl SqliteEngine { vec_created.push((index_id, create.clone())); } if let Some(delete) = &update.delete { - let existing: Option<(String,)> = sqlx::query_as( - "SELECT index_id FROM vector_indexes \ + let existing: Option<(String, String, Option)> = sqlx::query_as( + "SELECT index_id, index_status, backfilling FROM vector_indexes \ WHERE table_id = ? AND index_name = ?", ) .bind(&table_id) @@ -555,8 +554,29 @@ impl SqliteEngine { .fetch_optional(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let (del_id,) = existing + let (del_id, index_status, backfilling) = existing .ok_or_else(|| StorageError::IndexNotFound(delete.index_name.clone()))?; + + // Deleting an index that is still being created is + // phase-dependent, and the discriminator is the same + // `backfilling` flag the wire reports. While the index is + // allocating resources the service refuses the delete and asks + // the caller to retry; once the backfill is running it accepts. + // Measured against the service on 2026-08-19. + // + // The advice is followable in every state that reaches here: a + // first build flips the flag as its scan starts, and a build that + // died before its own flip is repaired by a rebuild, which + // re-asserts the phase before scanning. + if index_status == "CREATING" && backfilling == Some(false) { + return Err(StorageError::ResourceInUse( + extenddb_core::types::vector_index_delete_in_allocation_phase( + &input.table_name, + &delete.index_name, + ), + )); + } + sqlx::query("DELETE FROM vector_indexes WHERE table_id = ? AND index_name = ?") .bind(&table_id) .bind(&delete.index_name) @@ -763,11 +783,6 @@ impl SqliteEngine { meta: None, }; - // The scan is about to start, so the member becomes true. Set outside the - // backfill transaction, otherwise no observer could see it: the whole point - // of the flag is to be readable while the scan is in progress. - ops.set_backfilling().await?; - let mut meta_tx = self .pool .begin_with("BEGIN IMMEDIATE") @@ -800,6 +815,9 @@ impl SqliteEngine { let min_creating = std::time::Duration::from_millis(self.vector_index_min_creating_ms().await); let created_at = tokio::time::Instant::now(); + + let allocation_delay = + std::time::Duration::from_millis(self.vector_allocation_phase_delay().await); let owned_index_id = index_id.to_owned(); let owned_index_name = create.index_name.clone(); @@ -840,6 +858,37 @@ impl SqliteEngine { } } let _deregister = Deregister(registry, owned_index_id); + + // Nothing waits and no branch is taken when the lever is unset, which is + // the same shape the shared driver uses for its own inter-batch pause. + if !allocation_delay.is_zero() { + tokio::time::sleep(allocation_delay).await; + } + // The scan is about to start, so the member becomes true. Set outside the + // backfill transaction, otherwise no observer could see it: the whole + // point of the flag is to be readable while the scan is in progress. + // Inside the task rather than before the spawn, so the caller returns + // while the index is still allocating, which is the state the service + // reports first. + // + // A failure here leaves the index CREATING and allocating, which reads as + // undeletable until it is repaired. Nothing has to be released on the way + // out on this backend: the registry guard above fires on every exit path, + // and the propagation worker's stuck-build sweep rebuilds a CREATING index + // with no registry entry within one worker loop, which re-asserts the + // phase and unblocks both the delete and the table's queued index writes. + let mut ops = ops; + if let Err(e) = + extenddb_storage::vector_lifecycle::VectorIndexBuild::set_backfilling(&mut ops) + .await + { + tracing::error!( + index_name = %owned_index_name, + "could not mark the vector index as backfilling, leaving it CREATING \ + for recovery: {e}" + ); + return; + } extenddb_storage::vector_lifecycle::complete_build( ops, &owned_index_name, @@ -1635,4 +1684,252 @@ mod reconciler_tests { .expect("status"); assert_eq!(status, "CREATING", "the live build must be untouched"); } + + /// The measured delete rule, both halves, on this backend. + /// + /// While an index reports `CREATING` with `Backfilling: false` it is allocating + /// resources and the service refuses the delete, telling the caller to retry. + /// Once it reports `Backfilling: true` the same request is accepted. Asserted on + /// the whole message, because the wording is what makes the retry advice + /// followable rather than merely signalling that something was wrong. + #[tokio::test] + async fn a_vector_delete_is_refused_while_allocating_and_accepted_while_backfilling() { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + let account = "000000000000"; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind(account) + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl(account, input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-1', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 0)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert an allocating index"); + // The simulated CREATING window is the propagation worker's job to end, and + // no worker runs in this test. + sqlx::query("UPDATE tables SET table_status = 'ACTIVE', status_transition_at = NULL") + .execute(&engine.pool) + .await + .expect("activate the table"); + + let err = engine + .update_table_impl(account, delete_vidx_input()) + .await + .expect_err("a delete during resource allocation must be refused"); + match err { + extenddb_storage::error::StorageError::ResourceInUse(message) => assert_eq!( + message, + extenddb_core::types::vector_index_delete_in_allocation_phase("t", "vidx"), + "the refusal must carry the measured wording" + ), + other => panic!("expected ResourceInUse, got {other:?}"), + } + let (still_there,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM vector_indexes WHERE index_id = 'vidx-1'") + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!( + still_there, 1, + "a refused delete must not have half-removed the index" + ); + + sqlx::query("UPDATE vector_indexes SET backfilling = 1 WHERE index_id = 'vidx-1'") + .execute(&engine.pool) + .await + .expect("advance to the backfilling phase"); + engine + .update_table_impl(account, delete_vidx_input()) + .await + .expect("the same delete must be accepted once the backfill is running"); + let (gone,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM vector_indexes WHERE index_id = 'vidx-1'") + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(gone, 0, "the accepted delete must remove the catalog row"); + } + + /// A delete that lands DURING a recovery rebuild, which is a different + /// interleaving from a delete during a first build. + /// + /// A rebuild re-asserts `Backfilling: true` before it scans, so a delete arriving + /// mid-rebuild is accepted rather than refused with advice about a phase that + /// already passed. What must then hold is that the rebuild cannot bring the index + /// back: the catalog row and the data table are both gone, and the rest of the + /// rebuild has to fail rather than recreate either. + /// + /// The crashed state is also asserted, because it is the state a caller meets + /// first: a build that died before its own phase flip reads as allocating, so the + /// delete is refused until recovery re-asserts the phase, which is what makes the + /// retry advice honest instead of unfollowable. + #[tokio::test] + async fn a_delete_during_a_rebuild_is_accepted_and_the_rebuild_cannot_resurrect_the_index() { + use extenddb_storage::vector_lifecycle::VectorIndexBuild; + + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + let account = "000000000000"; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind(account) + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl(account, input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + let base_table = crate::data::data_table_name(&table_id); + sqlx::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES ('a', ?)" + )) + .bind(r#"{"pk":{"S":"a"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#) + .execute(&engine.pool) + .await + .expect("seed an item"); + // Same reason as the phase test: no worker runs here to end the simulated + // CREATING window. + sqlx::query("UPDATE tables SET table_status = 'ACTIVE', status_transition_at = NULL") + .execute(&engine.pool) + .await + .expect("activate the table"); + + // A build that died before its own phase flip: CREATING, allocating, no data + // table. + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-1', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 0)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert a crashed build"); + + engine + .update_table_impl(account, delete_vidx_input()) + .await + .expect_err("a crashed build still reads as allocating, so the delete is refused"); + + // The rebuild's first two steps, in the order the shared driver runs them. + // Stopping here leaves exactly the state a delete can arrive in. + let mut ops = crate::data::vector_index::SqliteVectorBuild { + pool: engine.pool.clone(), + write_lock: std::sync::Arc::clone(&engine.write_lock), + gsi_notify: engine.gsi_notify(), + table_id: table_id.clone(), + index_id: "vidx-1".to_owned(), + base_key_schema: vec![extenddb_core::types::KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: extenddb_core::types::KeyType::Hash, + }], + attribute_definitions: vec![extenddb_core::types::AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: extenddb_core::types::ScalarAttributeType::S, + }], + meta: None, + }; + ops.reset_data_table().await.expect("rebuild reset"); + ops.set_backfilling().await.expect("rebuild phase flip"); + + engine + .update_table_impl(account, delete_vidx_input()) + .await + .expect("a delete during a rebuild must be accepted"); + let (rows,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM vector_indexes WHERE index_id = 'vidx-1'") + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 0, "the catalog row must be gone"); + let vec_table = crate::data::vector_table_name(&table_id, "vidx-1"); + let (tables,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?") + .bind(&vec_table) + .fetch_one(&engine.pool) + .await + .expect("count tables"); + assert_eq!(tables, 0, "the delete must drop the index data table"); + + // The rest of the rebuild now runs against a deleted index. It must fail, and + // it must leave neither a catalog row nor a data table behind. + extenddb_storage::vector_lifecycle::rebuild_index( + &mut ops, + extenddb_storage::vector_lifecycle::BACKFILL_BATCH, + ) + .await + .expect_err("a rebuild of a deleted index must fail rather than recreate it"); + let (tables_after,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?") + .bind(&vec_table) + .fetch_one(&engine.pool) + .await + .expect("count tables again"); + assert_eq!( + tables_after, 0, + "an interrupted rebuild must not leave an orphan data table" + ); + assert_eq!( + engine + .reconcile_incomplete_vector_indexes() + .await + .expect("reconcile"), + 0, + "nothing is left for recovery to rebuild" + ); + } + + /// One `UpdateTable` request deleting the vector index named `vidx`. + fn delete_vidx_input() -> extenddb_core::types::UpdateTableInput { + serde_json::from_value(json!({ + "TableName": "t", + "VectorIndexUpdates": [{"Delete": {"IndexName": "vidx"}}] + })) + .expect("delete input") + } } diff --git a/docs/technical-debt.md b/docs/technical-debt.md index 3cb055de..c34bed44 100755 --- a/docs/technical-debt.md +++ b/docs/technical-debt.md @@ -30,7 +30,7 @@ Last updated: 2026-08-19 | F-15 | ~~TTL worker bypasses stream capture — expired item deletions don't generate REMOVE stream records~~ | `bin/cmd_serve.rs:ttl_cleanup_worker` | ~~High~~ | P26 | | F-16 | `transact_write_items.rs` passes `None` for `old_item` in stream capture — `OldImage` always `None` for transaction-originated stream records | `engine/transact_write_items.rs` | Medium | P27 | | F-17 | `validate_attribute_name_sizes` only checks top-level attribute names — nested map keys not validated | `core/validation/mod.rs` | Low | P30 | -| F-18 | UpdateTable Delete of a vector index in the resource-allocation phase (`CREATING`, `Backfilling: false`) is accepted; Amazon DynamoDB refuses it with `ResourceInUseException` until backfilling starts. The message constant, `StorageError::ResourceInUse`, and the engine mapping arms are in place with no runtime producer; enforcement belongs in the shared lifecycle, deferred so the extraction stayed behavior-preserving | `storage-sqlite/update_table.rs` (vector delete branch), `core/types/table.rs` (`vector_index_delete_in_allocation_phase`) | Medium | vector probe P2 | +| F-18 | ~~UpdateTable Delete of a vector index in the resource-allocation phase (`CREATING`, `Backfilling: false`) is accepted; Amazon DynamoDB refuses it with `ResourceInUseException` until backfilling starts~~ Both backends now enforce the phase rule with the measured message, and both hold the phase open under `vector_allocation_phase_delay_ms` so a client can observe it | ~~`storage-sqlite/update_table.rs` (vector delete branch), `core/types/table.rs` (`vector_index_delete_in_allocation_phase`)~~ | ~~Medium~~ | vector probe P2 | ## Cleanup diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index df3bf151..1899612b 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1080,6 +1080,151 @@ async fn an_empty_search_schema_is_reported_as_absent() { let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } +/// Deleting an index that is still being created is two behaviours, and the +/// discriminator is the `Backfilling` flag the wire already reports. +/// +/// Measured against the service: while the index reports `Backfilling: false` it is +/// allocating resources and the delete is refused with a byte-exact +/// `ResourceInUseException` telling the caller to retry; once it reports +/// `Backfilling: true` the same delete is accepted. Both halves are here because +/// both are measured, and a backend that implemented only the acceptance would fail +/// a client that retries exactly as the message tells it to. +/// +/// The allocation phase is held open by a test-only settings lever. Without it that +/// phase exists only between the catalog row's insert and the flip to backfilling, +/// both inside one UpdateTable call, so a client could only race it, and a race +/// asserting a whole measured string fails for reasons unrelated to the rule. +#[tokio::test] +async fn deleting_a_vector_index_is_refused_while_allocating_and_accepted_while_backfilling() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_phase_delete"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + call("CreateTable", &body).await; + wait_for_active(&name).await; + put_vector(&name, "seed", None, &[1.0, 0.0]).await; + + // Long enough to observe the refusal without making the test slow. + set_allocation_phase_delay(4000).await; + let delete_body = format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Delete": {{"IndexName": "vidx"}}}}]}}"# + ); + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Dimensions": 2, + "DistanceFunction": "COSINE", + "Projection": {{"ProjectionType": "ALL"}} + }}}}] + }}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable create failed: {text}"); + + // First half: still allocating, so the delete is refused. Asserted on the whole + // string, because the wording is what tells a caller to retry rather than that + // the request was wrong. + let (status, text) = call("UpdateTable", &delete_body).await; + assert_eq!( + status, 400, + "a delete during resource allocation must be refused: {text}" + ); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + assert_eq!( + json.pointer("/__type").and_then(|v| v.as_str()), + Some("com.amazonaws.dynamodb.v20120810#ResourceInUseException"), + "wrong error type: {text}" + ); + let expected = format!( + "Attempt to change a resource which is still in use: Index creation is in resource \ + allocation phase. Retry deletion during backfilling phase or when the index is \ + active. Table: {name} Index: vidx" + ); + assert_eq!( + json.pointer("/message").and_then(|v| v.as_str()), + Some(expected.as_str()), + "wrong message: {text}" + ); + + // The refusal must not have half-deleted anything: the index is still reported. + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + assert_eq!(status, 200, "DescribeTable failed: {text}"); + assert!( + text.contains("\"IndexName\":\"vidx\""), + "the refused delete must leave the index in place: {text}" + ); + + // Second half: wait for the phase to advance past allocation, then the same + // request is accepted. + // + // Waiting for "no longer allocating" rather than for `Backfilling: true` + // specifically. The true state is not deterministically observable on a table + // this size: the shared driver pauses between batches, and a handful of rows is + // one batch, so the flag flips to true and then to absent within milliseconds. + // Making it observable would mean seeding more than a full batch of items purely + // to slow a test down. What the rule actually distinguishes is the allocation + // phase from everything after it, and that boundary is what this asserts. + wait_past_allocation_phase(&name, "vidx").await; + let (status, text) = call("UpdateTable", &delete_body).await; + assert_eq!( + status, 200, + "the same delete must be accepted once the backfill is running: {text}" + ); + + // And the table is healthy afterwards: the index is gone and writes still work, + // which is what a leaked build hold would break. + for _ in 0..100 { + let (_, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + if !text.contains("\"IndexName\":\"vidx\"") { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + put_vector(&name, "after", None, &[0.0, 1.0]).await; + + set_allocation_phase_delay(0).await; + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Poll until a named vector index has left the resource-allocation phase. +/// +/// Left means anything other than `Backfilling: false`: either the scan is running +/// or the index is already ACTIVE. Both are past the boundary the delete rule turns +/// on, and neither is a state the refusal applies to. +async fn wait_past_allocation_phase(table: &str, index: &str) { + for _ in 0..600 { + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{table}"}}"#)).await; + if status == 200 { + let d: serde_json::Value = serde_json::from_str(&text).expect("json"); + let allocating = d["Table"]["VectorIndexes"] + .as_array() + .into_iter() + .flatten() + .any(|vi| vi["IndexName"] == index && vi["Backfilling"] == false); + if !allocating { + return; + } + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + panic!("vector index {index} on {table} never left the resource-allocation phase"); +} + /// The same collapse on the other path a client can reach: an index added by /// UpdateTable. /// @@ -1707,6 +1852,12 @@ async fn set_backfill_delay(ms: u64) { set_vector_setting("vector_backfill_batch_delay_ms", ms).await; } +/// Hold a new index in the resource-allocation phase for `ms`, so a client can +/// observe the phase at all. Zero in production; see the settings key's own docs. +pub(crate) async fn set_allocation_phase_delay(ms: u64) { + set_vector_setting("vector_allocation_phase_delay_ms", ms).await; +} + /// Write one vector test lever through the management API. async fn set_vector_setting(key: &str, ms: u64) { let user = std::env::var("EXTENDDB_ADMIN_USER").unwrap_or_else(|_| "admin".into()); From c8730e98f713f1f7b9e3306fcafb3d6859411b81 Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Mon, 24 Aug 2026 20:15:41 +0000 Subject: [PATCH 06/13] fix: keep every vector score finite, on both backends Query norms compute in f64 on both backends and pgvector's NaN cannot reach the client. Also covers the delayed queue path for a vector-only table and closes the remaining review nits. --- .../storage-postgres/src/data/delete_item.rs | 18 +- crates/storage-postgres/src/data/index.rs | 37 ++- crates/storage-postgres/src/data/put_item.rs | 18 +- .../storage-postgres/src/data/transactions.rs | 7 +- .../storage-postgres/src/data/update_item.rs | 18 +- .../storage-postgres/src/data/vector_index.rs | 242 ++++++++++------- crates/storage-postgres/src/delete_table.rs | 4 +- crates/storage-postgres/src/gsi_queue.rs | 2 +- crates/storage-postgres/src/lib.rs | 8 + crates/storage-postgres/src/update_table.rs | 7 +- crates/storage-postgres/src/vector_search.rs | 243 ++++++++++++++--- .../tests/vector_control_plane.rs | 116 +++++++-- crates/storage-sqlite/src/update_table.rs | 22 +- crates/storage-sqlite/src/vector_search.rs | 193 +++++++++----- docs/differences-from-dynamodb.md | 1 + tests/rust/src/vector_index_search.rs | 244 ++++++++++++++++++ 16 files changed, 922 insertions(+), 258 deletions(-) diff --git a/crates/storage-postgres/src/data/delete_item.rs b/crates/storage-postgres/src/data/delete_item.rs index 34050d18..1ea9dd1e 100755 --- a/crates/storage-postgres/src/data/delete_item.rs +++ b/crates/storage-postgres/src/data/delete_item.rs @@ -9,7 +9,7 @@ use extenddb_storage::StreamCapture; use extenddb_storage::error::StorageError; use extenddb_storage::util::{SortKeyValue, parse_sk, pk_to_text, sk_column, sk_info}; -use super::index::{enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::index::{enqueue_async_indexes, fetch_write_path_indexes, sync_indexes}; use super::query::check_condition; use super::tx_helpers::write_stream_record_in_tx; use super::{data_table_name, json_to_item}; @@ -34,10 +34,11 @@ impl PostgresEngine { .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; let pk_text = pk_to_text(pk_value)?; - // Fetch indexes for GSI/LSI updates (D-4: sync + async split). - let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; - // Vector indexes come from the same fresh read rather than from the cached - // key info, and the answer decides two things: whether this write needs a + // Both index families in one catalog visit (D-4: sync + async split for the + // secondary indexes). + // + // Vector indexes come from this fresh read rather than from the cached key + // info, and the answer decides two things: whether this write needs a // transaction at all, and what maintenance runs inside it. A cached empty // set would send a write down the no-maintenance fast path and silently // leave an index missing a row. @@ -51,11 +52,8 @@ impl PostgresEngine { // also exactly the window the secondary indexes have, for the same reason // and with the same read: parity with a GSI is the bar, and the backfill // that publishes a new index is what covers writes older than it. - let vector_metas = crate::data::vector_index::fetch_vector_indexes_for_table( - &self.pool, - &key_info.table_id, - ) - .await?; + let (indexes, vector_metas) = + fetch_write_path_indexes(&key_info.table_id, &self.pool).await?; // Read whenever anything can propagate, secondary or vector. Gating this on // the secondary set alone made a vector-only table ignore the configured // delay and apply its vector index inline, while a TransactWriteItems on the diff --git a/crates/storage-postgres/src/data/index.rs b/crates/storage-postgres/src/data/index.rs index 1a37d3f5..551861db 100644 --- a/crates/storage-postgres/src/data/index.rs +++ b/crates/storage-postgres/src/data/index.rs @@ -52,16 +52,45 @@ pub(crate) struct IndexMeta { pub(super) propagation_delay_ms: Option, } -/// Fetch all index metadata for a table from the catalog. -pub(crate) async fn fetch_indexes_for_table( +/// Both index families a write has to maintain, read in one catalog visit. +/// +/// One acquired connection for both statements rather than two trips through the +/// pool, because every write takes this path and most tables have no vector index +/// at all. Also the single place the two reads are ordered together, so the six +/// write sites cannot end up disagreeing about which membership they saw. +pub(crate) async fn fetch_write_path_indexes( table_id: &str, pool: &sqlx::PgPool, -) -> Result, StorageError> { +) -> Result< + ( + Vec, + Vec<(extenddb_storage::vector_lifecycle::VectorIndexMeta, String)>, + ), + StorageError, +> { + let mut conn = pool + .acquire() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let indexes = fetch_indexes_for_table(table_id, &mut *conn).await?; + let vector_metas = + crate::data::vector_index::fetch_vector_indexes_for_table(&mut *conn, table_id).await?; + Ok((indexes, vector_metas)) +} + +/// Fetch all index metadata for a table from the catalog. +pub(crate) async fn fetch_indexes_for_table<'e, E>( + table_id: &str, + executor: E, +) -> Result, StorageError> +where + E: sqlx::PgExecutor<'e>, +{ let rows: Vec<(String, String, String, serde_json::Value, serde_json::Value, Option)> = sqlx::query_as( "SELECT index_id, index_name, index_type, key_schema, projection, propagation_delay_ms FROM indexes WHERE table_id = $1", ) .bind(table_id) - .fetch_all(pool) + .fetch_all(executor) .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-postgres/src/data/put_item.rs b/crates/storage-postgres/src/data/put_item.rs index cb901f94..e307df66 100755 --- a/crates/storage-postgres/src/data/put_item.rs +++ b/crates/storage-postgres/src/data/put_item.rs @@ -9,7 +9,7 @@ use extenddb_storage::StreamCapture; use extenddb_storage::error::StorageError; use extenddb_storage::util::{composite_pk_to_text, parse_sk, pk_to_text, sk_column, sk_info}; -use super::index::{enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::index::{enqueue_async_indexes, fetch_write_path_indexes, sync_indexes}; use super::query::check_condition; use super::tx_helpers::write_stream_record_in_tx; use super::{data_table_name, json_to_item}; @@ -33,10 +33,11 @@ impl PostgresEngine { let item_json = serde_json::to_value(&item).map_err(|e| StorageError::Internal(e.to_string()))?; - // Fetch indexes for GSI/LSI updates (D-4: sync + async split). - let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; - // Vector indexes come from the same fresh read rather than from the cached - // key info, and the answer decides two things: whether this write needs a + // Both index families in one catalog visit (D-4: sync + async split for the + // secondary indexes). + // + // Vector indexes come from this fresh read rather than from the cached key + // info, and the answer decides two things: whether this write needs a // transaction at all, and what maintenance runs inside it. A cached empty // set would send a write down the no-maintenance fast path and silently // leave an index missing a row. @@ -50,11 +51,8 @@ impl PostgresEngine { // also exactly the window the secondary indexes have, for the same reason // and with the same read: parity with a GSI is the bar, and the backfill // that publishes a new index is what covers writes older than it. - let vector_metas = crate::data::vector_index::fetch_vector_indexes_for_table( - &self.pool, - &key_info.table_id, - ) - .await?; + let (indexes, vector_metas) = + fetch_write_path_indexes(&key_info.table_id, &self.pool).await?; // Index key attributes present in the item must match their declared // scalar type and be non-empty, matching real DynamoDB. This is up-front diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index 5d39d385..990c292e 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -13,7 +13,7 @@ use extenddb_core::validation; use extenddb_storage::error::StorageError; use extenddb_storage::{IdempotencyKey, TransactGetOp, TransactWriteOp}; -use super::index::{IndexMeta, enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::index::{IndexMeta, enqueue_async_indexes, fetch_write_path_indexes, sync_indexes}; use super::tx_helpers::{ check_idempotency_token_in_tx, delete_item_in_tx, fetch_item_for_update, fetch_item_in_tx, upsert_item_in_tx, write_stream_record_in_tx, @@ -88,11 +88,8 @@ impl PostgresEngine { let name = transact_op_table_name(op); if !table_indexes.contains_key(name) { let tid = transact_op_table_id(op); - let indexes = fetch_indexes_for_table(tid, &self.pool).await?; + let (indexes, vector_metas) = fetch_write_path_indexes(tid, &self.pool).await?; table_indexes.insert(name.to_owned(), indexes); - let vector_metas = - crate::data::vector_index::fetch_vector_indexes_for_table(&self.pool, tid) - .await?; table_vector_metas.insert(name.to_owned(), vector_metas); } } diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index d77b0d96..72ce6edb 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -10,7 +10,7 @@ use extenddb_storage::StreamCapture; use extenddb_storage::error::StorageError; use extenddb_storage::util::{parse_sk, pk_to_text, sk_column, sk_info}; -use super::index::{enqueue_async_indexes, fetch_indexes_for_table, sync_indexes}; +use super::index::{enqueue_async_indexes, fetch_write_path_indexes, sync_indexes}; use super::query::check_condition; use super::tx_helpers::write_stream_record_in_tx; use super::{data_table_name, json_to_item}; @@ -45,10 +45,11 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - // Fetch indexes for GSI/LSI updates (D-4: sync + async split). - let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; - // Vector indexes come from the same fresh read rather than from the cached - // key info, and the answer decides two things: whether this write needs a + // Both index families in one catalog visit (D-4: sync + async split for the + // secondary indexes). + // + // Vector indexes come from this fresh read rather than from the cached key + // info, and the answer decides two things: whether this write needs a // transaction at all, and what maintenance runs inside it. A cached empty // set would send a write down the no-maintenance fast path and silently // leave an index missing a row. @@ -62,11 +63,8 @@ impl PostgresEngine { // also exactly the window the secondary indexes have, for the same reason // and with the same read: parity with a GSI is the bar, and the backfill // that publishes a new index is what covers writes older than it. - let vector_metas = crate::data::vector_index::fetch_vector_indexes_for_table( - &self.pool, - &key_info.table_id, - ) - .await?; + let (indexes, vector_metas) = + fetch_write_path_indexes(&key_info.table_id, &self.pool).await?; // Read whenever anything can propagate, secondary or vector. Gating this on // the secondary set alone made a vector-only table ignore the configured // delay and apply its vector index inline, while a TransactWriteItems on the diff --git a/crates/storage-postgres/src/data/vector_index.rs b/crates/storage-postgres/src/data/vector_index.rs index 89c46913..1867be95 100644 --- a/crates/storage-postgres/src/data/vector_index.rs +++ b/crates/storage-postgres/src/data/vector_index.rs @@ -49,10 +49,13 @@ use super::{all_sort_key_info, vector_table_name}; /// /// Returns the rows with their status, because the status decides inline versus /// enqueue and only this read can see it. -pub(crate) async fn fetch_vector_indexes_for_table( - catalog: &sqlx::PgPool, +pub(crate) async fn fetch_vector_indexes_for_table<'e, E>( + catalog: E, table_id: &str, -) -> Result, StorageError> { +) -> Result, StorageError> +where + E: sqlx::PgExecutor<'e>, +{ let rows: Vec<( String, i32, @@ -105,7 +108,7 @@ pub(crate) async fn fetch_vector_indexes_for_table( } /// The base table's key columns for a vector data table, in insert order. -fn base_key_columns(base_sks: &[(&str, ScalarAttributeType)]) -> Vec { +pub(crate) fn base_key_columns(base_sks: &[(&str, ScalarAttributeType)]) -> Vec { let mut cols = vec!["base_pk".to_owned()]; for (i, &(_, sk_type)) in base_sks.iter().enumerate() { let col = if i == 0 { @@ -242,83 +245,124 @@ pub(crate) async fn insert_vector_row( base_key_schema: &[KeySchemaElement], attr_defs: &[AttributeDefinition], ) -> Result<(), StorageError> { - if !item_is_indexable(item, meta) { - return Ok(()); - } - let Some(value) = item.get(&meta.vector_attribute_name) else { - return Ok(()); - }; - let Some(components) = vector_components(value) else { - tracing::warn!( - index_id = %meta.index_id, - attribute = %meta.vector_attribute_name, - "stored vector attribute cannot be read as a vector; leaving the item unindexed" - ); - return Ok(()); - }; - if components.len() != meta.dimensions { - tracing::warn!( - index_id = %meta.index_id, - found = components.len(), - declared = meta.dimensions, - "stored vector has the wrong dimension count; leaving the item unindexed" + VectorInsertPlan::new(meta, base_key_schema, attr_defs) + .insert(tx, item) + .await +} + +/// Everything about writing one index's rows that does not change per row: the +/// data table's name, the base key columns, and the INSERT statement over them. +/// +/// A backfill writes up to a full batch of rows through one plan, so the schema +/// work and the statement text are computed once rather than per row. The write +/// path builds a plan for its single row, which costs the same as before. +pub(crate) struct VectorInsertPlan<'a> { + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + base_sks: Vec<(&'a str, ScalarAttributeType)>, + sql: String, +} + +impl<'a> VectorInsertPlan<'a> { + pub(crate) fn new( + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], + ) -> Self { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let key_cols = base_key_columns(&base_sks); + let mut cols = vec!["part".to_owned()]; + cols.extend(key_cols); + cols.extend([ + "embedding".to_owned(), + "nrm".to_owned(), + "item_data".to_owned(), + ]); + let placeholders: Vec = (1..=cols.len()).map(|i| format!("${i}")).collect(); + let sql = format!( + "INSERT INTO {} ({}) VALUES ({})", + vector_table_name(&meta.index_id), + cols.join(", "), + placeholders.join(", ") ); - return Ok(()); + Self { + meta, + base_key_schema, + base_sks, + sql, + } } - let vec_table = vector_table_name(&meta.index_id); - let base_sks = all_sort_key_info(base_key_schema, attr_defs); - let key_cols = base_key_columns(&base_sks); - let part = item_partition(item, meta)?; - let norm = vector_norm(&components); - // Projection, the always-projected SearchSchema attributes, and the stripped - // vector attribute are the shared payload rules, so a live-written row and a - // backfilled one cannot differ in shape. - let projected = projected_payload(item, base_key_schema, meta); - let item_json = - serde_json::to_value(&projected).map_err(|e| StorageError::Internal(e.to_string()))?; - - // A plain INSERT, deliberately, where the GSI sibling upserts. Every caller - // reaches this through `apply_vector_index`, which deletes the base key's row - // first, so no live row can exist here. Keeping it plain means that if a - // future change ever makes that delete conditional, this fails loudly on the - // primary key rather than quietly replacing a row and hiding the break. - let mut cols = vec!["part".to_owned()]; - cols.extend(key_cols.iter().cloned()); - cols.extend([ - "embedding".to_owned(), - "nrm".to_owned(), - "item_data".to_owned(), - ]); - let placeholders: Vec = (1..=cols.len()).map(|i| format!("${i}")).collect(); - let sql = format!( - "INSERT INTO {vec_table} ({}) VALUES ({})", - cols.join(", "), - placeholders.join(", ") - ); - - // As bytes: the column is BYTEA because the unscoped sentinel carries a NUL, - // which PostgreSQL rejects in a text column. - let mut query = sqlx::query(&sql) - .bind(part.into_bytes()) - .bind(composite_pk_to_text(item, base_key_schema)?); - for &(sk_name, sk_type) in &base_sks { - if let Some(value) = item.get(sk_name) { - query = match parse_sk(value, sk_type)? { - SortKeyValue::S(s) => query.bind(s), - SortKeyValue::N(n) => query.bind(n), - SortKeyValue::B(b) => query.bind(b), - }; + /// Write one item's row, or skip it for the reasons the doc above gives. + pub(crate) async fn insert( + &self, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + item: &Item, + ) -> Result<(), StorageError> { + let meta = self.meta; + let base_key_schema = self.base_key_schema; + if !item_is_indexable(item, meta) { + return Ok(()); + } + let Some(value) = item.get(&meta.vector_attribute_name) else { + return Ok(()); + }; + let Some(components) = vector_components(value) else { + tracing::warn!( + index_id = %meta.index_id, + attribute = %meta.vector_attribute_name, + "stored vector attribute cannot be read as a vector; leaving the item unindexed" + ); + return Ok(()); + }; + if components.len() != meta.dimensions { + tracing::warn!( + index_id = %meta.index_id, + found = components.len(), + declared = meta.dimensions, + "stored vector has the wrong dimension count; leaving the item unindexed" + ); + return Ok(()); + } + + let part = item_partition(item, meta)?; + let norm = vector_norm(&components); + // Projection, the always-projected SearchSchema attributes, and the stripped + // vector attribute are the shared payload rules, so a live-written row and a + // backfilled one cannot differ in shape. + let projected = projected_payload(item, base_key_schema, meta); + let item_json = + serde_json::to_value(&projected).map_err(|e| StorageError::Internal(e.to_string()))?; + + // A plain INSERT, deliberately, where the GSI sibling upserts. Every caller + // reaches this through `apply_vector_index`, which deletes the base key's row + // first, so no live row can exist here. Keeping it plain means that if a + // future change ever makes that delete conditional, this fails loudly on the + // primary key rather than quietly replacing a row and hiding the break. + // + // As bytes: the column is BYTEA because the unscoped sentinel carries a NUL, + // which PostgreSQL rejects in a text column. + let mut query = sqlx::query(&self.sql) + .bind(part.into_bytes()) + .bind(composite_pk_to_text(item, base_key_schema)?); + for &(sk_name, sk_type) in &self.base_sks { + if let Some(value) = item.get(sk_name) { + query = match parse_sk(value, sk_type)? { + SortKeyValue::S(s) => query.bind(s), + SortKeyValue::N(n) => query.bind(n), + SortKeyValue::B(b) => query.bind(b), + }; + } } + query + .bind(Vector::from(components)) + .bind(f64::from(norm)) + .bind(item_json) + .execute(&mut **tx) + .await + .map_err(crate::vector::map_vector_sql_error)?; + Ok(()) } - query - .bind(Vector::from(components)) - .bind(f64::from(norm)) - .bind(item_json) - .execute(&mut **tx) - .await - .map_err(crate::vector::map_vector_sql_error)?; - Ok(()) } /// Apply one claimed queue row to its vector index, from the row's own context. @@ -487,6 +531,10 @@ impl extenddb_storage::vector_lifecycle::VectorIndexBuild for PostgresVectorBuil let mut skipped = 0usize; let mut next_cursor = None; + // One plan for the whole batch: the table name, the key columns and the + // statement text are the same for every row it writes. + let plan = VectorInsertPlan::new(&meta, &self.base_key_schema, &self.attribute_definitions); + for row in &rows { use sqlx::Row as _; let pk: String = row @@ -528,14 +576,7 @@ impl extenddb_storage::vector_lifecycle::VectorIndexBuild for PostgresVectorBuil BackfillRow::Index(item) => { // The classifier parsed the row already, so the item comes from // it rather than being deserialised a second time. - insert_vector_row( - &mut tx, - &meta, - &item, - &self.base_key_schema, - &self.attribute_definitions, - ) - .await?; + plan.insert(&mut tx, &item).await?; written += 1; } BackfillRow::Poison => skipped += 1, @@ -739,12 +780,19 @@ fn build_owner_label() -> String { /// Namespace for ExtendDB advisory locks, so a lock taken here cannot collide /// with one taken by another feature that hashed a different string. +/// +/// The namespace is only half of a lock's identity: an advisory lock is scoped to the +/// DATABASE its session is connected to, so the same namespace and key taken on the +/// catalog database and on the data database are two different locks and neither +/// excludes the other. Every vector-side call site takes ownership on the data pool, +/// which is what makes this work. Moving one of them to the catalog pool for tidiness +/// would break mutual exclusion with no error message anywhere. const ADVISORY_LOCK_NAMESPACE: i32 = 0x0045_4442; -/// A held build-ownership lock. Dropping it returns the connection to the pool, -/// which releases the session-scoped lock. -pub(crate) struct BuildOwner { - _conn: sqlx::pool::PoolConnection, +/// A held build-ownership lock. Dropping it ends the session the lock lives in, +/// which is what releases the lock. +pub struct BuildOwner { + _conn: sqlx::PgConnection, } /// Try to take ownership of one index's build. @@ -758,11 +806,21 @@ pub(crate) struct BuildOwner { /// /// Returns `None` when another process owns the build, which is not an error: the /// other process is doing the work. -pub(crate) async fn build_ownership(data: &sqlx::PgPool, index_id: &str) -> Option { - let mut conn = match data.acquire().await { +/// +/// The session is opened directly rather than taken from the data pool, for two +/// reasons. A build lasts as long as its scan, and the data pool serves every +/// write, so a pinned pool connection would spend the whole build competing with +/// the writes the batched backfill exists to keep flowing. And a pooled connection +/// cannot release this lock: the lock is session-scoped, returning a connection to +/// the pool does not end its session, so an abandoned owner would leave the lock +/// held on an idle pooled connection and no peer could recover that index until the +/// connection was recycled. Its own session dies with the owner. +pub async fn build_ownership(data: &sqlx::PgPool, index_id: &str) -> Option { + let options = data.connect_options(); + let mut conn = match ::connect_with(&options).await { Ok(conn) => conn, Err(e) => { - tracing::warn!("could not acquire a connection for vector build ownership: {e}"); + tracing::warn!("could not open a session for vector build ownership: {e}"); return None; } }; @@ -772,7 +830,7 @@ pub(crate) async fn build_ownership(data: &sqlx::PgPool, index_id: &str) -> Opti sqlx::query_scalar("SELECT pg_try_advisory_lock($1, hashtext($2))") .bind(ADVISORY_LOCK_NAMESPACE) .bind(index_id) - .fetch_one(&mut *conn) + .fetch_one(&mut conn) .await; match taken { Ok(true) => Some(BuildOwner { _conn: conn }), diff --git a/crates/storage-postgres/src/delete_table.rs b/crates/storage-postgres/src/delete_table.rs index b5192bb2..5a55231f 100755 --- a/crates/storage-postgres/src/delete_table.rs +++ b/crates/storage-postgres/src/delete_table.rs @@ -89,8 +89,8 @@ impl PostgresEngine { // Vector index rows cascade the same way, through // vector_indexes_table_id_fkey, so a table with vector indexes needs // no extra catalog cleanup. Their data tables do need dropping, and - // by then the rows that named them are gone, which is why the sweep - // matches on the table id prefix instead of reading the catalog. + // the cascade takes away the rows that name them, which is why the + // index ids are collected BEFORE the catalog row goes. sqlx::query("DELETE FROM tags WHERE resource_arn = $1") .bind(&row.table_arn) .execute(&mut *tx) diff --git a/crates/storage-postgres/src/gsi_queue.rs b/crates/storage-postgres/src/gsi_queue.rs index fcda7976..39e60d44 100644 --- a/crates/storage-postgres/src/gsi_queue.rs +++ b/crates/storage-postgres/src/gsi_queue.rs @@ -73,7 +73,7 @@ fn partition_for(base_pk_text: &str) -> i32 { /// /// This occurs when an index table is dropped (table deleted) while an async /// GSI update is still queued. The pending row is consumed rather than retried. -fn is_undefined_table(err: &StorageError) -> bool { +pub(crate) fn is_undefined_table(err: &StorageError) -> bool { match err { StorageError::Internal(msg) => msg.contains(PG_UNDEFINED_TABLE), _ => false, diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 4b3da645..63740fe9 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -46,6 +46,14 @@ pub use credential_store::DbCredentialStore; /// it, unlike the two recovery entry points above. #[doc(hidden)] pub use data::vector_index::apply_claimed_vector_row; +/// Try to take ownership of one vector index's build. +/// +/// Reachable so an integration test can assert that ownership is held in a session +/// of its own and given back when the owner is dropped, which is only observable +/// from a second session. Hidden for the same reason as the row applier: no +/// deployment path calls it. +#[doc(hidden)] +pub use data::vector_index::build_ownership; /// Rebuild vector index builds whose heartbeat has gone stale. The runtime half of /// the same repair, exported for the same reason and for its test. pub use data::vector_index::rebuild_stuck_vector_indexes; diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index 5ab10053..647eca1e 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -931,9 +931,10 @@ impl PostgresEngine { ); continue; } - // The queue rows for this table can outlive the index. They are - // tolerated by the worker (a missing table is a routine race), but - // removing them here saves the claim-and-skip cycle and the log noise. + // The queue rows for this index outlive it, deliberately. The worker + // consumes them when it finds the data table gone, which is the route + // that keeps a partition moving; deleting them here would need the same + // transaction as the catalog change to be safe, and it is not. if let Err(e) = data_tx .commit() .await diff --git a/crates/storage-postgres/src/vector_search.rs b/crates/storage-postgres/src/vector_search.rs index 7ccb4e06..31c5ebdc 100644 --- a/crates/storage-postgres/src/vector_search.rs +++ b/crates/storage-postgres/src/vector_search.rs @@ -36,7 +36,36 @@ use crate::data::vector_table_name; /// `$1` is the query vector and `${norm_param}` the caller's precomputed norm, bound /// rather than interpolated: a norm formatted into the statement can render as `inf` /// for a large query vector, which PostgreSQL then reads as a column name and the -/// search fails with a 500 for input the service answers. +/// search fails with a 500 for input the service answers. The norm is bound for every +/// metric so one statement layout serves all three, and only the cosine expression +/// reads it; an unused trailing bind is accepted by the protocol, as is the resulting +/// gap in the numbering when filters follow. +/// +/// # Every metric is kept finite here, in SQL +/// +/// pgvector accumulates in single precision, so all three operators return a value +/// that cannot be serialised for magnitudes well below `f32::MAX`, measured on 0.8.0: +/// Euclidean overflows to `Infinity` above about 9.2e18 (its difference is doubled +/// before squaring, so it goes first), dot product returns `-Infinity` above about +/// 1.8e19, and cosine returns `NaN` at both ends, above about 1.8e19 and below about +/// 3.7e-23, because it divides two values that have both overflowed or both underflowed. +/// `serde_json` renders NaN and Infinity alike as `null`, so each of those reaches a +/// client as `"Score": null` on a 200 response. +/// +/// The repair belongs here rather than where the score is reported, and that is not a +/// stylistic preference. PostgreSQL's ordering is already correct in the overflowed +/// cases: `Infinity` sorts last, so an overflowed Euclidean distance is correctly the +/// farthest, and `-Infinity` sorts first, correctly the most similar. Substituting a +/// value in Rust after the database has ordered and cut the rows would leave that +/// ordering untouched, so a hit reported as nearer could appear after one reported as +/// farther, breaking the most-similar-first contract. Clamping in the expression keeps +/// the value and the order consistent by construction. +/// +/// `1e308` is a serialisation bound, not a measured answer, and it is not the true +/// distance: the real value is representable in `f64` and only pgvector's `f32` +/// accumulator loses it. Recovering it would mean unpacking every candidate vector and +/// recomputing outside the operator, which costs a per-row scan and gives up the index. +/// The difference from Amazon DynamoDB is recorded in `docs/differences-from-dynamodb.md`. fn score_expression(function: DistanceFunction, norm_param: usize) -> String { match function { // The CASE is a conformance requirement, not a nicety. pgvector's cosine @@ -44,15 +73,60 @@ fn score_expression(function: DistanceFunction, norm_param: usize) -> String { // answers exactly 1.0 with a zero vector on either side (measured // 2026-08-19), which is also what the SQLite backend produces. Removing // the CASE would make a zero vector sort unpredictably and report NaN. + // + // The wrapper is the same rule applied to the cases the guard cannot see: a + // query vector whose f32 squares underflow (components around 1e-30, which + // validation accepts) or magnitudes above about 1.8e19, where pgvector's own + // norms underflow or overflow even though ours is correct. Both give NaN, and + // no probed magnitude gave an infinity, so the NaN filter is the whole fix for + // this metric: cosine distance has domain [0, 2] and the operator clamps. + // NULLIF works as a NaN filter only because `NaN = NaN` is TRUE in PostgreSQL. + // 1.0 is right under every reading: the true distance for orthogonal vectors, + // the measured answer for a zero vector on either side, and what SQLite + // returns for the same input. + // + // The wrapper also makes the two sides safe independently. The stored side is + // masked today only because `nrm` comes from the shared `vector_norm`, which + // accumulates in f32, so a tiny stored vector reads as zero and takes the + // guard. Widening that function, the obvious later tidy-up, would open the + // identical hole on the stored side; with this wrapper it cannot. DistanceFunction::Cosine => format!( "CASE WHEN nrm = 0 OR ${norm_param} = 0 THEN 1.0 \ - ELSE (embedding <=> $1)::float8 END" + ELSE COALESCE(NULLIF((embedding <=> $1)::float8, 'NaN'::float8), 1.0) END" ), - DistanceFunction::Euclidean => "(embedding <-> $1)::float8".to_owned(), - DistanceFunction::DotProduct => "(embedding <#> $1)::float8".to_owned(), + // A distance, so the overflow is at the top: cap it and the farthest row stays + // farthest. + DistanceFunction::Euclidean => "LEAST((embedding <-> $1)::float8, 1e308)".to_owned(), + // The operator returns the NEGATED inner product, so its overflow is at the + // bottom: floor it and the most similar row stays most similar, both here and + // after `report_score` undoes the sign. + DistanceFunction::DotProduct => "GREATEST((embedding <#> $1)::float8, -1e308)".to_owned(), } } +/// The query vector's Euclidean norm, in double precision. +/// +/// Each component is widened BEFORE it is squared, and the sum is accumulated as an +/// `f64`. Squaring in single precision and widening the result afterwards is the +/// same expression to read and a different function: `1e-30` squared underflows to +/// zero in `f32`, so the norm of a small-but-valid vector came out as zero, the +/// zero-vector guard fired, and every row scored exactly the same. That is a wrong +/// answer rather than an error, which is why it is a named function with its own +/// test rather than an inline expression. +/// +/// Over the whole valid input domain the result is finite, and zero only for a +/// genuinely zero vector. Both ends have room to spare rather than being close +/// calls: the smallest positive `f32` subnormal squared is 1.96e-90, which is 234 +/// orders of magnitude above the smallest positive `f64`, and 4096 components (the +/// dimension cap) at `f32::MAX` squared sum to 4.7e80, well inside `f64`'s range. +fn query_norm(values: &[f32]) -> f64 { + values + .iter() + .map(|x| f64::from(*x) * f64::from(*x)) + .sum::() + .sqrt() +} + /// Turn the ordered SQL value into the score the engine contract defines. /// /// Cosine and Euclidean report the distance itself, lower being more similar. @@ -78,6 +152,8 @@ impl VectorSearchEngine for PostgresEngine { .find(|vi| vi.index_name == index_name) .map(|vi| vi.dimensions); let top_k = req.top_k; + let base_key_schema = req.key_info.key_schema.clone(); + let base_attr_defs = req.key_info.attribute_definitions.clone(); let partition = partition_value(req.hash_key); let filters: Vec<(String, AttributeValue)> = req .filters @@ -136,7 +212,7 @@ impl VectorSearchEngine for PostgresEngine { ))); } - let query_norm = f64::from(query_vector.iter().map(|x| x * x).sum::().sqrt()); + let query_norm = query_norm(&query_vector); let vec_table = vector_table_name(&index_id); // Filters are equality over the index's inline-filter attributes, @@ -158,10 +234,21 @@ impl VectorSearchEngine for PostgresEngine { )); } + // Ties break on the whole base key, not on the partition key alone: on a + // composite-key table several rows share a `base_pk`, so ordering by it + // alone leaves their relative order up to the plan, and two identical + // searches can disagree about which of them the top_k cut keeps. + let base_sks = crate::data::all_sort_key_info(&base_key_schema, &base_attr_defs); + let tie_break = crate::data::vector_index::base_key_columns(&base_sks) + .into_iter() + .map(|col| format!("{col} ASC")) + .collect::>() + .join(", "); + let sql = format!( "SELECT {score} AS score, embedding, item_data \ FROM {vec_table} WHERE part = $2{predicates} \ - ORDER BY score ASC, base_pk ASC LIMIT $3", + ORDER BY score ASC, {tie_break} LIMIT $3", score = score_expression(function, 4), ); @@ -180,10 +267,18 @@ impl VectorSearchEngine for PostgresEngine { query = query.bind(name).bind(value_json); } - let rows = query - .fetch_all(&self.data_pool) - .await - .map_err(crate::vector::map_vector_sql_error)?; + let rows = query.fetch_all(&self.data_pool).await.map_err(|e| { + let mapped = crate::vector::map_vector_sql_error(e); + // The data table is gone, which happens when the index is deleted + // between the catalog read above and this statement. That is a + // deleted index, so it answers like one instead of reporting an + // internal failure the caller can do nothing about. + if crate::gsi_queue::is_undefined_table(&mapped) { + StorageError::IndexNotFound(index_name.clone()) + } else { + mapped + } + })?; let mut hits = Vec::with_capacity(rows.len()); for (ordered, embedding, item_json) in rows { @@ -226,6 +321,64 @@ mod tests { assert!(sql.contains("<=>"), "{sql}"); } + /// The guard cannot see pgvector's own single-precision norms, so the NaN it can + /// return for a small-but-valid query vector is filtered in SQL as well. + /// + /// Without this, that NaN reaches a client as `"Score": null` on a 200 response, + /// because a non-finite double serialises as null rather than failing. The filter + /// is `NULLIF`, which works on NaN only because `NaN = NaN` is TRUE in PostgreSQL, + /// so the assertion names it: an equivalent-looking rewrite with a comparison + /// operator would silently stop filtering. + #[test] + fn cosine_substitutes_the_measured_answer_for_a_nan_distance() { + let sql = score_expression(DistanceFunction::Cosine, 4); + assert!( + sql.contains("NULLIF") && sql.contains("'NaN'::float8"), + "the NaN filter must survive: {sql}" + ); + assert!( + sql.contains("COALESCE"), + "a filtered NaN must fall back to the measured answer: {sql}" + ); + // Cosine is the only metric that divides, so it is the only one that can + // produce a NaN; the other two overflow to an infinity and are bounded instead. + for metric in [DistanceFunction::Euclidean, DistanceFunction::DotProduct] { + let sql = score_expression(metric, 4); + assert!( + !sql.contains("NULLIF"), + "only cosine divides, so only cosine needs the filter: {sql}" + ); + } + } + + /// Every metric's score stays finite, and each is bounded at the end its own + /// accumulator overflows towards. + /// + /// A distance overflows upwards and is capped; the negated inner product overflows + /// downwards and is floored. Getting the direction wrong would turn the farthest + /// row into the nearest, which is why the bound is asserted per metric rather than + /// as "a bound exists somewhere". + #[test] + fn each_metric_is_bounded_at_the_end_it_overflows_towards() { + let euclidean = score_expression(DistanceFunction::Euclidean, 4); + assert!( + euclidean.contains("LEAST") && euclidean.contains("1e308"), + "a distance must be capped above: {euclidean}" + ); + let dot = score_expression(DistanceFunction::DotProduct, 4); + assert!( + dot.contains("GREATEST") && dot.contains("-1e308"), + "the negated inner product must be floored below: {dot}" + ); + // Cosine has domain [0, 2] and the operator clamps, so no probed magnitude + // produced an infinity there: a bound would be dead code. + let cosine = score_expression(DistanceFunction::Cosine, 4); + assert!( + !cosine.contains("LEAST") && !cosine.contains("GREATEST"), + "cosine needs no magnitude bound: {cosine}" + ); + } + #[test] fn each_metric_uses_its_own_operator() { assert!(score_expression(DistanceFunction::Euclidean, 4).contains("<->")); @@ -262,34 +415,58 @@ mod tests { !sql.contains("inf"), "a rendered norm reached the statement: {sql}" ); + // Both literals are named rather than counted, because both are deliberate and + // a global count cannot say which one went missing. + assert!( + sql.contains("THEN 1.0"), + "the zero-norm guard's measured answer is missing: {sql}" + ); + assert!( + sql.contains("), 1.0)"), + "the NaN substitute is missing: {sql}" + ); assert_eq!( sql.matches("1.0").count(), - 1, - "the only literal is the measured zero-norm score: {sql}" + 2, + "those two are the only literals the expression may carry: {sql}" ); } + /// The norm is finite for every valid input and zero only for a zero vector. + /// + /// Calls the function the search path calls. An earlier version of this test + /// recomputed the arithmetic it was asserting, so it passed while the search path + /// still accumulated in single precision: the test proved a property of itself. + /// + /// The magnitudes are the domain's extremes rather than samples. The smallest + /// positive subnormal is the worst case for underflow and the dimension cap at + /// `f32::MAX` is the worst case for overflow, so nothing between them can + /// misbehave. #[test] - fn a_large_query_vector_does_not_overflow_the_norm() { - // f32 accumulation overflowed to infinity here; f64 does not. The values are - // ones the service accepts, so this is a search that must work rather than an - // edge nobody reaches. - let big = [1e38f32; 4]; - let norm = big - .iter() - .map(|x| f64::from(*x) * f64::from(*x)) - .sum::() - .sqrt(); - assert!(norm.is_finite(), "norm overflowed: {norm}"); - - // And the other end: f32 squares of 1e-30 underflow to zero, which made the - // guard fire and every row score exactly 1.0, silently. - let tiny = [1e-30f32; 4]; - let norm = tiny - .iter() - .map(|x| f64::from(*x) * f64::from(*x)) - .sum::() - .sqrt(); - assert!(norm > 0.0, "norm underflowed to zero: {norm}"); + fn the_query_norm_is_finite_and_only_zero_for_a_zero_vector() { + assert!( + query_norm(&[1e38f32; 4]).is_finite(), + "single-precision accumulation overflowed to infinity here" + ); + assert!( + query_norm(&[f32::MAX; 4096]).is_finite(), + "the dimension cap at f32::MAX must stay inside f64's range" + ); + assert!( + query_norm(&[1e-30f32; 4]) > 0.0, + "a small but valid vector must not read as zero, or the guard fires and \ + every row scores the same" + ); + // The smallest positive f32 subnormal: squaring it in f32 gives zero, in f64 + // gives 1.96e-90. + assert!( + query_norm(&[f32::from_bits(1)]) > 0.0, + "the worst case for underflow must still be non-zero" + ); + assert_eq!( + query_norm(&[0.0f32; 4]), + 0.0, + "a genuinely zero vector is the only input that may read as zero" + ); } } diff --git a/crates/storage-postgres/tests/vector_control_plane.rs b/crates/storage-postgres/tests/vector_control_plane.rs index d96b5ba0..ea999729 100644 --- a/crates/storage-postgres/tests/vector_control_plane.rs +++ b/crates/storage-postgres/tests/vector_control_plane.rs @@ -871,7 +871,7 @@ async fn update_table_creates_a_vector_index_and_backfills_what_is_already_there let holds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") .bind(&id) - .fetch_one(&s.catalog) + .fetch_one(s.engine.data_pool()) .await .expect("count the holds"); assert_eq!(holds, 0, "the hold must be released after the ACTIVE flip"); @@ -1385,12 +1385,12 @@ async fn describe_table_refuses_a_vector_index_with_a_corrupt_payload() { /// here belongs to the table under test. Deliberately does not read the catalog, /// because the case worth checking is the one where the catalog rows are already /// gone and an orphaned data table would be invisible. -async fn vector_data_tables(catalog: &PgPool) -> Vec { +async fn vector_data_tables(data: &PgPool) -> Vec { sqlx::query_scalar( "SELECT tablename FROM pg_tables WHERE schemaname = 'public' \ AND tablename LIKE '_ddb\\_vec\\_%' ORDER BY tablename", ) - .fetch_all(catalog) + .fetch_all(data) .await .expect("list the vector data tables") } @@ -1412,7 +1412,7 @@ async fn create_table_builds_the_vector_data_table_and_delete_table_sweeps_it() ) .await .expect("create a table with a vector index"); - let tables = vector_data_tables(&s.catalog).await; + let tables = vector_data_tables(s.engine.data_pool()).await; assert_eq!( tables.len(), 1, @@ -1452,10 +1452,10 @@ async fn create_table_builds_the_vector_data_table_and_delete_table_sweeps_it() .await .expect("delete the table"); - // Swept by prefix, because the catalog rows that named these tables cascade - // away with the table row before the sweep runs. + // The ids are read before the catalog row cascades away, so the sweep knows + // exactly which tables to drop rather than matching a name pattern. assert!( - vector_data_tables(&s.catalog).await.is_empty(), + vector_data_tables(s.engine.data_pool()).await.is_empty(), "DeleteTable must sweep the vector data tables" ); @@ -1486,7 +1486,7 @@ async fn update_table_delete_drops_the_index_data_table() { .await .expect("create a table with two vector indexes"); let id = table_id(&s.catalog, "t_dropone").await; - assert_eq!(vector_data_tables(&s.catalog).await.len(), 2); + assert_eq!(vector_data_tables(s.engine.data_pool()).await.len(), 2); s.engine .update_table( @@ -1501,7 +1501,7 @@ async fn update_table_delete_drops_the_index_data_table() { // One table gone, one left: an index delete must not take the survivor's // storage with it, which is the failure a prefix sweep would cause here. - let remaining = vector_data_tables(&s.catalog).await; + let remaining = vector_data_tables(s.engine.data_pool()).await; assert_eq!(remaining.len(), 1, "{remaining:?}"); let keep_id: String = sqlx::query_scalar("SELECT index_id FROM vector_indexes WHERE table_id = $1") @@ -2085,7 +2085,7 @@ async fn a_failed_update_table_gives_back_every_hold_it_took() { let holds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") .bind(&id) - .fetch_one(&s.catalog) + .fetch_one(s.engine.data_pool()) .await .expect("count the holds"); assert_eq!( @@ -2143,7 +2143,7 @@ async fn a_stale_heartbeat_is_rebuilt_at_runtime_and_a_fresh_one_is_left_alone() sqlx::query("INSERT INTO vector_index_holds (table_id, index_id) VALUES ($1, $2)") .bind(&id) .bind(&index_id) - .execute(&s.catalog) + .execute(s.engine.data_pool()) .await .expect("restore the hold the dead build held"); @@ -2170,7 +2170,7 @@ async fn a_stale_heartbeat_is_rebuilt_at_runtime_and_a_fresh_one_is_left_alone() let holds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") .bind(&id) - .fetch_one(&s.catalog) + .fetch_one(s.engine.data_pool()) .await .expect("count the holds"); assert_eq!(holds, 0, "the rebuild must release the hold it repaired"); @@ -2234,7 +2234,7 @@ async fn a_hold_with_no_building_index_is_swept_at_runtime_not_only_at_startup() VALUES ($1, 'no-such-build', NOW() - INTERVAL '1 hour')", ) .bind(&id) - .execute(&s.catalog) + .execute(s.engine.data_pool()) .await .expect("leave an orphan hold"); @@ -2251,7 +2251,7 @@ async fn a_hold_with_no_building_index_is_swept_at_runtime_not_only_at_startup() let holds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") .bind(&id) - .fetch_one(&s.catalog) + .fetch_one(s.engine.data_pool()) .await .expect("count the holds"); assert_eq!( @@ -2369,13 +2369,13 @@ async fn deleting_an_index_mid_backfill_leaves_nothing_orphaned() { .expect("count the catalog rows"); assert_eq!(rows, 0, "the catalog row must be gone"); assert!( - vector_data_tables(&s.catalog).await.is_empty(), + vector_data_tables(s.engine.data_pool()).await.is_empty(), "the data table must be dropped, not left orphaned" ); let holds: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1") .bind(&id) - .fetch_one(&s.catalog) + .fetch_one(s.engine.data_pool()) .await .expect("count the holds"); assert_eq!( @@ -2447,7 +2447,7 @@ async fn startup_rebuilds_a_half_built_index_and_frees_a_stale_hold() { sqlx::query("INSERT INTO vector_index_holds (table_id, index_id) VALUES ($1, $2)") .bind(&id) .bind(&index_id) - .execute(&s.catalog) + .execute(s.engine.data_pool()) .await .expect("restore the hold the dead build held"); @@ -2462,13 +2462,13 @@ async fn startup_rebuilds_a_half_built_index_and_frees_a_stale_hold() { "INSERT INTO vector_index_holds (table_id, index_id, created_at) \ VALUES ('ghost-old', 'ghost-old', NOW() - INTERVAL '1 hour')", ) - .execute(&s.catalog) + .execute(s.engine.data_pool()) .await .expect("add an aged orphan hold"); sqlx::query( "INSERT INTO vector_index_holds (table_id, index_id) VALUES ('ghost-new', 'ghost-new')", ) - .execute(&s.catalog) + .execute(s.engine.data_pool()) .await .expect("add a just-taken hold"); @@ -2490,7 +2490,7 @@ async fn startup_rebuilds_a_half_built_index_and_frees_a_stale_hold() { assert_eq!(status, "ACTIVE"); let remaining: Vec = sqlx::query_scalar("SELECT table_id FROM vector_index_holds ORDER BY table_id") - .fetch_all(&s.catalog) + .fetch_all(s.engine.data_pool()) .await .expect("list the holds"); assert_eq!( @@ -2702,3 +2702,79 @@ async fn losing_pgvector_after_startup_refuses_a_vector_index_rather_than_record s.cleanup().await; } + +/// Build ownership must survive as a session of its own, and must let go when the +/// owner is dropped. +/// +/// A build runs for as long as the scan takes, so where its session comes from is a +/// write-path question: the data pool serves every write, and a connection pinned +/// for the whole build is one fewer connection for the writes the build is +/// deliberately not blocking. +/// +/// The release half is the part a pooled connection cannot give. An advisory lock is +/// session-scoped, and returning a pooled connection does not end its session, so the +/// lock would stay held on an idle pooled connection after the owner is gone: the +/// index becomes unrecoverable by any peer until that connection is recycled. This +/// asserts release from a SEPARATE session, which is the only observer that can tell +/// a released lock from a re-entrant one. +#[tokio::test] +async fn build_ownership_uses_its_own_session_and_releases_on_drop() { + let test = "build_ownership_uses_its_own_session_and_releases_on_drop"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + // The namespace and key are the implementation's, restated here so this test + // observes the lock exactly as a peer front-end would. + const NAMESPACE: i32 = 0x0045_4442; + let probe_lock = |pool: PgPool| async move { + let taken: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock($1, hashtext($2))") + .bind(NAMESPACE) + .bind("vidx-owned") + .fetch_one(&pool) + .await + .expect("probe the lock"); + if taken { + sqlx::query("SELECT pg_advisory_unlock($1, hashtext($2))") + .bind(NAMESPACE) + .bind("vidx-owned") + .execute(&pool) + .await + .expect("give the probe's lock back"); + } + taken + }; + + // A peer's session, one connection so a probe cannot accidentally land on the + // owner's own session. + let peer = PgPoolOptions::new() + .max_connections(1) + .connect(&format!( + "{}/{}", + base_conn().expect("base connection"), + s.db_name + )) + .await + .expect("a peer connection"); + + let owner = extenddb_storage_postgres::build_ownership(s.engine.data_pool(), "vidx-owned") + .await + .expect("ownership must be available on an unowned index"); + assert!( + !probe_lock(peer.clone()).await, + "a peer must not be able to take a build another process owns" + ); + + drop(owner); + assert!( + probe_lock(peer.clone()).await, + "dropping the owner must end its session and release the lock, or the index \ + cannot be recovered by any peer until the connection is recycled" + ); + + peer.close().await; + s.cleanup().await; +} diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index cd266c53..7467182d 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -67,9 +67,25 @@ impl SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; if vector_count > 0 { - return Err(StorageError::Validation( - extenddb_core::types::VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST.to_owned(), - )); + // Two measured shapes, two strings. A plain switch reports the + // create-side rule; a switch that also carries VectorIndexUpdates + // reports its own message, measured 2026-08-19 on a switch combined + // with deleting the last vector index, which the service refuses + // even though the net state would carry none. + // + // Only those two shapes are measured. Which of the two fires for a + // switch combined with a vector index CREATE is unmapped, and that + // shape is refused by the rule below, so it cannot reach this choice. + let message = if input + .vector_index_updates + .as_ref() + .is_some_and(|u| !u.is_empty()) + { + extenddb_core::types::VECTOR_TABLE_REQUIRES_PAY_PER_REQUEST_MODE + } else { + extenddb_core::types::VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST + }; + return Err(StorageError::Validation(message.to_owned())); } } diff --git a/crates/storage-sqlite/src/vector_search.rs b/crates/storage-sqlite/src/vector_search.rs index 1c144967..a7529eee 100644 --- a/crates/storage-sqlite/src/vector_search.rs +++ b/crates/storage-sqlite/src/vector_search.rs @@ -48,53 +48,79 @@ fn decode_vector(bytes: &[u8], dimensions: usize) -> Result, StorageErr .collect()) } +/// The Euclidean norm of a vector, in double precision. +/// +/// Not the shared `vector_norm`, which returns an `f32` and is what the stored `nrm` +/// column holds: squaring an `f32` component in `f32` overflows above about 1.8e19 and +/// underflows below about 1e-22, both well inside the range of components validation +/// accepts, so the stored norm is unusable for deciding whether a vector is zero. +fn norm_f64(components: &[f32]) -> f64 { + components + .iter() + .map(|x| f64::from(*x) * f64::from(*x)) + .sum::() + .sqrt() +} + +/// The inner product of two vectors, in double precision. +fn dot_f64(a: &[f32], b: &[f32]) -> f64 { + a.iter() + .zip(b) + .map(|(x, y)| f64::from(*x) * f64::from(*y)) + .sum() +} + /// Score one candidate under the index's distance function. /// /// Cosine and Euclidean are distances, so smaller is more similar; dot product is /// a similarity, so larger is. The caller must not compare scores across /// functions, which is why the output reports which one was used. -fn score( - function: DistanceFunction, - query: &[f32], - query_norm: f32, - candidate: &[f32], - candidate_norm: f32, -) -> f64 { +/// +/// Every accumulation is `f64`, and that is a correctness requirement rather than +/// precision hygiene. In `f32` these overflow for components validation accepts, and +/// the result was a score that cannot be serialised at all: a Euclidean difference is +/// doubled before squaring, so `1e19` against `-1e19` gave `inf`; a dot product of +/// `2e19` gave `inf`; and cosine at `3.4e38` divided two overflowed values and gave +/// `NaN`. `serde_json` renders both as `null`, so a client received `"Score": null` on +/// a 200 response. Widened, the worst case in the whole domain is 4096 components at +/// `f32::MAX` squared, which is 4.7e80 and finite, so no clamping is needed here: the +/// values are simply correct. The PostgreSQL backend cannot do this, because the +/// arithmetic happens inside pgvector, so it bounds the result in SQL instead. +fn score(function: DistanceFunction, query: &[f32], candidate: &[f32]) -> f64 { match function { DistanceFunction::Cosine => { + let query_norm = norm_f64(query); + let candidate_norm = norm_f64(candidate); if query_norm == 0.0 || candidate_norm == 0.0 { // Undefined angle. Reported as maximally distant rather than as // an error, matching how a zero vector is treated elsewhere. + // + // Genuinely zero, not merely small: the norms above are computed in + // f64 for exactly this decision. With an f32 norm a vector of 1e-30 + // components read as zero, so every row scored 1.0 and the search + // silently returned a ranking it had not computed. return 1.0; } - let mut dot = 0.0f32; - for i in 0..query.len() { - dot += query[i] * candidate[i]; - } // Clamped because the quotient can exceed 1 by a float epsilon when the // vectors are identical, which made an exact self-match report a // NEGATIVE distance (-1.19e-07 was measured against a stored item's own // vector). Cosine distance has domain [0, 2], so a consumer that // clamps, or takes a square root of the score, sees a value the metric // cannot produce. The service returned +1.49e-08 for the same query. - let similarity = (dot / (query_norm * candidate_norm)).clamp(-1.0, 1.0); - f64::from(1.0 - similarity) - } - DistanceFunction::Euclidean => { - let mut sum = 0.0f32; - for i in 0..query.len() { - let d = query[i] - candidate[i]; - sum += d * d; - } - f64::from(sum.sqrt()) - } - DistanceFunction::DotProduct => { - let mut dot = 0.0f32; - for i in 0..query.len() { - dot += query[i] * candidate[i]; - } - f64::from(dot) + let similarity = + (dot_f64(query, candidate) / (query_norm * candidate_norm)).clamp(-1.0, 1.0); + 1.0 - similarity } + DistanceFunction::Euclidean => query + .iter() + .zip(candidate) + .map(|(x, y)| { + let d = f64::from(*x) - f64::from(*y); + d * d + }) + .sum::() + .sqrt(), + DistanceFunction::DotProduct => dot_f64(query, candidate), } } @@ -217,13 +243,11 @@ impl VectorSearchEngine for SqliteEngine { } let vec_table = vector_table_name(&table_id, &index_id); - let sql = format!("SELECT vec, nrm, item_data FROM {vec_table} WHERE part = ?"); - - let mut query_norm = 0.0f32; - for x in &query_vector { - query_norm += x * x; - } - let query_norm = query_norm.sqrt(); + // `nrm` is deliberately not selected. The stored norm is an f32 and cannot + // be trusted for either the zero test or the cosine denominator, so the + // scorer recomputes both sides in f64 from the vector it already decoded. + // The column stays for operator inspection and for the other backend. + let sql = format!("SELECT vec, item_data FROM {vec_table} WHERE part = ?"); let k = usize::try_from(top_k.max(0)).unwrap_or(0); let mut top = TopK::new(k, function); @@ -232,11 +256,11 @@ impl VectorSearchEngine for SqliteEngine { // not allocate proportionally to its size. This is the reason the // row-per-vector layout was chosen over a packed blob per partition. use futures::TryStreamExt; - let mut stream = sqlx::query_as::<_, (Vec, f64, String)>(&sql) + let mut stream = sqlx::query_as::<_, (Vec, String)>(&sql) .bind(&partition) .fetch(&self.pool); - while let Some((blob, norm, item_json)) = stream + while let Some((blob, item_json)) = stream .try_next() .await .map_err(|e| StorageError::Internal(e.to_string()))? @@ -256,15 +280,7 @@ impl VectorSearchEngine for SqliteEngine { continue; } - #[allow(clippy::cast_possible_truncation)] - let candidate_norm = norm as f32; - let candidate_score = score( - function, - &query_vector, - query_norm, - &candidate, - candidate_norm, - ); + let candidate_score = score(function, &query_vector, &candidate); top.offer(candidate_score, item, candidate); } @@ -303,8 +319,7 @@ mod tests { #[test] fn cosine_of_identical_vectors_is_zero() { let v = [1.0f32, 2.0, 3.0]; - let n = (14.0f32).sqrt(); - let s = score(DistanceFunction::Cosine, &v, n, &v, n); + let s = score(DistanceFunction::Cosine, &v, &v); assert!(s.abs() < 1e-6, "expected ~0.0, got {s}"); } @@ -335,11 +350,10 @@ mod tests { s = s.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); v.push(((s >> 33) as f32 / u32::MAX as f32) - 0.5); } - let norm = v.iter().map(|x| x * x).sum::().sqrt(); - if norm == 0.0 { + if norm_f64(&v) == 0.0 { continue; } - let d = score(DistanceFunction::Cosine, &v, norm, &v, norm); + let d = score(DistanceFunction::Cosine, &v, &v); assert!( d >= 0.0, "cosine distance left its domain for a self-match: {d} (dim {dim})" @@ -348,24 +362,73 @@ mod tests { } } - /// The clamp must hold even when the norms handed in understate the true ones, - /// which is the mechanism that pushed the quotient above 1 in the first place: - /// `norm * norm` can be strictly less than `sum(x*x)` in f32. + /// Every metric returns a FINITE score for the extremes of the input domain. + /// + /// This replaces a test that forced the cosine clamp by handing in a norm 1% below + /// the true one. That mechanism is gone: the scorer computes both norms itself now, + /// so a caller cannot understate them, and the clamp is exercised by the self-match + /// property above instead. + /// + /// These are the magnitudes at which single-precision accumulation broke, measured + /// before the widening: `1e19` against `-1e19` gave `inf` under Euclidean, because + /// the difference is doubled before squaring; `2e19` gave `inf` under dot product; + /// and `3.4e38` gave `NaN` under cosine, dividing two overflowed values. Each + /// reached a client as `"Score": null` on a 200 response, because that is how + /// `serde_json` renders a non-finite double. + /// + /// The JSON rendering is asserted rather than only `is_finite`, because null is what + /// the client actually saw and it is what a regression would reintroduce. #[test] - fn cosine_clamps_when_the_supplied_norms_understate() { - let v = [0.6f32, 0.8]; - // Deliberately 1% low, far beyond any real rounding error, so the - // unclamped expression would return roughly -0.02. - let understated = 0.99f32; - let d = score(DistanceFunction::Cosine, &v, understated, &v, understated); - assert!(d >= 0.0, "expected the clamp to hold, got {d}"); + fn every_metric_scores_the_extremes_of_the_domain_as_a_finite_number() { + let cases = [ + ( + DistanceFunction::Euclidean, + [1e19f32, 0.0, 0.0, 0.0], + [-1e19f32, 0.0, 0.0, 0.0], + ), + ( + DistanceFunction::DotProduct, + [2e19f32, 0.0, 0.0, 0.0], + [2e19f32, 0.0, 0.0, 0.0], + ), + ( + DistanceFunction::Cosine, + [3.4e38f32, 1e38, 0.0, 0.0], + [3.4e38f32, 0.0, 0.0, 0.0], + ), + ]; + for (function, query, candidate) in cases { + let s = score(function, &query, &candidate); + assert!( + s.is_finite(), + "{function:?} produced a non-finite score: {s}" + ); + assert_ne!( + serde_json::to_string(&s).expect("serialise the score"), + "null", + "{function:?} produced a score that serialises as null: {s}" + ); + } + + // The other end: components small enough that their f32 squares underflow must + // NOT read as a zero vector, or the guard fires and every row scores 1.0. + let tiny = [1e-30f32, 0.0, 0.0, 0.0]; + assert!( + score(DistanceFunction::Cosine, &tiny, &[1.0, 0.0, 0.0, 0.0]) < 1e-9, + "a tiny vector parallel to the candidate is a near-zero distance, not the \ + zero-vector answer" + ); + assert!( + (score(DistanceFunction::Cosine, &tiny, &[0.0, 1.0, 0.0, 0.0]) - 1.0).abs() < 1e-9, + "and orthogonal to it is exactly 1.0, by the metric rather than by the guard" + ); } #[test] fn cosine_of_opposite_vectors_is_two() { let a = [1.0f32, 0.0]; let b = [-1.0f32, 0.0]; - let s = score(DistanceFunction::Cosine, &a, 1.0, &b, 1.0); + let s = score(DistanceFunction::Cosine, &a, &b); assert!((s - 2.0).abs() < 1e-6, "expected ~2.0, got {s}"); } @@ -373,14 +436,14 @@ mod tests { fn a_zero_vector_is_maximally_distant_rather_than_an_error() { let a = [1.0f32, 0.0]; let z = [0.0f32, 0.0]; - assert!((score(DistanceFunction::Cosine, &a, 1.0, &z, 0.0) - 1.0).abs() < 1e-6); + assert!((score(DistanceFunction::Cosine, &a, &z) - 1.0).abs() < 1e-6); } #[test] fn euclidean_is_the_straight_line_distance() { let a = [0.0f32, 0.0]; let b = [3.0f32, 4.0]; - let s = score(DistanceFunction::Euclidean, &a, 0.0, &b, 5.0); + let s = score(DistanceFunction::Euclidean, &a, &b); assert!((s - 5.0).abs() < 1e-6, "expected 5.0, got {s}"); } @@ -388,7 +451,7 @@ mod tests { fn dot_product_is_reported_raw_and_can_be_negative() { let a = [1.0f32, 0.0]; let b = [-2.0f32, 0.0]; - let s = score(DistanceFunction::DotProduct, &a, 1.0, &b, 2.0); + let s = score(DistanceFunction::DotProduct, &a, &b); assert!((s + 2.0).abs() < 1e-6, "expected -2.0, got {s}"); } diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index f0c07d30..44468861 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -65,6 +65,7 @@ adaptation when switching between ExtendDB and the real service. |------|----------|------| | GSI update propagation | Eventually consistent (milliseconds to seconds) | Per-GSI propagation delay. System default: `index_propagation_delay_ms` setting (default 10ms). Each GSI can override with its own `propagation_delay_ms` (stored in catalog). A value of 0 means synchronous (future sync GSI feature). | | Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance synchronously in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. | +| `SearchVectors` score at extreme magnitudes | Returns a number for any vector of finite components | Both backends return a number, and the two differ in the extreme. The SQLite backend computes distances itself and does so in double precision, so its score is the true value across the whole domain. The PostgreSQL backend cannot: pgvector accumulates in single precision, so magnitudes far below `f32::MAX` overflow inside the extension (Euclidean above about 9.2e18, dot product above about 1.8e19, cosine at both ends, above about 1.8e19 and below about 3.7e-23). Since a non-finite score cannot be serialised as JSON at all, that backend bounds the result in SQL: `1e308` for an overflowed distance, `-1e308` for an overflowed negated inner product, and 1.0 for a cosine NaN. Those bounds are not measured service answers. Ranking is unaffected on either backend, because each bound sits at the end its metric overflows towards, so the farthest row stays farthest and the most similar stays most similar; two rows that both overflow tie, and the tie breaks on the base key. | | Multi-part base table keys | Not supported | Preview extension (opt-in via `enable_multipart_keys` setting). Standard single/composite keys work identically. | ## Capacity and Throttling diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 1899612b..099a2559 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1225,6 +1225,250 @@ async fn wait_past_allocation_phase(table: &str, index: &str) { panic!("vector index {index} on {table} never left the resource-allocation phase"); } +/// Switching a table that holds a vector index to PROVISIONED is refused, and the +/// two measured shapes of that refusal carry DIFFERENT messages. +/// +/// A plain switch reports the create-side rule. A switch that also carries +/// `VectorIndexUpdates` reports its own message, measured on a switch combined with +/// deleting the last vector index, which the service refuses even though the net +/// state would hold none. Both are asserted on the whole string here, because a +/// backend that emitted one message for both shapes would pass a substring check +/// and still tell a caller the wrong rule. It is also the pair that had drifted +/// between the two backends, which nothing at the wire could see until now. +#[tokio::test] +async fn switching_a_vector_table_to_provisioned_is_refused_with_the_measured_message() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_switch"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 2, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name).await; + + // ProvisionedThroughput is supplied throughout, so a refusal cannot be + // attributed to a missing-throughput error instead of the rule under test. + let switch = format!( + r#""TableName": "{name}", "BillingMode": "PROVISIONED", "ProvisionedThroughput": {{"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}}"# + ); + + let (status, text) = call("UpdateTable", &format!("{{{switch}}}")).await; + assert_eq!(status, 400, "a plain switch must be refused: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + assert_eq!( + json.pointer("/message").and_then(|v| v.as_str()), + Some( + "One or more parameter values were invalid: Vector indexes are only supported for \ + PAY_PER_REQUEST tables" + ), + "wrong message for a plain switch: {text}" + ); + + let (status, text) = call( + "UpdateTable", + &format!(r#"{{{switch}, "VectorIndexUpdates": [{{"Delete": {{"IndexName": "vidx"}}}}]}}"#), + ) + .await; + assert_eq!( + status, 400, + "a switch carrying vector index updates must be refused too: {text}" + ); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + assert_eq!( + json.pointer("/message").and_then(|v| v.as_str()), + Some( + "One or more parameter values were invalid: Tables with vector indexes must be in \ + PAY_PER_REQUEST mode" + ), + "wrong message for a switch carrying vector index updates: {text}" + ); + + // The refusals changed nothing: the index is still there and the table is still + // on demand. + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + assert_eq!(status, 200, "DescribeTable failed: {text}"); + assert!( + text.contains("\"IndexName\":\"vidx\"") && text.contains("PAY_PER_REQUEST"), + "a refused switch must leave the table as it was: {text}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// The delayed propagation path for vector rows, end to end, on a default +/// deployment's shape. +/// +/// Vector maintenance used to ignore `index_propagation_delay_ms` unless the table +/// also had a secondary index, so a vector-only table applied every write inline and +/// no suite ever exercised the queued path for a vector row: not the delay, not the +/// jitter, not the monotonic clamp on `ready_at`. Fixing that made the queued path +/// the ONLY path on a vector-only table, because the default delay is non-zero, so +/// the queue is now what every such deployment relies on. +/// +/// Convergence is the assertion rather than absence-then-presence: a delay is a lower +/// bound, so checking that the hit is missing immediately after the write is a race +/// that would fail for timing reasons rather than for the behaviour under test. +#[tokio::test] +async fn a_vector_only_table_converges_through_the_delayed_queue() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_delayed"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 2, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + call("CreateTable", &body).await; + wait_for_active(&name).await; + + // Well above the default, so the row really does go through the queue with a + // future `ready_at` rather than being applied while the write is still in flight. + set_vector_setting("index_propagation_delay_ms", 200).await; + put_vector(&name, "delayed", None, &[1.0, 0.0]).await; + search_until_pks(&name, &[1.0, 0.0], 5, None, &["delayed"]).await; + + // Back to the default, because this setting is deployment-wide and every later + // test in this process would otherwise inherit it. + set_vector_setting("index_propagation_delay_ms", 10).await; + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// A query vector small enough to underflow single-precision arithmetic must still +/// produce a NUMBER for every hit's score. +/// +/// The search engine computes its own zero test in double precision, so a vector with +/// components around 1e-30 is correctly treated as non-zero. pgvector's cosine +/// operator, however, accumulates both norms in single precision, so on the same input +/// its own norm underflows to zero and the distance comes back NaN for any stored +/// vector that is not parallel to the query. `serde_json` renders a non-finite double +/// as `null` rather than failing, so the response is a 200 carrying +/// `"Score": null`, which is off-contract for a typed client deserialising a double. +/// +/// Both stored vectors are here for a reason: the parallel one gives an infinity that +/// the operator clamps, which is already correct, so only the orthogonal one exposes +/// the NaN. Asserting on both is what makes this a test of the whole result set rather +/// than of the lucky row. +/// +/// The components are client input, narrowed to f32 by validation, so any embedding +/// with tiny activations reaches this. It is not a hostile input and the service +/// accepts it. +#[tokio::test] +async fn a_tiny_query_vector_still_scores_every_hit_as_a_number() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_tiny_query"); + create_vector_table(&name, 4, "COSINE", false).await; + + put_vector(&name, "parallel", None, &[1.0, 0.0, 0.0, 0.0]).await; + put_vector(&name, "orthogonal", None, &[0.0, 1.0, 0.0, 0.0]).await; + search_until_count(&name, &[1.0, 0.0, 0.0, 0.0], 10, None, 2).await; + + let response = search(&name, &[1e-30, 0.0, 0.0, 0.0], 10, None).await; + let hits = response + .get("SearchResults") + .and_then(|v| v.as_array()) + .unwrap_or_else(|| panic!("no results array in: {response}")) + .clone(); + assert_eq!(hits.len(), 2, "both rows must come back: {response}"); + for hit in &hits { + let score = hit.get("Score").unwrap_or_else(|| panic!("no score: {hit}")); + assert!( + score.is_number(), + "every score must be a number, not null: {hit}" + ); + let score = score.as_f64().expect("a numeric score"); + assert!( + score.is_finite(), + "a non-finite score reaches the client as null: {hit}" + ); + } + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Extreme but valid magnitudes must still score as NUMBERS, under every metric. +/// +/// pgvector accumulates in single precision, so a query at these magnitudes overflows +/// its accumulators even though every component is a finite f32 that validation +/// accepts, and every metric has its own threshold and its own failure value: the +/// Euclidean difference is doubled before squaring, so it overflows first (around +/// 9.2e18) and gives Infinity; dot product gives negative Infinity above about 1.8e19, +/// which the score contract negates into positive Infinity; and cosine divides two +/// overflowed values and gives NaN. Measured on pgvector 0.8.0. +/// +/// `serde_json` renders both NaN and Infinity as `null`, so all three reach a client as +/// `"Score": null` on a 200 response, which no typed SDK can deserialise into a double. +/// +/// The ordering is already correct in the overflowed cases, which is why the repair +/// belongs in SQL next to the operator rather than in the Rust that reports the score: +/// substituting a value after the database has ordered the rows would let a hit +/// reported as nearer appear after one reported as farther. +#[tokio::test] +async fn an_extreme_magnitude_query_scores_as_a_number_under_every_metric() { + if skip_unless_supported().await { + return; + } + + // Metric, stored vector, query vector: each pair is a measured overflow for that + // metric rather than a round number. + let cases: [(&str, [f32; 4], [f32; 4]); 3] = [ + ("EUCLIDEAN", [1e19, 0.0, 0.0, 0.0], [-1e19, 0.0, 0.0, 0.0]), + ("DOT_PRODUCT", [2e19, 0.0, 0.0, 0.0], [2e19, 0.0, 0.0, 0.0]), + ("COSINE", [3.4e38, 1e38, 0.0, 0.0], [3.4e38, 0.0, 0.0, 0.0]), + ]; + + for (metric, stored, query) in cases { + let name = table_name(&format!("pos_huge_{}", metric.to_lowercase())); + create_vector_table(&name, 4, metric, false).await; + put_vector(&name, "extreme", None, &stored).await; + search_until_count(&name, &stored, 10, None, 1).await; + + let response = search(&name, &query, 10, None).await; + let hit = response + .pointer("/SearchResults/0") + .unwrap_or_else(|| panic!("no hit under {metric}: {response}")); + let score = hit + .get("Score") + .unwrap_or_else(|| panic!("no score under {metric}: {hit}")); + assert!( + score.is_number(), + "{metric}: an overflowed score must not reach the client as null: {hit}" + ); + assert!( + score.as_f64().expect("a numeric score").is_finite(), + "{metric}: the score must be finite, or it serialises as null: {hit}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + } +} + /// The same collapse on the other path a client can reach: an index added by /// UpdateTable. /// From f4a30afbd53832e75e5069d48233fab0ce632fc8 Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Mon, 24 Aug 2026 20:15:41 +0000 Subject: [PATCH 07/13] docs: accuracy pass over the vector documentation Corrects comments and manual sections that described code as it used to be: the out-of-pool session count, the underflow degradation, the backend split hidden by two vague rows, and a backend-author section that was wrong about a shipped backend. Adds a test asserting the ranking a tiny query vector must produce. --- .agents/skills/extenddb/SKILL.md | 2 +- .../troubleshooting/01-symptom-index.md | 10 +- .../05-feature-gate-symptoms.md | 22 ++++ AGENTS.md | 2 +- crates/storage-postgres/src/update_table.rs | 18 +-- crates/storage-postgres/src/vector_search.rs | 11 ++ crates/storage-sqlite/src/data/ddl.rs | 15 ++- crates/storage/src/server_components.rs | 6 +- crates/storage/src/vector_lifecycle/meta.rs | 13 +- docs/adr/0003-catalog-migration-mechanism.md | 2 +- docs/adr/0004-vector-search-exact-scan.md | 2 +- docs/adr/0006-pgvector-storage-and-scoring.md | 112 +++++++++++++++++ docs/adr/README.md | 6 +- docs/design/04-component-storage.md | 115 ++++++++++++++++-- docs/differences-from-dynamodb.md | 8 +- docs/getting-started.md | 13 +- docs/manuals/01-architecture-guide.md | 6 +- docs/manuals/04-quickstart-setup-guide.md | 6 +- docs/manuals/05-admin-guide.md | 2 + docs/manuals/08-install-linux.md | 2 +- docs/manuals/09-install-macos.md | 2 +- docs/manuals/11-deployment-guide.md | 2 +- docs/technical-debt.md | 3 +- docs/troubleshooting.md | 40 ++++++ extenddb.sample.toml | 20 ++- tests/rust/src/vector_index_search.rs | 32 +++++ 26 files changed, 415 insertions(+), 57 deletions(-) create mode 100644 docs/adr/0006-pgvector-storage-and-scoring.md diff --git a/.agents/skills/extenddb/SKILL.md b/.agents/skills/extenddb/SKILL.md index 366044e2..1626ec3c 100644 --- a/.agents/skills/extenddb/SKILL.md +++ b/.agents/skills/extenddb/SKILL.md @@ -143,7 +143,7 @@ This skill presents commands but does not execute state-changing operations (`ex | `references/samples/02-sample-app.md` | Nine-stage lifecycle walkthrough | | `references/samples/03-stream-consumer.md` | Streams demo, two-client pattern | | **Troubleshooting** | | -| `references/troubleshooting/01-symptom-index.md` | 16-symptom keyword-to-category lookup | +| `references/troubleshooting/01-symptom-index.md` | 17-symptom keyword-to-category lookup | | `references/troubleshooting/02-postgres-symptoms.md` | Connection refused, password auth, migration | | `references/troubleshooting/03-catalog-symptoms.md` | Version mismatch, not initialized, already exists | | `references/troubleshooting/04-startup-symptoms.md` | Address in use, TLS, permissions, daemonize | diff --git a/.agents/skills/extenddb/references/troubleshooting/01-symptom-index.md b/.agents/skills/extenddb/references/troubleshooting/01-symptom-index.md index 38b59d07..fd8c4d67 100644 --- a/.agents/skills/extenddb/references/troubleshooting/01-symptom-index.md +++ b/.agents/skills/extenddb/references/troubleshooting/01-symptom-index.md @@ -2,7 +2,7 @@ ## 1. Purpose -This index maps each of the 16 known extenddb symptoms to the category file that holds the verbatim Cause and Fix from `docs/troubleshooting.md`. To use it, grep this file for the user's error text, follow the link to the category file, and present the entry to the user. The skill never executes a remediation command on the user's behalf. Requirement 14.4 applies to every entry. +This index maps each of the 17 known extenddb symptoms to the category file that holds the verbatim Cause and Fix from `docs/troubleshooting.md`. To use it, grep this file for the user's error text, follow the link to the category file, and present the entry to the user. The skill never executes a remediation command on the user's behalf. Requirement 14.4 applies to every entry. ## 2. Symptom table @@ -24,6 +24,7 @@ This index maps each of the 16 known extenddb symptoms to the category file that | 14 | UnrecognizedClientException | `06-auth-symptoms.md#unrecognizedclientexception` | `UnrecognizedClientException: The security token included in the request is invalid` | | 15 | AccessDeniedException | `06-auth-symptoms.md#accessdeniedexception` | `AccessDeniedException: User: is not authorized to perform: ` | | 16 | Connection pool exhausted / HTTP 500 under load | `07-runtime-symptoms.md#connection-pool-exhausted` | `HTTP 500 on all requests under heavy load` | +| 17 | Vector indexes are not supported / SearchVectors is not supported | `05-feature-gate-symptoms.md#vector-unsupported` | `Vector indexes are not supported by this storage backend` | ## 3. Per-entry summaries @@ -142,6 +143,13 @@ The cause and fix summaries below are paraphrased for quick scanning. The catego **Fix summary:** Raise `pool_size` under `[storage.postgres]` in `extenddb.toml` and investigate long-running queries in `pg_stat_activity`. **Full entry:** `references/07-runtime-symptoms.md#connection-pool-exhausted` +### Vector indexes are not supported + +**Error text:** `Vector indexes are not supported by this storage backend` (or `SearchVectors is not supported by this storage backend`) +**Cause summary:** Vector indexes need the pgvector extension on the PostgreSQL data database, and the server probes for it once at startup and caches the answer. +**Fix summary:** Install the extension for the server version, run `CREATE EXTENSION vector;` on the data database, then restart ExtendDB, because the probe result is cached at startup. +**Full entry:** `references/05-feature-gate-symptoms.md#vector-unsupported` + ## 4. Unknown-symptom fallback If the user's error text does not match any entry above, ask the user to pull the last 100 lines of the extenddb log, then retry the lookup on the new text. diff --git a/.agents/skills/extenddb/references/troubleshooting/05-feature-gate-symptoms.md b/.agents/skills/extenddb/references/troubleshooting/05-feature-gate-symptoms.md index 1823f3ed..3015d9cf 100644 --- a/.agents/skills/extenddb/references/troubleshooting/05-feature-gate-symptoms.md +++ b/.agents/skills/extenddb/references/troubleshooting/05-feature-gate-symptoms.md @@ -39,3 +39,25 @@ paths = ["/path/to/exports"] ``` **Source:** `docs/troubleshooting.md`, section "`Export is disabled. Configure [export] paths in extenddb.toml to enable.`", last synced 2026-05-12. + +### Vector indexes are not supported by this storage backend + + + +**Error text:** +``` +Vector indexes are not supported by this storage backend +SearchVectors is not supported by this storage backend +``` + +**Cause:** Vector indexes need the pgvector extension on the PostgreSQL **data** +database. Support is a property of the server, not of the ExtendDB build, and the +server probes for the extension once at startup and caches the answer. + +**Fix:** Install the extension for the server version (for example +`postgresql-16-pgvector`), run `CREATE EXTENSION vector;` on the data database, then +**restart ExtendDB**, because the probe result is cached at startup. The startup log +line `pgvector ... detected` or `pgvector not installed ...` says which answer the +running server is serving. + +**Source:** `docs/troubleshooting.md`, section "`Vector indexes are not supported by this storage backend`", last synced 2026-08-20. diff --git a/AGENTS.md b/AGENTS.md index be249af5..b9db1b15 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -430,7 +430,7 @@ walkthroughs, and troubleshooting. It dispatches to domain-specific reference fi ├── postgres/ PostgreSQL readiness and installation ├── first-request/ AWS CLI/SDK configuration, first CRUD ├── samples/ sample_app.py and stream_consumer.py - └── troubleshooting/ Symptom-to-fix lookup (16 indexed errors) + └── troubleshooting/ Symptom-to-fix lookup (17 indexed errors) ``` Activate when the user asks about installing, configuring, running, or debugging ExtendDB. diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index 647eca1e..413e493d 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -535,16 +535,16 @@ impl PostgresEngine { )> = Vec::new(); // Vector index create/delete. // - // Delete is implemented; Create is refused. Creating an index means - // building and maintaining its data table, which this backend cannot yet - // do, and a catalog row with no storage behind it would be an index that - // reports ACTIVE and answers nothing. Refusing is the same fail-closed - // posture the backend takes for every other vector operation. + // Both are implemented and both are reachable over the wire: this backend + // declares vector search capability whenever pgvector is present, so the + // engine's gate passes and a client can create or delete a vector index + // here. Create records the catalog row in this transaction and starts the + // build after the commit, which is why the created ids are collected + // rather than acted on inline. // - // Neither branch is reachable over the wire yet: the engine refuses - // vector index updates while this backend declares no vector search - // capability, so these paths are exercised below the wire. They are - // implemented now because the catalog state they act on is created here. + // A catalog row with no storage behind it would be an index that reports + // ACTIVE and answers nothing, so the create path's failure story is to + // leave the index CREATING for recovery rather than to publish it. // Wrapped so a failure anywhere in this block gives the holds back. // // `take_hold` writes to the data database, a different database from the diff --git a/crates/storage-postgres/src/vector_search.rs b/crates/storage-postgres/src/vector_search.rs index 31c5ebdc..b13d10b8 100644 --- a/crates/storage-postgres/src/vector_search.rs +++ b/crates/storage-postgres/src/vector_search.rs @@ -85,6 +85,17 @@ fn score_expression(function: DistanceFunction, norm_param: usize) -> String { // the measured answer for a zero vector on either side, and what SQLite // returns for the same input. // + // Worth knowing what the underflow end actually degrades to, because it is + // narrower than "returns 1.0". Measured through this expression with a 1e-30 + // query vector at five angles: parallel 0, 45 degrees 0, orthogonal 1, + // 135 degrees 2, antiparallel 2. Only the exactly-orthogonal case reaches the + // substitute, because only there is the inner product also zero and the + // quotient 0/0; the other four give pgvector an infinity that it clamps to + // plus or minus 1 before we see it. So the distance collapses into three + // values by the SIGN of the inner product: ranking still separates + // nearer-than-orthogonal from farther, and loses all resolution inside each + // half. + // // The wrapper also makes the two sides safe independently. The stored side is // masked today only because `nrm` comes from the shared `vector_norm`, which // accumulates in f32, so a tiny stored vector reads as zero and takes the diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs index 0041972a..4052c973 100644 --- a/crates/storage-sqlite/src/data/ddl.rs +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -102,8 +102,19 @@ impl SqliteEngine { /// /// `part` is the search-schema HASH value when one is declared, and a single /// constant otherwise, so an unscoped index is one partition rather than a - /// separate code path. `nrm` is the vector's precomputed L2 norm, so cosine - /// costs one dot product at query time instead of two passes. + /// separate code path. + /// + /// `nrm` holds the vector's precomputed L2 norm and **this backend's search path + /// no longer reads it**. It is an `f32` from the shared norm helper, which + /// overflows and underflows inside the range of components validation accepts, so + /// the scorer recomputes both norms in `f64` from the vector it has already + /// decoded. The column is still written, and kept rather than dropped because + /// removing it would need a data migration for no benefit. + /// + /// The same column name is load-bearing on the PostgreSQL backend, which is the + /// asymmetry to know about: its zero test runs in SQL, where the stored vector's + /// norm cannot be recomputed, so it reads `nrm` and its scoring expression + /// compensates for the `f32` range separately. /// /// # Safety (SQL injection) /// `index_id` is a server-generated UUID and column names are constants, so diff --git a/crates/storage/src/server_components.rs b/crates/storage/src/server_components.rs index 86f1156f..0755b4e6 100644 --- a/crates/storage/src/server_components.rs +++ b/crates/storage/src/server_components.rs @@ -4,8 +4,10 @@ //! Backend factory infrastructure for creating server components. //! //! This module provides the factory pattern for creating storage backends. -//! Backends register themselves via the inventory crate, allowing `cmd_serve` -//! to remain backend-agnostic. +//! A binary serves exactly one backend, installed from its thin `main` via +//! [`crate::set_backend`]; this factory is reached through the installed +//! [`crate::Backend`] value, which is what keeps `cmd_serve` backend-agnostic +//! without a registry to look anything up in. use std::future::Future; use std::pin::Pin; diff --git a/crates/storage/src/vector_lifecycle/meta.rs b/crates/storage/src/vector_lifecycle/meta.rs index 5d7fe093..5cdfa98c 100644 --- a/crates/storage/src/vector_lifecycle/meta.rs +++ b/crates/storage/src/vector_lifecycle/meta.rs @@ -50,11 +50,14 @@ pub struct VectorIndexMeta { /// serialized into the pending queue row's context column. /// /// `table_id` is carried here even though the queue row has a `table_id` column of -/// its own, because a vector data table is named from the table id *and* the index -/// id. Reading it from the context preserves the invariant that the context alone -/// is sufficient, rather than splitting one apply's inputs across a column and a -/// JSON blob. Both are written from the same variable in the same statement, so -/// they cannot disagree. +/// its own, because a backend may need it to name the index's data table: SQLite +/// names one from the table id *and* the index id, while PostgreSQL names one from +/// the index id alone, since the two ids together exceed its 63-byte identifier limit +/// and were being silently truncated into collisions. So this field is load-bearing on +/// one backend and unread on the other, and it stays in the context regardless: the +/// invariant worth keeping is that the context alone is sufficient to apply the row, +/// rather than splitting one apply's inputs across a column and a JSON blob. Both are +/// written from the same variable in the same statement, so they cannot disagree. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct VectorApplyContext { pub base_key_schema: Vec, diff --git a/docs/adr/0003-catalog-migration-mechanism.md b/docs/adr/0003-catalog-migration-mechanism.md index e27102db..8136635b 100644 --- a/docs/adr/0003-catalog-migration-mechanism.md +++ b/docs/adr/0003-catalog-migration-mechanism.md @@ -1,6 +1,6 @@ # ADR-0003: Adopt sqlx::migrate for PostgreSQL catalog and data schema migrations -- Status: Proposed +- Status: Accepted - Date: 2026-06-17 - Deciders: ExtendDB CODEOWNERS diff --git a/docs/adr/0004-vector-search-exact-scan.md b/docs/adr/0004-vector-search-exact-scan.md index 63e33e77..cda78693 100644 --- a/docs/adr/0004-vector-search-exact-scan.md +++ b/docs/adr/0004-vector-search-exact-scan.md @@ -1,6 +1,6 @@ # ADR-0004: Vector search is an exact scan over one row per vector -- Status: Proposed +- Status: Accepted - Date: 2026-08-06 - Deciders: @LeeroyHannigan diff --git a/docs/adr/0006-pgvector-storage-and-scoring.md b/docs/adr/0006-pgvector-storage-and-scoring.md new file mode 100644 index 00000000..e0e13e60 --- /dev/null +++ b/docs/adr/0006-pgvector-storage-and-scoring.md @@ -0,0 +1,112 @@ +# ADR-0006: Vector storage on PostgreSQL uses pgvector's type, and the engine decides what it cannot compute + +- Status: Accepted +- Date: 2026-08-20 +- Deciders: @yesyayen + +## Context + +The PostgreSQL backend had to serve the vector surface the engine already +defines: a `vector(N)` payload per indexed item, a `SearchVectors` operation with +a top-k ordered by one of three distance functions, and the build lifecycle +shared with the SQLite backend. The decisions below are the ones a reader will +otherwise have to reconstruct from the code, and three of them were reversed or +narrowed by measurement during the work. + +The measurements referenced here were taken against Amazon DynamoDB (August +2026) and against pgvector 0.8.0 on PostgreSQL 15.18. + +## Decision + +**1. Embeddings are stored in pgvector's `vector(N)` column, not as `BYTEA`.** + +The alternative was a byte-packed blob decoded in Rust, which is what the SQLite +backend does because SQLite has no vector type. On PostgreSQL that would put the +distance computation in the process rather than in the database, so every +candidate row would cross the wire for every search, and the index types pgvector +provides could never be used. The cost of the choice is that the arithmetic is no +longer ours (see decision 4). + +**2. Vector support is detected at runtime and fails closed.** + +Whether an ExtendDB build can serve vector indexes is a property of the +PostgreSQL server it is pointed at, not of the binary. The engine probes for the +extension once at startup and caches the answer, and a server without it refuses +every vector operation with a message naming the extension rather than failing +somewhere inside a query. The consequence, which is documented in the admin +guide: installing pgvector under a running server needs an ExtendDB restart, +because the probe is not re-run. + +Failing closed rather than degrading is the same reasoning as the restore +refusal. A silent downgrade produces a table that looks like it has an index and +answers every search with nothing. + +**3. Search is an exact scan first; the approximate index is a follow-up.** + +ADR-0004 decided this for the SQLite backend and it holds here for a different +reason: correctness of the scan is a precondition for measuring recall against +it. An HNSW index changes which rows are considered, so landing it before the +exact path is verified would make a recall regression indistinguishable from a +scan defect. The partition predicate and the inline filters are evaluated in SQL +so that `LIMIT` applies after filtering rather than before. + +**4. A score that PostgreSQL cannot compute is bounded in SQL, not in Rust.** + +pgvector accumulates distances in single precision, so for vectors of ordinary +finite `f32` components the operators return values that cannot be serialised: +Euclidean overflows to infinity above about 9.2e18 (its difference is doubled +before squaring, so it goes first), dot product returns negative infinity above +about 1.8e19, and cosine returns NaN at both ends. Every one of those reaches a client as `"Score": null` +on a 200 response, because that is how a non-finite double serialises. + +The scoring expression therefore bounds each metric at the end its own +accumulator overflows towards, and substitutes the measured zero-vector answer +for a cosine NaN. + +The location is the decision, and it is decided by the cosine case alone. Being +specific matters, because the uniform version of this argument is false for two of +the three metrics. + +The two magnitude bounds could equally run in Rust. `LEAST(x, 1e308)` is monotone +non-decreasing, and every finite value the operator can return is at most about +3.4e38, far below the bound, so the top-k set, its order and the reported values +are identical wherever the clamp is applied. The same holds for the floor on the +negated inner product, including after the score contract negates it. They are in +SQL for consistency with the third case, not because Rust would break them. + +Cosine is different because its repair is a substitution rather than a clamp. +PostgreSQL sorts NaN as greater than every other value, so a NaN row sorts last +and may be cut by `LIMIT`. Reporting that row as 1.0 in Rust would place a value +of 1.0 after values of 2.0, and would keep or drop the wrong rows at the cut, +because both the order and the truncation were already decided by the unrepaired +value. So the substitution has to happen before the cut, which means in the +expression. + +## Consequences + +The bounds are not measured service answers, and the reported distance at the +underflow end is not the true one: with a query vector whose `f32` squares +underflow, pgvector's own norms collapse and the reported cosine distance takes +one of three values following the sign of the inner product. Ranking still +separates nearer-than-orthogonal from farther and loses resolution within each +half. Both facts are recorded in `docs/differences-from-dynamodb.md` rather than +left for a user to discover. + +The SQLite backend does not need any of this. It owns its arithmetic, computes in +`f64`, and reports the true value across the whole domain, which is why the +differences row is scoped to PostgreSQL. Where the two backends can differ in a +reported number, the doc says so. + +Removing the difference would mean computing distances outside pgvector's +operators, which costs a per-row unpack of every candidate and gives up any index +on the column. That trade is worse than the documented asymmetry for an input +class no real embedding produces. + +## License + +Copyright 2026 ExtendDB contributors. Licensed under the Apache License, Version 2.0. +See [LICENSE](../../LICENSE) for the full text. + +This software is provided "as is" without warranty of any kind. ExtendDB is not +affiliated with, endorsed by, or sponsored by Amazon Web Services. "DynamoDB" is +a trademark of Amazon.com, Inc. diff --git a/docs/adr/README.md b/docs/adr/README.md index 11afd8d3..568ce0f2 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -32,5 +32,7 @@ decision, write a new ADR. |---|-------|--------| | [0001](0001-documentation-format.md) | Documentation format — Markdown over LaTeX | Accepted | | [0002](0002-sql-injection-defense.md) | SQL injection defense | Accepted | -| [0003](0003-catalog-migration-mechanism.md) | Adopt sqlx::migrate for PostgreSQL catalog and data schema migrations | Proposed | -| [0004](0004-vector-search-exact-scan.md) | Vector search is an exact scan over one row per vector | Proposed | +| [0003](0003-catalog-migration-mechanism.md) | Adopt sqlx::migrate for PostgreSQL catalog and data schema migrations | Accepted | +| [0004](0004-vector-search-exact-scan.md) | Vector search is an exact scan over one row per vector | Accepted | +| [0005](0005-index-build-lifecycle-ownership.md) | Index-build lifecycle stays in the backend until a second backend needs it | Accepted | +| [0006](0006-pgvector-storage-and-scoring.md) | Vector storage on PostgreSQL uses pgvector's type, and the engine decides what it cannot compute | Accepted | diff --git a/docs/design/04-component-storage.md b/docs/design/04-component-storage.md index 102fa8cf..a8b4d18b 100755 --- a/docs/design/04-component-storage.md +++ b/docs/design/04-component-storage.md @@ -9,7 +9,7 @@ The storage layer provides a trait-based abstraction for all persistent data operations. Traits are defined in the `storage` crate with no database-specific dependencies. Backend implementations live in separate crates (e.g., -`storage-postgres`) and register themselves via a factory pattern using the `inventory` crate. +`storage-postgres`) and are installed once from a thin `main` via `set_backend`, one backend per binary. The trait-based design allows new storage backends to be added by implementing the storage traits and registering a factory function, with no changes needed to the `engine` or `server` crates. The factory pattern enables runtime @@ -809,13 +809,41 @@ The factory receives: It returns a `Future` that resolves to `ServerComponents` or `BackendError`. -### 10.3 Backend Registration with inventory +### 10.3 Backend Installation from a Thin `main` -Backends register themselves using the `inventory` crate for compile-time registration: +There is no registry and no auto-registration. **A binary serves exactly one +backend**, and its `main` installs that backend once before dispatching any +subcommand: + +```rust +// crates/bin/src/main.rs, the reference thin bin +fn main() -> anyhow::Result<()> { + extenddb_storage::set_backend(extenddb_storage_postgres::backend())?; + extenddb_app::run(extenddb_app::BuildInfo { /* ... */ }) +} +``` + +`backend()` returns a `Backend` value carrying the backend's name and its +factories (`bootstrapper`, `storage_config`, `operations`, `settings_store`, +`diagnostics_store`, and the server components factory). `set_backend` stores it in +a `OnceLock`, so a second call returns `BackendAlreadySet` and the first one wins. +The config file's `backend` key is validated against the installed backend's name +rather than driving dispatch, so a mistyped name is a startup error rather than an +unknown-backend failure later. + +This replaced two earlier mechanisms, and the reasons are worth knowing before +reintroducing either. `inventory`-based auto-registration relied on the linker +preserving `submit!` statics, which only happened if the binary referenced the +backend crate: an invisible, compiles-fine failure mode. A name-keyed registry +fixed that but kept string dispatch, which allows a class of runtime error that +cannot exist now, because there is nothing to look up. + +The components factory itself is unchanged in shape, and this is the body a +backend supplies: ```rust // In crates/storage-postgres/src/lib.rs -inventory::submit! { +{ ServerComponentsRegistration { backend: "postgres", factory: |config, region| { @@ -876,9 +904,9 @@ inventory::submit! { } ``` -The `inventory` crate collects all registrations at compile time. The `storage` -crate provides `create_server_components(backend_name, config, region)` which -looks up the matching factory and invokes it. +The installed `Backend` carries this factory, and the `storage` crate invokes it +through the installed value. There is no lookup by name: the only backend a +process can reach is the one its `main` installed. ### 10.4 Backend Selection in cmd_serve @@ -1092,7 +1120,7 @@ use extenddb_storage::{ }; use extenddb_auth::{BuiltinAuthProvider, CredentialStore}; -inventory::submit! { +{ ServerComponentsRegistration { backend: "sqlite", factory: |config, region| { @@ -1154,7 +1182,7 @@ Add the new backend as a dependency: extenddb-storage-sqlite = { path = "../storage-sqlite" } ``` -This ensures the backend's `inventory::submit!` registration is linked into the binary. +The dependency is what lets the thin `main` name the backend's `backend()` function; there is no linker-visibility requirement, because nothing is registered implicitly. ### 11.5 Test @@ -1176,7 +1204,72 @@ cargo test --workspace ./devtools/run-tests --extenddb --all ``` -### 11.6 RuntimeHooks Decision Tree +### 11.6 Vector Search Capability (Optional) + +Vector search is the one capability a backend may decline. Declining is a +supported end state, not a stub: the engine refuses every vector operation with a +message naming what is missing, and no other operation is affected. + +**The opt-out is a single method.** `DataEngine::as_vector_search` returns +`Option<&dyn VectorSearchEngine>` and defaults to `None`, so a backend that +implements nothing declines by construction. A backend that participates returns +`Some(self)` and implements `VectorSearchEngine::search_vectors`. + +**Fail closed, and decide it once.** Whether a backend can serve vector search may +depend on the server it is connected to rather than on the build: the PostgreSQL +backend probes for the pgvector extension at startup and caches the answer, then +declines for the process lifetime if it is absent. Two rules follow from the +engine's contract. A backend must not accept a vector index it cannot serve, and +it must not answer a search from a partially built index. Both refusals belong at +the storage boundary, because degrading instead produces a table that reports an +index and returns nothing from every search. + +**The build lifecycle is shared; the SQL is not.** `extenddb_storage::vector_lifecycle` +owns the ordering rules for the whole build: the batched backfill, the +`CREATING` to `ACTIVE` state machine, the poison-row policy, and the rebuild that +crash recovery uses. A backend supplies storage primitives through +`VectorIndexBuild` (one transactional batch, the phase flip, the publish, the +data-table reset, the wake, and an optional heartbeat) and keeps three things of +its own: the backfill cursor type, the propagation queue's claim predicate, and +its stuck-build detection policy. ADR-0005 records why the split is where it is, +and the module docs carry the measured status sequence the primitives must +produce. + +**One requirement that is easy to miss: an unscoped index needs a partition +column that can hold its sentinel.** An index with no declared search-schema HASH +attribute stores every row under one shared sentinel value, so the scan needs no +second code path. + +What makes that safe is **not** that the sentinel is unguessable. A client can +supply the identical string as a partition key, because a string attribute is +stored verbatim. The guarantee is structural: the partition is chosen from the +*index's* schema rather than from the item, and each index has its own data table, +so within one table either every row is keyed by a real hash value or every row +uses the sentinel. The two never coexist, so there is nothing for a collision to +leak into. Read the constant's own documentation before relying on any other +reading of this; it is written to prevent exactly the wrong one. + +The sentinel begins with a NUL byte as defence in depth for the day that +invariant changes, and that byte is what constrains the column type. It is not a +single rule across backends: PostgreSQL rejects a NUL byte in a `TEXT` +value outright, with an encoding error naming the untranslatable byte, so its +column is `BYTEA` and the value is bound as bytes on both the write path and +the search path; SQLite keeps the same value in `TEXT`, because its text bindings +are length-delimited and tolerate an embedded NUL. Both shipped backends are +correct and they differ, so copy the one whose engine you are targeting rather +than assuming byte columns everywhere. + +The failure to know about is PostgreSQL-specific: a `TEXT` partition column there +rejects every row of every unscoped index with SQLSTATE `22021`, and nothing +reveals it until the first unscoped index exists. + +**What to test first.** The engine-level refusals are cheap to verify and are the +contract a declining backend must satisfy; after that, the write path matters more +than the search path, because a row that never reaches the index cannot be found +by any query. The shared lifecycle's own tests cover the ordering rules, so a +backend's tests are about its primitives producing the states those rules assume. + +### 11.7 RuntimeHooks Decision Tree **Does your backend need `ServerRuntimeHooks`?** @@ -1198,7 +1291,7 @@ indexes for TTL). because DynamoDB handles all background work internally (TTL, streams, backups). -### 11.7 Design Rationale +### 11.8 Design Rationale **Why factory pattern instead of direct construction?** - `cmd_serve` remains backend-agnostic (no PostgreSQL imports) diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 44468861..f81d351c 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -8,7 +8,7 @@ adaptation when switching between ExtendDB and the real service. | Area | DynamoDB | ExtendDB | |------|----------|------| -| Storage backend | Proprietary distributed storage | PostgreSQL (default) or MongoDB (feature flag) | +| Storage backend | Proprietary distributed storage | PostgreSQL (default), SQLite, or MongoDB, selected by mutually exclusive Cargo features at build time; one backend per binary | | Global Tables | CreateGlobalTable, replication | Not implemented (returns UnknownOperationException) | | DAX (Accelerator) | In-memory caching layer | Not applicable | | PartiQL | ExecuteStatement, BatchExecuteStatement | Not implemented (returns UnknownOperationException) | @@ -65,7 +65,11 @@ adaptation when switching between ExtendDB and the real service. |------|----------|------| | GSI update propagation | Eventually consistent (milliseconds to seconds) | Per-GSI propagation delay. System default: `index_propagation_delay_ms` setting (default 10ms). Each GSI can override with its own `propagation_delay_ms` (stored in catalog). A value of 0 means synchronous (future sync GSI feature). | | Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance synchronously in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. | -| `SearchVectors` score at extreme magnitudes | Returns a number for any vector of finite components | Both backends return a number, and the two differ in the extreme. The SQLite backend computes distances itself and does so in double precision, so its score is the true value across the whole domain. The PostgreSQL backend cannot: pgvector accumulates in single precision, so magnitudes far below `f32::MAX` overflow inside the extension (Euclidean above about 9.2e18, dot product above about 1.8e19, cosine at both ends, above about 1.8e19 and below about 3.7e-23). Since a non-finite score cannot be serialised as JSON at all, that backend bounds the result in SQL: `1e308` for an overflowed distance, `-1e308` for an overflowed negated inner product, and 1.0 for a cosine NaN. Those bounds are not measured service answers. Ranking is unaffected on either backend, because each bound sits at the end its metric overflows towards, so the farthest row stays farthest and the most similar stays most similar; two rows that both overflow tie, and the tie breaks on the base key. | +| `SearchVectors` score at extreme magnitudes (PostgreSQL backend only) | Returns the true distance as a number for any vector of finite components | The score is bounded to a finite value instead, and the bound is not a measured service answer. pgvector accumulates distances in single precision, so magnitudes far below `f32::MAX` overflow inside the extension: Euclidean above about 9.2e18, dot product above about 1.8e19, and cosine at both ends, above about 1.8e19 and below about 3.7e-23. A non-finite score cannot be serialised as JSON at all, so the result is bounded in SQL: `1e308` for an overflowed distance, `-1e308` for an overflowed negated inner product, which the score contract negates so a client sees `1e308` in `Score`, and 1.0 for a cosine that comes back NaN. Ranking is unaffected at the overflow end, because each bound sits at the end its metric overflows towards, so the farthest row stays farthest and the most similar stays most similar; two rows that both overflow tie, and the tie breaks on the base key. At the underflow end cosine loses resolution rather than being bounded: pgvector's underflowed norms yield infinities that it clamps before the value is read, so the reported distance collapses to one of 0, 1 or 2, following the sign of the inner product, with the 1.0 substitute firing only when the vectors are exactly orthogonal (measured). Ranking there still separates nearer-than-orthogonal from farther, and loses all resolution within each half. The SQLite backend owns its own arithmetic, computes in double precision, and reports the true value, which is why this row is scoped to PostgreSQL. | +| Vector index deletion window | `UpdateTable` Delete leaves the index in `DELETING` long enough to observe, then removes it | No observable `DELETING` window on either backend: the catalog row is removed inside the `UpdateTable` transaction, so a `DescribeTable` immediately afterwards already omits the index. The index's data table is dropped after that commit, in a separate transaction, best effort: on PostgreSQL it is a different database entirely, and a failure there is logged and skipped rather than failing the request. So an operator debugging a leftover `_ddb_vec_*` table should look for that warning rather than assume the delete was incomplete. | +| Restoring a backup of a table that had vector indexes | Restores the table with its vector indexes intact: the configuration survives, items keep their vector attributes, and `SearchVectors` works as soon as the table is `ACTIVE` (measured) | Neither backend restores the indexes, and the two fail differently. **PostgreSQL refuses the restore** with a `ValidationException` naming the backup and the index count, because restore does not carry index data across and a table that looks restored while answering every search with nothing is worse than a refusal a caller can act on. **SQLite does not refuse**: its backup path does not capture vector indexes at all, so a restore silently produces the table without them. That silence is tracked as a defect rather than intended, and it is the reason the PostgreSQL path refuses instead of matching it. A backup taken from a table with no vector indexes restores normally on both. | +| `SearchVectors` endpoint | Served only on `search-dynamodb..amazonaws.com`. The standard `dynamodb..amazonaws.com` endpoint answers the same request with HTTP 400 `UnknownOperationException` ("This operation is not supported by this endpoint"); every control-plane and item operation stays on the standard endpoint. Signing is unchanged either way (service name `dynamodb`, target prefix `DynamoDB_20120810`) | Served on the same endpoint as every other operation, so a client is pointed at one ExtendDB endpoint for all of them. Two consequences worth knowing: an SDK that resolves a separate search hostname from its endpoint ruleset needs its endpoint overridden to reach ExtendDB, and ExtendDB does **not** reproduce the service's refusal, so a test asserting `UnknownOperationException` for vector search on the base endpoint passes against Amazon DynamoDB and fails here. | +| `SearchVectors` result order for equal distances | Measured unstable: three identical searches returned tied rows in three different orders, and a top-k that truncates a tie group keeps an arbitrary subset of it | Deterministic on both backends, by different means. **PostgreSQL** sorts explicitly on the base table's full primary key after the score, so the order is a property of the query. **SQLite** issues no `ORDER BY` and resolves ties by scan order through a stable top-k, so its order is a property of the plan rather than something the query guarantees. Either way a client re-issuing an identical search sees the same order, which is stricter than the service rather than divergent in outcome. Do not rely on the two backends agreeing on which subset of a truncated tie group they keep. | | Multi-part base table keys | Not supported | Preview extension (opt-in via `enable_multipart_keys` setting). Standard single/composite keys work identically. | ## Capacity and Throttling diff --git a/docs/getting-started.md b/docs/getting-started.md index 8bd4db94..cdb92be0 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -199,7 +199,7 @@ You should see all checks pass: --- Checking catalog connection... OK: Connected to catalog. --- Checking catalog version... - OK: Catalog version 0.0.2 + OK: Catalog version 0.0.3 --- Checking data database... OK: Connected to data database 'extenddb_catalog_data'. --- Enumerating tables... @@ -211,11 +211,12 @@ You should see all checks pass: ## 4. Start the server -extenddb runs as a daemon (background process) and logs to syslog. On startup it prints a banner to stdout confirming the version, catalog version, and bind address, then forks to background. +extenddb runs as a daemon (background process) and logs to syslog. On startup it prints a two-line banner to stdout, the version, catalog version and bind address followed by the storage backend and its redacted connection string, then forks to background. It says "starting", not "listening": the socket is not accepting connections yet at that point. ```bash ./target/release/extenddb serve --config extenddb.toml -# extenddb 0.0.2 (catalog 0.0.2) listening on 127.0.0.1:18443 +# extenddb 0.1.6 (catalog 0.0.3) starting on 127.0.0.1:18443 +# storage: postgres (postgresql://extenddb:***@localhost:5432/extenddb_catalog) ``` Check status (includes the daemon PID): @@ -1292,8 +1293,8 @@ Each runner requires its tools to be installed. The runner checks prerequisites ```bash ./target/release/extenddb version -# extenddb 0.0.2 -# catalog 0.0.2 +# extenddb 0.1.6 +# catalog 0.0.3 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z ``` @@ -1324,7 +1325,7 @@ The `storage.postgres.catalog_pool_size` setting controls the maximum number of **When to increase:** If you see elevated latency under concurrent load, the pool may be saturated. Requests queue at the pool level when all connections are in use. Increase `pool_size` (and `catalog_pool_size` if auth is enabled) to allow more concurrent transactions. -**Relationship to PostgreSQL `max_connections`:** The total connection footprint is `pool_size + catalog_pool_size + 1` (the extra 1 is for the log-level poller). PostgreSQL's default `max_connections` is 100. Ensure `pool_size + catalog_pool_size + 1` does not exceed your PostgreSQL `max_connections` setting. +**Relationship to PostgreSQL `max_connections`:** The total connection footprint is `pool_size + catalog_pool_size + 1` (the extra 1 is for the log-level poller), plus one connection per vector index build running on the server and one while a schema migration runs. Those extra sessions are opened outside both pools on purpose, for two different reasons. A build's ownership lock is session-scoped, so it needs a session that ends when the build does: a pooled connection would return to the pool still holding the lock. The migration lock is transaction-scoped, and needs a pinned connection because its explicit transaction has to retain one PostgreSQL backend until COMMIT, which is what stops a transaction-pooling proxy from letting the lock move between backends. PostgreSQL's default `max_connections` is 100. Ensure `pool_size + catalog_pool_size + 1`, plus the number of vector index builds you expect to overlap, does not exceed your PostgreSQL `max_connections` setting. **Example:** To support 50 concurrent data operations with auth enabled, set both pools to 50 in `extenddb.toml` and ensure PostgreSQL allows at least 101 connections. diff --git a/docs/manuals/01-architecture-guide.md b/docs/manuals/01-architecture-guide.md index 4af34045..fa885866 100755 --- a/docs/manuals/01-architecture-guide.md +++ b/docs/manuals/01-architecture-guide.md @@ -84,7 +84,7 @@ Trait definitions for the storage layer. Thirteen storage traits partition backe - **AuthorizationStore**: Policy evaluation cache - **Bootstrapper**: Initial database setup -Traits use `BoxFuture` for object safety. Backends register at compile time via the `inventory` crate and are selected at startup by name. The `RuntimeHooks` trait allows backends to spawn backend-specific workers (PostgreSQL spawns 7). +Traits use `BoxFuture` for object safety. A binary serves exactly one backend, installed from its thin `main` via `set_backend`. The `RuntimeHooks` trait allows backends to spawn backend-specific workers (PostgreSQL spawns 7). ### storage-postgres @@ -170,13 +170,13 @@ extenddb uses a dual-database architecture: - **Catalog database** (e.g., `extenddb_catalog`): Stores table metadata, account/user/group/role/policy definitions, access keys, settings, stream metadata, and metrics. Shared across all accounts. - **Data database** (e.g., `extenddb_catalog_data`): Stores user items, GSI/LSI data, and stream records. Each table gets its own PostgreSQL table. -The catalog version (currently 0.0.2) is stored in the `catalog_metadata` table and checked at startup. Version mismatches prevent the server from starting — run `extenddb migrate` to upgrade. +The catalog version is stored in the `settings` table under the key `catalog_version` and checked at startup. It is per backend: PostgreSQL and SQLite are at 0.0.3, MongoDB at 0.0.2. A mismatch between the compiled-in version and the stored one prevents the server from starting; run `extenddb migrate` to upgrade. ## Pluggable Architecture ### Storage -Storage backends implement thirteen traits (see **storage** section above). The traits use `BoxFuture` for object safety. Backends register at compile time via the `inventory` crate, and the `bin` crate selects the backend by name at startup. Currently only PostgreSQL is implemented. +Storage backends implement thirteen traits (see **storage** section above). The traits use `BoxFuture` for object safety. A binary serves exactly one backend: the thin `bin` crate installs it via `set_backend` before any subcommand runs, and the config file's `backend` key is validated against the installed name rather than selecting it. PostgreSQL, SQLite and MongoDB backends are implemented, selected by mutually exclusive Cargo features at build time. ### Authentication diff --git a/docs/manuals/04-quickstart-setup-guide.md b/docs/manuals/04-quickstart-setup-guide.md index 3a881a3d..d629a8a1 100755 --- a/docs/manuals/04-quickstart-setup-guide.md +++ b/docs/manuals/04-quickstart-setup-guide.md @@ -129,8 +129,8 @@ Check the version: ```bash ./target/release/extenddb version -# extenddb 0.0.2 -# catalog 0.0.2 +# extenddb 0.1.6 +# catalog 0.0.3 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z ``` @@ -171,7 +171,7 @@ Expected output: --- Checking catalog connection... OK: Connected to catalog. --- Checking catalog version... - OK: Catalog version 0.0.2 + OK: Catalog version 0.0.3 --- Checking data database... OK: Connected to data database 'extenddb_catalog_data'. --- Enumerating tables... diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index fff2d8de..45b844ae 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -176,6 +176,8 @@ Managed via `extenddb settings set`. Changes take effect within 30 seconds witho | `log_level` | `info` | Log level: trace, debug, info, warn, error | | `control_plane_delay_seconds` | `5` | Delay for table status transitions (0 = instant) | | `allow_credential_import` | `true` | Whether `import-access-key` is allowed | +| `vector_backfill_batch_delay_ms` | `0` | **Test-oriented.** Milliseconds to pause between batches while a vector index backfills. Zero in production. A test sets it so a write is guaranteed to land while the index is still building; the pause is outside any lock, so the table stays writable throughout either way. | +| `vector_allocation_phase_delay_ms` | `0` | **Test-oriented.** Milliseconds to hold a new vector index in the resource-allocation phase (`CREATING` with `Backfilling: false`) before the scan starts. Zero in production. Both transitions otherwise happen inside one `UpdateTable` call, so without this a client cannot observe the phase that the delete rule turns on. | ```bash # View current settings diff --git a/docs/manuals/08-install-linux.md b/docs/manuals/08-install-linux.md index 6208193b..1512201b 100755 --- a/docs/manuals/08-install-linux.md +++ b/docs/manuals/08-install-linux.md @@ -118,7 +118,7 @@ Expected: ``` === extenddb verify === ... - OK: Catalog version 0.0.2 + OK: Catalog version 0.0.3 ... === HEALTHY: All checks passed === ``` diff --git a/docs/manuals/09-install-macos.md b/docs/manuals/09-install-macos.md index 5a22f190..4e2d87e8 100755 --- a/docs/manuals/09-install-macos.md +++ b/docs/manuals/09-install-macos.md @@ -96,7 +96,7 @@ Expected: ``` === extenddb verify === ... - OK: Catalog version 0.0.2 + OK: Catalog version 0.0.3 ... === HEALTHY: All checks passed === ``` diff --git a/docs/manuals/11-deployment-guide.md b/docs/manuals/11-deployment-guide.md index 612cf112..6035fbd1 100755 --- a/docs/manuals/11-deployment-guide.md +++ b/docs/manuals/11-deployment-guide.md @@ -250,7 +250,7 @@ Key PostgreSQL settings for extenddb workloads: - `shared_buffers`: 25% of available RAM - `effective_cache_size`: 75% of available RAM - `work_mem`: 64MB (for sort operations in Query/Scan) -- `max_connections`: ≥ extenddb pool_size + 10 +- `max_connections`: ≥ extenddb pool_size + 10, and add one per vector index build you expect to overlap (a build holds its own session outside the pools) ### Monitoring Queries diff --git a/docs/technical-debt.md b/docs/technical-debt.md index c34bed44..39d440e9 100755 --- a/docs/technical-debt.md +++ b/docs/technical-debt.md @@ -1,6 +1,6 @@ # Technical Debt Tracker -Last updated: 2026-08-19 +Last updated: 2026-08-20 ## Categories @@ -31,6 +31,7 @@ Last updated: 2026-08-19 | F-16 | `transact_write_items.rs` passes `None` for `old_item` in stream capture — `OldImage` always `None` for transaction-originated stream records | `engine/transact_write_items.rs` | Medium | P27 | | F-17 | `validate_attribute_name_sizes` only checks top-level attribute names — nested map keys not validated | `core/validation/mod.rs` | Low | P30 | | F-18 | ~~UpdateTable Delete of a vector index in the resource-allocation phase (`CREATING`, `Backfilling: false`) is accepted; Amazon DynamoDB refuses it with `ResourceInUseException` until backfilling starts~~ Both backends now enforce the phase rule with the measured message, and both hold the phase open under `vector_allocation_phase_delay_ms` so a client can observe it | ~~`storage-sqlite/update_table.rs` (vector delete branch), `core/types/table.rs` (`vector_index_delete_in_allocation_phase`)~~ | ~~Medium~~ | vector probe P2 | +| F-19 | SQLite backup does not capture vector indexes, so `RestoreTableFromBackup` silently produces the table without them. Amazon DynamoDB preserves vector state through backup and restore (measured). The PostgreSQL backend refuses the restore rather than matching this, so the two backends fail differently: one refuses with a reason, one loses declared indexes quietly. Fix: capture the index set in the SQLite backup row and either restore it or refuse, matching PostgreSQL | `storage-sqlite/backup.rs` (no `vector_indexes` capture), `storage-postgres/backup_engine.rs:467-497` (the refusal to match) | Medium | PR-2 docs stage | ## Cleanup diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 50376067..6bf08365 100755 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -850,6 +850,46 @@ If the problem persists, check for long-running queries or connection leaks with --- +### `Vector indexes are not supported by this storage backend` + +**Error text:** +``` +Vector indexes are not supported by this storage backend +``` + +Also `SearchVectors is not supported by this storage backend` for a search request. + +**Cause:** Vector indexes are stored in `vector(N)` columns, a type the +[pgvector](https://github.com/pgvector/pgvector) extension provides, so support is a +property of the PostgreSQL server rather than of the ExtendDB build. The server probes +for the extension once at startup and refuses every vector operation for the rest of +its life if it was absent. Other operations are unaffected. + +**Fix:** Confirm what the running server decided, by looking for one of these lines in +its startup log: + +``` +pgvector 0.8.0 detected on the data database; vector index storage available +pgvector not installed on the data database; vector indexes are not supported +``` + +If it is absent, install the extension for your server version (for example +`postgresql-16-pgvector`), then create it on the **data** database, not the catalog: + +```sql +CREATE EXTENSION vector; +``` + +`extenddb init` and `extenddb migrate` both attempt this and print a notice rather than +failing when they cannot; a role without permission to create extensions needs a +superuser or the database owner to run it once. + +**Then restart ExtendDB.** The probe result is cached at startup, so a server that +started without the extension keeps refusing vector operations until it is restarted, +even after the extension exists. + +--- + ## License Copyright 2026 ExtendDB contributors. Licensed under the Apache License, Version 2.0. diff --git a/extenddb.sample.toml b/extenddb.sample.toml index 3c641d7b..a4404f3a 100755 --- a/extenddb.sample.toml +++ b/extenddb.sample.toml @@ -50,9 +50,23 @@ # Minimum: 10 (smaller values are clamped with # a startup warning). Total PostgreSQL # connections used: pool_size + catalog_pool_size + 1 - # (log-level poller: 1). Increase for - # higher concurrency; ensure PostgreSQL - # max_connections >= pool_size + catalog_pool_size + 1. + # (log-level poller: 1), plus one per vector + # index build running on this server and one + # while a schema migration runs. Both sit + # OUTSIDE both pools, for two different reasons. + # A build's ownership lock is SESSION-scoped, so + # it needs a session that ends when the build + # ends: a pooled connection returns to the pool + # still holding the lock. The migration lock is + # TRANSACTION-scoped and needs a pinned + # connection instead, because its explicit + # transaction must retain one PostgreSQL backend + # until COMMIT; a transaction-pooling proxy could + # otherwise let the lock move between backends. + # Increase for higher concurrency; ensure + # PostgreSQL max_connections >= pool_size + + # catalog_pool_size + 1 + the number of vector + # index builds you expect to overlap. # catalog_pool_size = 20 # Maximum concurrent connections for the # management/catalog pool (authz, IAM, console). # Defaults to pool_size if not set. diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 099a2559..a1503a58 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1409,6 +1409,38 @@ async fn a_tiny_query_vector_still_scores_every_hit_as_a_number() { ); } + // The semantic half, and the only observable that distinguishes a computed + // ranking from the zero-vector answer. A tiny vector is not a zero vector: the + // row parallel to it is nearly identical, so its cosine distance is ~0, and the + // orthogonal row is exactly 1.0. A backend that treats the query as zero returns + // 1.0 for BOTH and still passes every assertion above, which is precisely the + // state one backend was in: it reported a ranking it had not computed. + let pks = hit_pks(&response); + assert_eq!( + pks, + vec!["parallel", "orthogonal"], + "the parallel row must rank first: {response}" + ); + let score_of = |pk: &str| -> f64 { + hits.iter() + .find(|h| h.pointer("/Item/pk/S").and_then(|v| v.as_str()) == Some(pk)) + .and_then(|h| h.get("Score")) + .and_then(serde_json::Value::as_f64) + .unwrap_or_else(|| panic!("no score for {pk}: {response}")) + }; + assert!( + score_of("parallel") < 1e-6, + "a tiny query parallel to the stored vector is a ~0 cosine distance, not the \ + zero-vector answer: got {}", + score_of("parallel") + ); + assert!( + (score_of("orthogonal") - 1.0).abs() < 1e-6, + "and orthogonal to it is exactly 1.0, by the metric rather than by the guard: \ + got {}", + score_of("orthogonal") + ); + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } From 117fbe40ec79d0af248fc5664f7aa2b8d184d7c7 Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Mon, 24 Aug 2026 20:15:41 +0000 Subject: [PATCH 08/13] test: guard the documentation's version literals against the compiled constants The guard sees every literal form it claims to cover, reaches every file, and does not arm on an inline marker. Includes the documentation corrections the guard and the audit caught, and tells users the feature exists. --- AGENTS.md | 14 +- Cargo.lock | 1 - README.md | 3 + crates/app/Cargo.toml | 1 - crates/cache/src/lib.rs | 2 +- crates/core/src/settings_keys.rs | 4 +- .../storage-postgres/src/data/vector_index.rs | 17 +- crates/storage-postgres/src/update_table.rs | 2 +- crates/storage-postgres/src/vector_search.rs | 9 +- .../tests/doc_version_literals.rs | 356 ++++++++++++++++++ crates/storage-sqlite/src/vector_search.rs | 3 +- docs/adr/0006-pgvector-storage-and-scoring.md | 16 +- docs/design/02-high-level-design.md | 21 +- docs/design/04-component-storage.md | 49 ++- docs/differences-from-dynamodb.md | 7 +- docs/getting-started.md | 6 +- docs/manuals/01-architecture-guide.md | 10 +- docs/manuals/03-usage-guide.md | 49 +++ docs/manuals/05-admin-guide.md | 5 +- docs/manuals/11-deployment-guide.md | 2 +- docs/technical-debt.md | 3 +- extenddb.sample.toml | 26 +- tests/rust/src/vector_index_search.rs | 62 ++- 23 files changed, 608 insertions(+), 60 deletions(-) create mode 100644 crates/storage-postgres/tests/doc_version_literals.rs diff --git a/AGENTS.md b/AGENTS.md index b9db1b15..56ec6117 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ by AWS engineers. It is not a fork of DynamoDB and contains no DynamoDB source c protocol: any AWS SDK, CLI, or tool that works with DynamoDB works with ExtendDB, unchanged. - **Language:** Rust (edition 2024, MSRV 1.88+) -- **Storage backends:** PostgreSQL 14+ (default), MongoDB 7.0+ (feature flag `mongodb`) +- **Storage backends:** PostgreSQL 14+ (default), SQLite (bundled, no server or client install required; feature flag `sqlite`, also `sqlite-memory` for dev and CI), MongoDB 7.0+ (feature flag `mongodb`). One backend per binary, selected by mutually exclusive Cargo features. - **Architecture:** Async (tokio), trait-based storage abstraction - **Authentication:** Mandatory SigV4 with built-in IAM (users, groups, roles, policies) - **TLS:** Mandatory (self-signed cert generated by default) @@ -32,6 +32,11 @@ extenddb/ │ ├── engine/ # DynamoDB operation handlers (PutItem, Query, etc.) │ ├── storage/ # Storage trait definitions (TableEngine trait) │ ├── storage-postgres/ # PostgreSQL implementation of TableEngine +│ ├── storage-sqlite/ # SQLite implementation (dev, CI, and embedded use) +│ ├── storage-mongodb/ # MongoDB implementation (feature flag) +│ ├── cache/ # Auth and authz caches +│ ├── config/ # Config parsing shared by the CLI and the server +│ ├── app/ # Shared CLI surface the thin bin dispatches to │ ├── auth/ # SigV4 verification, IAM policy engine │ ├── server/ # HTTP server, management API, web console │ └── bin/ # CLI, config, daemon lifecycle (extenddb binary) @@ -52,6 +57,7 @@ extenddb/ The `TableEngine` trait in `crates/storage/src/lib.rs` defines the storage interface. All storage backends implement this trait: - `storage-postgres` (PostgreSQL) — default backend +- `storage-sqlite` (SQLite) — feature flag `sqlite`, and `sqlite-memory` for dev and CI - `storage-mongodb` (MongoDB) — feature flag `mongodb` The trait uses RPITIT (return-position impl Trait in traits) for async methods — no `#[async_trait]` macro. @@ -68,12 +74,15 @@ extenddb (bin) │ └─> extenddb-storage (trait definitions) ├─> extenddb-auth ├─> extenddb-storage-postgres (feature: postgres) + ├─> extenddb-storage-sqlite (feature: sqlite / sqlite-memory) └─> extenddb-storage-mongodb (feature: mongodb) ``` - **extenddb-core:** Pure synchronous Rust. No async, no I/O. Types, validation, expression parsing. - **extenddb-storage:** Trait definitions only. No implementation. - **extenddb-storage-postgres:** Concrete PostgreSQL implementation. +- **extenddb-storage-sqlite:** Concrete SQLite implementation, the dev and CI backend. + It serves the same wire surface, vector search included, and needs no server. - **extenddb-storage-mongodb:** Concrete MongoDB implementation. - **extenddb-engine:** Operation handlers that call storage traits. - **extenddb-server:** HTTP server, management API, web console. @@ -329,6 +338,9 @@ Query, Scan (key conditions, filters, projections, pagination, index selection) ### Batch & Transactions BatchGetItem (100 keys), BatchWriteItem (25 ops), TransactGetItems (100 items), TransactWriteItems (100 ops) +### Vector Search +SearchVectors, and vector indexes on CreateTable and UpdateTable (COSINE, EUCLIDEAN, DOT_PRODUCT; on-demand tables only; five per table). Served by the PostgreSQL and SQLite backends; MongoDB provides no implementation and refuses every vector operation. PostgreSQL additionally requires the pgvector extension on the data database, probed once at startup, and refuses without it. The refusal is two strings, which matters when grepping logs: `Vector indexes are not supported by this storage backend` for CreateTable and UpdateTable, and `SearchVectors is not supported by this storage backend` for a search. + ### Streams ListStreams, DescribeStream, GetShardIterator, GetRecords diff --git a/Cargo.lock b/Cargo.lock index 28228c43..fbe53dac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1127,7 +1127,6 @@ dependencies = [ "clap", "daemonize", "extenddb-auth", - "extenddb-cache", "extenddb-config", "extenddb-core", "extenddb-engine", diff --git a/README.md b/README.md index 3053c17c..c22aeca1 100755 --- a/README.md +++ b/README.md @@ -196,6 +196,9 @@ Query, Scan (key conditions, filters, projections, pagination, index selection) ### Batch & Transactions BatchGetItem (100 keys), BatchWriteItem (25 ops), TransactGetItems (100 items), TransactWriteItems (100 ops) +### Vector Search +SearchVectors, and vector indexes on CreateTable and UpdateTable (cosine, Euclidean and dot product; on-demand tables only). On PostgreSQL this requires the pgvector extension on the data database; without it every vector operation is refused and nothing else is affected. + ### Streams ListStreams, DescribeStream, GetShardIterator, GetRecords diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index b9f2f802..e9a3ef86 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -15,7 +15,6 @@ dev-mode = ["extenddb-config/dev-mode"] [dependencies] extenddb-auth = { workspace = true } -extenddb-cache = { workspace = true } extenddb-core = { workspace = true } extenddb-engine = { workspace = true } extenddb-storage = { workspace = true } diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 8e2b50d4..c1860f5e 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -1,7 +1,7 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Stale-while-revalidate (SWR) cache primitive used by the auth and storage layers. +//! Stale-while-revalidate (SWR) cache primitive used by the auth and server layers. //! //! See `docs/design/12-auth-authz-cache.md` for the full design rationale. //! diff --git a/crates/core/src/settings_keys.rs b/crates/core/src/settings_keys.rs index 4b325bd1..75303b8e 100644 --- a/crates/core/src/settings_keys.rs +++ b/crates/core/src/settings_keys.rs @@ -70,8 +70,8 @@ pub const VECTOR_INDEX_MIN_CREATING_MS: &str = "vector_index_min_creating_ms"; /// /// Without this there is no deterministic way to observe the first half from a /// client: the allocation phase exists only between the catalog row's insert and -/// the flip to `Backfilling: true`, both inside one `UpdateTable` call, so a test -/// could only race it. A race that asserts a whole measured string is worse than +/// the flip to `Backfilling: true`, the second of them inside the detached build task; the window is therefore a task +/// spawn wide, so a test could only race it. A race that asserts a whole measured string is worse than /// no test, because it fails for reasons unrelated to the rule. /// /// Unset, nothing waits and no branch is taken. diff --git a/crates/storage-postgres/src/data/vector_index.rs b/crates/storage-postgres/src/data/vector_index.rs index 1867be95..98b66b99 100644 --- a/crates/storage-postgres/src/data/vector_index.rs +++ b/crates/storage-postgres/src/data/vector_index.rs @@ -334,11 +334,18 @@ impl<'a> VectorInsertPlan<'a> { let item_json = serde_json::to_value(&projected).map_err(|e| StorageError::Internal(e.to_string()))?; - // A plain INSERT, deliberately, where the GSI sibling upserts. Every caller - // reaches this through `apply_vector_index`, which deletes the base key's row - // first, so no live row can exist here. Keeping it plain means that if a - // future change ever makes that delete conditional, this fails loudly on the - // primary key rather than quietly replacing a row and hiding the break. + // A plain INSERT, deliberately, where the GSI sibling upserts. Two callers reach + // it and only one deletes first: the live write path through + // `apply_vector_index` does, the backfill does not. So the caller set is not + // what makes a conflict unreachable. + // + // What makes it unreachable on this backend is the inline gate in + // `maintain_vector_indexes`, `delay_ms == 0 && index_status == "ACTIVE"`: a + // write to an index that is still CREATING is always queued, so it cannot put a + // row here while the backfill is scanning. Keeping the INSERT plain means that + // if that gate is ever weakened, this fails loudly on the primary key rather + // than quietly replacing a row and hiding the break. The SQLite backend has no + // such gate, which is tech-debt item F-20. // // As bytes: the column is BYTEA because the unscoped sentinel carries a NUL, // which PostgreSQL rejects in a text column. diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index 413e493d..16151c70 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -1006,7 +1006,7 @@ impl PostgresEngine { // Zero in production. A test sets it to hold the index in the // resource-allocation phase, which is the only way a client can observe that // phase: it otherwise exists only between the catalog row's insert and the - // flip below, both inside this one call. + // flip below, the second inside the detached task this call spawns. let allocation_delay = std::time::Duration::from_millis(self.vector_allocation_phase_delay().await); let ownership_pool = self.data_pool.clone(); diff --git a/crates/storage-postgres/src/vector_search.rs b/crates/storage-postgres/src/vector_search.rs index b13d10b8..5b278d46 100644 --- a/crates/storage-postgres/src/vector_search.rs +++ b/crates/storage-postgres/src/vector_search.rs @@ -47,8 +47,13 @@ use crate::data::vector_table_name; /// that cannot be serialised for magnitudes well below `f32::MAX`, measured on 0.8.0: /// Euclidean overflows to `Infinity` above about 9.2e18 (its difference is doubled /// before squaring, so it goes first), dot product returns `-Infinity` above about -/// 1.8e19, and cosine returns `NaN` at both ends, above about 1.8e19 and below about -/// 3.7e-23, because it divides two values that have both overflowed or both underflowed. +/// 1.8e19, and cosine fails at both ends, above about 1.8e19 and below about +/// 3.7e-23. The two ends fail differently. Above, both the norms and the inner +/// product overflow and the quotient is `NaN`. Below, 3.7e-23 is where a component's +/// `f32` square rounds to zero, so it is the query norm that underflows while the +/// inner product usually does not: the quotient is then an infinity that pgvector +/// clamps, and only an exactly zero inner product gives the 0/0 that is `NaN`. The +/// measurement below spells out what that produces. /// `serde_json` renders NaN and Infinity alike as `null`, so each of those reaches a /// client as `"Score": null` on a 200 response. /// diff --git a/crates/storage-postgres/tests/doc_version_literals.rs b/crates/storage-postgres/tests/doc_version_literals.rs new file mode 100644 index 00000000..8328f874 --- /dev/null +++ b/crates/storage-postgres/tests/doc_version_literals.rs @@ -0,0 +1,356 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Version literals in the documentation must match the compiled constants. +//! +//! The documentation carries sample output containing the binary version and the +//! catalog version. Those literals go stale silently: nothing fails, nothing warns, +//! and the next reader trusts them. This stage found five such blocks still showing +//! catalog `0.0.2` long after `0.0.3` shipped, and the binary version wrong in the +//! same lines, so the fix has to leave a check behind rather than five fresh +//! literals that expire at the next release. +//! +//! The same guard pattern already exists for the schema side: the migration runner +//! asserts the final migration writes the expected version, and the SQLite schema +//! asserts its seeded literal matches the constant. This is that pattern applied to +//! prose. +//! +//! Two escapes are deliberate. Files that document a past version, an upgrade +//! between versions, or another backend's version are exempt by path, each with a +//! reason. And a line may opt out with a trailing `` +//! marker, so a genuinely historical example inside an otherwise live document does +//! not force the whole file onto the exemption list. + +use std::path::{Path, PathBuf}; + +/// Documents whose version literals are historical rather than current. +/// +/// Each entry states why, because an unexplained exemption is how a check becomes +/// decoration. +const EXEMPT: &[(&str, &str)] = &[ + ( + "docs/manuals/07-upgrade-manual.md", + "documents the 0.0.2 to 0.0.3 upgrade, so both versions appear on purpose", + ), + ( + "docs/backlog.md", + "records completed history, including the version current at the time", + ), + ( + "docs/design/13-storage-mongodb.md", + "describes the MongoDB backend's own expected catalog version, which is 0.0.2", + ), + ( + "docs/design/01-requirements.md", + "illustrative version numbers in a requirement example, not sample output", + ), +]; + +fn repo_root() -> PathBuf { + // CARGO_MANIFEST_DIR is crates/storage-postgres. + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("workspace root above crates/storage-postgres") + .to_path_buf() +} + +fn markdown_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + // `rendered` is build output, not source. + if path.file_name().is_some_and(|n| n == "rendered") { + continue; + } + markdown_files(&path, out); + } else if path.extension().is_some_and(|e| e == "md") { + out.push(path); + } + } +} + +/// How far past a keyword a version literal may sit and still be about it. +/// +/// Wide enough for the real forms, `Catalog version 0.0.3` and +/// `catalog 0.0.3 (postgres)`, and narrow enough that an unrelated number later in a +/// sentence is not attributed to the keyword. +const LOOKAHEAD: usize = 24; + +/// Pull version literals that belong to `keyword` out of one line. +/// +/// Deliberately not a prefix match. The first version of this guard matched +/// `"catalog "` immediately followed by digits, which is one form the documentation +/// uses and not the one this stage had to fix: every stale literal it corrected read +/// `OK: Catalog version 0.0.3`, with a capital letter and a word in between, so the +/// check could not see the lines it existed for, and two files carried no other +/// literal at all, which made their coverage zero while the test passed. The lesson is +/// the one this suite applies elsewhere: ask whether the check would pass if the thing +/// it checks were broken. +/// +/// So the match is case-insensitive and scans a short window after the keyword rather +/// than requiring adjacency. +fn versions_near(line: &str, keyword: &str) -> Vec { + let haystack = line.to_ascii_lowercase(); + let chars: Vec = haystack.chars().collect(); + let key: Vec = keyword.chars().collect(); + let mut found = Vec::new(); + + for start in 0..chars.len() { + if !chars[start..].starts_with(&key[..]) { + continue; + } + // Whole word only. Without this, `crc-catalog | 2.4.0` in the dependency + // table reads as a claim about the catalog version. + let before_ok = start == 0 || !is_word_char(chars[start - 1]); + let after = start + key.len(); + let after_ok = after >= chars.len() || !chars[after].is_ascii_alphanumeric(); + if !before_ok || !after_ok { + continue; + } + + // Find where a number starts, within the window, then read the WHOLE run of + // digits and dots rather than the part of it that fits. Truncating the run is + // how an IP address in `--bind-addr 10.0.1.5` became the version `10.0.1`. + let limit = (after + LOOKAHEAD).min(chars.len()); + if let Some(digit_at) = (after..limit).find(|&i| chars[i].is_ascii_digit()) { + let mut j = digit_at; + while j < chars.len() && (chars[j].is_ascii_digit() || chars[j] == '.') { + j += 1; + } + let run: String = chars[digit_at..j].iter().collect(); + if let Some(version) = exact_version(&run) { + found.push(version); + } + } + } + found +} + +/// True for characters that make a keyword part of a longer word. +fn is_word_char(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '-' || c == '_' +} + +/// `text` as a version, or `None`. The whole run must be `X.Y.Z`. +/// +/// Requiring the entire run rather than a prefix of it is what rejects an IP address: +/// `10.0.1.5` has four components and is not a version, where its first three +/// characters-worth would have passed. +fn exact_version(text: &str) -> Option { + let parts: Vec<&str> = text.split('.').collect(); + if parts.len() == 3 + && parts + .iter() + .all(|p| !p.is_empty() && p.chars().all(|c| c.is_ascii_digit())) + { + Some(text.to_owned()) + } else { + None + } +} + +/// Which literal a finding is about. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Keyword { + Catalog, + Binary, +} + +/// Every stale literal in one document, as `(line number, which, value)`. +/// +/// Extracted from the corpus test so the rules below can be fed the forms they claim +/// to cover. The reason is a finding against this file: it had been "verified failing +/// first" by mutating a sample of the one form the matcher could already see, which +/// could not reveal that four of the five documents it existed for had zero coverage. +/// **An instrument is only proven on the cases you feed it**, so the cases are +/// enumerated in `the_matcher_sees_every_form_it_claims_to_cover` and +/// `a_marker_above_a_fence_covers_the_block` rather than left to a corpus that happens +/// to contain them today. +fn stale_literals( + text: &str, + expected_catalog: &str, + expected_binary: &str, +) -> Vec<(usize, Keyword, String)> { + // A marker inside a fenced block would render as sample output rather than as a + // comment, and one did: it shipped in a manual as part of an error message an + // operator would try to match. So a marker on the line immediately before a fence + // covers that whole block instead, and the fence state has to be tracked to know + // where the block ends. Fences are not skipped in general, because most of the + // literals worth checking are inside them. + let mut out = Vec::new(); + let mut in_marked_fence = false; + let mut marker_pending = false; + for (index, line) in text.lines().enumerate() { + let number = index + 1; + if line.trim_start().starts_with("```") { + if in_marked_fence { + in_marked_fence = false; + } else if marker_pending { + in_marked_fence = true; + } + marker_pending = false; + continue; + } + if line.contains("version-literal-ok:") { + // A marker alone on its line covers the block that follows. One at the end + // of a sentence covers that sentence only: otherwise inserting a code block + // after such a sentence would silence it, with no marker visible above the + // block and nothing to fail. + marker_pending = line.trim_start().starts_with("\n```\ncatalog 0.0.2\n```\n"; + assert_eq!( + stale_literals(marked, "0.0.3", "0.1.6"), + vec![], + "a marker on the line before a fence must cover the whole block" + ); + + let unmarked = "```\ncatalog 0.0.2\n```\n"; + assert_eq!( + stale_literals(unmarked, "0.0.3", "0.1.6"), + vec![(2, Keyword::Catalog, "0.0.2".to_owned())], + "an unmarked fence must still be checked, or the escape becomes a blanket" + ); + + // The marker must stop at the closing fence. Asserting the exact finding rather + // than "not empty" is what makes this fail if the exemption leaks past the block. + let leaky = "\n```\ncatalog 0.0.2\n```\ncatalog 0.0.1\n"; + assert_eq!( + stale_literals(leaky, "0.0.3", "0.1.6"), + vec![(5, Keyword::Catalog, "0.0.1".to_owned())], + "exactly the line after the fence is reported: the block is covered, the rest is not" + ); + + // An INLINE marker exempts its own line and must not arm the block scope. The + // in-tree inline marker is followed by prose today, so nothing was exempted, but a + // code block inserted after that sentence would have been silenced with no marker + // visible above it and nothing to fail. + let inline = + "Some prose. \n```\ncatalog 0.0.2\n```\n"; + assert_eq!( + stale_literals(inline, "0.0.3", "0.1.6"), + vec![(3, Keyword::Catalog, "0.0.2".to_owned())], + "a marker at the end of a sentence covers that sentence, not the next block" + ); +} + +#[test] +fn documentation_version_literals_match_the_compiled_constants() { + let expected_catalog = extenddb_storage_postgres::CATALOG_VERSION.to_string(); + let expected_binary = env!("CARGO_PKG_VERSION"); // inherited from the workspace + let root = repo_root(); + + let mut files = Vec::new(); + markdown_files(&root.join("docs"), &mut files); + let readme = root.join("README.md"); + if readme.exists() { + files.push(readme); + } + assert!( + files.len() > 10, + "expected to find the documentation set, found {} files under {}", + files.len(), + root.display() + ); + + let mut stale = Vec::new(); + for file in files { + let rel = file + .strip_prefix(&root) + .unwrap_or(&file) + .to_string_lossy() + .replace('\\', "/"); + if EXEMPT.iter().any(|(path, _)| *path == rel) { + continue; + } + let text = std::fs::read_to_string(&file).expect("read a documentation file"); + for (number, kind, found) in stale_literals(&text, &expected_catalog, expected_binary) { + let (label, expected) = match kind { + Keyword::Catalog => ("catalog", expected_catalog.as_str()), + Keyword::Binary => ("extenddb", expected_binary), + }; + stale.push(format!( + "{rel}:{number}: {label} {found}, compiled constant is {expected}" + )); + } + } + + assert!( + stale.is_empty(), + "documentation version literals are stale. Update them, add a \ + `` marker to a deliberately historical \ + line, or exempt the file with a reason in EXEMPT:\n {}", + stale.join("\n ") + ); +} diff --git a/crates/storage-sqlite/src/vector_search.rs b/crates/storage-sqlite/src/vector_search.rs index a7529eee..17b0c5fb 100644 --- a/crates/storage-sqlite/src/vector_search.rs +++ b/crates/storage-sqlite/src/vector_search.rs @@ -246,7 +246,8 @@ impl VectorSearchEngine for SqliteEngine { // `nrm` is deliberately not selected. The stored norm is an f32 and cannot // be trusted for either the zero test or the cosine denominator, so the // scorer recomputes both sides in f64 from the vector it already decoded. - // The column stays for operator inspection and for the other backend. + // The column stays because dropping it would need a data migration for + // no benefit; see the reasoning on `create_vector_data_table`. let sql = format!("SELECT vec, item_data FROM {vec_table} WHERE part = ?"); let k = usize::try_from(top_k.max(0)).unwrap_or(0); diff --git a/docs/adr/0006-pgvector-storage-and-scoring.md b/docs/adr/0006-pgvector-storage-and-scoring.md index e0e13e60..7e09ec67 100644 --- a/docs/adr/0006-pgvector-storage-and-scoring.md +++ b/docs/adr/0006-pgvector-storage-and-scoring.md @@ -30,10 +30,18 @@ longer ours (see decision 4). **2. Vector support is detected at runtime and fails closed.** Whether an ExtendDB build can serve vector indexes is a property of the -PostgreSQL server it is pointed at, not of the binary. The engine probes for the -extension once at startup and caches the answer, and a server without it refuses -every vector operation with a message naming the extension rather than failing -somewhere inside a query. The consequence, which is documented in the admin +PostgreSQL server it is pointed at, not of the binary. The PostgreSQL **backend** +probes for the extension once at startup and caches the answer, and a server +without it refuses every vector operation rather than failing somewhere inside a +query. + +The refusal does not name pgvector, and that is deliberate rather than an +oversight: the engine's capability gate is backend-agnostic by design, so it says +only that the capability is absent, and it is the same string whichever backend is +installed. Naming the cause is the startup log's job and the troubleshooting +entry's. A second refusal does name the extension, but it covers the narrower case +of an extension that disappears after the probe said yes, so it is unreachable when +a server started without pgvector. The consequence, which is documented in the admin guide: installing pgvector under a running server needs an ExtendDB restart, because the probe is not re-run. diff --git a/docs/design/02-high-level-design.md b/docs/design/02-high-level-design.md index dd91a29d..6503509a 100755 --- a/docs/design/02-high-level-design.md +++ b/docs/design/02-high-level-design.md @@ -365,11 +365,22 @@ The server runs on a tokio multi-thread runtime. Each incoming HTTP request is h All database access goes through an sqlx `PgPool`. The pool size is configurable via `storage.postgres.pool_size` in `extenddb.toml` (default: 20). When all connections are in use, new requests queue at the pool level until a connection is returned or the acquire timeout expires. If the timeout expires, the request fails with an internal server error (HTTP 500). Total connection footprint on the PostgreSQL server: -- `pool_size` connections for DynamoDB data operations (shared by all background workers) -- +2 for the management API (separate pool, `max_connections(2)`) -- +1 for the log-level poller (separate pool, `max_connections(1)`) - -So the total is `pool_size + 3`. The default PostgreSQL `max_connections` is 100, which comfortably supports the default pool_size of 20. +- `pool_size` for the catalog pool +- `pool_size` again for the data pool, because `extenddb init` always puts the + catalog and the data in separate databases and the engine opens a pool against + each. The two collapse into one only where a deployment shares a database. +- `catalog_pool_size` for the management pool (authorization, IAM, console), + which defaults to `pool_size` and has a minimum of 10 +- one session per vector index build running on the server, opened outside every + pool because its ownership lock is session-scoped +- one connection while a schema migration runs, pinned outside the pools so its + transaction-scoped lock cannot move between backends + +So the total is `2 * pool_size + catalog_pool_size`, plus builds and migrations. +There is no separate poller pool: the log-level poller borrows from the management +pool like every other background poller. `max_connections` is per cluster rather +than per database, so both `pool_size` pools draw on the same budget: the defaults +need 60 of PostgreSQL's default 100. ### 6.3 Row-Level Locking diff --git a/docs/design/04-component-storage.md b/docs/design/04-component-storage.md index a8b4d18b..0a0f30e1 100755 --- a/docs/design/04-component-storage.md +++ b/docs/design/04-component-storage.md @@ -838,13 +838,27 @@ backend crate: an invisible, compiles-fine failure mode. A name-keyed registry fixed that but kept string dispatch, which allows a class of runtime error that cannot exist now, because there is nothing to look up. -The components factory itself is unchanged in shape, and this is the body a -backend supplies: +A backend supplies one `backend()` function returning that value. Its fields are +factories, and the server-components one carries the body that used to live in the +registration: ```rust // In crates/storage-postgres/src/lib.rs -{ - ServerComponentsRegistration { +pub fn backend() -> extenddb_storage::Backend { + extenddb_storage::Backend { + name: "postgres", + bootstrapper: |config_path, cli_args| { /* ... */ }, + storage_config: |table| { /* ... */ }, + operations: &operations::PostgresOperationsEngine, + settings_store: |config| { /* ... */ }, + diagnostics_store: |config| { /* ... */ }, + server_components: server_components_factory, + } +} + +// and the factory itself, a free function with the body the registry used to hold +fn server_components_factory(config: &StorageConfig, region: &str) -> /* ... */ { + { backend: "postgres", factory: |config, region| { Box::pin(async move { @@ -1115,13 +1129,15 @@ In `lib.rs`: ```rust use extenddb_storage::{ - ServerComponents, ServerComponentsRegistration, BackendError, + Backend, ServerComponents, BackendError, StorageConfig, StorageEngine, CatalogStore, }; use extenddb_auth::{BuiltinAuthProvider, CredentialStore}; -{ - ServerComponentsRegistration { +// The same shape as the PostgreSQL backend above: a `backend()` returning +// `Backend { name, .., server_components }`, and the factory as a free function. +fn server_components_factory(config: &StorageConfig, region: &str) -> /* ... */ { + { backend: "sqlite", factory: |config, region| { Box::pin(async move { @@ -1218,11 +1234,20 @@ implements nothing declines by construction. A backend that participates returns **Fail closed, and decide it once.** Whether a backend can serve vector search may depend on the server it is connected to rather than on the build: the PostgreSQL backend probes for the pgvector extension at startup and caches the answer, then -declines for the process lifetime if it is absent. Two rules follow from the -engine's contract. A backend must not accept a vector index it cannot serve, and -it must not answer a search from a partially built index. Both refusals belong at -the storage boundary, because degrading instead produces a table that reports an -index and returns nothing from every search. +declines for the process lifetime if it is absent. + +Two refusals matter here and **neither is yours to write**. Refusing a vector index +on a backend that cannot serve one is the engine's capability gate, which needs no +code from a backend that simply returns `None`. Refusing to answer a search from an +index that is still building is also the engine's, on the search path, by filtering +to indexes that report `ACTIVE`; no backend has a status predicate in its search +query, and adding one would duplicate a decision that already exists. + +**What is yours** is the case a cached answer cannot cover: re-check any +environmental capability at the moment you persist catalog state. The PostgreSQL +backend runs a live `SELECT NULL::vector` before recording an index, because the +extension can be dropped after the startup probe said yes, and a catalog row with +no storage behind it is an index that reports itself and answers nothing. **The build lifecycle is shared; the SQL is not.** `extenddb_storage::vector_lifecycle` owns the ordering rules for the whole build: the batched backfill, the diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index f81d351c..1efa430d 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -64,12 +64,13 @@ adaptation when switching between ExtendDB and the real service. | Area | DynamoDB | ExtendDB | |------|----------|------| | GSI update propagation | Eventually consistent (milliseconds to seconds) | Per-GSI propagation delay. System default: `index_propagation_delay_ms` setting (default 10ms). Each GSI can override with its own `propagation_delay_ms` (stored in catalog). A value of 0 means synchronous (future sync GSI feature). | -| Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance synchronously in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. | -| `SearchVectors` score at extreme magnitudes (PostgreSQL backend only) | Returns the true distance as a number for any vector of finite components | The score is bounded to a finite value instead, and the bound is not a measured service answer. pgvector accumulates distances in single precision, so magnitudes far below `f32::MAX` overflow inside the extension: Euclidean above about 9.2e18, dot product above about 1.8e19, and cosine at both ends, above about 1.8e19 and below about 3.7e-23. A non-finite score cannot be serialised as JSON at all, so the result is bounded in SQL: `1e308` for an overflowed distance, `-1e308` for an overflowed negated inner product, which the score contract negates so a client sees `1e308` in `Score`, and 1.0 for a cosine that comes back NaN. Ranking is unaffected at the overflow end, because each bound sits at the end its metric overflows towards, so the farthest row stays farthest and the most similar stays most similar; two rows that both overflow tie, and the tie breaks on the base key. At the underflow end cosine loses resolution rather than being bounded: pgvector's underflowed norms yield infinities that it clamps before the value is read, so the reported distance collapses to one of 0, 1 or 2, following the sign of the inner product, with the 1.0 substitute firing only when the vectors are exactly orthogonal (measured). Ranking there still separates nearer-than-orthogonal from farther, and loses all resolution within each half. The SQLite backend owns its own arithmetic, computes in double precision, and reports the true value, which is why this row is scoped to PostgreSQL. | -| Vector index deletion window | `UpdateTable` Delete leaves the index in `DELETING` long enough to observe, then removes it | No observable `DELETING` window on either backend: the catalog row is removed inside the `UpdateTable` transaction, so a `DescribeTable` immediately afterwards already omits the index. The index's data table is dropped after that commit, in a separate transaction, best effort: on PostgreSQL it is a different database entirely, and a failure there is logged and skipped rather than failing the request. So an operator debugging a leftover `_ddb_vec_*` table should look for that warning rather than assume the delete was incomplete. | +| Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance inline in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. That zero behaves differently while an index is still building, and the two backends differ: **PostgreSQL** applies inline only to an index that is already `ACTIVE`, and defers a write to a building index whatever the delay says, because the write must not reach the index ahead of the backfill's older snapshot of the same item; **SQLite** does not check the status on this path, so with a zero delay a write during a backfill is applied inline and bypasses the hold that keeps the two ordered. The SQLite behaviour is tracked as a defect (F-20), not intended, and it is reachable only with the zero delay, which is a test setting. | +| `SearchVectors` score at extreme magnitudes (PostgreSQL backend only) | Returns the true distance as a number for any vector of finite components | The score is bounded to a finite value instead, and the bound is not a measured service answer. pgvector accumulates distances in single precision, so magnitudes far below `f32::MAX` overflow inside the extension: Euclidean above about 9.2e18, dot product above about 1.8e19, and cosine at both ends, above about 1.8e19 and below about 3.7e-23, which is where a component's single-precision square rounds to zero. A non-finite score cannot be serialised as JSON at all, so the result is bounded in SQL: `1e308` for an overflowed distance, `-1e308` for an overflowed negated inner product, which the score contract negates so a client sees `1e308` in `Score`, and 1.0 for a cosine that comes back NaN. Ranking is unaffected at the overflow end, because each bound sits at the end its metric overflows towards, so the farthest row stays farthest and the most similar stays most similar; two rows that both overflow tie, and the tie breaks on the base key. At the underflow end cosine loses resolution rather than being bounded, and which side is tiny decides how much. For a tiny **query** vector, its norm underflows while the inner product usually does not, so the quotient is an infinity that pgvector clamps and the reported distance collapses to one of 0, 1 or 2 following the sign of the inner product, with the 1.0 substitute firing only when the vectors are exactly orthogonal and the quotient is therefore 0/0 (measured). For a tiny **stored** vector it is worse: the stored norm is computed in single precision and reaches zero, so the zero-vector guard fires and that row reports 1.0 at every angle, parallel included. A corpus of tiny embeddings therefore loses ranking altogether rather than losing resolution. For a tiny query vector, by contrast, ranking still separates nearer-than-orthogonal from farther and loses resolution only within each half. The SQLite backend owns its own arithmetic, computes in double precision, and reports the true value, which is why this row is scoped to PostgreSQL. | +| Vector index deletion window | `UpdateTable` Delete leaves the index in `DELETING` long enough to observe, then removes it | No observable `DELETING` window on either backend: the catalog row is removed inside the `UpdateTable` transaction, so a `DescribeTable` immediately afterwards already omits the index. The index's data table is dropped after that commit, in a separate transaction, and the two backends handle a failure there differently. **PostgreSQL** treats it as best effort, because the data table lives in a different database entirely: a failure is logged and skipped rather than failing the request. **SQLite** propagates it, so a failed drop returns an error from an `UpdateTable` whose catalog change has already committed, which is the more surprising outcome of the two: the index is gone from the catalog and the caller saw a failure. So an operator debugging a leftover `_ddb_vec_*` table should look for that warning rather than assume the delete was incomplete. | | Restoring a backup of a table that had vector indexes | Restores the table with its vector indexes intact: the configuration survives, items keep their vector attributes, and `SearchVectors` works as soon as the table is `ACTIVE` (measured) | Neither backend restores the indexes, and the two fail differently. **PostgreSQL refuses the restore** with a `ValidationException` naming the backup and the index count, because restore does not carry index data across and a table that looks restored while answering every search with nothing is worse than a refusal a caller can act on. **SQLite does not refuse**: its backup path does not capture vector indexes at all, so a restore silently produces the table without them. That silence is tracked as a defect rather than intended, and it is the reason the PostgreSQL path refuses instead of matching it. A backup taken from a table with no vector indexes restores normally on both. | | `SearchVectors` endpoint | Served only on `search-dynamodb..amazonaws.com`. The standard `dynamodb..amazonaws.com` endpoint answers the same request with HTTP 400 `UnknownOperationException` ("This operation is not supported by this endpoint"); every control-plane and item operation stays on the standard endpoint. Signing is unchanged either way (service name `dynamodb`, target prefix `DynamoDB_20120810`) | Served on the same endpoint as every other operation, so a client is pointed at one ExtendDB endpoint for all of them. Two consequences worth knowing: an SDK that resolves a separate search hostname from its endpoint ruleset needs its endpoint overridden to reach ExtendDB, and ExtendDB does **not** reproduce the service's refusal, so a test asserting `UnknownOperationException` for vector search on the base endpoint passes against Amazon DynamoDB and fails here. | | `SearchVectors` result order for equal distances | Measured unstable: three identical searches returned tied rows in three different orders, and a top-k that truncates a tie group keeps an arbitrary subset of it | Deterministic on both backends, by different means. **PostgreSQL** sorts explicitly on the base table's full primary key after the score, so the order is a property of the query. **SQLite** issues no `ORDER BY` and resolves ties by scan order through a stable top-k, so its order is a property of the plan rather than something the query guarantees. Either way a client re-issuing an identical search sees the same order, which is stricter than the service rather than divergent in outcome. Do not rely on the two backends agreeing on which subset of a truncated tie group they keep. | +| Vector indexes on the MongoDB backend | Vector indexes and `SearchVectors` are available | Not supported at all. That backend provides no vector search implementation, so the engine's capability gate refuses `CreateTable` and `UpdateTable` carrying `VectorIndexes` and every `SearchVectors` request, with the same capability message a PostgreSQL deployment without pgvector returns. Every other vector row in this document describes the PostgreSQL and SQLite backends. | | Multi-part base table keys | Not supported | Preview extension (opt-in via `enable_multipart_keys` setting). Standard single/composite keys work identically. | ## Capacity and Throttling diff --git a/docs/getting-started.md b/docs/getting-started.md index cdb92be0..4c4e4c08 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1319,15 +1319,15 @@ The `--yes` flag is required to confirm destruction. Without it, the command exi ### Connection pool size -The `storage.postgres.pool_size` setting (default: 20, minimum: 10) controls the maximum number of concurrent PostgreSQL connections used for DynamoDB data operations. Each in-flight request that touches the database holds one connection for the duration of its transaction. Values below 10 are clamped at startup with a warning. +The `storage.postgres.pool_size` setting (default: 20, minimum: 10) sizes the connections used for DynamoDB data operations. It sizes **two** pools, not one: a deployment created by `extenddb init` keeps its catalog and its data in separate databases, and the engine opens a pool of `pool_size` against each. Both count against the same server-wide `max_connections`, because that limit is per cluster rather than per database. Each in-flight request that touches the database holds one connection for the duration of its transaction. Values below 10 are clamped at startup with a warning. The `storage.postgres.catalog_pool_size` setting controls the maximum number of concurrent connections for the management/catalog pool (authorization queries, IAM operations, console). Defaults to `pool_size` if not set, minimum: 10. With auth enabled (`provider = "builtin"`), each DynamoDB request makes concurrent authorization queries against this pool — size it to match expected concurrency. Values below 10 are clamped at startup with a warning. **When to increase:** If you see elevated latency under concurrent load, the pool may be saturated. Requests queue at the pool level when all connections are in use. Increase `pool_size` (and `catalog_pool_size` if auth is enabled) to allow more concurrent transactions. -**Relationship to PostgreSQL `max_connections`:** The total connection footprint is `pool_size + catalog_pool_size + 1` (the extra 1 is for the log-level poller), plus one connection per vector index build running on the server and one while a schema migration runs. Those extra sessions are opened outside both pools on purpose, for two different reasons. A build's ownership lock is session-scoped, so it needs a session that ends when the build does: a pooled connection would return to the pool still holding the lock. The migration lock is transaction-scoped, and needs a pinned connection because its explicit transaction has to retain one PostgreSQL backend until COMMIT, which is what stops a transaction-pooling proxy from letting the lock move between backends. PostgreSQL's default `max_connections` is 100. Ensure `pool_size + catalog_pool_size + 1`, plus the number of vector index builds you expect to overlap, does not exceed your PostgreSQL `max_connections` setting. +**Relationship to PostgreSQL `max_connections`:** The total connection footprint is `2 * pool_size + catalog_pool_size`, plus one connection per vector index build running on the server and one while a schema migration runs. The doubled term is the catalog pool and the data pool, both sized by `pool_size`; the third pool is the management pool sized by `catalog_pool_size`. A deployment whose catalog and data share one database is the only shape where the two `pool_size` pools collapse into one, and `extenddb init` does not produce it. Those extra sessions are opened outside both pools on purpose, for two different reasons. A build's ownership lock is session-scoped, so it needs a session that ends when the build does: a pooled connection would return to the pool still holding the lock. The migration lock is transaction-scoped, and needs a pinned connection because its explicit transaction has to retain one PostgreSQL backend until COMMIT, which is what stops a transaction-pooling proxy from letting the lock move between backends. PostgreSQL's default `max_connections` is 100, and the default pool sizes already need 60 of it. Ensure `2 * pool_size + catalog_pool_size`, plus the number of vector index builds you expect to overlap, does not exceed your PostgreSQL `max_connections` setting. -**Example:** To support 50 concurrent data operations with auth enabled, set both pools to 50 in `extenddb.toml` and ensure PostgreSQL allows at least 101 connections. +**Example:** To support 50 concurrent data operations with auth enabled, set both pools to 50 in `extenddb.toml`. That configuration can open 150 connections, so PostgreSQL needs `max_connections` raised to at least 150 plus headroom for overlapping vector index builds. The default of 100 is not enough for it: an operator who sizes for 100 here meets `FATAL: sorry, too many clients already` at exactly the concurrency the setting was chosen to support. ```toml [storage.postgres] diff --git a/docs/manuals/01-architecture-guide.md b/docs/manuals/01-architecture-guide.md index fa885866..bd40427c 100755 --- a/docs/manuals/01-architecture-guide.md +++ b/docs/manuals/01-architecture-guide.md @@ -68,7 +68,7 @@ The `dispatch` function routes `X-Amz-Target` operation names to handlers. ### storage -Trait definitions for the storage layer. Thirteen storage traits partition backend responsibilities: +Trait definitions for the storage layer. Thirteen required storage traits partition backend responsibilities: - **TableEngine**: Table lifecycle (create, delete, describe, list, update) - **DataEngine**: Item CRUD (put, get, update, delete, query, scan, batch, transact) @@ -82,9 +82,11 @@ Trait definitions for the storage layer. Thirteen storage traits partition backe - **MetricsStore**: Metrics collection and retrieval - **RateLimitStore**: Rate limiting state - **AuthorizationStore**: Policy evaluation cache + +Two further traits are optional and belong to vector search: **VectorSearchEngine**, which a backend hands over through `as_vector_search` or declines by returning `None`, and **VectorIndexBuild**, the storage primitives the shared build lifecycle drives. A backend that implements neither refuses every vector operation and is otherwise unaffected. - **Bootstrapper**: Initial database setup -Traits use `BoxFuture` for object safety. A binary serves exactly one backend, installed from its thin `main` via `set_backend`. The `RuntimeHooks` trait allows backends to spawn backend-specific workers (PostgreSQL spawns 7). +Traits use `BoxFuture` for object safety. A binary serves exactly one backend, installed from its thin `main` via `set_backend`. The `ServerRuntimeHooks` trait allows backends to spawn backend-specific workers (PostgreSQL spawns 7). ### storage-postgres @@ -170,7 +172,9 @@ extenddb uses a dual-database architecture: - **Catalog database** (e.g., `extenddb_catalog`): Stores table metadata, account/user/group/role/policy definitions, access keys, settings, stream metadata, and metrics. Shared across all accounts. - **Data database** (e.g., `extenddb_catalog_data`): Stores user items, GSI/LSI data, and stream records. Each table gets its own PostgreSQL table. -The catalog version is stored in the `settings` table under the key `catalog_version` and checked at startup. It is per backend: PostgreSQL and SQLite are at 0.0.3, MongoDB at 0.0.2. A mismatch between the compiled-in version and the stored one prevents the server from starting; run `extenddb migrate` to upgrade. +The catalog version is 0.0.3 on PostgreSQL and SQLite, stored in the `settings` table under the key `catalog_version` and checked at startup. +The MongoDB backend tracks its own catalog version, 0.0.2. +A mismatch between the compiled-in version and the stored one prevents the server from starting; run `extenddb migrate` to upgrade. ## Pluggable Architecture diff --git a/docs/manuals/03-usage-guide.md b/docs/manuals/03-usage-guide.md index 4aecc29b..861f6bba 100755 --- a/docs/manuals/03-usage-guide.md +++ b/docs/manuals/03-usage-guide.md @@ -399,6 +399,54 @@ aws dynamodbstreams get-records \ Both DynamoDB and DynamoDB Streams endpoints use the same extenddb server URL. See `samples/stream_consumer.py` for a complete working example. +## Vector Search + +Available on the PostgreSQL backend when the pgvector extension is present on the +data database, and on the SQLite backend. A deployment without pgvector refuses +every vector operation and is otherwise unaffected. The refusal comes in two +strings, which matters if you grep logs for one of them. `CreateTable` and +`UpdateTable` carrying `VectorIndexes` are refused with: + +`Vector indexes are not supported by this storage backend` + +and a search is refused with: + +`SearchVectors is not supported by this storage backend` + +See the admin guide for the detection and the restart requirement. + +A vector index is declared on a table that uses on-demand capacity, either at +CreateTable or by UpdateTable on an existing table. The index names the attribute +holding the vector, its dimension count, and one of three distance functions +(`COSINE`, `EUCLIDEAN`, `DOT_PRODUCT`). Five vector indexes per table is the limit. + +Items are written normally: the vector attribute is a list of numbers with exactly +the declared number of dimensions. An item that omits it is stored and simply does +not appear in that index, the same way a GSI ignores an item missing its key. + +`SearchVectors` returns the nearest items with a `Score` per hit, ordered nearest +first, and takes an optional `TopK` up to 100. + +Four things are worth knowing before building on it. + +**Amazon DynamoDB serves SearchVectors on a separate endpoint**, +`search-dynamodb..amazonaws.com`, and rejects it on the standard endpoint. +ExtendDB serves every operation on one endpoint, so an SDK that resolves a distinct +search hostname from its endpoint ruleset needs its endpoint overridden. + +**A vector index is eventually consistent**, like a GSI: a search immediately after +a write may not see the item. Adding an index to a table that already holds items +starts a backfill, during which the index reports `CREATING`, the table stays +writable, and searches against that index are refused until it is `ACTIVE`. + +**On-demand capacity is required.** A table holding a vector index cannot be +switched to provisioned mode, and a vector index cannot be added to a provisioned +table. + +**Scores and ranking differ from the service in documented ways** at extreme +component magnitudes, and tie ordering is deterministic here where the service's is +measured unstable. Both are in `docs/differences-from-dynamodb.md`. + ## Other Operations ### DescribeEndpoints @@ -509,6 +557,7 @@ extenddb reproduces DynamoDB error responses exactly. Common errors: | BatchWriteItem / BatchGetItem | ✓ | | TransactWriteItems / TransactGetItems | ✓ | | Global Secondary Indexes (GSI) | ✓ | +| Vector indexes and SearchVectors | ✓ (PostgreSQL requires the pgvector extension) | | Local Secondary Indexes (LSI) | ✓ | | DynamoDB Streams | ✓ | | ConditionExpression / FilterExpression / UpdateExpression / ProjectionExpression | ✓ | diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index 45b844ae..90fe53f8 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -176,8 +176,8 @@ Managed via `extenddb settings set`. Changes take effect within 30 seconds witho | `log_level` | `info` | Log level: trace, debug, info, warn, error | | `control_plane_delay_seconds` | `5` | Delay for table status transitions (0 = instant) | | `allow_credential_import` | `true` | Whether `import-access-key` is allowed | -| `vector_backfill_batch_delay_ms` | `0` | **Test-oriented.** Milliseconds to pause between batches while a vector index backfills. Zero in production. A test sets it so a write is guaranteed to land while the index is still building; the pause is outside any lock, so the table stays writable throughout either way. | -| `vector_allocation_phase_delay_ms` | `0` | **Test-oriented.** Milliseconds to hold a new vector index in the resource-allocation phase (`CREATING` with `Backfilling: false`) before the scan starts. Zero in production. Both transitions otherwise happen inside one `UpdateTable` call, so without this a client cannot observe the phase that the delete rule turns on. | +| `vector_backfill_batch_delay_ms` | `0` | **Test-oriented.** Milliseconds to pause between batches while a vector index backfills. Zero in production. A test sets it so a write is guaranteed to land while the index is still building. The pause is outside the batch transaction, so writes are still accepted throughout, but it does extend the per-table propagation hold: no index on that table advances while the build runs, GSIs included, and the accepted range goes up to 60 s per batch. | +| `vector_allocation_phase_delay_ms` | `0` | **Test-oriented.** Milliseconds to hold a new vector index in the resource-allocation phase (`CREATING` with `Backfilling: false`) before the scan starts. Zero in production. Without it the phase lasts only from the `UpdateTable` transaction, which inserts the row as `CREATING` with `Backfilling: false`, until the detached build task flips the flag, which is a window no client can time reliably rather than one that cannot exist. | ```bash # View current settings @@ -544,6 +544,7 @@ Check that PostgreSQL is running and the connection string in `extenddb.toml` is **Catalog version mismatch:** + ``` Error: catalog version mismatch: found 1.0.0, expected 0.0.3 ``` diff --git a/docs/manuals/11-deployment-guide.md b/docs/manuals/11-deployment-guide.md index 6035fbd1..cd229b80 100755 --- a/docs/manuals/11-deployment-guide.md +++ b/docs/manuals/11-deployment-guide.md @@ -250,7 +250,7 @@ Key PostgreSQL settings for extenddb workloads: - `shared_buffers`: 25% of available RAM - `effective_cache_size`: 75% of available RAM - `work_mem`: 64MB (for sort operations in Query/Scan) -- `max_connections`: ≥ extenddb pool_size + 10, and add one per vector index build you expect to overlap (a build holds its own session outside the pools) +- `max_connections`: ≥ `2 * pool_size + catalog_pool_size`, plus one per vector index build you expect to overlap and one during a schema migration. `pool_size` sizes both the catalog and the data pool, and `max_connections` is per cluster, so the default pool sizes need 60 of PostgreSQL's default 100 ### Monitoring Queries diff --git a/docs/technical-debt.md b/docs/technical-debt.md index 39d440e9..b4b531ef 100755 --- a/docs/technical-debt.md +++ b/docs/technical-debt.md @@ -31,7 +31,8 @@ Last updated: 2026-08-20 | F-16 | `transact_write_items.rs` passes `None` for `old_item` in stream capture — `OldImage` always `None` for transaction-originated stream records | `engine/transact_write_items.rs` | Medium | P27 | | F-17 | `validate_attribute_name_sizes` only checks top-level attribute names — nested map keys not validated | `core/validation/mod.rs` | Low | P30 | | F-18 | ~~UpdateTable Delete of a vector index in the resource-allocation phase (`CREATING`, `Backfilling: false`) is accepted; Amazon DynamoDB refuses it with `ResourceInUseException` until backfilling starts~~ Both backends now enforce the phase rule with the measured message, and both hold the phase open under `vector_allocation_phase_delay_ms` so a client can observe it | ~~`storage-sqlite/update_table.rs` (vector delete branch), `core/types/table.rs` (`vector_index_delete_in_allocation_phase`)~~ | ~~Medium~~ | vector probe P2 | -| F-19 | SQLite backup does not capture vector indexes, so `RestoreTableFromBackup` silently produces the table without them. Amazon DynamoDB preserves vector state through backup and restore (measured). The PostgreSQL backend refuses the restore rather than matching this, so the two backends fail differently: one refuses with a reason, one loses declared indexes quietly. Fix: capture the index set in the SQLite backup row and either restore it or refuse, matching PostgreSQL | `storage-sqlite/backup.rs` (no `vector_indexes` capture), `storage-postgres/backup_engine.rs:467-497` (the refusal to match) | Medium | PR-2 docs stage | +| F-19 | SQLite backup does not capture vector indexes, so `RestoreTableFromBackup` silently produces the table without them. Amazon DynamoDB preserves vector state through backup and restore (measured). The PostgreSQL backend refuses the restore rather than matching this, so the two backends fail differently: one refuses with a reason, one loses declared indexes quietly. Fix: capture the index set in the SQLite backup row and either restore it or refuse, matching PostgreSQL | `storage-sqlite/backup.rs:313-317` (the restore reads three columns: `key_schema`, `attribute_definitions`, `billing_mode`, none of which can carry an index), `storage-postgres/backup_engine.rs:140-149` and `:167` (captures the index set into the backup row), `:467-497` (refuses the restore) | Medium | PR-2 docs stage | +| F-20 | SQLite applies vector index maintenance inline whenever `index_propagation_delay_ms` is 0, without checking the index status, so a write landing during a backfill reaches the index data table directly and bypasses the claim-time hold that keeps it behind the backfill's older snapshot of the same item. The shared lifecycle contract requires that hold (`storage/vector_lifecycle/mod.rs:31-38`), and PostgreSQL enforces it with `delay_ms == 0 && index_status == "ACTIVE"`. Reachable only with the zero delay, which is a test setting. The symptom is not a failed build: the backfill's plain INSERT raises a primary key violation, which propagates out of the batch, and the detached build deliberately leaves the index in `CREATING` because there is no failure state on the wire. So an operator sees an index parked in `CREATING` with every search against it refused, until the stuck-build sweep or the startup reconciler rebuilds it; that rebuild drops and recreates the data table, so the retry starts clean and the state self-heals unless the interleaving repeats. The self-heal is why this is Medium rather than High. Fix: gate the inline branch on the index being ACTIVE, matching PostgreSQL | `storage-sqlite/data/vector_index.rs:176` (inline branch), `storage-postgres/data/vector_index.rs:161` (the shape to match) | Medium | PR-2 docs stage | ## Cleanup diff --git a/extenddb.sample.toml b/extenddb.sample.toml index a4404f3a..eaf6cbe2 100755 --- a/extenddb.sample.toml +++ b/extenddb.sample.toml @@ -48,12 +48,18 @@ # pool_size = 20 # Maximum concurrent database connections # for DynamoDB data operations. Default: 20. # Minimum: 10 (smaller values are clamped with - # a startup warning). Total PostgreSQL - # connections used: pool_size + catalog_pool_size + 1 - # (log-level poller: 1), plus one per vector - # index build running on this server and one - # while a schema migration runs. Both sit - # OUTSIDE both pools, for two different reasons. + # a startup warning). This value sizes TWO + # pools: one against the catalog database and + # one against the data database, which `init` + # always creates separately. Total PostgreSQL + # connections used: 2 * pool_size + + # catalog_pool_size, plus one per vector index + # build running on this server and one while a + # schema migration runs. max_connections is + # per cluster, not per database, so both pools + # draw on the same budget. The build session and + # the migration connection sit OUTSIDE every + # pool, for two different reasons. # A build's ownership lock is SESSION-scoped, so # it needs a session that ends when the build # ends: a pooled connection returns to the pool @@ -64,9 +70,11 @@ # until COMMIT; a transaction-pooling proxy could # otherwise let the lock move between backends. # Increase for higher concurrency; ensure - # PostgreSQL max_connections >= pool_size + - # catalog_pool_size + 1 + the number of vector - # index builds you expect to overlap. + # PostgreSQL max_connections >= 2 * pool_size + # + catalog_pool_size + the number of vector + # index builds you expect to overlap. The + # defaults already need 60 of PostgreSQL's + # default 100. # catalog_pool_size = 20 # Maximum concurrent connections for the # management/catalog pool (authz, IAM, console). # Defaults to pool_size if not set. diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index a1503a58..b87cd3bf 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1415,6 +1415,13 @@ async fn a_tiny_query_vector_still_scores_every_hit_as_a_number() { // orthogonal row is exactly 1.0. A backend that treats the query as zero returns // 1.0 for BOTH and still passes every assertion above, which is precisely the // state one backend was in: it reported a ranking it had not computed. + // + // These two values are safe to pin forever, and the reason is worth stating for + // whoever is tempted to weaken them: they are the metric's DEFINITION at 0 and 90 + // degrees, not an implementation's current behaviour. Cosine distance is exactly 0 + // for a parallel pair and exactly 1 for an orthogonal one whatever the magnitudes + // involved, so no correct implementation can fail this, and widening the shared + // norm helper to f64, which is the obvious later tidy-up, changes neither number. let pks = hit_pks(&response); assert_eq!( pks, @@ -1436,8 +1443,11 @@ async fn a_tiny_query_vector_still_scores_every_hit_as_a_number() { ); assert!( (score_of("orthogonal") - 1.0).abs() < 1e-6, - "and orthogonal to it is exactly 1.0, by the metric rather than by the guard: \ - got {}", + "and orthogonal to it is exactly 1.0 on both backends: by the metric on \ + SQLite, which computes in f64, and by the NaN substitute on PostgreSQL, \ + where pgvector's own f32 norm underflows and the operator divides zero by \ + zero. Either way not by the zero-vector guard, which cannot fire because \ + the query norm is computed in f64: got {}", score_of("orthogonal") ); @@ -1501,6 +1511,54 @@ async fn an_extreme_magnitude_query_scores_as_a_number_under_every_metric() { } } +/// The other side of the tiny-vector boundary: an all-zero query vector. +/// +/// The pair is what makes either half meaningful from outside the process. A tiny +/// query is not a zero vector, so it must produce a computed ranking, 0 for the +/// parallel row and 1.0 for the orthogonal one. A genuinely zero query has no +/// direction, so every row must score the same 1.0, and the difference between the two +/// results is what shows the zero-vector guard fires where it should and not where it +/// should not. With only the tiny case pinned, a backend that treated every small +/// vector as zero would still pass one test and fail the pair. +/// +/// 1.0 is the measured service answer rather than a choice: writing and cosine +/// searching an all-zeros vector both succeed against Amazon DynamoDB, and it defines +/// the score involving a zero vector as exactly 1.0 on either side. The expected +/// rejection does not exist. +#[tokio::test] +async fn a_zero_query_vector_scores_every_hit_as_the_zero_vector_answer() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_zero_query"); + create_vector_table(&name, 4, "COSINE", false).await; + + put_vector(&name, "parallel", None, &[1.0, 0.0, 0.0, 0.0]).await; + put_vector(&name, "orthogonal", None, &[0.0, 1.0, 0.0, 0.0]).await; + search_until_count(&name, &[1.0, 0.0, 0.0, 0.0], 10, None, 2).await; + + let response = search(&name, &[0.0, 0.0, 0.0, 0.0], 10, None).await; + let hits = response + .get("SearchResults") + .and_then(|v| v.as_array()) + .unwrap_or_else(|| panic!("no results array in: {response}")) + .clone(); + assert_eq!(hits.len(), 2, "both rows must come back: {response}"); + for hit in &hits { + let score = hit + .get("Score") + .and_then(serde_json::Value::as_f64) + .unwrap_or_else(|| panic!("no numeric score: {hit}")); + assert!( + (score - 1.0).abs() < 1e-6, + "a zero query vector has no direction, so every row is the zero-vector \ + answer of exactly 1.0, whatever its own direction: got {score} for {hit}" + ); + } + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + /// The same collapse on the other path a client can reach: an index added by /// UpdateTable. /// From 5d6aedbbcdd25a4bcb4cc1c36cd4fba7f83afbfb Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Mon, 24 Aug 2026 20:15:41 +0000 Subject: [PATCH 09/13] docs: audit findings on refusal strings, capability claims, and version samples The capability refusal is two strings, kept greppable in the source and stated in the ADR; F-19 and F-20 carry measured bounds; the extension-install comments claim only what the measurement supports; sample version output follows the workspace version. --- crates/storage-postgres/src/bootstrapper.rs | 10 ++++++---- docs/adr/0006-pgvector-storage-and-scoring.md | 4 ++-- docs/design/04-component-storage.md | 5 ++++- docs/getting-started.md | 4 ++-- docs/manuals/04-quickstart-setup-guide.md | 2 +- docs/technical-debt.md | 15 +++++---------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/storage-postgres/src/bootstrapper.rs b/crates/storage-postgres/src/bootstrapper.rs index 7ea69bb1..b7f5a4d0 100755 --- a/crates/storage-postgres/src/bootstrapper.rs +++ b/crates/storage-postgres/src/bootstrapper.rs @@ -148,10 +148,12 @@ impl PostgresBootstrapper { /// Try to install pgvector on the data database, tolerating refusal. /// - /// Init and migrate are the moments when this process holds the privileges - /// that `CREATE EXTENSION` needs: the data database is owned by the - /// application role, and pgvector is a trusted extension, so its owner can - /// install it without being a superuser. Serve-time code never attempts + /// This is an attempt, not a guarantee: pgvector's control file does not + /// mark the extension trusted, so `CREATE EXTENSION` is refused for a + /// non-superuser even on a database its role owns ("Must be superuser", + /// measured on a stock install). On refusal the printed hint tells the + /// operator to create it once as a superuser or as the database owner. + /// Serve-time code never attempts /// this, because a request path must not carry data-definition privileges it /// only needs once. /// diff --git a/docs/adr/0006-pgvector-storage-and-scoring.md b/docs/adr/0006-pgvector-storage-and-scoring.md index 7e09ec67..fe0fd554 100644 --- a/docs/adr/0006-pgvector-storage-and-scoring.md +++ b/docs/adr/0006-pgvector-storage-and-scoring.md @@ -37,8 +37,8 @@ query. The refusal does not name pgvector, and that is deliberate rather than an oversight: the engine's capability gate is backend-agnostic by design, so it says -only that the capability is absent, and it is the same string whichever backend is -installed. Naming the cause is the startup log's job and the troubleshooting +only that the capability is absent, in the same two strings whichever backend is +installed: one for the index operations and one for a search. Naming the cause is the startup log's job and the troubleshooting entry's. A second refusal does name the extension, but it covers the narrower case of an extension that disappears after the probe said yes, so it is unreachable when a server started without pgvector. The consequence, which is documented in the admin diff --git a/docs/design/04-component-storage.md b/docs/design/04-component-storage.md index 0a0f30e1..887b1e2f 100755 --- a/docs/design/04-component-storage.md +++ b/docs/design/04-component-storage.md @@ -1224,7 +1224,10 @@ cargo test --workspace Vector search is the one capability a backend may decline. Declining is a supported end state, not a stub: the engine refuses every vector operation with a -message naming what is missing, and no other operation is affected. +message naming the missing capability rather than its cause, and no other operation is +affected. The refusal deliberately does not name the environmental reason: that is the +startup log's job and the troubleshooting entry's, which is why a backend needs no +hook to produce it. **The opt-out is a single method.** `DataEngine::as_vector_search` returns `Option<&dyn VectorSearchEngine>` and defaults to `None`, so a backend that diff --git a/docs/getting-started.md b/docs/getting-started.md index 4c4e4c08..8b862d37 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -215,7 +215,7 @@ extenddb runs as a daemon (background process) and logs to syslog. On startup it ```bash ./target/release/extenddb serve --config extenddb.toml -# extenddb 0.1.6 (catalog 0.0.3) starting on 127.0.0.1:18443 +# extenddb 0.1.8 (catalog 0.0.3) starting on 127.0.0.1:18443 # storage: postgres (postgresql://extenddb:***@localhost:5432/extenddb_catalog) ``` @@ -1293,7 +1293,7 @@ Each runner requires its tools to be installed. The runner checks prerequisites ```bash ./target/release/extenddb version -# extenddb 0.1.6 +# extenddb 0.1.8 # catalog 0.0.3 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z diff --git a/docs/manuals/04-quickstart-setup-guide.md b/docs/manuals/04-quickstart-setup-guide.md index d629a8a1..4fef43b4 100755 --- a/docs/manuals/04-quickstart-setup-guide.md +++ b/docs/manuals/04-quickstart-setup-guide.md @@ -129,7 +129,7 @@ Check the version: ```bash ./target/release/extenddb version -# extenddb 0.1.6 +# extenddb 0.1.8 # catalog 0.0.3 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z diff --git a/docs/technical-debt.md b/docs/technical-debt.md index b4b531ef..ec062611 100755 --- a/docs/technical-debt.md +++ b/docs/technical-debt.md @@ -31,8 +31,8 @@ Last updated: 2026-08-20 | F-16 | `transact_write_items.rs` passes `None` for `old_item` in stream capture — `OldImage` always `None` for transaction-originated stream records | `engine/transact_write_items.rs` | Medium | P27 | | F-17 | `validate_attribute_name_sizes` only checks top-level attribute names — nested map keys not validated | `core/validation/mod.rs` | Low | P30 | | F-18 | ~~UpdateTable Delete of a vector index in the resource-allocation phase (`CREATING`, `Backfilling: false`) is accepted; Amazon DynamoDB refuses it with `ResourceInUseException` until backfilling starts~~ Both backends now enforce the phase rule with the measured message, and both hold the phase open under `vector_allocation_phase_delay_ms` so a client can observe it | ~~`storage-sqlite/update_table.rs` (vector delete branch), `core/types/table.rs` (`vector_index_delete_in_allocation_phase`)~~ | ~~Medium~~ | vector probe P2 | -| F-19 | SQLite backup does not capture vector indexes, so `RestoreTableFromBackup` silently produces the table without them. Amazon DynamoDB preserves vector state through backup and restore (measured). The PostgreSQL backend refuses the restore rather than matching this, so the two backends fail differently: one refuses with a reason, one loses declared indexes quietly. Fix: capture the index set in the SQLite backup row and either restore it or refuse, matching PostgreSQL | `storage-sqlite/backup.rs:313-317` (the restore reads three columns: `key_schema`, `attribute_definitions`, `billing_mode`, none of which can carry an index), `storage-postgres/backup_engine.rs:140-149` and `:167` (captures the index set into the backup row), `:467-497` (refuses the restore) | Medium | PR-2 docs stage | -| F-20 | SQLite applies vector index maintenance inline whenever `index_propagation_delay_ms` is 0, without checking the index status, so a write landing during a backfill reaches the index data table directly and bypasses the claim-time hold that keeps it behind the backfill's older snapshot of the same item. The shared lifecycle contract requires that hold (`storage/vector_lifecycle/mod.rs:31-38`), and PostgreSQL enforces it with `delay_ms == 0 && index_status == "ACTIVE"`. Reachable only with the zero delay, which is a test setting. The symptom is not a failed build: the backfill's plain INSERT raises a primary key violation, which propagates out of the batch, and the detached build deliberately leaves the index in `CREATING` because there is no failure state on the wire. So an operator sees an index parked in `CREATING` with every search against it refused, until the stuck-build sweep or the startup reconciler rebuilds it; that rebuild drops and recreates the data table, so the retry starts clean and the state self-heals unless the interleaving repeats. The self-heal is why this is Medium rather than High. Fix: gate the inline branch on the index being ACTIVE, matching PostgreSQL | `storage-sqlite/data/vector_index.rs:176` (inline branch), `storage-postgres/data/vector_index.rs:161` (the shape to match) | Medium | PR-2 docs stage | +| F-19 | **Reachable in default configuration, no setting required.** SQLite backup does not capture vector indexes, so `CreateBackup` followed by `RestoreTableFromBackup` silently produces the table without them and reports success. Its `backups` table has no column that could carry an index set, where PostgreSQL added one specifically so it could detect the case and refuse, and neither SQLite entry point is gated by anything. That makes this the more reachable of the two vector gaps: F-20 needs `index_propagation_delay_ms` at zero, which is a test setting, and this needs nothing. Raised to High on that reachability: severity attaches to what a default configuration can reach, and the earlier Medium rested on a belief about whether operators back up vector tables rather than on any barrier in the code. Amazon DynamoDB preserves vector state through backup and restore (measured). The PostgreSQL backend refuses the restore rather than matching this, so the two backends fail differently: one refuses with a reason, one loses declared indexes quietly. Fix: capture the index set in the SQLite backup row and either restore it or refuse, matching PostgreSQL | `storage-sqlite/backup.rs:313-317` (the restore reads three columns: `key_schema`, `attribute_definitions`, `billing_mode`, none of which can carry an index), `storage-postgres/backup_engine.rs:140-149` and `:167` (captures the index set into the backup row), `:467-497` (refuses the restore) | High | PR-2 docs stage | +| F-20 | SQLite applies vector index maintenance inline whenever `index_propagation_delay_ms` is 0, without checking the index status, so a write landing during a backfill reaches the index data table directly and bypasses the claim-time hold that keeps it behind the backfill's older snapshot of the same item. The shared lifecycle contract requires that hold (`storage/vector_lifecycle/mod.rs:31-38`), and PostgreSQL enforces it with `delay_ms == 0 && index_status == "ACTIVE"`. Reachable only with the zero delay, which is a test setting. The symptom is not a failed build: the backfill's plain INSERT raises a primary key violation, which propagates out of the batch, and the detached build deliberately leaves the index in `CREATING` because there is no failure state on the wire. So an operator sees an index parked in `CREATING` with every search against it refused, until the stuck-build sweep or the startup reconciler rebuilds it; that rebuild drops and recreates the data table, so the retry starts clean and the state self-heals unless the interleaving repeats. On SQLite, which is the backend this item is about, that recovery is bounded at roughly two seconds: the propagation worker sweeps every pass with a maximum sleep of one second and requires two consecutive sightings before rebuilding. Do not generalise that figure: PostgreSQL's runtime sweep polls every 60 seconds and only treats a build as stuck once its heartbeat is 300 seconds stale, so an equivalent park there would persist for up to about six minutes. The bounded SQLite recovery is why this is Medium rather than High. Fix: gate the inline branch on the index being ACTIVE, matching PostgreSQL | `storage-sqlite/data/vector_index.rs:176` (inline branch), `storage-postgres/data/vector_index.rs:161` (the shape to match) | Medium | PR-2 docs stage | ## Cleanup @@ -69,20 +69,15 @@ Last updated: 2026-08-20 | # | Item | Location | Priority | Origin | |---|------|----------|----------|--------| -| A-1 | Catalog/data database separation not implemented (REQ-CAT-001/002) | `storage-postgres/src/lib.rs` | High | P40 | +| A-1 | ~~Catalog/data database separation not implemented (REQ-CAT-001/002)~~ Implemented at the location this row cites: the runtime reads the data connection string from the catalog and opens a second pool, and the item, GSI-queue, and vector-index paths run on it. Residual: the pool selection falls back to the catalog pool when the setting is absent, so a hand-written configuration can run both roles on one database without a warning, which makes the separation enforced by `extenddb init` rather than by the code path | `storage-postgres/src/lib.rs:238-256` (the second pool and its fallback) | ~~High~~ Low | P40 | ### A-1: Catalog/Data Database Separation **Design requirement:** Two databases — catalog (`extenddb`) for metadata, data (`extenddb_data`) for user items (REQ-CAT-001, REQ-CAT-002). -**Current state:** `extenddb init` correctly creates both databases and stores the data connection string in the settings table. However, the runtime (`PostgresEngine`) only opens one connection pool to the catalog database. All `_ddb_*` item tables are created in the catalog database. The `extenddb_data` database exists but sits empty. The settings table has `data_database_connection_string` and `data_database_name` but the code never reads them at runtime. +**Current state:** Implemented on PostgreSQL. `extenddb init` creates both databases, records the data connection string in the catalog `settings` table (`storage/src/bootstrapper.rs:139`, called at `app/src/cmd_init.rs:334`), and initialises the data schema. The runtime reads that setting and opens a second pool (`storage-postgres/src/lib.rs:238-256`). Table creation, the GSI queue, the item-write paths, vector index data, and the backup and stream engines all take that pool (`create_table.rs:311`, `gsi_queue.rs`, `data/put_item.rs`, `data/vector_index.rs`, `backup_engine.rs`, `stream_engine.rs`), and `create_table.rs:300` commits catalog metadata before creating data tables. `extenddb destroy` drops both databases (`storage-postgres/src/bootstrapper.rs:725` and `:734`). SQLite co-locates catalog and data by design, which its worker module states (`storage-sqlite/src/workers.rs:11`). -**What needs to change:** -1. Open a second connection pool for the data database at startup -2. Route item storage operations (`_ddb_*` tables) to the data pool -3. Update transaction boundaries — catalog metadata and data writes may need coordinated commits -4. Update GSI queue to use the data pool for item data -5. Update `extenddb destroy` to drop both databases +**Residual, which is a note rather than an architectural gap:** the pool selection ends in `_ => pool.clone()`, so a catalog with no `data_database_connection_string` runs both roles on one database and nothing says so. The separation is enforced by `init` rather than by the code path. That is also what the connection formula in `docs/getting-started.md` assumes: it says the collapsed shape is the only one where the two `pool_size` pools become one, and that `extenddb init` does not produce it. Fix, if wanted: warn or refuse when the setting is absent, instead of collapsing silently. ## Unenforced DynamoDB Limits From 5048cf13d93ebe36b71d568a10caa64dde400ce9 Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Tue, 25 Aug 2026 02:57:42 +0000 Subject: [PATCH 10/13] fix: review findings on the vector index surface Five behavior fixes from the review of this PR, on both backends wherever the rule is shared. A write no longer applies inline while its table has queued rows. Queue order only ordered queued rows against each other, so once an index flipped to ACTIVE and released its hold, an older queued row could be applied after a newer inline write to the same item and leave the index disagreeing with the base table until that item was written again. Completing a build whose index was deleted now drops the data table the build recreated. A rebuild recreates that table after reloading the definition, so a delete landing in between left a table nothing referenced. UpdateTable refuses an index name another index family already holds. CreateTable already enforced this across families; UpdateTable consulted only its own catalog table, so a GSI and a vector index could share a name and therefore an ARN. A write whose vector data table vanishes mid-apply is skipped under a savepoint. The failed statement used to abort the whole transaction, so the write answered InternalServerError where the service answers normally. update_table reads the stored billing mode once, under the row lock it already holds, rather than re-querying it twice. --- .../storage-postgres/src/data/vector_index.rs | 104 +++++++++++++++++- crates/storage-postgres/src/update_table.rs | 102 ++++++++--------- .../storage-sqlite/src/data/vector_index.rs | 23 +++- crates/storage-sqlite/src/update_table.rs | 61 ++++++---- 4 files changed, 214 insertions(+), 76 deletions(-) diff --git a/crates/storage-postgres/src/data/vector_index.rs b/crates/storage-postgres/src/data/vector_index.rs index 98b66b99..e729cb3e 100644 --- a/crates/storage-postgres/src/data/vector_index.rs +++ b/crates/storage-postgres/src/data/vector_index.rs @@ -151,6 +151,49 @@ pub(crate) async fn maintain_vector_indexes( return Ok(0); } + // Inline apply is only safe while this table's propagation queue is empty. + // + // Queue order protects queued rows against each other, never a queued row + // against an inline one. Once an index flips to ACTIVE and its hold goes, rows + // enqueued while it was CREATING drain at the same time as new writes take the + // inline path. A new write to an item can then apply inline, and the worker can + // afterwards apply an older queued row for that same item, leaving the index + // row permanently disagreeing with the base item until the item is written + // again. + // + // The probe closes the in-flight window because of WHERE it runs, and that + // ordering is load-bearing rather than incidental: this function is called + // after the base row's own write in the same transaction, so two concurrent + // writes to one item serialize on the base row lock. READ COMMITTED takes a + // fresh snapshot per statement, so the later writer's probe runs after the + // earlier writer committed and therefore sees the row it queued. A writer that + // saw CREATING and enqueued cannot be invisible to a writer that sees ACTIVE + // and wants to inline. + // + // A claimed but uncommitted queue row is still visible here, because the worker + // has not committed its delete, so this write queues behind it and the two + // apply in id order within one worker partition. A consumed and committed row + // is gone, and its writes to the index landed before these statements, so an + // inline apply lands last. Both interleavings are safe. + // + // Deliberately an over-approximation. A GSI row pending under a per-index delay + // while the global delay is zero also forces this write to queue. The queued + // row is ready immediately and applies in order, so the cost is eventualness, + // never a wrong index row. Inline only exists at delay zero, where the queue + // holds build-era rows and drains to empty, so the inline path resumes; there + // is no steady-state traffic that could strand it. + let queue_empty = if delay_ms == 0 && metas.iter().any(|(_, status)| status == "ACTIVE") { + let pending: Option<(i32,)> = + sqlx::query_as("SELECT 1 FROM gsi_pending WHERE table_id = $1 LIMIT 1") + .bind(table_id) + .fetch_optional(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + pending.is_none() + } else { + false + }; + let mut enqueued = 0usize; for (meta, index_status) in metas { // A CREATING index never takes an inline write, whatever the delay. The @@ -158,9 +201,38 @@ pub(crate) async fn maintain_vector_indexes( // same item; writing the new one now would let the backfill overwrite it, // and its deliberately plain INSERT would collide with the row this put // there. The queue hold parks the row until the index is published. - let inline = delay_ms == 0 && index_status == "ACTIVE"; + let inline = delay_ms == 0 && index_status == "ACTIVE" && queue_empty; if inline { - apply_vector_index(tx, meta, base_key_schema, attr_defs, old_item, new_item).await?; + // Guarded by a savepoint so a concurrent delete of this index cannot + // fail the caller's write. The data table vanishing under an inline + // apply is the same routine race the propagation worker already + // tolerates. Without the savepoint the failed statement aborts the + // whole write subtransaction, and the write answers + // InternalServerError where the service answers 200. + sqlx::query("SAVEPOINT vector_inline_apply") + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + match apply_vector_index(tx, meta, base_key_schema, attr_defs, old_item, new_item).await + { + Ok(()) => { + sqlx::query("RELEASE SAVEPOINT vector_inline_apply") + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Err(e) if crate::gsi_queue::is_undefined_table(&e) => { + sqlx::query("ROLLBACK TO SAVEPOINT vector_inline_apply") + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + tracing::debug!( + index_id = %meta.index_id, + "vector index data table gone during inline apply, skipping" + ); + } + Err(e) => return Err(e), + } continue; } // Enqueued even when the new image carries no vector: the removal is the @@ -626,7 +698,7 @@ impl extenddb_storage::vector_lifecycle::VectorIndexBuild for PostgresVectorBuil // One transition: ACTIVE, the member cleared to absent rather than false, // and the skip count recorded so an index that deliberately omits rows says // so. The build columns are cleared because ownership ends here. - sqlx::query( + let flipped = sqlx::query( "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL, \ skipped_item_count = $3, build_owner = NULL, build_heartbeat_at = NULL \ WHERE table_id = $1 AND index_id = $2", @@ -638,6 +710,32 @@ impl extenddb_storage::vector_lifecycle::VectorIndexBuild for PostgresVectorBuil .await .map_err(|e| StorageError::Internal(e.to_string()))?; + if flipped.rows_affected() == 0 { + // No row to flip means a delete committed while this build was + // running. That is only a leak for a rebuild: it recreates its data + // table after reloading the definition, so a delete landing in + // between drops the table it could see and this build then recreates + // one nothing references. The absent catalog row is the proof the + // index is gone, so drop the table here rather than leaving it for an + // operator to find. + let mut tx = self + .data + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + crate::PostgresEngine::drop_vector_data_table(&mut tx, &self.index_id).await?; + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + tracing::info!( + index_id = %self.index_id, + table_id = %self.table_id, + "vector index was deleted while its build ran; dropped the rebuilt data table" + ); + release_hold(&self.data, &self.table_id, &self.index_id).await?; + return Ok(()); + } + // The hold goes only after the flip has committed. Held slightly too long // is harmless, because the queue rows simply wait; released early would let // the worker apply a write against an index that is not yet published. diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index 16151c70..bd369ffa 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -36,6 +36,38 @@ async fn release_taken_holds( } } +/// Refuse a new index whose name is already taken on this table, whatever index +/// family holds it. +/// +/// CreateTable enforces uniqueness across secondary and vector index names +/// together, in one place. UpdateTable builds each family's create path +/// separately, and each one used to consult only its own catalog table, so a GSI +/// and a vector index could end up sharing a name on the same table, and with it +/// a single index ARN. +/// +/// The error wording is the existing duplicate-index message. The service's own +/// wording for the cross-family case is not measured, so it is not claimed here. +async fn ensure_index_name_free( + conn: &mut sqlx::PgConnection, + table_id: &str, + index_name: &str, +) -> Result<(), StorageError> { + let taken: Option<(String,)> = sqlx::query_as( + "SELECT index_name FROM indexes WHERE table_id = $1 AND index_name = $2 \ + UNION ALL \ + SELECT index_name FROM vector_indexes WHERE table_id = $1 AND index_name = $2", + ) + .bind(table_id) + .bind(index_name) + .fetch_optional(&mut *conn) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if taken.is_some() { + return Err(StorageError::IndexAlreadyExists(index_name.to_owned())); + } + Ok(()) +} + impl PostgresEngine { /// Core implementation of `update_table` (REQ-CTRL-003). pub(crate) async fn update_table_impl( @@ -50,9 +82,21 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - // Lock the row and fetch table_id, key_schema, attribute_definitions. - let row: Option<(String, String, serde_json::Value, serde_json::Value)> = sqlx::query_as( - "SELECT table_status, table_id, key_schema, attribute_definitions FROM tables WHERE account_id = $1 AND table_name = $2 FOR UPDATE", + // Lock the row and fetch table_id, key_schema, attribute_definitions and + // the stored billing mode. + // + // `stored_billing_mode` is read once here and threaded through every + // later check that needs it, matching the SQLite backend. Re-querying it + // per check cost extra round trips and, because the row is already + // locked, could never return a different answer. + let row: Option<( + String, + String, + serde_json::Value, + serde_json::Value, + Option, + )> = sqlx::query_as( + "SELECT table_status, table_id, key_schema, attribute_definitions, billing_mode FROM tables WHERE account_id = $1 AND table_name = $2 FOR UPDATE", ) .bind(account_id) .bind(&input.table_name) @@ -60,7 +104,7 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let (status, table_id, ks_json, ad_json) = + let (status, table_id, ks_json, ad_json, stored_billing_mode) = row.ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; if status != "ACTIVE" { return Err(StorageError::TableNotActive(input.table_name.clone())); @@ -136,15 +180,6 @@ impl PostgresEngine { .as_ref() .is_some_and(|updates| updates.iter().any(|u| u.create.is_some())); if creates_vector_index { - let stored_billing_mode: Option = sqlx::query_scalar( - "SELECT billing_mode FROM tables WHERE account_id = $1 AND table_name = $2", - ) - .bind(account_id) - .bind(&input.table_name) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - .flatten(); let net_pay_per_request = match input.billing_mode { Some(mode) => mode == BillingMode::PayPerRequest, None => stored_billing_mode.as_deref() == Some("PAY_PER_REQUEST"), @@ -171,17 +206,7 @@ impl PostgresEngine { let effective_ppr = match input.billing_mode { Some(BillingMode::PayPerRequest) => true, Some(BillingMode::Provisioned) => false, - None => { - let current_bm: Option> = sqlx::query_scalar( - "SELECT billing_mode FROM tables WHERE account_id = $1 AND table_name = $2", - ) - .bind(account_id) - .bind(&input.table_name) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - current_bm.flatten().as_deref() == Some("PAY_PER_REQUEST") - } + None => stored_billing_mode.as_deref() == Some("PAY_PER_REQUEST"), }; if effective_ppr { return Err(StorageError::Validation( @@ -390,19 +415,9 @@ impl PostgresEngine { if let Some(updates) = &input.global_secondary_index_updates { for update in updates { if let Some(create) = &update.create { - // Check for duplicate index name. - let existing: Option<(String,)> = sqlx::query_as( - "SELECT index_name FROM indexes WHERE table_id = $1 AND index_name = $2", - ) - .bind(&table_id) - .bind(&create.index_name) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if existing.is_some() { - return Err(StorageError::IndexAlreadyExists(create.index_name.clone())); - } + // Across families, not just this one: see + // `ensure_index_name_free`. + ensure_index_name_free(&mut tx, &table_id, &create.index_name).await?; let gsi_ks = serde_json::to_value(&create.key_schema) .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -627,18 +642,7 @@ impl PostgresEngine { )); } - let dup: Option<(String,)> = sqlx::query_as( - "SELECT index_name FROM vector_indexes \ - WHERE table_id = $1 AND index_name = $2", - ) - .bind(&table_id) - .bind(&create.index_name) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if dup.is_some() { - return Err(StorageError::IndexAlreadyExists(create.index_name.clone())); - } + ensure_index_name_free(&mut tx, &table_id, &create.index_name).await?; let index_id = uuid::Uuid::new_v4().to_string(); let vec_attr = serde_json::to_value(&create.vector_attribute) diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 042c3dbc..54c45d18 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -606,7 +606,7 @@ impl VectorIndexBuild for SqliteVectorBuild { // `backfilling` is cleared to NULL rather than set to 0, because the // service removes the member once ACTIVE and the catalog CHECK // constraint enforces that pairing. - sqlx::query( + let flipped = sqlx::query( "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL, \ skipped_item_count = ? \ WHERE table_id = ? AND index_id = ?", @@ -617,6 +617,27 @@ impl VectorIndexBuild for SqliteVectorBuild { .execute(&self.pool) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + + if flipped.rows_affected() == 0 { + // No row to flip means a delete committed while this build was + // running, and a rebuild recreates its data table after reading the + // definition back. A delete landing in between drops the table it + // could see, and this build then recreates one nothing references. + // The absent catalog row is the proof the index is gone. + let _writer = self.write_lock.lock().await; + crate::SqliteEngine::drop_vector_data_table_by_id( + &self.pool, + &self.table_id, + &self.index_id, + ) + .await?; + tracing::info!( + index_id = %self.index_id, + table_id = %self.table_id, + "vector index was deleted while its build ran; dropped the rebuilt data table" + ); + return Ok(()); + } Ok(()) } diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 7467182d..c331c41e 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -22,6 +22,40 @@ use extenddb_storage::util::effective_attribute_definitions; use crate::store::SqliteEngine; +/// Refuse a new index whose name is already taken on this table, whatever index +/// family holds it. +/// +/// CreateTable enforces uniqueness across secondary and vector index names +/// together, in one place. UpdateTable builds each family's create path +/// separately, and each one used to consult only its own catalog table, so a GSI +/// and a vector index could end up sharing a name on the same table, and with it +/// a single index ARN. +/// +/// The error wording is the existing duplicate-index message. The service's own +/// wording for the cross-family case is not measured, so it is not claimed here. +async fn ensure_index_name_free( + conn: &mut sqlx::SqliteConnection, + table_id: &str, + index_name: &str, +) -> Result<(), StorageError> { + let taken: Option<(String,)> = sqlx::query_as( + "SELECT index_name FROM indexes WHERE table_id = ? AND index_name = ? \ + UNION ALL \ + SELECT index_name FROM vector_indexes WHERE table_id = ? AND index_name = ?", + ) + .bind(table_id) + .bind(index_name) + .bind(table_id) + .bind(index_name) + .fetch_optional(&mut *conn) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if taken.is_some() { + return Err(StorageError::IndexAlreadyExists(index_name.to_owned())); + } + Ok(()) +} + impl SqliteEngine { pub(crate) async fn update_table_impl( &self, @@ -282,17 +316,9 @@ impl SqliteEngine { if let Some(updates) = &input.global_secondary_index_updates { for update in updates { if let Some(create) = &update.create { - let dup: Option<(String,)> = sqlx::query_as( - "SELECT index_name FROM indexes WHERE table_id = ? AND index_name = ?", - ) - .bind(&table_id) - .bind(&create.index_name) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if dup.is_some() { - return Err(StorageError::IndexAlreadyExists(create.index_name.clone())); - } + // Across families, not just this one: see + // `ensure_index_name_free`. + ensure_index_name_free(&mut tx, &table_id, &create.index_name).await?; let ks = serde_json::to_string(&create.key_schema) .map_err(|e| StorageError::Internal(e.to_string()))?; let proj = serde_json::to_string(&create.projection) @@ -511,18 +537,7 @@ impl SqliteEngine { )); } - let dup: Option<(String,)> = sqlx::query_as( - "SELECT index_name FROM vector_indexes \ - WHERE table_id = ? AND index_name = ?", - ) - .bind(&table_id) - .bind(&create.index_name) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if dup.is_some() { - return Err(StorageError::IndexAlreadyExists(create.index_name.clone())); - } + ensure_index_name_free(&mut tx, &table_id, &create.index_name).await?; let index_id = uuid::Uuid::new_v4().to_string(); let vec_attr = serde_json::to_string(&create.vector_attribute) .map_err(|e| StorageError::Internal(e.to_string()))?; From dbc6beffc5ed13ca7af81396faecb0b4c978e08b Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Tue, 25 Aug 2026 02:57:42 +0000 Subject: [PATCH 11/13] test: cover the review fixes, and repair assertions that could not fail Six tests for the fixes in the previous commit, each proven to fail against the unfixed code first: the queue ordering gate and its empty-queue control, deleted-index build completion on both backends, and the two cross-family index name directions. Build completion is reachable from a test through a hidden entry point, alongside the two that already exist for the same reason, because the branch it exercises only runs when the catalog row is already gone and the alternative is timing the race. Two SQLite assertions compared sqlite_master.name against a name that arrives already quoted for DDL use, so they matched nothing and passed whatever the real state was. Both now use the bare name through one helper, and each carries a positive control asserting presence where the table must exist, so an assertion that cannot fail would itself fail. The allocation-phase test resets its global lever even when the body panics, which previously left every later test in the serial run with a four second phase and failures pointing at the wrong cause. Ledger: F-20's fix is coupled to the queue gate, since gating SQLite's inline branch on ACTIVE is what first produces the queued rows the gate protects, and landing it alone would swap one race for another. --- .../storage-postgres/src/data/vector_index.rs | 30 ++ crates/storage-postgres/src/lib.rs | 5 + .../tests/vector_control_plane.rs | 464 +++++++++++++++++- crates/storage-sqlite/src/update_table.rs | 161 +++++- docs/technical-debt.md | 4 +- tests/rust/src/vector_index_search.rs | 17 +- 6 files changed, 657 insertions(+), 24 deletions(-) diff --git a/crates/storage-postgres/src/data/vector_index.rs b/crates/storage-postgres/src/data/vector_index.rs index e729cb3e..0bc0a18b 100644 --- a/crates/storage-postgres/src/data/vector_index.rs +++ b/crates/storage-postgres/src/data/vector_index.rs @@ -479,6 +479,36 @@ pub(crate) struct PostgresVectorBuild { pub(crate) meta: Option, } +/// Flip one vector index to ACTIVE the way a finished build does. +/// +/// Reachable so an integration test can drive the completion step against a +/// catalog whose row for this index is already gone, which is the state a delete +/// racing a rebuild leaves and the only way to reach the branch that cleans up +/// after it. The alternative is timing the race, which does not belong in a suite. +/// Hidden for the same reason as the other two build entry points: no deployment +/// path calls it. +pub async fn mark_vector_index_active( + catalog: &sqlx::PgPool, + data: &sqlx::PgPool, + table_id: &str, + index_id: &str, +) -> Result<(), StorageError> { + // The fields the completion step does not read are left empty rather than + // invented: it works from the two ids and the two pools. + let mut ops = PostgresVectorBuild { + catalog: catalog.clone(), + data: data.clone(), + queue_notify: None, + table_id: table_id.to_owned(), + index_id: index_id.to_owned(), + base_key_schema: Vec::new(), + attribute_definitions: Vec::new(), + dimensions: 0, + meta: None, + }; + extenddb_storage::vector_lifecycle::VectorIndexBuild::mark_active(&mut ops, 0).await +} + /// The backfill's position in the base table: the whole primary key. /// /// A keyset cursor rather than an offset, and the FULL key rather than the diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 63740fe9..0e65d7cd 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -54,6 +54,11 @@ pub use data::vector_index::apply_claimed_vector_row; /// deployment path calls it. #[doc(hidden)] pub use data::vector_index::build_ownership; +/// Flip one vector index to ACTIVE the way a finished build does. Reachable so a +/// test can drive completion against a catalog row that is already gone, and +/// hidden for the same reason as the two entry points above. +#[doc(hidden)] +pub use data::vector_index::mark_vector_index_active; /// Rebuild vector index builds whose heartbeat has gone stale. The runtime half of /// the same repair, exported for the same reason and for its test. pub use data::vector_index::rebuild_stuck_vector_indexes; diff --git a/crates/storage-postgres/tests/vector_control_plane.rs b/crates/storage-postgres/tests/vector_control_plane.rs index ea999729..0f47e78b 100644 --- a/crates/storage-postgres/tests/vector_control_plane.rs +++ b/crates/storage-postgres/tests/vector_control_plane.rs @@ -24,11 +24,12 @@ use std::collections::{BTreeMap, HashMap}; use extenddb_core::expression::{self, ExpressionMaps}; use extenddb_core::types::TableKeyInfo; use extenddb_core::types::{ - AttributeDefinition, AttributeValue, BillingMode, CreateTableInput, DeleteTableInput, - DeleteVectorIndexAction, DescribeTableInput, DistanceFunction, IndexStatus, Item, - KeySchemaElement, KeyType, Projection, ProjectionType, ProvisionedThroughput, - ScalarAttributeType, SearchSchemaElement, SearchSchemaElementType, UpdateTableInput, - VectorAttribute, VectorIndexSpecification, VectorIndexUpdate, + AttributeDefinition, AttributeValue, BillingMode, CreateGsiAction, CreateTableInput, + DeleteTableInput, DeleteVectorIndexAction, DescribeTableInput, DistanceFunction, + GlobalSecondaryIndexUpdate, GsiInput, IndexStatus, Item, KeySchemaElement, KeyType, Projection, + ProjectionType, ProvisionedThroughput, ScalarAttributeType, SearchSchemaElement, + SearchSchemaElementType, UpdateTableInput, VectorAttribute, VectorIndexSpecification, + VectorIndexUpdate, }; use extenddb_storage::error::StorageError; use extenddb_storage::{BackupEngine, DataEngine, TableEngine}; @@ -2778,3 +2779,456 @@ async fn build_ownership_uses_its_own_session_and_releases_on_drop() { peer.close().await; s.cleanup().await; } + +/// A vector index create must not reuse a name a secondary index already holds. +/// +/// CreateTable rejects the collision across families in one place. UpdateTable +/// builds each family's create path separately, and the vector path used to +/// consult only `vector_indexes`, so this pair of requests produced a table whose +/// GSI and vector index shared one name, and with it one index ARN. +#[tokio::test] +async fn update_table_refuses_a_vector_index_named_like_an_existing_gsi() { + let test = "update_table_refuses_a_vector_index_named_like_an_existing_gsi"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + let mut input = create_input("t_name_gsi_first", vec![]); + input.attribute_definitions = vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "gsipk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ]; + input.global_secondary_indexes = Some(vec![GsiInput { + index_name: "shared".to_owned(), + key_schema: hash_key("gsipk"), + projection: projection(ProjectionType::All), + provisioned_throughput: None, + }]); + s.engine + .create_table(ACCOUNT, input) + .await + .expect("create a table carrying a GSI named 'shared'"); + + let err = s + .engine + .update_table( + ACCOUNT, + UpdateTableInput { + vector_index_updates: Some(vec![VectorIndexUpdate { + create: Some(vector_spec("shared", 2, Some("pk"))), + delete: None, + }]), + ..update_input("t_name_gsi_first") + }, + ) + .await + .expect_err("a vector index may not take a name a GSI already holds"); + + match err { + StorageError::IndexAlreadyExists(name) => assert_eq!(name, "shared"), + other => panic!("expected IndexAlreadyExists, got {other:?}"), + } + + // The refusal must not have written a half-made index on the way out. + let id = table_id(&s.catalog, "t_name_gsi_first").await; + assert_eq!(vector_row_count(&s.catalog, &id).await, 0); + + s.cleanup().await; +} + +/// And the mirror image: a GSI create must not reuse a vector index's name. +#[tokio::test] +async fn update_table_refuses_a_gsi_named_like_an_existing_vector_index() { + let test = "update_table_refuses_a_gsi_named_like_an_existing_vector_index"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input( + "t_name_vec_first", + vec![vector_spec("shared", 2, Some("pk"))], + ), + ) + .await + .expect("create a table carrying a vector index named 'shared'"); + + let err = s + .engine + .update_table( + ACCOUNT, + UpdateTableInput { + global_secondary_index_updates: Some(vec![GlobalSecondaryIndexUpdate { + create: Some(CreateGsiAction { + index_name: "shared".to_owned(), + key_schema: hash_key("pk"), + projection: projection(ProjectionType::All), + provisioned_throughput: None, + }), + update: None, + delete: None, + }]), + ..update_input("t_name_vec_first") + }, + ) + .await + .expect_err("a GSI may not take a name a vector index already holds"); + + match err { + StorageError::IndexAlreadyExists(name) => assert_eq!(name, "shared"), + other => panic!("expected IndexAlreadyExists, got {other:?}"), + } + + s.cleanup().await; +} + +/// An inline write whose index data table has gone must still succeed. +/// +/// The window is a delete committing between a write's catalog read, which still +/// lists the index, and its inline apply, by which time the data table is gone. +/// Dropping the table directly is the same end state and is deterministic where +/// the race is not. Before the savepoint the failed statement aborted the write's +/// transaction, so the caller saw InternalServerError on a request the service +/// answers normally. The propagation worker already tolerated this shape. +#[tokio::test] +async fn an_inline_write_survives_the_index_data_table_disappearing() { + let test = "an_inline_write_survives_the_index_data_table_disappearing"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_table_gone", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_table_gone") + .await + .expect("key info"); + let table = only_index_table(&s.catalog).await; + + sqlx::query(&format!("DROP TABLE \"{table}\"")) + .execute(&s.catalog) + .await + .expect("drop the vector data table out from under the write path"); + + let maps = ExpressionMaps::new(HashMap::new(), HashMap::new()); + s.engine + .put_item( + &key_info, + vector_item("a", None, &["1", "0"]), + false, + None, + &maps, + None, + ) + .await + .expect("a write racing an index delete must succeed, not fail internally"); + + s.cleanup().await; +} + +/// An item with a generation marker, so old and new images are distinguishable in +/// the index payload. The vector attribute is stripped on the way in, so the +/// embedding cannot serve as the marker. +fn marked_item(pk: &str, generation: &str, values: &[&str]) -> Item { + let mut item = vector_item(pk, None, values); + item.insert("gen".to_owned(), AttributeValue::S(generation.to_owned())); + item +} + +/// Drain every queued row for a table, oldest first, the way the worker would. +/// +/// The engine under test runs no workers, so the rows are applied through the same +/// entry point the worker uses, in the id order the worker claims them in. +async fn drain_queue(engine: &PostgresEngine, catalog: &PgPool, table_id: &str) -> usize { + let rows: Vec<( + i64, + Option, + Option, + serde_json::Value, + )> = sqlx::query_as( + "SELECT id, old_item, new_item, index_context FROM gsi_pending \ + WHERE table_id = $1 ORDER BY id", + ) + .bind(table_id) + .fetch_all(catalog) + .await + .expect("read the queued rows"); + + let drained = rows.len(); + for (id, old_json, new_json, context) in rows { + let vector_context: extenddb_storage::vector_lifecycle::VectorApplyContext = + serde_json::from_value(context).expect("a vector context"); + let old_item: Option = old_json.map(|v| serde_json::from_value(v).expect("old item")); + let new_item: Option = new_json.map(|v| serde_json::from_value(v).expect("new item")); + let mut tx = engine.data_pool().begin().await.expect("begin"); + extenddb_storage_postgres::apply_claimed_vector_row( + &mut tx, + &vector_context, + old_item.as_ref(), + new_item.as_ref(), + ) + .await + .expect("apply a queued row"); + tx.commit().await.expect("commit the applied row"); + sqlx::query("DELETE FROM gsi_pending WHERE id = $1") + .bind(id) + .execute(catalog) + .await + .expect("consume the row"); + } + drained +} + +async fn queue_depth(catalog: &PgPool, table_id: &str) -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM gsi_pending WHERE table_id = $1") + .bind(table_id) + .fetch_one(catalog) + .await + .expect("count the queued rows") +} + +async fn set_propagation_delay(catalog: &PgPool, ms: u64) { + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', $1) \ + ON CONFLICT (key) DO UPDATE SET value = $1", + ) + .bind(ms.to_string()) + .execute(catalog) + .await + .expect("set the propagation delay"); +} + +/// A write must not apply inline while the table still has queued rows, or it +/// overwrites its own newer image with an older queued one. +/// +/// The window is the moment after an index flips ACTIVE and gives up its hold: +/// rows queued while it was CREATING drain at the same time as new writes take the +/// inline path. Queue order only ever ordered queued rows against each other. So +/// an inline write could land the new image and the worker could then apply an +/// older queued row for the same item, leaving the index permanently disagreeing +/// with the base table until that item was written again. +/// +/// The pre-flip row is produced by writing under a delay rather than by hand, so +/// the queued context and images are the ones the write path really produces. +#[tokio::test] +async fn a_write_queues_behind_pending_rows_instead_of_overtaking_them() { + let test = "a_write_queues_behind_pending_rows_instead_of_overtaking_them"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + // A delay first, so this write queues: it stands in for a write that arrived + // while the index was still CREATING and was parked by the build's hold. + set_propagation_delay(&s.catalog, 50).await; + s.engine + .create_table( + ACCOUNT, + create_input("t_overtake", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_overtake").await; + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_overtake") + .await + .expect("key info"); + let table = only_index_table(&s.catalog).await; + + put(&s.engine, &key_info, marked_item("x", "old", &["1", "0"])).await; + assert_eq!( + queue_depth(&s.catalog, &id).await, + 1, + "the first write must be queued at a non-zero delay" + ); + assert!( + index_rows(&s.catalog, &table).await.is_empty(), + "a queued write must not have reached the index yet" + ); + + // The index is ACTIVE and the delay is now zero, which is the state the inline + // path runs in. The queued row above is still pending. + set_propagation_delay(&s.catalog, 0).await; + + put(&s.engine, &key_info, marked_item("x", "new", &["0", "1"])).await; + + // The second write must have queued behind the first rather than applying + // inline ahead of it. + assert_eq!( + queue_depth(&s.catalog, &id).await, + 2, + "a write must queue while the table has pending rows, not apply inline" + ); + assert!( + index_rows(&s.catalog, &table).await.is_empty(), + "the second write must not have applied inline" + ); + + // Draining in order leaves the newest image, which is the property the whole + // gate exists to preserve. Without the gate the inline apply happened first and + // the older queued row overwrote it, leaving "old" here forever. + assert_eq!(drain_queue(&s.engine, &s.catalog, &id).await, 2); + let rows = index_rows(&s.catalog, &table).await; + assert_eq!(rows.len(), 1, "one item, one index row: {rows:?}"); + assert_eq!( + rows[0].1.get("gen").and_then(|v| v.get("S")), + Some(&serde_json::Value::String("new".to_owned())), + "the index must hold the newest image after the queue drains: {:?}", + rows[0].1 + ); + + s.cleanup().await; +} + +/// The other half of the same gate: with nothing pending, a write still applies +/// inline. Without this the fix could pass by queueing everything forever. +#[tokio::test] +async fn a_write_applies_inline_when_the_queue_is_empty() { + let test = "a_write_applies_inline_when_the_queue_is_empty"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_inline", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_inline").await; + let key_info = s + .engine + .table_key_info(ACCOUNT, "t_inline") + .await + .expect("key info"); + let table = only_index_table(&s.catalog).await; + + put(&s.engine, &key_info, marked_item("x", "only", &["1", "0"])).await; + + assert_eq!( + queue_depth(&s.catalog, &id).await, + 0, + "an empty queue must leave the write on the inline path" + ); + let rows = index_rows(&s.catalog, &table).await; + assert_eq!(rows.len(), 1, "the inline write must reach the index"); + assert_eq!( + rows[0].1.get("gen").and_then(|v| v.get("S")), + Some(&serde_json::Value::String("only".to_owned())), + "{:?}", + rows[0].1 + ); + + s.cleanup().await; +} + +/// Completing a build whose index was deleted must not leave its data table behind. +/// +/// A rebuild recreates the data table after reloading the definition, so a delete +/// committing in between drops the table it could see and the rebuild then creates +/// one nothing references. Driven by deleting the catalog row and completing the +/// build directly, because the alternative is timing the race. +#[tokio::test] +async fn completing_a_build_whose_index_was_deleted_drops_the_rebuilt_table() { + let test = "completing_a_build_whose_index_was_deleted_drops_the_rebuilt_table"; + if base_conn().is_none() { + return skip(test); + } + let Some(s) = vector_scratch(test).await else { + return; + }; + + s.engine + .create_table( + ACCOUNT, + create_input("t_orphan_table", vec![vector_spec("vidx", 2, Some("pk"))]), + ) + .await + .expect("create a table with a vector index"); + let id = table_id(&s.catalog, "t_orphan_table").await; + let table = only_index_table(&s.catalog).await; + let index_id = table.trim_start_matches("_ddb_vec_").to_owned(); + + // A hold, so the release half is observable too. This is what a real build + // takes for the duration of its backfill. + sqlx::query( + "INSERT INTO vector_index_holds (table_id, index_id) VALUES ($1, $2) \ + ON CONFLICT DO NOTHING", + ) + .bind(&id) + .bind(&index_id) + .execute(s.engine.data_pool()) + .await + .expect("take a build hold"); + + // The delete lands: the catalog row goes, the data table this build recreated + // stays, which is exactly the interleaving the fix is for. + sqlx::query("DELETE FROM vector_indexes WHERE table_id = $1 AND index_id = $2") + .bind(&id) + .bind(&index_id) + .execute(&s.catalog) + .await + .expect("delete the catalog row"); + assert!( + vector_data_tables(&s.catalog).await.contains(&table), + "the data table must still exist before completion" + ); + + extenddb_storage_postgres::mark_vector_index_active( + s.engine.data_pool(), + s.engine.data_pool(), + &id, + &index_id, + ) + .await + .expect("completing a build for a deleted index must not fail"); + + assert!( + !vector_data_tables(&s.catalog).await.contains(&table), + "the rebuilt data table must be dropped once the index is known to be gone" + ); + let holds: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM vector_index_holds WHERE table_id = $1 AND index_id = $2", + ) + .bind(&id) + .bind(&index_id) + .fetch_one(s.engine.data_pool()) + .await + .expect("count the holds"); + assert_eq!( + holds, 0, + "the hold must be released, or the table's propagation stays paused" + ); + + s.cleanup().await; +} diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index c331c41e..516c83de 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -1906,6 +1906,21 @@ mod reconciler_tests { meta: None, }; ops.reset_data_table().await.expect("rebuild reset"); + + // Positive control, and the point of it rather than a nicety. These counts + // compare against `sqlite_master.name`, which holds the bare name, so binding + // the DDL-ready name straight from `vector_table_name` can never match and + // every absence assertion below would pass whatever the real state was. + // Proving the count is 1 while the table must exist is what makes the later + // zeroes mean the table went away. + let vec_table = vector_table_lookup_name(&table_id, "vidx-1"); + assert_eq!( + data_table_count(&engine.pool, &vec_table).await, + 1, + "the rebuild must have recreated its data table, or the assertions below \ + cannot tell a dropped table from a name that never matches" + ); + ops.set_backfilling().await.expect("rebuild phase flip"); engine @@ -1918,14 +1933,11 @@ mod reconciler_tests { .await .expect("count"); assert_eq!(rows, 0, "the catalog row must be gone"); - let vec_table = crate::data::vector_table_name(&table_id, "vidx-1"); - let (tables,): (i64,) = - sqlx::query_as("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?") - .bind(&vec_table) - .fetch_one(&engine.pool) - .await - .expect("count tables"); - assert_eq!(tables, 0, "the delete must drop the index data table"); + assert_eq!( + data_table_count(&engine.pool, &vec_table).await, + 0, + "the delete must drop the index data table" + ); // The rest of the rebuild now runs against a deleted index. It must fail, and // it must leave neither a catalog row nor a data table behind. @@ -1935,14 +1947,9 @@ mod reconciler_tests { ) .await .expect_err("a rebuild of a deleted index must fail rather than recreate it"); - let (tables_after,): (i64,) = - sqlx::query_as("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?") - .bind(&vec_table) - .fetch_one(&engine.pool) - .await - .expect("count tables again"); assert_eq!( - tables_after, 0, + data_table_count(&engine.pool, &vec_table).await, + 0, "an interrupted rebuild must not leave an orphan data table" ); assert_eq!( @@ -1955,6 +1962,130 @@ mod reconciler_tests { ); } + /// Completing a build whose index was deleted must not leave its data table. + /// + /// The sibling of the rebuild test above, at the other end of the same race. That + /// one has the delete arrive after `reset_data_table`, so the rebuild fails before + /// it can finish. This one has the build reach its completion step: the data table + /// it recreated exists, the catalog row is already gone, and the status flip + /// therefore matches no row. Without the cleanup the recreated table survives with + /// nothing referencing it. + /// + /// Driven by calling the completion step directly, because the alternative is + /// timing the window between the definition reload and the CREATE TABLE. + #[tokio::test] + async fn completing_a_build_whose_index_was_deleted_drops_the_rebuilt_table() { + use extenddb_storage::vector_lifecycle::VectorIndexBuild; + + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + let account = "000000000000"; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind(account) + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl(account, input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-1', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 1)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert a build in progress"); + + let mut ops = crate::data::vector_index::SqliteVectorBuild { + pool: engine.pool.clone(), + write_lock: std::sync::Arc::clone(&engine.write_lock), + gsi_notify: engine.gsi_notify(), + table_id: table_id.clone(), + index_id: "vidx-1".to_owned(), + base_key_schema: vec![extenddb_core::types::KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: extenddb_core::types::KeyType::Hash, + }], + attribute_definitions: vec![extenddb_core::types::AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: extenddb_core::types::ScalarAttributeType::S, + }], + meta: None, + }; + + // The build recreates its data table, exactly as a rebuild does. + ops.reset_data_table().await.expect("rebuild reset"); + let vec_table = vector_table_lookup_name(&table_id, "vidx-1"); + assert_eq!( + data_table_count(&engine.pool, &vec_table).await, + 1, + "the build must have recreated its data table" + ); + + // The delete lands: catalog row gone, recreated data table still there. + sqlx::query("DELETE FROM vector_indexes WHERE table_id = ? AND index_id = 'vidx-1'") + .bind(&table_id) + .execute(&engine.pool) + .await + .expect("delete the catalog row"); + + ops.mark_active(0) + .await + .expect("completing a build for a deleted index must not fail"); + + assert_eq!( + data_table_count(&engine.pool, &vec_table).await, + 0, + "the rebuilt data table must be dropped once the index is known to be gone" + ); + } + + /// The bare data-table name for a vector index, for comparing against + /// `sqlite_master.name`. + /// + /// `vector_table_name` returns the name already quoted, which is what its callers + /// interpolating into DDL and DML need. `sqlite_master.name` holds the bare name, + /// so a bound comparison against the quoted form matches nothing, and an absence + /// assertion written that way passes whatever the real state is. Written once here + /// so no test has to remember the difference. + fn vector_table_lookup_name(table_id: &str, index_id: &str) -> String { + crate::data::vector_table_name(table_id, index_id) + .trim_matches('"') + .to_owned() + } + + /// How many tables carry this bare name. Zero or one. + async fn data_table_count(pool: &sqlx::SqlitePool, bare_name: &str) -> i64 { + let (count,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?") + .bind(bare_name) + .fetch_one(pool) + .await + .expect("count the data tables carrying this name"); + count + } + /// One `UpdateTable` request deleting the vector index named `vidx`. fn delete_vidx_input() -> extenddb_core::types::UpdateTableInput { serde_json::from_value(json!({ diff --git a/docs/technical-debt.md b/docs/technical-debt.md index ec062611..f5f4a8e6 100755 --- a/docs/technical-debt.md +++ b/docs/technical-debt.md @@ -32,7 +32,7 @@ Last updated: 2026-08-20 | F-17 | `validate_attribute_name_sizes` only checks top-level attribute names — nested map keys not validated | `core/validation/mod.rs` | Low | P30 | | F-18 | ~~UpdateTable Delete of a vector index in the resource-allocation phase (`CREATING`, `Backfilling: false`) is accepted; Amazon DynamoDB refuses it with `ResourceInUseException` until backfilling starts~~ Both backends now enforce the phase rule with the measured message, and both hold the phase open under `vector_allocation_phase_delay_ms` so a client can observe it | ~~`storage-sqlite/update_table.rs` (vector delete branch), `core/types/table.rs` (`vector_index_delete_in_allocation_phase`)~~ | ~~Medium~~ | vector probe P2 | | F-19 | **Reachable in default configuration, no setting required.** SQLite backup does not capture vector indexes, so `CreateBackup` followed by `RestoreTableFromBackup` silently produces the table without them and reports success. Its `backups` table has no column that could carry an index set, where PostgreSQL added one specifically so it could detect the case and refuse, and neither SQLite entry point is gated by anything. That makes this the more reachable of the two vector gaps: F-20 needs `index_propagation_delay_ms` at zero, which is a test setting, and this needs nothing. Raised to High on that reachability: severity attaches to what a default configuration can reach, and the earlier Medium rested on a belief about whether operators back up vector tables rather than on any barrier in the code. Amazon DynamoDB preserves vector state through backup and restore (measured). The PostgreSQL backend refuses the restore rather than matching this, so the two backends fail differently: one refuses with a reason, one loses declared indexes quietly. Fix: capture the index set in the SQLite backup row and either restore it or refuse, matching PostgreSQL | `storage-sqlite/backup.rs:313-317` (the restore reads three columns: `key_schema`, `attribute_definitions`, `billing_mode`, none of which can carry an index), `storage-postgres/backup_engine.rs:140-149` and `:167` (captures the index set into the backup row), `:467-497` (refuses the restore) | High | PR-2 docs stage | -| F-20 | SQLite applies vector index maintenance inline whenever `index_propagation_delay_ms` is 0, without checking the index status, so a write landing during a backfill reaches the index data table directly and bypasses the claim-time hold that keeps it behind the backfill's older snapshot of the same item. The shared lifecycle contract requires that hold (`storage/vector_lifecycle/mod.rs:31-38`), and PostgreSQL enforces it with `delay_ms == 0 && index_status == "ACTIVE"`. Reachable only with the zero delay, which is a test setting. The symptom is not a failed build: the backfill's plain INSERT raises a primary key violation, which propagates out of the batch, and the detached build deliberately leaves the index in `CREATING` because there is no failure state on the wire. So an operator sees an index parked in `CREATING` with every search against it refused, until the stuck-build sweep or the startup reconciler rebuilds it; that rebuild drops and recreates the data table, so the retry starts clean and the state self-heals unless the interleaving repeats. On SQLite, which is the backend this item is about, that recovery is bounded at roughly two seconds: the propagation worker sweeps every pass with a maximum sleep of one second and requires two consecutive sightings before rebuilding. Do not generalise that figure: PostgreSQL's runtime sweep polls every 60 seconds and only treats a build as stuck once its heartbeat is 300 seconds stale, so an equivalent park there would persist for up to about six minutes. The bounded SQLite recovery is why this is Medium rather than High. Fix: gate the inline branch on the index being ACTIVE, matching PostgreSQL | `storage-sqlite/data/vector_index.rs:176` (inline branch), `storage-postgres/data/vector_index.rs:161` (the shape to match) | Medium | PR-2 docs stage | +| F-20 | SQLite applies vector index maintenance inline whenever `index_propagation_delay_ms` is 0, without checking the index status, so a write landing during a backfill reaches the index data table directly and bypasses the claim-time hold that keeps it behind the backfill's older snapshot of the same item. The shared lifecycle contract requires that hold (`storage/vector_lifecycle/mod.rs:31-38`), and PostgreSQL enforces it with `delay_ms == 0 && index_status == "ACTIVE"`. Reachable only with the zero delay, which is a test setting. The symptom is not a failed build: the backfill's plain INSERT raises a primary key violation, which propagates out of the batch, and the detached build deliberately leaves the index in `CREATING` because there is no failure state on the wire. So an operator sees an index parked in `CREATING` with every search against it refused, until the stuck-build sweep or the startup reconciler rebuilds it; that rebuild drops and recreates the data table, so the retry starts clean and the state self-heals unless the interleaving repeats. On SQLite, which is the backend this item is about, that recovery is bounded at roughly two seconds: the propagation worker sweeps every pass with a maximum sleep of one second and requires two consecutive sightings before rebuilding. Do not generalise that figure: PostgreSQL's runtime sweep polls every 60 seconds and only treats a build as stuck once its heartbeat is 300 seconds stale, so an equivalent park there would persist for up to about six minutes. The bounded SQLite recovery is why this is Medium rather than High. Fix: gate the inline branch on the index being ACTIVE, and it MUST land together with the queue-emptiness gate the PostgreSQL backend now applies in `maintain_vector_indexes`. Gating on ACTIVE is what first makes SQLite produce pre-flip queued rows, and those rows are then exposed to the same stale-overwrite race: once the index flips and its hold goes, a queued row can be applied after a newer inline write to the same item and leave the index row permanently disagreeing with the base item. Landing the ACTIVE gate alone would swap one race for the other | `storage-sqlite/data/vector_index.rs:176` (inline branch), `storage-postgres/data/vector_index.rs:161` (the shape to match) | Medium | PR-2 docs stage | ## Cleanup @@ -63,6 +63,8 @@ Last updated: 2026-08-20 |---|------|----------|----------|--------| | T-1 | `test_disable_ttl` flaky due to TTL modification cooldown | `tests/test_ttl.py` | Medium | P23 | | T-2 | ~~No code coverage tooling configured (Rust or Python)~~ | — | ~~Medium~~ | P25 review | +| T-4 | Cross-family index-name collision wording is unmeasured. UpdateTable now refuses a vector index name already held by a secondary index (and the reverse) using the existing duplicate-index message, but Amazon DynamoDB's own wording for the cross-family case has not been captured. Probe: create a table with a GSI, then UpdateTable-create a vector index of the same name, and the reverse | `storage-postgres/update_table.rs`, `storage-sqlite/update_table.rs` (`ensure_index_name_free`) | Low | PR-2 review | +| T-5 | ~~Two SQLite vector tests assert a data table is absent by binding the name returned from `vector_table_name`, which is already quoted for DDL use, so it can never match `sqlite_master.name` and the assertion passes whatever the real state is~~ Both tests now compare the bare name through one helper pair, and each carries a positive control asserting the table is present where it must be, so an absence assertion that cannot fail would itself fail (`e7fc11c`) | ~~`storage-sqlite/update_table.rs` (`a_delete_during_a_rebuild_...`)~~ | ~~Low~~ | PR-2 review | | T-3 | External Java tests lack `waitForGSI` helpers — 5 GSI tests fail intermittently due to propagation timing | `tests/external/` | Medium | P26 | ## Architecture diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index b87cd3bf..0528f14f 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1099,6 +1099,20 @@ async fn deleting_a_vector_index_is_refused_while_allocating_and_accepted_while_ if skip_unless_supported().await { return; } + // Long enough to observe the refusal without making the test slow. + set_allocation_phase_delay(4000).await; + // The body runs as a task so that a panic inside it cannot skip the reset + // below. This lever is global and the suite is serial, so a leaked 4s + // allocation phase would slow every later test and make their failures point + // at the wrong cause. + let outcome = tokio::spawn(phase_dependent_delete_body()).await; + set_allocation_phase_delay(0).await; + if let Err(e) = outcome { + std::panic::resume_unwind(e.into_panic()); + } +} + +async fn phase_dependent_delete_body() { let name = table_name("pos_phase_delete"); let body = format!( r#"{{ @@ -1112,8 +1126,6 @@ async fn deleting_a_vector_index_is_refused_while_allocating_and_accepted_while_ wait_for_active(&name).await; put_vector(&name, "seed", None, &[1.0, 0.0]).await; - // Long enough to observe the refusal without making the test slow. - set_allocation_phase_delay(4000).await; let delete_body = format!( r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Delete": {{"IndexName": "vidx"}}}}]}}"# ); @@ -1197,7 +1209,6 @@ async fn deleting_a_vector_index_is_refused_while_allocating_and_accepted_while_ } put_vector(&name, "after", None, &[0.0, 1.0]).await; - set_allocation_phase_delay(0).await; let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } From ec7263b7251af9156347dbbdc5a41029e87c14ee Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Tue, 25 Aug 2026 05:55:55 +0000 Subject: [PATCH 12/13] fix(postgres): install pgvector as the admin role during init and migrate The extension attempt ran as the application role, which the code's own doc comment proves can never succeed on a stock server: pgvector's control file does not mark it trusted, so a non-superuser is refused even on a database its role owns. init and migrate hold admin credentials the whole time and never used them for this, so a deployment initialised with a superuser admin still came up with vector search off, and every vector wire test failed in CI while the suite pinned EXTENDDB_EXPECT_VECTORS=1. The attempt now runs as the admin role first and falls back to the application role, which is the shape a managed platform allowlisting pgvector for the database owner needs. Refusal of both stays a notice with the same hint, and serve-time code still never attempts it. --- crates/storage-postgres/src/bootstrapper.rs | 51 +++++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/crates/storage-postgres/src/bootstrapper.rs b/crates/storage-postgres/src/bootstrapper.rs index b7f5a4d0..a8827df3 100755 --- a/crates/storage-postgres/src/bootstrapper.rs +++ b/crates/storage-postgres/src/bootstrapper.rs @@ -148,21 +148,51 @@ impl PostgresBootstrapper { /// Try to install pgvector on the data database, tolerating refusal. /// - /// This is an attempt, not a guarantee: pgvector's control file does not - /// mark the extension trusted, so `CREATE EXTENSION` is refused for a - /// non-superuser even on a database its role owns ("Must be superuser", - /// measured on a stock install). On refusal the printed hint tells the - /// operator to create it once as a superuser or as the database owner. - /// Serve-time code never attempts - /// this, because a request path must not carry data-definition privileges it - /// only needs once. + /// The attempt runs as the admin role first: pgvector's control file does + /// not mark the extension trusted, so a stock server refuses `CREATE + /// EXTENSION` from any non-superuser, even on a database its role owns + /// ("Must be superuser", measured on a stock install). The application role + /// can therefore never succeed there, and the admin credentials this command + /// already holds are exactly the ones that can. The application role is + /// still tried on admin failure, which is the shape managed platforms + /// allowlisting pgvector for the database owner need. Serve-time code never + /// attempts this, because a request path must not carry data-definition + /// privileges it only needs once. /// /// Failure is a notice, not an error. A deployment that does not want vector /// indexes, or a managed PostgreSQL that does not offer pgvector, must still /// initialise and upgrade normally; the server then refuses vector /// operations, which is the fail-closed half of the same decision. async fn try_create_vector_extension(&self, pool: &PgPool) { + use sqlx::Connection as _; + println!("--- Checking pgvector extension on the data database..."); + let admin_opts = PgConnectOptions::new() + .host(&self.config.host) + .port(self.config.port) + .username(&self.config.admin_user) + .database(&self.config.data_db); + let admin_opts = if let Some(ref pass) = self.config.admin_password { + admin_opts.password(pass) + } else { + admin_opts + }; + let admin_error = match sqlx::PgConnection::connect_with(&admin_opts).await { + Ok(mut conn) => { + let created = sqlx::query("CREATE EXTENSION IF NOT EXISTS vector") + .execute(&mut conn) + .await; + let _ = conn.close().await; + match created { + Ok(_) => { + println!(" pgvector available; vector indexes are supported."); + return; + } + Err(e) => e.to_string(), + } + } + Err(e) => e.to_string(), + }; match sqlx::query("CREATE EXTENSION IF NOT EXISTS vector") .execute(pool) .await @@ -170,7 +200,10 @@ impl PostgresBootstrapper { Ok(_) => println!(" pgvector available; vector indexes are supported."), Err(e) => { let hint = crate::vector::create_extension_hint(&e); - println!(" NOTICE: could not create the pgvector extension ({e}). {hint}."); + println!( + " NOTICE: could not create the pgvector extension (as the admin role: \ + {admin_error}; as the application role: {e}). {hint}." + ); } } } From cba355071296619b77dfc9231cf1fb84e3502b2c Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Wed, 26 Aug 2026 22:39:24 +0000 Subject: [PATCH 13/13] fix: reconcile the postgres branch with the rebased groundwork and main Carried across the rebase onto the reviewed groundwork tip, which itself moved onto a main that gained the vector lifecycle and capacity fixes: - the vector phase refusal produces IndexesInUse, the folded in-use variant that carries the whole wire message - the PostgreSQL UpdateTable-create path applies the same min-CREATING floor SQLite gained on main, passed to the shared driver beside the flip it delays - CreateTable's response initializer reports restore_summary: None, the field the restore work added to TableDescription - doc sample version literals follow the workspace to 0.1.10, caught by the branch's own doc guard - the F-20 ledger entry follows main: the wedge it originally tracked is fixed by KeepExisting, and what remains is the ordering coupling to the queue-emptiness gate - the propagation row in the differences doc states the queue-emptiness consequence plainly: a search-after-write window exists even at delay zero while queued rows drain, and it is ordering, not data loss --- crates/storage-postgres/src/create_table.rs | 4 +++ crates/storage-postgres/src/lib.rs | 25 +++++++++++++++++++ crates/storage-postgres/src/update_table.rs | 10 +++++++- .../tests/vector_control_plane.rs | 10 ++++---- crates/storage-sqlite/src/store.rs | 3 +++ crates/storage-sqlite/src/update_table.rs | 4 +-- docs/differences-from-dynamodb.md | 2 +- docs/getting-started.md | 4 +-- docs/manuals/04-quickstart-setup-guide.md | 2 +- docs/technical-debt.md | 2 +- 10 files changed, 53 insertions(+), 13 deletions(-) diff --git a/crates/storage-postgres/src/create_table.rs b/crates/storage-postgres/src/create_table.rs index f640ea0a..343e6fdb 100755 --- a/crates/storage-postgres/src/create_table.rs +++ b/crates/storage-postgres/src/create_table.rs @@ -569,6 +569,10 @@ impl PostgresEngine { // break this site and force a decision about whether create must // report it, rather than silently defaulting. vector_indexes: vector_index_descs, + // A freshly created table was not restored from anything: the + // service reports no RestoreSummary member on it. See the field's + // measured provenance on `TableDescription`. + restore_summary: None, }) } } diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 0e65d7cd..d792fc6a 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -451,6 +451,31 @@ impl PostgresEngine { } } + /// Minimum milliseconds an UpdateTable-created vector index stays `CREATING` + /// before its `ACTIVE` flip. See + /// [`extenddb_core::settings_keys::VECTOR_INDEX_MIN_CREATING_MS`] for why the + /// hold exists; the SQLite backend applies the same floor. Defaults to 1000 + /// when unset or unparseable; zero disables it. + pub(crate) async fn vector_index_min_creating_ms(&self) -> u64 { + const DEFAULT_MS: u64 = 1_000; + let live: Result, _> = + sqlx::query_as("SELECT value FROM settings WHERE key = $1") + .bind(extenddb_core::settings_keys::VECTOR_INDEX_MIN_CREATING_MS) + .fetch_optional(&self.pool) + .await; + match live { + Ok(row) => row + .and_then(|(v,)| v.parse::().ok()) + .unwrap_or(DEFAULT_MS), + Err(e) => { + tracing::debug!( + "vector_index_min_creating_ms: live read failed, using {DEFAULT_MS}: {e:?}" + ); + DEFAULT_MS + } + } + } + /// Milliseconds to hold a new vector index in the resource-allocation phase. /// /// A test lever, zero in production, read live for the same reason the batch diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index bd369ffa..3cbdada3 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -715,7 +715,7 @@ impl PostgresEngine { // asks the caller to retry; once the backfill is running it // accepts. Measured against the service on 2026-08-19. if index_status == "CREATING" && backfilling == Some(false) { - return Err(StorageError::ResourceInUse( + return Err(StorageError::IndexesInUse( extenddb_core::types::vector_index_delete_in_allocation_phase( &input.table_name, &delete.index_name, @@ -1013,6 +1013,13 @@ impl PostgresEngine { // flip below, the second inside the detached task this call spawns. let allocation_delay = std::time::Duration::from_millis(self.vector_allocation_phase_delay().await); + // The floor on how long the index stays CREATING, captured with the + // creation instant here so the hold measures from when the caller could + // first observe the index. The wait itself lives in the shared driver, + // beside the flip it delays; the SQLite backend applies the same floor. + let min_creating = + std::time::Duration::from_millis(self.vector_index_min_creating_ms().await); + let created_at = tokio::time::Instant::now(); let ownership_pool = self.data_pool.clone(); let ownership_id = index_id.to_owned(); let hold_table_id = table_id.to_owned(); @@ -1081,6 +1088,7 @@ impl PostgresEngine { &index_name, extenddb_storage::vector_lifecycle::BACKFILL_BATCH, batch_delay, + Some(created_at + min_creating), ) .await; }); diff --git a/crates/storage-postgres/tests/vector_control_plane.rs b/crates/storage-postgres/tests/vector_control_plane.rs index 0f47e78b..265638c7 100644 --- a/crates/storage-postgres/tests/vector_control_plane.rs +++ b/crates/storage-postgres/tests/vector_control_plane.rs @@ -689,15 +689,15 @@ async fn deleting_a_vector_index_in_the_allocation_phase_is_refused() { .await .expect_err("a delete during resource allocation must be refused"); - // ResourceInUse, not Validation: the request is well formed and the resource + // IndexesInUse, not Validation: the request is well formed and the resource // exists, so the client should retry rather than change the request. The // whole string is the measured one, including both resource names. match err { - StorageError::ResourceInUse(msg) => assert_eq!( + StorageError::IndexesInUse(msg) => assert_eq!( msg, extenddb_core::types::vector_index_delete_in_allocation_phase("t_phase", "vidx") ), - other => panic!("expected ResourceInUse, got {other:?}"), + other => panic!("expected IndexesInUse, got {other:?}"), } // The refusal must not have deleted anything on the way out. @@ -1900,11 +1900,11 @@ async fn a_real_build_holds_the_allocation_phase_and_the_delete_rule_follows_it( .await .expect_err("a delete during resource allocation must be refused"); match err { - StorageError::ResourceInUse(msg) => assert_eq!( + StorageError::IndexesInUse(msg) => assert_eq!( msg, extenddb_core::types::vector_index_delete_in_allocation_phase("t_phase_real", "vidx") ), - other => panic!("expected ResourceInUse, got {other:?}"), + other => panic!("expected IndexesInUse, got {other:?}"), } // Second half: once the phase advances, the same request is accepted and the diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index f84fe3d6..e1a7dbbb 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -315,6 +315,9 @@ impl SqliteEngine { "vector_index_min_creating_ms: live read failed, using {DEFAULT_MS}: {e:?}" ); DEFAULT_MS + } + } + } /// Milliseconds to hold a new vector index in the resource-allocation phase. /// diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 516c83de..7f54febb 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -600,7 +600,7 @@ impl SqliteEngine { // died before its own flip is repaired by a rebuild, which // re-asserts the phase before scanning. if index_status == "CREATING" && backfilling == Some(false) { - return Err(StorageError::ResourceInUse( + return Err(StorageError::IndexesInUse( extenddb_core::types::vector_index_delete_in_allocation_phase( &input.table_name, &delete.index_name, @@ -1776,7 +1776,7 @@ mod reconciler_tests { .await .expect_err("a delete during resource allocation must be refused"); match err { - extenddb_storage::error::StorageError::ResourceInUse(message) => assert_eq!( + extenddb_storage::error::StorageError::IndexesInUse(message) => assert_eq!( message, extenddb_core::types::vector_index_delete_in_allocation_phase("t", "vidx"), "the refusal must carry the measured wording" diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 1efa430d..75b0af02 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -64,7 +64,7 @@ adaptation when switching between ExtendDB and the real service. | Area | DynamoDB | ExtendDB | |------|----------|------| | GSI update propagation | Eventually consistent (milliseconds to seconds) | Per-GSI propagation delay. System default: `index_propagation_delay_ms` setting (default 10ms). Each GSI can override with its own `propagation_delay_ms` (stored in catalog). A value of 0 means synchronous (future sync GSI feature). | -| Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance inline in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. That zero behaves differently while an index is still building, and the two backends differ: **PostgreSQL** applies inline only to an index that is already `ACTIVE`, and defers a write to a building index whatever the delay says, because the write must not reach the index ahead of the backfill's older snapshot of the same item; **SQLite** does not check the status on this path, so with a zero delay a write during a backfill is applied inline and bypasses the hold that keeps the two ordered. The SQLite behaviour is tracked as a defect (F-20), not intended, and it is reachable only with the zero delay, which is a test setting. | +| Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance inline in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. Inline additionally requires the table's propagation queue to be empty: while rows queued during an index build are still draining, a new write queues behind them instead of overtaking them, so a brief search-after-write window exists even at 0 until the queue drains. That window is ordering, not data loss; the write is applied, in order, by the queue worker. That zero behaves differently while an index is still building, and the two backends differ: **PostgreSQL** applies inline only to an index that is already `ACTIVE`, and defers a write to a building index whatever the delay says, because the write must not reach the index ahead of the backfill's older snapshot of the same item; **SQLite** does not check the status on this path, so with a zero delay a write during a backfill is applied inline and bypasses the hold that keeps the two ordered. The SQLite behaviour is tracked as a defect (F-20), not intended, and it is reachable only with the zero delay, which is a test setting. | | `SearchVectors` score at extreme magnitudes (PostgreSQL backend only) | Returns the true distance as a number for any vector of finite components | The score is bounded to a finite value instead, and the bound is not a measured service answer. pgvector accumulates distances in single precision, so magnitudes far below `f32::MAX` overflow inside the extension: Euclidean above about 9.2e18, dot product above about 1.8e19, and cosine at both ends, above about 1.8e19 and below about 3.7e-23, which is where a component's single-precision square rounds to zero. A non-finite score cannot be serialised as JSON at all, so the result is bounded in SQL: `1e308` for an overflowed distance, `-1e308` for an overflowed negated inner product, which the score contract negates so a client sees `1e308` in `Score`, and 1.0 for a cosine that comes back NaN. Ranking is unaffected at the overflow end, because each bound sits at the end its metric overflows towards, so the farthest row stays farthest and the most similar stays most similar; two rows that both overflow tie, and the tie breaks on the base key. At the underflow end cosine loses resolution rather than being bounded, and which side is tiny decides how much. For a tiny **query** vector, its norm underflows while the inner product usually does not, so the quotient is an infinity that pgvector clamps and the reported distance collapses to one of 0, 1 or 2 following the sign of the inner product, with the 1.0 substitute firing only when the vectors are exactly orthogonal and the quotient is therefore 0/0 (measured). For a tiny **stored** vector it is worse: the stored norm is computed in single precision and reaches zero, so the zero-vector guard fires and that row reports 1.0 at every angle, parallel included. A corpus of tiny embeddings therefore loses ranking altogether rather than losing resolution. For a tiny query vector, by contrast, ranking still separates nearer-than-orthogonal from farther and loses resolution only within each half. The SQLite backend owns its own arithmetic, computes in double precision, and reports the true value, which is why this row is scoped to PostgreSQL. | | Vector index deletion window | `UpdateTable` Delete leaves the index in `DELETING` long enough to observe, then removes it | No observable `DELETING` window on either backend: the catalog row is removed inside the `UpdateTable` transaction, so a `DescribeTable` immediately afterwards already omits the index. The index's data table is dropped after that commit, in a separate transaction, and the two backends handle a failure there differently. **PostgreSQL** treats it as best effort, because the data table lives in a different database entirely: a failure is logged and skipped rather than failing the request. **SQLite** propagates it, so a failed drop returns an error from an `UpdateTable` whose catalog change has already committed, which is the more surprising outcome of the two: the index is gone from the catalog and the caller saw a failure. So an operator debugging a leftover `_ddb_vec_*` table should look for that warning rather than assume the delete was incomplete. | | Restoring a backup of a table that had vector indexes | Restores the table with its vector indexes intact: the configuration survives, items keep their vector attributes, and `SearchVectors` works as soon as the table is `ACTIVE` (measured) | Neither backend restores the indexes, and the two fail differently. **PostgreSQL refuses the restore** with a `ValidationException` naming the backup and the index count, because restore does not carry index data across and a table that looks restored while answering every search with nothing is worse than a refusal a caller can act on. **SQLite does not refuse**: its backup path does not capture vector indexes at all, so a restore silently produces the table without them. That silence is tracked as a defect rather than intended, and it is the reason the PostgreSQL path refuses instead of matching it. A backup taken from a table with no vector indexes restores normally on both. | diff --git a/docs/getting-started.md b/docs/getting-started.md index 8b862d37..9a48427f 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -215,7 +215,7 @@ extenddb runs as a daemon (background process) and logs to syslog. On startup it ```bash ./target/release/extenddb serve --config extenddb.toml -# extenddb 0.1.8 (catalog 0.0.3) starting on 127.0.0.1:18443 +# extenddb 0.1.10 (catalog 0.0.3) starting on 127.0.0.1:18443 # storage: postgres (postgresql://extenddb:***@localhost:5432/extenddb_catalog) ``` @@ -1293,7 +1293,7 @@ Each runner requires its tools to be installed. The runner checks prerequisites ```bash ./target/release/extenddb version -# extenddb 0.1.8 +# extenddb 0.1.10 # catalog 0.0.3 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z diff --git a/docs/manuals/04-quickstart-setup-guide.md b/docs/manuals/04-quickstart-setup-guide.md index 4fef43b4..faa7ceda 100755 --- a/docs/manuals/04-quickstart-setup-guide.md +++ b/docs/manuals/04-quickstart-setup-guide.md @@ -129,7 +129,7 @@ Check the version: ```bash ./target/release/extenddb version -# extenddb 0.1.8 +# extenddb 0.1.10 # catalog 0.0.3 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z diff --git a/docs/technical-debt.md b/docs/technical-debt.md index f5f4a8e6..f1fa849a 100755 --- a/docs/technical-debt.md +++ b/docs/technical-debt.md @@ -32,7 +32,7 @@ Last updated: 2026-08-20 | F-17 | `validate_attribute_name_sizes` only checks top-level attribute names — nested map keys not validated | `core/validation/mod.rs` | Low | P30 | | F-18 | ~~UpdateTable Delete of a vector index in the resource-allocation phase (`CREATING`, `Backfilling: false`) is accepted; Amazon DynamoDB refuses it with `ResourceInUseException` until backfilling starts~~ Both backends now enforce the phase rule with the measured message, and both hold the phase open under `vector_allocation_phase_delay_ms` so a client can observe it | ~~`storage-sqlite/update_table.rs` (vector delete branch), `core/types/table.rs` (`vector_index_delete_in_allocation_phase`)~~ | ~~Medium~~ | vector probe P2 | | F-19 | **Reachable in default configuration, no setting required.** SQLite backup does not capture vector indexes, so `CreateBackup` followed by `RestoreTableFromBackup` silently produces the table without them and reports success. Its `backups` table has no column that could carry an index set, where PostgreSQL added one specifically so it could detect the case and refuse, and neither SQLite entry point is gated by anything. That makes this the more reachable of the two vector gaps: F-20 needs `index_propagation_delay_ms` at zero, which is a test setting, and this needs nothing. Raised to High on that reachability: severity attaches to what a default configuration can reach, and the earlier Medium rested on a belief about whether operators back up vector tables rather than on any barrier in the code. Amazon DynamoDB preserves vector state through backup and restore (measured). The PostgreSQL backend refuses the restore rather than matching this, so the two backends fail differently: one refuses with a reason, one loses declared indexes quietly. Fix: capture the index set in the SQLite backup row and either restore it or refuse, matching PostgreSQL | `storage-sqlite/backup.rs:313-317` (the restore reads three columns: `key_schema`, `attribute_definitions`, `billing_mode`, none of which can carry an index), `storage-postgres/backup_engine.rs:140-149` and `:167` (captures the index set into the backup row), `:467-497` (refuses the restore) | High | PR-2 docs stage | -| F-20 | SQLite applies vector index maintenance inline whenever `index_propagation_delay_ms` is 0, without checking the index status, so a write landing during a backfill reaches the index data table directly and bypasses the claim-time hold that keeps it behind the backfill's older snapshot of the same item. The shared lifecycle contract requires that hold (`storage/vector_lifecycle/mod.rs:31-38`), and PostgreSQL enforces it with `delay_ms == 0 && index_status == "ACTIVE"`. Reachable only with the zero delay, which is a test setting. The symptom is not a failed build: the backfill's plain INSERT raises a primary key violation, which propagates out of the batch, and the detached build deliberately leaves the index in `CREATING` because there is no failure state on the wire. So an operator sees an index parked in `CREATING` with every search against it refused, until the stuck-build sweep or the startup reconciler rebuilds it; that rebuild drops and recreates the data table, so the retry starts clean and the state self-heals unless the interleaving repeats. On SQLite, which is the backend this item is about, that recovery is bounded at roughly two seconds: the propagation worker sweeps every pass with a maximum sleep of one second and requires two consecutive sightings before rebuilding. Do not generalise that figure: PostgreSQL's runtime sweep polls every 60 seconds and only treats a build as stuck once its heartbeat is 300 seconds stale, so an equivalent park there would persist for up to about six minutes. The bounded SQLite recovery is why this is Medium rather than High. Fix: gate the inline branch on the index being ACTIVE, and it MUST land together with the queue-emptiness gate the PostgreSQL backend now applies in `maintain_vector_indexes`. Gating on ACTIVE is what first makes SQLite produce pre-flip queued rows, and those rows are then exposed to the same stale-overwrite race: once the index flips and its hold goes, a queued row can be applied after a newer inline write to the same item and leave the index row permanently disagreeing with the base item. Landing the ACTIVE gate alone would swap one race for the other | `storage-sqlite/data/vector_index.rs:176` (inline branch), `storage-postgres/data/vector_index.rs:161` (the shape to match) | Medium | PR-2 docs stage | +| F-20 | SQLite applies vector index maintenance inline whenever `index_propagation_delay_ms` is 0, without checking the index status, so a write landing during a backfill reaches the CREATING index's data table directly, bypassing the claim-time hold that keeps queued rows behind the backfill. The shared lifecycle contract requires that hold (`storage/vector_lifecycle/mod.rs`), and PostgreSQL enforces it with `delay_ms == 0 && index_status == "ACTIVE"` plus the queue-emptiness gate. The WEDGE this item originally tracked is fixed on main: the backfill inserts with `VectorRowConflict::KeepExisting` (`INSERT OR IGNORE`), so the collision no longer errors the build (regression-tested). What remains is ordering, not liveness: an inline write to a CREATING index can be overwritten by nothing today, but once SQLite gains the ACTIVE gate it produces pre-flip queued rows, and those rows meet the same stale-overwrite race the PostgreSQL queue-emptiness gate closes. Fix: gate the inline branch on the index being ACTIVE, and it MUST land together with the queue-emptiness gate, or the change swaps one race for another | `storage-sqlite/data/vector_index.rs` (inline branch), `storage-postgres/data/vector_index.rs` (the shape to match) | Medium | PR-2 docs stage | ## Cleanup