From 505f0bdc7d4f2b6296c68b67b7f1e208bf8a5ae6 Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Fri, 21 Aug 2026 02:34:07 +0000 Subject: [PATCH 1/2] fix: make the Query/Scan refusal on a vector index reachable A vector index is not a row in the indexes catalog, so index_info_by_table_id reported it as missing and Query/Scan answered "The table does not have the specified index" for an index the table does have. The intended refusals were dead code. On the index-not-found path both handlers now re-resolve the name against the table's vector index metadata and refuse with the measured messages: Query with "Query operation not supported on this index type." (trailing period, measured 2026-08-20), Scan without one. A backfilling index gets Scan's backfilling wording while Query keeps the type refusal; an index CREATING before its backfill starts stays index-not-found, all as measured. Measured precedence is preserved: on Query the KeyConditionExpression and ConsistentRead checks fire before the refusal, on Scan the refusal fires first. --- crates/core/src/types/mod.rs | 6 +- crates/core/src/types/table.rs | 15 +++ crates/engine/src/index_helpers.rs | 181 ++++++++++++++++++++++++++++- crates/engine/src/query.rs | 73 ++++++++---- crates/engine/src/scan.rs | 65 ++++++++--- 5 files changed, 297 insertions(+), 43 deletions(-) diff --git a/crates/core/src/types/mod.rs b/crates/core/src/types/mod.rs index f17aae79..a7ed110e 100755 --- a/crates/core/src/types/mod.rs +++ b/crates/core/src/types/mod.rs @@ -61,8 +61,10 @@ pub use table::{ Tag, TagResourceInput, TimeToLiveDescription, TimeToLiveSpecification, TimeToLiveSpecificationOutput, TimeToLiveStatus, UntagResourceInput, UpdateGsiAction, UpdateTableInput, UpdateTableOutput, UpdateTimeToLiveInput, UpdateTimeToLiveOutput, - VECTOR_INDEX_ALREADY_EXISTS, VECTOR_INDEX_COUNT_LIMIT_CREATE, VECTOR_INDEX_COUNT_LIMIT_UPDATE, - VECTOR_INDEX_CREATE_IN_USE_PREFIX, VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST, + VECTOR_INDEX_ALREADY_EXISTS, VECTOR_INDEX_BACKFILLING_SCAN_PREFIX, + VECTOR_INDEX_COUNT_LIMIT_CREATE, VECTOR_INDEX_COUNT_LIMIT_UPDATE, + VECTOR_INDEX_CREATE_IN_USE_PREFIX, VECTOR_INDEX_QUERY_NOT_SUPPORTED, + VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST, VECTOR_INDEX_SCAN_NOT_SUPPORTED, VECTOR_SEARCH_SCHEMA_UNDECLARED, VECTOR_TABLE_REQUIRES_PAY_PER_REQUEST_MODE, VectorAttribute, VectorIndexDescription, VectorIndexSpecification, VectorIndexUpdate, vector_attribute_conflicting_definition, vector_attribute_redefines_key, diff --git a/crates/core/src/types/table.rs b/crates/core/src/types/table.rs index b895b2fa..a225e00b 100755 --- a/crates/core/src/types/table.rs +++ b/crates/core/src/types/table.rs @@ -663,6 +663,21 @@ pub fn vector_index_delete_in_allocation_phase(table_name: &str, index_name: &st ) } +/// Refusal for a Query naming a vector index. Measured 2026-08-20; note the +/// trailing period, which the Scan variant does not have. +pub const VECTOR_INDEX_QUERY_NOT_SUPPORTED: &str = + "Query operation not supported on this index type."; + +/// Refusal for a Scan naming a vector index that is past its backfill. +/// Measured 2026-08-20. +pub const VECTOR_INDEX_SCAN_NOT_SUPPORTED: &str = "Scan operation not supported on this index type"; + +/// Prefix of the Scan refusal while a vector index is still backfilling; +/// continues with the index name. The service reuses the GSI wording even +/// though the index is a vector index. Measured 2026-08-20. +pub const VECTOR_INDEX_BACKFILLING_SCAN_PREFIX: &str = + "Cannot read from backfilling global secondary index: "; + impl VectorIndexDescription { /// Reject a description whose reported state the service would never produce. /// diff --git a/crates/engine/src/index_helpers.rs b/crates/engine/src/index_helpers.rs index 3d3803e2..8b1fa70b 100755 --- a/crates/engine/src/index_helpers.rs +++ b/crates/engine/src/index_helpers.rs @@ -3,11 +3,14 @@ //! Shared helpers for secondary index operations in the engine layer. +use extenddb_core::error::DynamoDbError; use extenddb_core::types::{ - AttributeDefinition, AttributeValue, IndexInfo, Item, KeySchemaElement, ProjectionType, - ScalarAttributeType, + AttributeDefinition, AttributeValue, DescribeTableInput, IndexInfo, IndexStatus, Item, + KeySchemaElement, ProjectionType, ScalarAttributeType, TableKeyInfo, VectorIndexDescription, }; +use crate::OperationContext; + /// Build the combined key schema for `LastEvaluatedKey` extraction. /// /// For index queries/scans, the LEK includes both the base table key attributes @@ -162,3 +165,177 @@ fn attr_value_matches_scalar(value: &AttributeValue, scalar: ScalarAttributeType | (AttributeValue::B(_), ScalarAttributeType::B) ) } + +/// How Query and Scan must refuse an index name that is not a row in the +/// `indexes` catalog. A vector index never is one, so both handlers land here +/// for vector indexes and for genuinely absent names alike. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VectorIndexReadRefusal { + /// No vector index carries the name either, or one does but the service + /// reports it as absent (measured 2026-08-20: CREATING before the + /// backfill starts is indistinguishable from a nonexistent index). + NotFound, + /// A vector index mid-backfill. Scan gets the backfilling wording while + /// Query keeps the ordinary type refusal (measured 2026-08-20). + Backfilling, + /// A vector index past its backfill: the operation-specific type refusal. + NotSupported, +} + +/// Classify a vector index (or its absence) for a Query/Scan refusal. +/// +/// Mirrors the measured lifecycle: ACTIVE and CREATING-while-backfilling are +/// recognized as vector indexes; every other state reads as index-not-found, +/// matching how `SearchVectors` treats a non-ACTIVE index as absent. The +/// backfilling arm requires CREATING so a stale `Backfilling` member on any +/// other status cannot resurrect the backfilling wording. +pub fn classify_vector_index_read( + vector_index: Option<&VectorIndexDescription>, +) -> VectorIndexReadRefusal { + match vector_index { + Some(vi) if vi.index_status.is_active() => VectorIndexReadRefusal::NotSupported, + Some(vi) if vi.index_status == IndexStatus::Creating && vi.backfilling == Some(true) => { + VectorIndexReadRefusal::Backfilling + } + _ => VectorIndexReadRefusal::NotFound, + } +} + +/// Resolve an index name that `index_info_by_table_id` reported missing +/// against the table's vector index metadata. +/// +/// The `key_info` name pre-filter keeps the dominant case (a mistyped GSI +/// name on a table with no vector indexes) free of the extra round trip; the +/// `describe_table` read runs only when the name matches a known vector +/// index, because the key-info cache carries no lifecycle status and backfill +/// transitions never invalidate it. +/// +/// # Errors +/// Propagates storage failures from `describe_table`. +pub async fn classify_unresolved_index_read( + ctx: &OperationContext, + key_info: &TableKeyInfo, + index_name: &str, +) -> Result { + if !key_info + .vector_indexes + .iter() + .any(|vi| vi.index_name == index_name) + { + return Ok(VectorIndexReadRefusal::NotFound); + } + let table = ctx + .storage + .describe_table( + &ctx.account_id, + DescribeTableInput { + table_name: key_info.table_name.clone(), + }, + ) + .await + .map_err(crate::create_table::storage_err_to_dynamo)?; + Ok(classify_vector_index_read( + table + .vector_indexes + .as_deref() + .unwrap_or(&[]) + .iter() + .find(|vi| vi.index_name == index_name), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::types::{ + DistanceFunction, IndexStatus, Projection, VectorAttribute, VectorIndexDescription, + }; + + fn description(index_status: IndexStatus, backfilling: Option) -> VectorIndexDescription { + VectorIndexDescription { + index_name: "vidx".to_owned(), + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + dimensions: 4, + search_schema: None, + distance_function: DistanceFunction::Cosine, + index_status, + backfilling, + index_size_bytes: 0, + item_count: 0, + index_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/t/index/vidx".to_owned(), + projection: Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + } + } + + #[test] + fn absent_index_is_not_found() { + assert_eq!( + classify_vector_index_read(None), + VectorIndexReadRefusal::NotFound + ); + } + + #[test] + fn active_index_gets_the_type_refusal() { + let vi = description(IndexStatus::Active, None); + assert_eq!( + classify_vector_index_read(Some(&vi)), + VectorIndexReadRefusal::NotSupported + ); + } + + #[test] + fn creating_before_backfill_reads_as_not_found() { + // Measured 2026-08-20: CREATING with Backfilling=false answers with + // the index-not-found message, exactly like a nonexistent index. + let vi = description(IndexStatus::Creating, Some(false)); + assert_eq!( + classify_vector_index_read(Some(&vi)), + VectorIndexReadRefusal::NotFound + ); + let vi = description(IndexStatus::Creating, None); + assert_eq!( + classify_vector_index_read(Some(&vi)), + VectorIndexReadRefusal::NotFound + ); + } + + #[test] + fn creating_while_backfilling_is_backfilling() { + let vi = description(IndexStatus::Creating, Some(true)); + assert_eq!( + classify_vector_index_read(Some(&vi)), + VectorIndexReadRefusal::Backfilling + ); + } + + #[test] + fn stale_backfilling_on_a_non_creating_status_stays_not_found() { + // The backfilling wording is tied to CREATING; a stale Backfilling + // member on any other status must not resurrect it. + let vi = description(IndexStatus::Deleting, Some(true)); + assert_eq!( + classify_vector_index_read(Some(&vi)), + VectorIndexReadRefusal::NotFound + ); + let vi = description(IndexStatus::Updating, Some(true)); + assert_eq!( + classify_vector_index_read(Some(&vi)), + VectorIndexReadRefusal::NotFound + ); + } + + #[test] + fn deleting_reads_as_not_found() { + let vi = description(IndexStatus::Deleting, None); + assert_eq!( + classify_vector_index_read(Some(&vi)), + VectorIndexReadRefusal::NotFound + ); + } +} diff --git a/crates/engine/src/query.rs b/crates/engine/src/query.rs index 1693b9a6..98989d11 100755 --- a/crates/engine/src/query.rs +++ b/crates/engine/src/query.rs @@ -11,15 +11,19 @@ use extenddb_core::error::DynamoDbError; use extenddb_core::expression::PathElement; use extenddb_core::expression::{ExpressionKind, ExpressionMaps, Projection}; use extenddb_core::types::{ - IndexType, KeyType, ProjectionType, QueryInput, QueryOutput, Select, TableKeyInfo, extract_key, - item_size_bytes, + IndexType, KeyType, ProjectionType, QueryInput, QueryOutput, Select, TableKeyInfo, + VECTOR_INDEX_QUERY_NOT_SUPPORTED, extract_key, item_size_bytes, }; +use extenddb_storage::error::StorageError; use crate::OperationContext; use crate::capacity_helpers; use crate::create_table::storage_err_to_dynamo; use crate::expression_helpers::{build_expression_maps, parse_optional_filter}; -use crate::index_helpers::{combined_lek_key_schema, validate_query_exclusive_start_key}; +use crate::index_helpers::{ + VectorIndexReadRefusal, classify_unresolved_index_read, combined_lek_key_schema, + validate_query_exclusive_start_key, +}; use crate::legacy_filter::{desugar_filter, desugar_key_conditions}; use crate::read_helpers::apply_post_read; use crate::serialize_output; @@ -75,30 +79,48 @@ pub async fn handle_query( // GSI/LSI: resolve index metadata if querying a secondary index. // Uses table_id from pre-fetched key_info to skip redundant table lookup (P118 #4). - let index_info = if let Some(ref idx_name) = input.index_name { - Some( - ctx.storage + // A vector index is not a row in the `indexes` catalog, so a not-found + // result is re-resolved against the vector index metadata before the + // name is treated as absent. + let (index_info, vector_index_named) = match input.index_name { + Some(ref idx_name) => { + match ctx + .storage .index_info_by_table_id(&key_info.table_id, idx_name) .await - .map_err(storage_err_to_dynamo)?, - ) - } else { - None + { + // Defense in depth: no in-tree backend stores a vector index in + // `indexes`, but if one ever surfaces here it must be refused, + // not sent down the GSI/LSI data path. + Ok(info) if info.index_type == IndexType::Vector => (None, true), + Ok(info) => (Some(info), false), + Err(err @ StorageError::IndexNotFound(_)) => { + match classify_unresolved_index_read(ctx, &key_info, idx_name).await? { + VectorIndexReadRefusal::NotFound => { + return Err(storage_err_to_dynamo(err)); + } + // Query refuses a backfilling vector index with the same + // message as an active one (measured 2026-08-20; Scan + // differs). The refusal itself fires further down, after + // the KeyConditionExpression checks. + VectorIndexReadRefusal::Backfilling + | VectorIndexReadRefusal::NotSupported => (None, true), + } + } + Err(err) => return Err(storage_err_to_dynamo(err)), + } + } + None => (None, false), }; - // A vector index is searched only via the vector search API, never queried. - if let Some(ref idx) = index_info - && idx.index_type == IndexType::Vector - { - return Err(DynamoDbError::ValidationException( - "Query operation not supported on this index type".to_owned(), - )); - } - // ConsistentRead is not supported on GSI queries (tenet 1: fidelity). + // Measured 2026-08-20: it fires for a vector index too, with the same + // wording, and before the vector-index refusal. if input.consistent_read == Some(true) - && let Some(ref idx) = index_info - && idx.index_type == IndexType::Gsi + && (vector_index_named + || index_info + .as_ref() + .is_some_and(|idx| idx.index_type == IndexType::Gsi)) { return Err(DynamoDbError::ValidationException( "Consistent reads are not supported on global secondary indexes".to_owned(), @@ -232,6 +254,15 @@ pub async fn handle_query( )); }; + // A vector index is searched only via the vector search API, never + // queried. Measured 2026-08-20: the service refuses after the + // KeyConditionExpression presence and syntax checks, hence below the parse. + if vector_index_named { + return Err(DynamoDbError::ValidationException( + VECTOR_INDEX_QUERY_NOT_SUPPORTED.to_owned(), + )); + } + // Use legacy maps for key condition resolution if KeyConditions was used let effective_maps = if let Some(ref kc_maps) = legacy_kc_maps { kc_maps diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index df8b77a8..84c60cde 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -10,15 +10,20 @@ use serde_json::Value; use extenddb_core::error::DynamoDbError; use extenddb_core::expression::{ExpressionKind, ExpressionMaps, Projection}; use extenddb_core::types::{ - IndexType, ProjectionType, ScanInput, ScanOutput, Select, TableKeyInfo, extract_key, + IndexType, ProjectionType, ScanInput, ScanOutput, Select, TableKeyInfo, + VECTOR_INDEX_BACKFILLING_SCAN_PREFIX, VECTOR_INDEX_SCAN_NOT_SUPPORTED, extract_key, item_size_bytes, }; +use extenddb_storage::error::StorageError; use crate::OperationContext; use crate::capacity_helpers; use crate::create_table::storage_err_to_dynamo; use crate::expression_helpers::{build_expression_maps, parse_optional_filter}; -use crate::index_helpers::{combined_lek_key_schema, validate_scan_exclusive_start_key}; +use crate::index_helpers::{ + VectorIndexReadRefusal, classify_unresolved_index_read, combined_lek_key_schema, + validate_scan_exclusive_start_key, +}; use crate::legacy_filter::desugar_filter; use crate::read_helpers::apply_post_read; use crate::serialize_output; @@ -231,26 +236,50 @@ pub async fn handle_scan( // GSI/LSI: resolve index metadata if scanning a secondary index. // Uses table_id from pre-fetched key_info to skip redundant table lookup (P118 #4). - let index_info = if let Some(ref idx_name) = input.index_name { - Some( - ctx.storage + // A vector index is not a row in the `indexes` catalog, so a not-found + // result is re-resolved against the vector index metadata before the + // name is treated as absent. A vector index is searched only via the + // vector search API, never scanned; the refusal fires here, before the + // ConsistentRead check (measured 2026-08-20; Query is the opposite order). + let index_info = match input.index_name { + Some(ref idx_name) => { + match ctx + .storage .index_info_by_table_id(&key_info.table_id, idx_name) .await - .map_err(storage_err_to_dynamo)?, - ) - } else { - None + { + // Defense in depth: no in-tree backend stores a vector index in + // `indexes`, but if one ever surfaces here it must be refused, + // not sent down the GSI/LSI data path. + Ok(info) if info.index_type == IndexType::Vector => { + return Err(DynamoDbError::ValidationException( + VECTOR_INDEX_SCAN_NOT_SUPPORTED.to_owned(), + )); + } + Ok(info) => Some(info), + Err(err @ StorageError::IndexNotFound(_)) => { + return Err( + match classify_unresolved_index_read(ctx, &key_info, idx_name).await? { + VectorIndexReadRefusal::NotFound => storage_err_to_dynamo(err), + VectorIndexReadRefusal::Backfilling => { + DynamoDbError::ValidationException(format!( + "{VECTOR_INDEX_BACKFILLING_SCAN_PREFIX}{idx_name}" + )) + } + VectorIndexReadRefusal::NotSupported => { + DynamoDbError::ValidationException( + VECTOR_INDEX_SCAN_NOT_SUPPORTED.to_owned(), + ) + } + }, + ); + } + Err(err) => return Err(storage_err_to_dynamo(err)), + } + } + None => None, }; - // A vector index is searched only via the vector search API, never scanned. - if let Some(ref idx) = index_info - && idx.index_type == IndexType::Vector - { - return Err(DynamoDbError::ValidationException( - "Scan operation not supported on this index type".to_owned(), - )); - } - // ConsistentRead is not supported on GSI scans (tenet 1: fidelity). if input.consistent_read == Some(true) && let Some(ref idx) = index_info From 56340000f11a69a13affaefa0145d12baa60057e Mon Sep 17 00:00:00 2001 From: Anandh Somasundaram Date: Fri, 21 Aug 2026 02:34:07 +0000 Subject: [PATCH 2/2] test: Query and Scan refusal parity on a vector index Dual-target pytest and a wire test covering the refusals, the index-not-found contrast case, the measured precedence against the KeyConditionExpression and ConsistentRead checks, and base-table/GSI controls. --- tests/rust/src/vector_index_search.rs | 225 +++++++++++++++++ tests/test_vector_index_query_scan.py | 331 ++++++++++++++++++++++++++ 2 files changed, 556 insertions(+) create mode 100644 tests/test_vector_index_query_scan.py diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 0528f14f..78249cd5 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -2874,3 +2874,228 @@ async fn a_transaction_reports_an_invalid_vector_per_item() { let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } + +// --------------------------------------------------------------------------- +// Query and Scan naming a vector index +// --------------------------------------------------------------------------- + +/// A vector index is searched only via the vector search API: Query and Scan +/// naming it are refused with the operation-specific messages, while a name +/// that matches no index of any kind keeps the index-not-found message. +/// +/// Whole strings and precedence measured against Amazon DynamoDB 2026-08-20: +/// the Query refusal carries a trailing period, the Scan one does not; on +/// Query the KeyConditionExpression and ConsistentRead checks fire before the +/// refusal, on Scan the refusal fires before the ConsistentRead check. +#[tokio::test] +async fn query_and_scan_on_a_vector_index_are_refused() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_query_scan"); + create_vector_table(&name, 2, "COSINE", false).await; + + let key_condition = + r#""KeyConditionExpression": "pk = :v", "ExpressionAttributeValues": {":v": {"S": "a"}}"#; + + // Query: the type refusal, with the trailing period the service emits. + let (status, text) = call( + "Query", + &format!(r#"{{"TableName": "{name}", "IndexName": "vidx", {key_condition}}}"#), + ) + .await; + assert_eq!( + status, 400, + "Query on a vector index must be refused: {text}" + ); + assert!( + text.contains("Query operation not supported on this index type."), + "expected the measured whole-string refusal, trailing period included: {text}" + ); + + // Scan: the type refusal, without a trailing period. The closing quote is + // part of the match so a period cannot sneak in. + let (status, text) = call( + "Scan", + &format!(r#"{{"TableName": "{name}", "IndexName": "vidx"}}"#), + ) + .await; + assert_eq!( + status, 400, + "Scan on a vector index must be refused: {text}" + ); + assert!( + text.contains(r#"Scan operation not supported on this index type""#), + "expected the measured whole-string refusal, no trailing period: {text}" + ); + + // A genuinely absent index name keeps the index-not-found message on both. + for op in ["Query", "Scan"] { + let cond = if op == "Query" { + format!(", {key_condition}") + } else { + String::new() + }; + let (status, text) = call( + op, + &format!(r#"{{"TableName": "{name}", "IndexName": "nosuchindex"{cond}}}"#), + ) + .await; + assert_eq!( + status, 400, + "{op} on a nonexistent index must be refused: {text}" + ); + assert!( + text.contains("The table does not have the specified index: nosuchindex"), + "{op}: a nonexistent index must stay index-not-found: {text}" + ); + } + + // Precedence, Query side: ConsistentRead and a missing + // KeyConditionExpression both fire before the vector refusal. + let (status, text) = call( + "Query", + &format!( + r#"{{"TableName": "{name}", "IndexName": "vidx", "ConsistentRead": true, {key_condition}}}"# + ), + ) + .await; + assert_eq!(status, 400, "response: {text}"); + assert!( + text.contains("Consistent reads are not supported on global secondary indexes"), + "ConsistentRead must fire before the vector refusal, with the GSI wording: {text}" + ); + + let (status, text) = call( + "Query", + &format!(r#"{{"TableName": "{name}", "IndexName": "vidx"}}"#), + ) + .await; + assert_eq!(status, 400, "response: {text}"); + assert!( + text.contains( + "Either the KeyConditions or KeyConditionExpression parameter must be specified" + ), + "a missing KeyConditionExpression must fire before the vector refusal: {text}" + ); + + // Precedence, Scan side: the refusal fires before the ConsistentRead check. + let (status, text) = call( + "Scan", + &format!(r#"{{"TableName": "{name}", "IndexName": "vidx", "ConsistentRead": true}}"#), + ) + .await; + assert_eq!(status, 400, "response: {text}"); + assert!( + text.contains(r#"Scan operation not supported on this index type""#), + "the Scan refusal must fire before the ConsistentRead check: {text}" + ); + + // Control: the base table still answers. + let (status, text) = call( + "Query", + &format!(r#"{{"TableName": "{name}", {key_condition}}}"#), + ) + .await; + assert_eq!(status, 200, "base-table Query must be unaffected: {text}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Scan and Query against a vector index observed mid-backfill. +/// +/// Measured against Amazon DynamoDB 2026-08-20, on an index in CREATING with +/// `Backfilling: true`: Scan is refused with the GSI backfilling wording +/// ("Cannot read from backfilling global secondary index: vidx") while Query +/// keeps the ordinary type refusal, trailing period included. The window is +/// held open with the backfill batch delay, the same mechanism +/// `a_building_vector_index_is_visible_but_not_searchable` uses; the CREATING +/// phase before the backfill starts is too short to pin at the wire and is +/// covered by the classifier unit tests instead. +#[tokio::test] +async fn scan_during_backfill_gets_the_backfilling_wording() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_backfill_scan"); + let (status, text) = call( + "CreateTable", + &format!( + r#"{{"TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST"}}"# + ), + ) + .await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name).await; + + // More than one batch, so the delay is actually reached. + for i in 0..600 { + put_vector(&name, &format!("k{i:06}"), None, &[1.0, 0.0]).await; + } + + set_backfill_delay(3000).await; + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", "Dimensions": 2, "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}}}}}}]}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable failed: {text}"); + + // Wait for the backfill to be observably in flight, so the assertions + // below cannot race the pre-backfill CREATING phase. + let mut backfilling = false; + for _ in 0..100 { + let (_, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + let d: serde_json::Value = serde_json::from_str(&text).expect("json"); + let vi = &d["Table"]["VectorIndexes"][0]; + if vi["IndexStatus"] == "CREATING" && vi["Backfilling"] == true { + backfilling = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!( + backfilling, + "the index never reported CREATING with Backfilling: true; the batch \ + delay did not hold the window open" + ); + + let (status, text) = call( + "Scan", + &format!(r#"{{"TableName": "{name}", "IndexName": "vidx"}}"#), + ) + .await; + assert_eq!(status, 400, "Scan mid-backfill must be refused: {text}"); + assert!( + text.contains("Cannot read from backfilling global secondary index: vidx"), + "expected the measured GSI-worded backfilling refusal: {text}" + ); + + let (status, text) = call( + "Query", + &format!( + r#"{{"TableName": "{name}", "IndexName": "vidx", + "KeyConditionExpression": "pk = :v", + "ExpressionAttributeValues": {{":v": {{"S": "a"}}}}}}"# + ), + ) + .await; + assert_eq!(status, 400, "Query mid-backfill must be refused: {text}"); + assert!( + text.contains("Query operation not supported on this index type."), + "Query mid-backfill keeps the type refusal, not the backfilling wording: {text}" + ); + + // Release the window and leave the instance quiet before deleting. + set_backfill_delay(0).await; + wait_for_vector_index_active(&name, "vidx").await; + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} diff --git a/tests/test_vector_index_query_scan.py b/tests/test_vector_index_query_scan.py new file mode 100644 index 00000000..af75a783 --- /dev/null +++ b/tests/test_vector_index_query_scan.py @@ -0,0 +1,331 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Query and Scan naming a vector index — refusal message parity. + +A vector index can never serve a Query or a Scan; the service refuses both with +an operation-specific message. Measured against Amazon DynamoDB on 2026-08-20 +(whole strings, ACTIVE index): + +- ``Query`` -> ``"Query operation not supported on this index type."`` (note the + trailing period) +- ``Scan`` -> ``"Scan operation not supported on this index type"`` (no period) + +A name that matches no index of any kind keeps the ordinary +``"The table does not have the specified index: "`` message, so the two +cases must stay distinguishable. + +Measured precedence on the same capture run: + +- Query: a missing or syntactically invalid ``KeyConditionExpression`` fires + before the vector refusal, and ``ConsistentRead=true`` fires before it too, + reusing the GSI wording ("Consistent reads are not supported on global + secondary indexes") even though the index is a vector index. +- Scan: the vector refusal fires before the ``ConsistentRead`` check. + +Tests post raw SigV4-signed JSON because published SDK models do not carry +``VectorIndexes``. The whole module skips when the target backend does not +support vector indexes (it then cannot create the table these tests need). + +REQ-TEST-001, REQ-TEST-002, REQ-TEST-003 +""" + +from __future__ import annotations + +import json +import os +import time +import uuid + +import boto3 +import pytest +import requests +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest + +_ENDPOINT_VAR = os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip() +_REGION = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") +# Real DynamoDB when no local endpoint is configured. +ENDPOINT = _ENDPOINT_VAR or f"https://dynamodb.{_REGION}.amazonaws.com/" + +QUERY_REFUSAL = "Query operation not supported on this index type." +SCAN_REFUSAL = "Scan operation not supported on this index type" +CONSISTENT_READ_REFUSAL = ( + "Consistent reads are not supported on global secondary indexes" +) + + +def _signed_post(operation: str, body: dict) -> requests.Response: + """POST ``body`` under the DynamoDB JSON-1.0 protocol, SigV4-signed. + + Credentials come from the default boto3 chain, so the same call works + against real DynamoDB (profile or env creds) and against extenddb (the + provisioned test keys in the environment). + """ + body_bytes = json.dumps(body).encode("utf-8") + headers = { + "X-Amz-Target": f"DynamoDB_20120810.{operation}", + "Content-Type": "application/x-amz-json-1.0", + } + creds = boto3.Session().get_credentials() + if creds is not None: + aws_req = AWSRequest(method="POST", url=ENDPOINT, data=body_bytes, headers=headers) + SigV4Auth(creds.get_frozen_credentials(), "dynamodb", _REGION).add_auth(aws_req) + headers = dict(aws_req.headers) + # Real DynamoDB gets normal TLS verification; a local https extenddb + # endpoint uses the self-signed cert from `extenddb init`. + verify = True if not _ENDPOINT_VAR else not ENDPOINT.startswith("https://") + return requests.post( + ENDPOINT, + data=body_bytes, + headers=headers, + verify=verify, + ) + + +def _error_message(resp: requests.Response) -> str: + payload = resp.json() + return payload.get("message", payload.get("Message", "")) + + +def _wait_for_vector_table_active(name: str, timeout: float = 180.0) -> None: + """Poll until the table and its vector indexes are all ACTIVE.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + resp = _signed_post("DescribeTable", {"TableName": name}) + if resp.status_code == 200: + table = resp.json()["Table"] + vector_indexes = table.get("VectorIndexes", []) + if table["TableStatus"] == "ACTIVE" and all( + vi.get("IndexStatus") == "ACTIVE" for vi in vector_indexes + ): + return + time.sleep(2.0 if not _ENDPOINT_VAR else 0.05) + raise TimeoutError(f"table {name} did not become ACTIVE within {timeout}s") + + +@pytest.fixture(scope="module") +def vector_table(): + """A table with a GSI and an ACTIVE vector index, or skip if unsupported. + + Module-scoped: vector tables are slow to create against the real service, + and every test here is read-only against the same fixture. + """ + name = f"extenddb-test-{uuid.uuid4().hex[:12]}" + resp = _signed_post( + "CreateTable", + { + "TableName": name, + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, + ], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST", + "GlobalSecondaryIndexes": [ + { + "IndexName": "gidx", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + } + ], + "VectorIndexes": [ + { + "IndexName": "vidx", + "Dimensions": 4, + "DistanceFunction": "COSINE", + "VectorAttribute": {"AttributeName": "emb"}, + "Projection": {"ProjectionType": "ALL"}, + } + ], + }, + ) + if resp.status_code == 400 and "not supported" in _error_message(resp): + pytest.skip("target backend does not support vector indexes") + assert resp.status_code == 200, f"CreateTable failed: {resp.text}" + try: + _wait_for_vector_table_active(name) + # A target may accept CreateTable while silently ignoring the unknown + # VectorIndexes member (the wait above then passes vacuously). Without + # the index every refusal test would fail with confusing not-found + # diffs, so verify it exists and skip when it does not. + describe = _signed_post("DescribeTable", {"TableName": name}) + assert describe.status_code == 200, describe.text + index_names = [ + vi.get("IndexName") + for vi in describe.json()["Table"].get("VectorIndexes", []) + ] + if "vidx" not in index_names: + pytest.skip( + "target accepted CreateTable but did not create the vector index" + ) + put = _signed_post( + "PutItem", + { + "TableName": name, + "Item": { + "pk": {"S": "a"}, + "gpk": {"S": "g"}, + "emb": { + "L": [{"N": "0.1"}, {"N": "0.2"}, {"N": "0.3"}, {"N": "0.4"}] + }, + }, + }, + ) + assert put.status_code == 200, f"PutItem failed: {put.text}" + yield name + finally: + _signed_post("DeleteTable", {"TableName": name}) + + +_KEY_CONDITION = { + "KeyConditionExpression": "pk = :v", + "ExpressionAttributeValues": {":v": {"S": "a"}}, +} + + +class TestVectorIndexRefusal: + """Query/Scan naming the ACTIVE vector index get the type refusal.""" + + def test_query_on_vector_index_refused(self, vector_table): + resp = _signed_post( + "Query", {"TableName": vector_table, "IndexName": "vidx", **_KEY_CONDITION} + ) + assert resp.status_code == 400, resp.text + assert _error_message(resp) == QUERY_REFUSAL + + def test_scan_on_vector_index_refused(self, vector_table): + resp = _signed_post("Scan", {"TableName": vector_table, "IndexName": "vidx"}) + assert resp.status_code == 400, resp.text + assert _error_message(resp) == SCAN_REFUSAL + + +class TestNonexistentIndexUnchanged: + """A genuinely absent index name keeps the index-not-found message.""" + + def test_query_nonexistent_index(self, vector_table): + resp = _signed_post( + "Query", + {"TableName": vector_table, "IndexName": "nosuchindex", **_KEY_CONDITION}, + ) + assert resp.status_code == 400, resp.text + assert ( + _error_message(resp) + == "The table does not have the specified index: nosuchindex" + ) + + def test_scan_nonexistent_index(self, vector_table): + resp = _signed_post( + "Scan", {"TableName": vector_table, "IndexName": "nosuchindex"} + ) + assert resp.status_code == 400, resp.text + assert ( + _error_message(resp) + == "The table does not have the specified index: nosuchindex" + ) + + +class TestPrecedence: + """Measured ordering between the vector refusal and neighbouring checks.""" + + def test_query_consistent_read_fires_before_vector_refusal(self, vector_table): + resp = _signed_post( + "Query", + { + "TableName": vector_table, + "IndexName": "vidx", + "ConsistentRead": True, + **_KEY_CONDITION, + }, + ) + assert resp.status_code == 400, resp.text + assert _error_message(resp) == CONSISTENT_READ_REFUSAL + + def test_scan_vector_refusal_fires_before_consistent_read(self, vector_table): + resp = _signed_post( + "Scan", + {"TableName": vector_table, "IndexName": "vidx", "ConsistentRead": True}, + ) + assert resp.status_code == 400, resp.text + assert _error_message(resp) == SCAN_REFUSAL + + def test_query_missing_key_condition_fires_before_vector_refusal( + self, vector_table + ): + resp = _signed_post("Query", {"TableName": vector_table, "IndexName": "vidx"}) + assert resp.status_code == 400, resp.text + assert _error_message(resp) == ( + "Either the KeyConditions or KeyConditionExpression parameter " + "must be specified in the request." + ) + + def test_query_key_condition_syntax_fires_before_vector_refusal( + self, vector_table + ): + resp = _signed_post( + "Query", + { + "TableName": vector_table, + "IndexName": "vidx", + "KeyConditionExpression": "pk = = :v", + "ExpressionAttributeValues": {":v": {"S": "a"}}, + }, + ) + assert resp.status_code == 400, resp.text + # Only the prefix is asserted: the parse-failure suffix wording differs + # between implementations for this expression. The point here is the + # precedence, i.e. that a KeyConditionExpression parse error wins over + # the vector-index refusal. + assert _error_message(resp).startswith("Invalid KeyConditionExpression:") + + +class TestControlsUnaffected: + """Base-table and GSI reads on the same table still succeed.""" + + @staticmethod + def _read_until_count(operation: str, body: dict, want: int = 1) -> requests.Response: + """Re-issue an eventually-consistent read briefly until it sees the item. + + GSI propagation on the real service can lag the PutItem by a moment; + these controls assert reachability, not propagation latency. + """ + deadline = time.monotonic() + 10.0 + while True: + resp = _signed_post(operation, body) + if resp.status_code != 200 or resp.json().get("Count") == want: + return resp + if time.monotonic() >= deadline: + return resp + time.sleep(0.5) + + def test_query_base_table(self, vector_table): + resp = self._read_until_count( + "Query", {"TableName": vector_table, **_KEY_CONDITION} + ) + assert resp.status_code == 200, resp.text + assert resp.json()["Count"] == 1 + + def test_scan_base_table(self, vector_table): + resp = self._read_until_count("Scan", {"TableName": vector_table}) + assert resp.status_code == 200, resp.text + assert resp.json()["Count"] == 1 + + def test_query_gsi(self, vector_table): + resp = self._read_until_count( + "Query", + { + "TableName": vector_table, + "IndexName": "gidx", + "KeyConditionExpression": "gpk = :v", + "ExpressionAttributeValues": {":v": {"S": "g"}}, + }, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["Count"] == 1 + + def test_scan_gsi(self, vector_table): + resp = self._read_until_count( + "Scan", {"TableName": vector_table, "IndexName": "gidx"} + ) + assert resp.status_code == 200, resp.text + assert resp.json()["Count"] == 1