diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ac68cc65..aec1d8cc 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -256,6 +256,14 @@ jobs: AWS_DEFAULT_REGION: us-east-1 AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }} AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }} + # No in-tree backend implements vector search, so every vector request + # must be refused and the refusal suite must actually run. Without this + # the suite self-skips the moment a backend gains vector support, and the + # contract that non-participating backends refuse would stop being + # checked anywhere while still reporting green. Pinning the expectation + # turns that skip into a failure. The first backend to implement vector + # search sets this to 1 in its own job. + EXTENDDB_EXPECT_VECTORS: "0" run: | # Self-signed cert generated by init; trust it for the SDK client. export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)" diff --git a/.gitignore b/.gitignore index 8718591b..4d05e972 100755 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target +/target-* extenddb.toml __pycache__/ .coverage @@ -19,3 +20,8 @@ extenddb.toml.bak .env .env.* !.env.example + +# Local test/dev config files +extenddb-*.toml +extenddb-*.toml.bak +extenddb-*.toml.keep diff --git a/crates/core/src/error/mod.rs b/crates/core/src/error/mod.rs index 32c10117..1f9d0a06 100755 --- a/crates/core/src/error/mod.rs +++ b/crates/core/src/error/mod.rs @@ -25,6 +25,15 @@ pub enum DynamoDbError { BackupNotFoundException(String), #[error("{0}")] ResourceInUseException(String), + /// A per-table or per-account limit was exceeded. + /// + /// Distinct from `ValidationException` on purpose: the service uses this + /// class for the vector-index count limit when the index is added through + /// `UpdateTable`, while `CreateTable` reports the same limit as a + /// `ValidationException`. Measured 2026-08-13; see + /// `VECTOR_INDEX_COUNT_LIMIT_UPDATE`. + #[error("{0}")] + LimitExceededException(String), #[error("{0}")] ConditionalCheckFailedException(String, Option), #[error("{message}")] @@ -106,6 +115,7 @@ impl DynamoDbError { | Self::ResourceNotFoundException(_) | Self::BackupNotFoundException(_) | Self::ResourceInUseException(_) + | Self::LimitExceededException(_) | Self::ConditionalCheckFailedException(..) | Self::TransactionCanceledException { .. } | Self::IdempotentParameterMismatchException(_) @@ -146,6 +156,7 @@ impl DynamoDbError { Self::ResourceNotFoundException(_) => "ResourceNotFoundException", Self::BackupNotFoundException(_) => "BackupNotFoundException", Self::ResourceInUseException(_) => "ResourceInUseException", + Self::LimitExceededException(_) => "LimitExceededException", Self::ConditionalCheckFailedException(..) => "ConditionalCheckFailedException", Self::TransactionCanceledException { .. } => "TransactionCanceledException", Self::IdempotentParameterMismatchException(_) => "IdempotentParameterMismatchException", @@ -211,6 +222,7 @@ impl DynamoDbError { | Self::ResourceNotFoundException(m) | Self::BackupNotFoundException(m) | Self::ResourceInUseException(m) + | Self::LimitExceededException(m) | Self::ConditionalCheckFailedException(m, _) | Self::IdempotentParameterMismatchException(m) | Self::SerializationException(m) diff --git a/crates/core/src/expression/mod.rs b/crates/core/src/expression/mod.rs index 8a4df5b7..4935d345 100755 --- a/crates/core/src/expression/mod.rs +++ b/crates/core/src/expression/mod.rs @@ -17,6 +17,7 @@ mod parser_common; mod projection; mod reserved_words; mod resolver; +mod search_condition; mod tokenizer; mod update_evaluator; mod update_parser; @@ -33,6 +34,11 @@ pub use resolver::{ resolve_name_ref, resolve_path, validate_begins_with_operands, validate_expression_param_usage, validate_ordering_operand_types, validate_unused_attributes, }; +pub use search_condition::{ + MAX_INLINE_FILTER_CONDITIONS, MAX_PARTITION_KEY_CONDITIONS, MAX_SEARCH_CONDITIONS, + SearchCondition, validate_conditions_against_search_schema, + validate_search_condition_expression, +}; pub use tokenizer::{Token, tokenize, tokenize_for, tokenize_with_limit}; -pub use update_evaluator::apply_update; +pub use update_evaluator::{apply_update, apply_update_validated}; pub use update_parser::{parse_update, parse_update_from}; diff --git a/crates/core/src/expression/search_condition.rs b/crates/core/src/expression/search_condition.rs new file mode 100644 index 00000000..11bce142 --- /dev/null +++ b/crates/core/src/expression/search_condition.rs @@ -0,0 +1,535 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Structural validation for the vector search filter expression. +//! +//! The vector search API accepts an optional filter expression that restricts +//! candidates before ranking. The grammar is deliberately narrow: a +//! conjunction (`AND`) of equality conditions (`name = :value`) over the +//! attributes declared in the index search schema, with at most one partition +//! key and a bounded number of inline-filter keys. +//! +//! This module performs the structural checks that do not need table or index +//! metadata (comparator, logical operator, nesting, duplicates, count, and +//! placeholder resolution). Schema-aware checks (attribute membership, required +//! partition key, and value-type agreement) run in the engine once the index +//! search schema is known. +//! +//! Pure synchronous Rust: no async, no I/O. + +use std::collections::HashMap; + +use crate::error::DynamoDbError; +use crate::types::{ + AttributeDefinition, AttributeValue, ScalarAttributeType, SearchSchemaElement, + SearchSchemaElementType, +}; + +/// Maximum number of partition-key conditions allowed in a filter expression. +pub const MAX_PARTITION_KEY_CONDITIONS: usize = 1; + +/// Maximum number of inline-filter conditions allowed in a filter expression. +pub const MAX_INLINE_FILTER_CONDITIONS: usize = 20; + +/// Maximum total number of equality conditions in a filter expression. +pub const MAX_SEARCH_CONDITIONS: usize = + MAX_PARTITION_KEY_CONDITIONS + MAX_INLINE_FILTER_CONDITIONS; + +/// A single resolved equality condition: `attribute_name = value`. +#[derive(Debug, Clone, PartialEq)] +pub struct SearchCondition { + /// The resolved attribute name (expression-attribute-name aliases applied). + pub attribute_name: String, + /// The resolved attribute value (expression-attribute-value applied). + pub value: AttributeValue, +} + +fn invalid(msg: impl Into) -> DynamoDbError { + DynamoDbError::ValidationException(msg.into()) +} + +/// Validate a vector search filter expression and return its resolved +/// equality conditions. +/// +/// Performs only the schema-independent structural checks. The returned +/// conditions carry resolved attribute names and values for schema-aware +/// validation by the caller. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` when the expression uses a +/// disallowed comparator or logical operator, references a nested attribute, +/// repeats an attribute, exceeds the condition count, references an undefined +/// value placeholder, or leaves a supplied value placeholder unused. +pub fn validate_search_condition_expression( + expr: &str, + names: Option<&HashMap>, + values: Option<&HashMap>, +) -> Result, DynamoDbError> { + let trimmed = expr.trim(); + if trimmed.is_empty() { + return Err(invalid( + "Invalid SearchConditionExpression: the expression must not be empty", + )); + } + + // Only equality is supported. Every other comparator contains one of these. + if trimmed.contains(['<', '>', '!']) { + return Err(invalid( + "Invalid comparator used in SearchConditionExpression", + )); + } + + // Normalize `=` into its own whitespace-delimited token, then split. + let normalized = trimmed.replace('=', " = "); + let tokens: Vec<&str> = normalized.split_whitespace().collect(); + + // Only AND is allowed to join conditions. + for tok in &tokens { + let upper = tok.to_ascii_uppercase(); + if matches!(upper.as_str(), "OR" | "NOT" | "BETWEEN" | "IN") { + return Err(invalid( + "Invalid operator used in SearchConditionExpression", + )); + } + } + + let mut conditions: Vec = Vec::new(); + let mut referenced_values: Vec = Vec::new(); + let mut index = 0; + + loop { + let lhs = tokens + .get(index) + .ok_or_else(|| invalid("Invalid SearchConditionExpression"))?; + let eq = tokens + .get(index + 1) + .ok_or_else(|| invalid("Invalid SearchConditionExpression"))?; + let rhs = tokens + .get(index + 2) + .ok_or_else(|| invalid("Invalid SearchConditionExpression"))?; + if *eq != "=" { + return Err(invalid("Invalid SearchConditionExpression")); + } + + // Nested attribute access is not permitted in the filter expression. + if lhs.contains('.') || lhs.contains('[') { + return Err(invalid( + "SearchConditionExpression cannot have conditions on nested attributes", + )); + } + + // Resolve the attribute name (#alias via ExpressionAttributeNames). + let attr_name = if let Some(alias) = lhs.strip_prefix('#') { + names + .and_then(|m| m.get(&format!("#{alias}")).or_else(|| m.get(alias))) + .cloned() + .ok_or_else(|| { + invalid(format!( + "An expression attribute name used in the expression is not defined: #{alias}" + )) + })? + } else { + (*lhs).to_owned() + }; + if attr_name.contains('.') || attr_name.contains('[') { + return Err(invalid( + "SearchConditionExpression cannot have conditions on nested attributes", + )); + } + + // Resolve the value placeholder (:name via ExpressionAttributeValues). + let placeholder = rhs + .strip_prefix(':') + .ok_or_else(|| invalid("Invalid SearchConditionExpression"))?; + let value = values + .and_then(|m| { + m.get(&format!(":{placeholder}")) + .or_else(|| m.get(placeholder)) + }) + .cloned() + .ok_or_else(|| { + invalid(format!( + "Invalid SearchConditionExpression: An expression attribute value used in \ + expression is not defined; attribute value: :{placeholder}" + )) + })?; + referenced_values.push(format!(":{placeholder}")); + + // At most one condition per attribute. + if conditions.iter().any(|c| c.attribute_name == attr_name) { + return Err(invalid( + "SearchConditionExpression must only contain one condition per attribute", + )); + } + conditions.push(SearchCondition { + attribute_name: attr_name, + value, + }); + + match tokens.get(index + 3) { + None => break, + Some(tok) if tok.eq_ignore_ascii_case("AND") => index += 4, + Some(_) => return Err(invalid("Invalid SearchConditionExpression")), + } + } + + if conditions.len() > MAX_SEARCH_CONDITIONS { + return Err(invalid(format!( + "Invalid SearchConditionExpression: SearchConditionExpression cannot have more than \ + {MAX_PARTITION_KEY_CONDITIONS} partition key and more than \ + {MAX_INLINE_FILTER_CONDITIONS} inline filter key attributes" + ))); + } + + // Every supplied value placeholder must be referenced by the expression. + if let Some(vals) = values { + for key in vals.keys() { + let normalized_key = if key.starts_with(':') { + key.clone() + } else { + format!(":{key}") + }; + if !referenced_values.contains(&normalized_key) { + return Err(invalid( + "Value provided in ExpressionAttributeValues unused in expressions", + )); + } + } + } + + Ok(conditions) +} + +/// Validate resolved filter conditions against a vector index search schema. +/// +/// Enforces that every referenced attribute belongs to the search schema, that +/// all partition-key (`HASH`) attributes are present, and that each supplied +/// value agrees with the attribute type declared in the table. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` when a condition references an +/// attribute outside the search schema, omits a required partition key, or +/// supplies a value whose type disagrees with the search schema. +pub fn validate_conditions_against_search_schema( + conditions: &[SearchCondition], + search_schema: Option<&[SearchSchemaElement]>, + attribute_definitions: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + let schema = search_schema.unwrap_or(&[]); + + // Every referenced attribute must be part of the index search schema. + for condition in conditions { + let in_schema = schema + .iter() + .any(|element| element.attribute_name == condition.attribute_name); + if !in_schema { + return Err(invalid( + "SearchConditionExpression must not contain any attributes outside the vector \ + index search schema", + )); + } + } + + // Every partition-key (HASH) element must be present in the conditions. + for element in schema + .iter() + .filter(|element| element.element_type == SearchSchemaElementType::Hash) + { + let present = conditions + .iter() + .any(|condition| condition.attribute_name == element.attribute_name); + if !present { + return Err(invalid( + "SearchConditionExpression must have all HASH attributes", + )); + } + } + + // Each value type must agree with the declared attribute type. + for condition in conditions { + if let Some(definition) = attribute_definitions + .iter() + .find(|definition| definition.attribute_name == condition.attribute_name) + && !value_matches_scalar_type(&condition.value, definition.attribute_type) + { + return Err(invalid(format!( + "Search condition value for attribute '{}' does not match type in search schema", + condition.attribute_name + ))); + } + } + + Ok(()) +} + +/// Whether an attribute value matches a scalar key type (`S`, `N`, or `B`). +fn value_matches_scalar_type(value: &AttributeValue, scalar_type: ScalarAttributeType) -> bool { + matches!( + (value, scalar_type), + (AttributeValue::S(_), ScalarAttributeType::S) + | (AttributeValue::N(_), ScalarAttributeType::N) + | (AttributeValue::B(_), ScalarAttributeType::B) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn names(pairs: &[(&str, &str)]) -> HashMap { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect() + } + + fn values(pairs: &[(&str, &str)]) -> HashMap { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), AttributeValue::S((*v).to_owned()))) + .collect() + } + + fn err(r: Result) -> String { + match r.unwrap_err() { + DynamoDbError::ValidationException(m) => m, + other => panic!("expected ValidationException, got {other:?}"), + } + } + + #[test] + fn single_literal_equality_resolves() { + let v = values(&[(":cat", "Electronics")]); + let out = validate_search_condition_expression("Category = :cat", None, Some(&v)).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].attribute_name, "Category"); + assert_eq!(out[0].value, AttributeValue::S("Electronics".to_owned())); + } + + #[test] + fn resolves_name_and_value_aliases() { + let n = names(&[("#country", "Country")]); + let v = values(&[(":c", "USA")]); + let out = + validate_search_condition_expression("#country = :c", Some(&n), Some(&v)).unwrap(); + assert_eq!(out[0].attribute_name, "Country"); + } + + #[test] + fn empty_expression_is_invalid() { + assert!( + err(validate_search_condition_expression("", None, None)) + .contains("Invalid SearchConditionExpression") + ); + } + + #[test] + fn rejects_non_equality_comparator() { + let n = names(&[("#country", "Country")]); + let v = values(&[(":c", "USA")]); + assert_eq!( + err(validate_search_condition_expression( + "#country <> :c", + Some(&n), + Some(&v) + )), + "Invalid comparator used in SearchConditionExpression" + ); + } + + #[test] + fn rejects_logical_or() { + let n = names(&[("#country", "Country"), ("#category", "Category")]); + let v = values(&[(":c", "USA"), (":cat", "Electronics")]); + assert_eq!( + err(validate_search_condition_expression( + "#country = :c OR #category = :cat", + Some(&n), + Some(&v) + )), + "Invalid operator used in SearchConditionExpression" + ); + } + + #[test] + fn rejects_nested_attribute() { + let n = names(&[ + ("#country", "Country"), + ("#product", "Product"), + ("#category", "Category"), + ]); + let v = values(&[(":c", "USA"), (":cat", "Electronics")]); + assert!( + err(validate_search_condition_expression( + "#country = :c AND #product.#category = :cat", + Some(&n), + Some(&v) + )) + .contains("cannot have conditions on nested attributes") + ); + } + + #[test] + fn rejects_duplicate_attribute() { + let n = names(&[("#country", "Country"), ("#category", "Category")]); + let v = values(&[(":c", "USA"), (":cat1", "Electronics"), (":cat2", "Books")]); + assert!( + err(validate_search_condition_expression( + "#country = :c AND #category = :cat1 AND #category = :cat2", + Some(&n), + Some(&v) + )) + .contains("must only contain one condition per attribute") + ); + } + + #[test] + fn rejects_missing_value_placeholder() { + assert_eq!( + err(validate_search_condition_expression( + "Category = :cat", + None, + None + )), + "Invalid SearchConditionExpression: An expression attribute value used in \ + expression is not defined; attribute value: :cat" + ); + } + + #[test] + fn rejects_unused_value_placeholder() { + let v = values(&[(":cat", "Electronics"), (":unused", "x")]); + assert_eq!( + err(validate_search_condition_expression( + "Category = :cat", + None, + Some(&v) + )), + "Value provided in ExpressionAttributeValues unused in expressions" + ); + } + + #[test] + fn rejects_too_many_conditions() { + let expr = (0..=MAX_SEARCH_CONDITIONS) + .map(|i| format!("filter_{i} = :val_{i}")) + .collect::>() + .join(" AND "); + let v: HashMap = (0..=MAX_SEARCH_CONDITIONS) + .map(|i| (format!(":val_{i}"), AttributeValue::S(format!("v{i}")))) + .collect(); + let message = err(validate_search_condition_expression(&expr, None, Some(&v))); + assert!(message.contains("Invalid SearchConditionExpression")); + assert!( + message.contains( + "cannot have more than 1 partition key and more than 20 inline filter key" + ) + ); + } + + #[test] + fn accepts_partition_key_plus_inline_filters() { + let n = names(&[ + ("#country", "Country"), + ("#category", "Category"), + ("#brand", "Brand"), + ]); + let v = values(&[(":c", "USA"), (":cat", "Electronics"), (":brand", "Apple")]); + let out = validate_search_condition_expression( + "#country = :c AND #category = :cat AND #brand = :brand", + Some(&n), + Some(&v), + ) + .unwrap(); + assert_eq!(out.len(), 3); + } + + fn schema() -> Vec { + vec![ + SearchSchemaElement { + attribute_name: "Country".to_owned(), + element_type: SearchSchemaElementType::Hash, + }, + SearchSchemaElement { + attribute_name: "Category".to_owned(), + element_type: SearchSchemaElementType::InlineFilter, + }, + ] + } + + fn defs() -> Vec { + vec![ + AttributeDefinition { + attribute_name: "Country".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "Category".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ] + } + + fn cond(name: &str, value: AttributeValue) -> SearchCondition { + SearchCondition { + attribute_name: name.to_owned(), + value, + } + } + + #[test] + fn schema_accepts_valid_conditions() { + let conds = vec![ + cond("Country", AttributeValue::S("USA".to_owned())), + cond("Category", AttributeValue::S("Electronics".to_owned())), + ]; + validate_conditions_against_search_schema(&conds, Some(&schema()), &defs()).unwrap(); + } + + #[test] + fn schema_rejects_unknown_attribute() { + let conds = vec![ + cond("Country", AttributeValue::S("USA".to_owned())), + cond("Price", AttributeValue::N("100".to_owned())), + ]; + assert!( + err(validate_conditions_against_search_schema( + &conds, + Some(&schema()), + &defs() + )) + .contains("SearchConditionExpression must not contain any attributes") + ); + } + + #[test] + fn schema_requires_partition_key() { + let conds = vec![cond( + "Category", + AttributeValue::S("Electronics".to_owned()), + )]; + assert!( + err(validate_conditions_against_search_schema( + &conds, + Some(&schema()), + &defs() + )) + .contains("SearchConditionExpression must have all HASH attributes") + ); + } + + #[test] + fn schema_rejects_type_mismatch() { + let conds = vec![cond("Country", AttributeValue::N("123".to_owned()))]; + assert!( + err(validate_conditions_against_search_schema( + &conds, + Some(&schema()), + &defs() + )) + .contains("does not match type in search schema") + ); + } +} diff --git a/crates/core/src/expression/update_evaluator.rs b/crates/core/src/expression/update_evaluator.rs index 62d3691e..168cbbf1 100755 --- a/crates/core/src/expression/update_evaluator.rs +++ b/crates/core/src/expression/update_evaluator.rs @@ -10,6 +10,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::error::DynamoDbError; use crate::types::AttributeValue; +use crate::types::{AttributeDefinition, VectorIndexKeyInfo}; use super::ast::{ArithOp, Expr, PathElement, UpdateAction}; use super::resolver::{ExpressionMaps, resolve_element_name, resolve_path}; @@ -57,6 +58,42 @@ pub fn apply_update( Ok(()) } +/// Apply a list of update actions and validate the resulting item image +/// against the table's vector indexes. +/// +/// Every write path that mutates an item through an `UpdateExpression` must +/// use this rather than [`apply_update`], because vector validity is a +/// property of the *stored value*, not of the expression that produced it. +/// `SET emb = :v` can be checked from the expression, but +/// `SET emb = list_append(:a, :b)`, `SET emb = other_attr`, +/// `SET emb = if_not_exists(emb, :v)` and any SET syntax added later cannot: +/// the value only exists once the actions have been evaluated against the +/// pre-update image. Validating the image is therefore the only check no +/// expression form can bypass, now or in future. +/// +/// Getting this wrong is silent rather than loud, which is why it is enforced +/// here instead of being left to each caller: an unvalidated wrong-dimension +/// list is accepted, then either fails as a 500 when the index propagation +/// delay is 0, or returns 200 and is dropped by the propagation worker when it +/// is greater than 0, leaving the index permanently stale with no error +/// surfaced to anyone. +/// +/// # Errors +/// +/// Returns `ValidationException` for unresolvable placeholders or type errors +/// while applying the actions, or when the resulting image has a vector or +/// search-schema attribute that is invalid for one of `vector_indexes`. +pub fn apply_update_validated( + actions: &[UpdateAction], + item: &mut BTreeMap, + maps: &ExpressionMaps, + vector_indexes: &[VectorIndexKeyInfo], + attribute_definitions: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + apply_update(actions, item, maps)?; + crate::validation::validate_vector_write(item, vector_indexes, attribute_definitions) +} + /// Evaluate a SET value expression to produce an `AttributeValue`. fn evaluate_set_value( expr: &Expr, diff --git a/crates/core/src/expression/update_evaluator_tests.rs b/crates/core/src/expression/update_evaluator_tests.rs index c8a145b5..f1b78692 100755 --- a/crates/core/src/expression/update_evaluator_tests.rs +++ b/crates/core/src/expression/update_evaluator_tests.rs @@ -19,6 +19,187 @@ fn apply( apply_update(&actions, item, &maps) } +/// Vector-validated variants. These exist to prove that validation follows the +/// evaluated image rather than the expression form, so no SET syntax can +/// smuggle a malformed vector past the write path. +mod vector_validated { + use super::*; + use crate::types::{ScalarAttributeType, VectorIndexKeyInfo}; + + const DIMS: u32 = 3; + + fn index() -> VectorIndexKeyInfo { + VectorIndexKeyInfo { + index_name: "vidx".to_owned(), + dimensions: DIMS, + vector_attribute_name: "emb".to_owned(), + search_schema: Vec::new(), + } + } + + fn num_list(values: &[&str]) -> AttributeValue { + AttributeValue::L( + values + .iter() + .map(|v| AttributeValue::N((*v).to_owned())) + .collect(), + ) + } + + fn apply_validated( + expr_str: &str, + item: &mut BTreeMap, + values: HashMap, + ) -> Result<(), DynamoDbError> { + let tokens = tokenize(expr_str)?; + let actions = parse_update(&tokens)?; + let maps = ExpressionMaps::new(HashMap::new(), values); + apply_update_validated(&actions, item, &maps, &[index()], &[]) + } + + fn base_item() -> BTreeMap { + let mut item = BTreeMap::new(); + item.insert("pk".into(), AttributeValue::S("k1".into())); + item + } + + /// Control: the form the old expression-matching check did cover. + #[test] + fn bare_placeholder_wrong_dimension_is_rejected() { + let mut values = HashMap::new(); + values.insert("v".into(), num_list(&["1", "2"])); + let err = apply_validated("SET emb = :v", &mut base_item(), values).unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("Expected: 3, Actual: 2")), + "unexpected error: {err:?}" + ); + } + + /// `list_append` was invisible to the old check: the value it produces + /// exists only after evaluation. Two 2-element lists concatenate to 4, + /// which must be rejected against a 3-dimension index. + #[test] + fn list_append_wrong_dimension_is_rejected() { + let mut values = HashMap::new(); + values.insert("a".into(), num_list(&["1", "2"])); + values.insert("b".into(), num_list(&["3", "4"])); + let err = + apply_validated("SET emb = list_append(:a, :b)", &mut base_item(), values).unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("Expected: 3, Actual: 4")), + "unexpected error: {err:?}" + ); + } + + /// `list_append` reaching the declared dimension must still be accepted, + /// so the guard discriminates on the value rather than on the syntax. + #[test] + fn list_append_correct_dimension_is_accepted() { + let mut values = HashMap::new(); + values.insert("a".into(), num_list(&["1", "2"])); + values.insert("b".into(), num_list(&["3"])); + let mut item = base_item(); + apply_validated("SET emb = list_append(:a, :b)", &mut item, values).unwrap(); + assert_eq!(item.get("emb"), Some(&num_list(&["1", "2", "3"]))); + } + + /// `if_not_exists` was also invisible: on a fresh item it resolves to the + /// placeholder, which here is the wrong dimension. + #[test] + fn if_not_exists_wrong_dimension_is_rejected() { + let mut values = HashMap::new(); + values.insert("v".into(), num_list(&["1", "2", "3", "4"])); + let err = apply_validated("SET emb = if_not_exists(emb, :v)", &mut base_item(), values) + .unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("Actual: 4")), + "unexpected error: {err:?}" + ); + } + + /// Copying another attribute never mentions a placeholder at all, so the + /// old check saw nothing to validate. + #[test] + fn attribute_copy_of_non_vector_is_rejected() { + let mut item = base_item(); + item.insert("other".into(), AttributeValue::S("not-a-vector".into())); + let err = apply_validated("SET emb = other", &mut item, HashMap::new()).unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("Expected: a list of numbers")), + "unexpected error: {err:?}" + ); + } + + /// Removing the vector attribute leaves no vector in the image, which is + /// legal: the validator is presence-conditional and must not demand one. + #[test] + fn removing_the_vector_attribute_is_allowed() { + let mut item = base_item(); + item.insert("emb".into(), num_list(&["1", "2", "3"])); + apply_validated("REMOVE emb", &mut item, HashMap::new()).unwrap(); + assert!(!item.contains_key("emb")); + } + + /// An update that does not touch the vector attribute at all must pass + /// even when the stored vector is absent. + #[test] + fn unrelated_update_is_allowed() { + let mut values = HashMap::new(); + values.insert("v".into(), AttributeValue::S("x".into())); + let mut item = base_item(); + apply_validated("SET label = :v", &mut item, values).unwrap(); + assert_eq!(item.get("label"), Some(&AttributeValue::S("x".into()))); + } + + /// Appending to an already-full vector overflows the declared dimension. + /// `apply_update` alone succeeds here (both operands are valid lists), so + /// only image-level validation can catch it. This also pins the + /// pre-update-snapshot semantics: the RHS reads the stored vector. + #[test] + fn appending_to_a_full_vector_overflows_and_is_rejected() { + let mut values = HashMap::new(); + values.insert("one".into(), num_list(&["4"])); + let mut item = base_item(); + item.insert("emb".into(), num_list(&["1", "2", "3"])); + let err = + apply_validated("SET emb = list_append(emb, :one)", &mut item, values).unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("Expected: 3, Actual: 4")), + "unexpected error: {err:?}" + ); + } + + /// Search-schema attributes are validated on the image too, not just the + /// vector itself. + #[test] + fn search_schema_attribute_wrong_type_is_rejected() { + let index = VectorIndexKeyInfo { + index_name: "vidx".to_owned(), + dimensions: DIMS, + vector_attribute_name: "emb".to_owned(), + search_schema: vec![crate::types::SearchSchemaElement { + attribute_name: "tenant".to_owned(), + element_type: crate::types::SearchSchemaElementType::Hash, + }], + }; + let defs = [crate::types::AttributeDefinition { + attribute_name: "tenant".to_owned(), + attribute_type: ScalarAttributeType::S, + }]; + let mut values = HashMap::new(); + values.insert("t".into(), AttributeValue::N("1".into())); + let tokens = tokenize("SET tenant = :t").unwrap(); + let actions = parse_update(&tokens).unwrap(); + let maps = ExpressionMaps::new(HashMap::new(), values); + let mut item = base_item(); + let err = apply_update_validated(&actions, &mut item, &maps, &[index], &defs).unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(_)), + "unexpected error: {err:?}" + ); + } +} + #[test] fn set_new_attribute() { let mut item = BTreeMap::new(); diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 6d1ec5d3..f4c253ea 100755 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -15,4 +15,5 @@ pub mod serde_helpers; pub mod throttle; pub mod types; pub mod validation; +pub mod vector_capacity; pub mod version; diff --git a/crates/core/src/types/capacity.rs b/crates/core/src/types/capacity.rs index 471ef728..a4fb9c8a 100755 --- a/crates/core/src/types/capacity.rs +++ b/crates/core/src/types/capacity.rs @@ -50,6 +50,29 @@ pub struct Capacity { pub write_capacity_units: Option, } +/// Capacity consumed by a vector index. +/// +/// Vector indexes meter in their own units, separate from table read and write +/// capacity: `VectorSearchRequestBytes` for `SearchVectors`, and +/// `VectorWriteRequestBytes` for writes replicated into the index. Both are +/// byte figures, not unit figures, and each is omitted rather than reported as +/// zero when the operation does not consume it. +#[derive(Debug, Clone, Default, Serialize)] +pub struct VectorCapacity { + /// Bytes consumed by a `SearchVectors` operation. + #[serde( + rename = "VectorSearchRequestBytes", + skip_serializing_if = "Option::is_none" + )] + pub vector_search_request_bytes: Option, + /// Bytes consumed replicating a write into the index. + #[serde( + rename = "VectorWriteRequestBytes", + skip_serializing_if = "Option::is_none" + )] + pub vector_write_request_bytes: Option, +} + /// Consumed capacity information returned when requested. #[derive(Debug, Clone, Serialize)] pub struct ConsumedCapacity { @@ -80,6 +103,14 @@ pub struct ConsumedCapacity { skip_serializing_if = "Option::is_none" )] pub local_secondary_indexes: Option>, + /// Per-vector-index capacity breakdown, keyed by index name. + /// + /// Measured against the service 2026-08-13: reported only for `INDEXES`, + /// not for `TOTAL` (which returns `TableName` and `CapacityUnits` alone, + /// without even the `Table` breakdown), and absent entirely rather than + /// empty when the operation charged no vector capacity. + #[serde(rename = "VectorIndexes", skip_serializing_if = "Option::is_none")] + pub vector_indexes: Option>, } /// Controls whether the existing item is returned in the error response when a @@ -150,6 +181,42 @@ pub struct ItemCollectionMetrics { } impl ConsumedCapacity { + /// Attach a vector-index write charge, keyed by index name. + /// + /// A charge of `None` for an index means the operation did not touch that + /// index's projected entry and so consumed nothing; such indexes are left + /// out of the map entirely, and when no index is charged the whole + /// `VectorIndexes` field is omitted. That matches the service, which omits + /// rather than zero-fills (measured 2026-08-13). + /// + /// Only applied at `INDEXES` granularity: `TOTAL` does not carry the map. + #[must_use] + pub fn with_vector_writes( + mut self, + charges: impl IntoIterator, + indexes: bool, + ) -> Self { + if !indexes { + return self; + } + let map: HashMap = charges + .into_iter() + .map(|(name, bytes)| { + ( + name, + VectorCapacity { + vector_search_request_bytes: None, + vector_write_request_bytes: Some(bytes), + }, + ) + }) + .collect(); + if !map.is_empty() { + self.vector_indexes = Some(map); + } + self + } + /// Build a `ConsumedCapacity` for a read operation with real capacity units. #[must_use] pub fn read(table_name: &str, cu: f64, indexes: bool) -> Self { @@ -169,6 +236,7 @@ impl ConsumedCapacity { }, global_secondary_indexes: None, local_secondary_indexes: None, + vector_indexes: None, } } @@ -191,6 +259,7 @@ impl ConsumedCapacity { }, global_secondary_indexes: None, local_secondary_indexes: None, + vector_indexes: None, } } @@ -217,6 +286,7 @@ impl ConsumedCapacity { }, global_secondary_indexes: None, local_secondary_indexes: None, + vector_indexes: None, } } @@ -243,6 +313,7 @@ impl ConsumedCapacity { }, global_secondary_indexes: None, local_secondary_indexes: None, + vector_indexes: None, } } @@ -275,6 +346,7 @@ impl ConsumedCapacity { table: breakdown.then(|| Capacity::units(base_cu)), global_secondary_indexes: if breakdown { map_or_none(gsi) } else { None }, local_secondary_indexes: if breakdown { map_or_none(lsi) } else { None }, + vector_indexes: None, } } } diff --git a/crates/core/src/types/key_schema.rs b/crates/core/src/types/key_schema.rs index c7f38e6a..54a2a408 100755 --- a/crates/core/src/types/key_schema.rs +++ b/crates/core/src/types/key_schema.rs @@ -71,7 +71,7 @@ impl<'de> serde::Deserialize<'de> for ScalarAttributeType { /// Used by data operations (`PutItem`, `GetItem`) that need key metadata /// without the full `TableDescription` overhead. Includes stream specification /// so write operations can check stream status without an extra SQL round-trip. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct TableKeyInfo { pub table_name: String, pub account_id: String, @@ -98,6 +98,24 @@ pub struct TableKeyInfo { /// Stream specification for the table, if streams are configured. /// Cached here to avoid an extra `describe_table` call per write operation. pub stream_specification: Option, + /// Vector indexes on the table, if any. Carried so write operations can + /// validate vector-valued and search-schema attributes without an extra + /// catalog round-trip. + pub vector_indexes: Vec, +} + +/// Vector-index metadata needed by the write path to validate an item's +/// vector-valued attribute and its search-schema attributes. +#[derive(Debug, Clone)] +pub struct VectorIndexKeyInfo { + /// Name of the vector index. + pub index_name: String, + /// Declared vector dimension. + pub dimensions: u32, + /// Item attribute that carries the vector. + pub vector_attribute_name: String, + /// Search-schema elements (partition key and inline filters). + pub search_schema: Vec, } /// Extract all HASH key elements from a key schema (preserving order). @@ -139,6 +157,8 @@ pub enum IndexType { Gsi, /// Local secondary index โ€” same partition key as base table. Lsi, + /// Vector index โ€” searched only via the vector search API, not Scan/Query. + Vector, } /// Metadata for a secondary index, used by query/scan operations. @@ -156,6 +176,43 @@ pub struct IndexInfo { pub projection: super::table::Projection, } +/// Index rows grouped by kind. +/// +/// Returned by [`partition_indexes`]. Exists so a backend reading its own index +/// catalog does not match on [`IndexType`] itself: adding a variant would +/// otherwise break every such match, which is how adding the vector variant +/// broke a backend that will never serve one. New variants land here instead. +#[derive(Debug, Default, Clone)] +pub struct PartitionedIndexes { + /// Global secondary indexes. + pub gsis: Vec, + /// Local secondary indexes. + pub lsis: Vec, + /// Vector indexes. A backend that does not declare vector support will never + /// have created one, so this being non-empty in that case means the catalog + /// disagrees with the backend's capability. + pub vectors: Vec, +} + +/// Group index rows by kind. +/// +/// Callers take the groups they serve and ignore the rest, so a backend needs no +/// knowledge of index kinds it does not implement. +pub fn partition_indexes(indexes: I) -> PartitionedIndexes +where + I: IntoIterator, +{ + let mut out = PartitionedIndexes::default(); + for info in indexes { + match info.index_type { + IndexType::Gsi => out.gsis.push(info), + IndexType::Lsi => out.lsis.push(info), + IndexType::Vector => out.vectors.push(info), + } + } + out +} + #[cfg(test)] mod tests { use super::*; @@ -271,6 +328,7 @@ mod tests { global_secondary_indexes: vec![], local_secondary_indexes: vec![], stream_specification: None, + vector_indexes: vec![], }; assert_eq!(info.key_schema, info.base_key_schema); } @@ -290,6 +348,7 @@ mod tests { global_secondary_indexes: vec![], local_secondary_indexes: vec![], stream_specification: None, + vector_indexes: vec![], }; assert_eq!(info.key_schema, index_schema); assert_eq!(info.base_key_schema, base_schema); @@ -297,3 +356,79 @@ mod tests { assert_eq!(info.base_key_schema[0].attribute_name, "pk"); } } + +#[cfg(test)] +mod partition_tests { + use super::*; + use crate::types::table::{Projection, ProjectionType}; + + fn info(name: &str, index_type: IndexType) -> IndexInfo { + IndexInfo { + index_name: name.to_owned(), + index_id: format!("{name}-id"), + index_type, + key_schema: Vec::new(), + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + } + } + + /// The point of the helper: a caller takes the groups it serves and never + /// matches on `IndexType`, so a new variant cannot break it. + #[test] + fn groups_each_kind_separately() { + let out = partition_indexes(vec![ + info("g1", IndexType::Gsi), + info("l1", IndexType::Lsi), + info("v1", IndexType::Vector), + info("g2", IndexType::Gsi), + ]); + assert_eq!( + out.gsis + .iter() + .map(|i| i.index_name.as_str()) + .collect::>(), + ["g1", "g2"] + ); + assert_eq!( + out.lsis + .iter() + .map(|i| i.index_name.as_str()) + .collect::>(), + ["l1"] + ); + assert_eq!( + out.vectors + .iter() + .map(|i| i.index_name.as_str()) + .collect::>(), + ["v1"] + ); + } + + /// Input order is preserved within a group, so a caller that relied on + /// catalog ordering keeps it. + #[test] + fn preserves_input_order_within_a_group() { + let out = partition_indexes(vec![ + info("b", IndexType::Gsi), + info("a", IndexType::Gsi), + info("c", IndexType::Gsi), + ]); + assert_eq!( + out.gsis + .iter() + .map(|i| i.index_name.as_str()) + .collect::>(), + ["b", "a", "c"] + ); + } + + #[test] + fn empty_input_yields_empty_groups() { + let out = partition_indexes(Vec::new()); + assert!(out.gsis.is_empty() && out.lsis.is_empty() && out.vectors.is_empty()); + } +} diff --git a/crates/core/src/types/mod.rs b/crates/core/src/types/mod.rs index f4c9c32b..ab5b12db 100755 --- a/crates/core/src/types/mod.rs +++ b/crates/core/src/types/mod.rs @@ -23,7 +23,7 @@ pub use batch::{ }; pub use capacity::{ Capacity, ConsumedCapacity, ItemCollectionMetrics, ReturnConsumedCapacity, - ReturnItemCollectionMetrics, ReturnValuesOnConditionCheckFailure, + ReturnItemCollectionMetrics, ReturnValuesOnConditionCheckFailure, VectorCapacity, }; pub use import_export::{ CsvOptions, ExportDescription, ExportFormat, ExportStatus, ExportTableToPointInTimeInput, @@ -37,8 +37,9 @@ pub use item::{ item_size_bytes, }; pub use key_schema::{ - AttributeDefinition, IndexInfo, IndexType, KeySchemaElement, KeyType, ScalarAttributeType, - TableKeyInfo, hash_key_elements, is_multipart_key_schema, range_key_elements, + AttributeDefinition, IndexInfo, IndexType, KeySchemaElement, KeyType, PartitionedIndexes, + ScalarAttributeType, TableKeyInfo, VectorIndexKeyInfo, hash_key_elements, + is_multipart_key_schema, partition_indexes, range_key_elements, }; pub use query::{Condition, QueryInput, QueryOutput, ScanInput, ScanOutput, Select}; pub use stream::{ @@ -49,16 +50,22 @@ pub use stream::{ }; pub use table::{ BillingMode, BillingModeSummary, CreateGsiAction, CreateTableInput, CreateTableOutput, - DeleteGsiAction, DeleteTableInput, DeleteTableOutput, DescribeLimitsOutput, DescribeTableInput, - DescribeTableOutput, DescribeTimeToLiveInput, DescribeTimeToLiveOutput, - GlobalSecondaryIndexUpdate, GsiDescription, GsiInput, ListTablesInput, ListTablesOutput, - ListTagsOfResourceInput, ListTagsOfResourceOutput, LsiDescription, LsiInput, + DeleteGsiAction, DeleteTableInput, DeleteTableOutput, DeleteVectorIndexAction, + DescribeLimitsOutput, DescribeTableInput, DescribeTableOutput, DescribeTimeToLiveInput, + DescribeTimeToLiveOutput, DistanceFunction, GlobalSecondaryIndexUpdate, GsiDescription, + GsiInput, IndexStatus, ListTablesInput, ListTablesOutput, ListTagsOfResourceInput, + ListTagsOfResourceOutput, LsiDescription, LsiInput, MAX_VECTOR_INDEXES_PER_TABLE, OnDemandThroughput, Projection, ProjectionType, ProvisionedThroughput, - ProvisionedThroughputDescription, SseDescription, SseType, StreamSpecification, StreamViewType, - TableDescription, TableStatus, Tag, TagResourceInput, TimeToLiveDescription, - TimeToLiveSpecification, TimeToLiveSpecificationOutput, TimeToLiveStatus, UntagResourceInput, - UpdateGsiAction, UpdateTableInput, UpdateTableOutput, UpdateTimeToLiveInput, - UpdateTimeToLiveOutput, + ProvisionedThroughputDescription, SearchSchemaElement, SearchSchemaElementType, SseDescription, + SseType, StreamSpecification, StreamViewType, TableDescription, TableStatus, Tag, + TagResourceInput, TimeToLiveDescription, TimeToLiveSpecification, + TimeToLiveSpecificationOutput, TimeToLiveStatus, UntagResourceInput, UpdateGsiAction, + UpdateTableInput, UpdateTableOutput, UpdateTimeToLiveInput, UpdateTimeToLiveOutput, + VECTOR_INDEX_ALREADY_EXISTS, VECTOR_INDEX_COUNT_LIMIT_CREATE, VECTOR_INDEX_COUNT_LIMIT_UPDATE, + VECTOR_INDEX_CREATE_IN_USE_PREFIX, VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST, + VECTOR_SEARCH_SCHEMA_UNDECLARED, VectorAttribute, VectorIndexDescription, + VectorIndexSpecification, VectorIndexUpdate, vector_attribute_conflicting_definition, + vector_attribute_redefines_key, }; pub use transaction::{ CancellationReason, ItemResponse, TransactConditionCheck, TransactDelete, TransactGet, diff --git a/crates/core/src/types/table.rs b/crates/core/src/types/table.rs index dc2f3cf5..9b66390e 100755 --- a/crates/core/src/types/table.rs +++ b/crates/core/src/types/table.rs @@ -17,10 +17,19 @@ pub enum BillingMode { } /// Current status of a Virtual `DynamoDB` table. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +/// +/// `Creating` is the [`Default`] so that [`TableDescription`] can derive +/// [`Default`], which is what lets a storage backend build one with +/// `..Default::default()` and stay unaffected when a field is added to the +/// description. `Creating` rather than `Active` because a partially built +/// description has certainly not been confirmed ready, so if a default ever does +/// leak it understates rather than overstates readiness. Every real construction +/// site sets the status explicitly. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum TableStatus { /// Table is being created. + #[default] Creating, /// Table is ready for use. Active, @@ -42,6 +51,52 @@ pub enum ProjectionType { Include, } +/// Distance function for a vector index. Selects the similarity metric used by +/// SearchVectors. Valid values: COSINE, EUCLIDEAN, DOT_PRODUCT. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub enum DistanceFunction { + /// Cosine distance. SearchVectors reports the cosine distance. + #[serde(rename = "COSINE")] + Cosine, + /// Euclidean (L2) distance. SearchVectors reports the L2 distance. + #[serde(rename = "EUCLIDEAN")] + Euclidean, + /// Dot product (inner product). SearchVectors reports the inner product. + #[serde(rename = "DOT_PRODUCT")] + DotProduct, +} + +impl DistanceFunction { + /// Whether score `a` ranks ahead of score `b` under this distance function. + /// + /// Use this rather than comparing scores directly. Cosine and Euclidean are + /// distances, so smaller wins; dot product is a similarity, so larger wins. + /// Hand-rolled comparisons are the usual source of silently reversed + /// rankings, and the direction is not uniform across the three. + #[must_use] + pub fn ranks_before(self, a: f64, b: f64) -> bool { + match self { + Self::Cosine | Self::Euclidean => a < b, + Self::DotProduct => a > b, + } + } +} + +impl<'de> serde::Deserialize<'de> for DistanceFunction { + fn deserialize>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + match s.as_str() { + "COSINE" => Ok(Self::Cosine), + "EUCLIDEAN" => Ok(Self::Euclidean), + "DOT_PRODUCT" => Ok(Self::DotProduct), + other => Err(serde::de::Error::custom(format!( + "1 validation error detected: Value '{other}' at 'distanceFunction' \ + failed to satisfy constraint: Member must satisfy enum value set: [COSINE, DOT_PRODUCT, EUCLIDEAN]" + ))), + } + } +} + /// View type for Virtual `DynamoDB` Streams records. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] @@ -77,7 +132,11 @@ pub struct ProvisionedThroughput { } /// Provisioned throughput description returned in responses. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +/// +/// Derives [`Default`] so [`TableDescription`] can. All-zero is the value a +/// PAY_PER_REQUEST table reports, so the default is meaningful rather than +/// invented. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub struct ProvisionedThroughputDescription { #[serde(rename = "ReadCapacityUnits")] pub read_capacity_units: i64, @@ -224,9 +283,351 @@ pub struct LsiInput { pub projection: Projection, } +/// The item attribute that holds the vector for a vector index. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct VectorAttribute { + #[serde(rename = "AttributeName")] + pub attribute_name: String, +} + +/// Role of a vector-index search-schema element. +/// +/// A search schema declares the scalar attributes that a vector search may +/// filter on: at most one partition key (`HASH`) and any number of +/// inline-filter keys (`INLINE_FILTER`). +/// +/// The `HASH` element is OPTIONAL, measured against the live service. Declaring +/// one makes `SearchConditionExpression` required and scopes the search to a +/// partition; omitting it searches the whole table. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum SearchSchemaElementType { + #[serde(rename = "HASH")] + Hash, + #[serde(rename = "INLINE_FILTER")] + InlineFilter, +} + +/// A single element of a vector index search schema. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SearchSchemaElement { + #[serde(rename = "AttributeName")] + pub attribute_name: String, + #[serde(rename = "SearchSchemaElementType")] + pub element_type: SearchSchemaElementType, +} + +/// Vector index definition for `CreateTable` requests. +/// +/// A vector index is a specialized global secondary index that supports +/// similarity search over a vector-valued attribute via the SearchVectors API. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct VectorIndexSpecification { + #[serde(rename = "IndexName")] + pub index_name: String, + #[serde(rename = "Dimensions")] + pub dimensions: u32, + #[serde(rename = "DistanceFunction")] + pub distance_function: DistanceFunction, + #[serde(rename = "VectorAttribute")] + pub vector_attribute: VectorAttribute, + #[serde( + rename = "SearchSchema", + skip_serializing_if = "Option::is_none", + default + )] + pub search_schema: Option>, + /// Required by the service, but deserialised as `Option` so that omitting it + /// is reported with the service's own message rather than a serde failure. + /// See `validate_vector_indexes`. + #[serde( + rename = "Projection", + skip_serializing_if = "Option::is_none", + default + )] + pub projection: Option, +} + +/// Lifecycle status of a secondary index. +/// +/// The value set is the service's own: `CREATING`, `UPDATING`, `DELETING`, +/// `ACTIVE`, taken from the shared `IndexStatus` shape in the service model, so +/// it is not specific to vector indexes even though only they are typed here for +/// now. `GsiDescription` still carries a bare `String`; converting it would change +/// an existing wire-facing type and belongs in its own change. +/// +/// `Unknown` catches a value the service adds later. Without it a new status would +/// make `DescribeTable` fail to parse, turning a forward-compatible response into +/// an outage. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum IndexStatus { + /// Being created, and not yet able to serve. + #[default] + Creating, + /// Being modified. + Updating, + /// Being deleted. + Deleting, + /// Able to serve complete results. + Active, + /// A status this build does not recognise. + #[serde(other)] + Unknown, +} + +impl IndexStatus { + /// Whether the index can serve complete results. + #[must_use] + pub fn is_active(self) -> bool { + matches!(self, Self::Active) + } +} + +/// Vector index description returned in responses. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct VectorIndexDescription { + #[serde(rename = "IndexName")] + pub index_name: String, + #[serde(rename = "VectorAttribute")] + pub vector_attribute: VectorAttribute, + #[serde(rename = "Dimensions")] + pub dimensions: u32, + #[serde( + rename = "SearchSchema", + skip_serializing_if = "Option::is_none", + default + )] + pub search_schema: Option>, + #[serde(rename = "DistanceFunction")] + pub distance_function: DistanceFunction, + #[serde(rename = "IndexStatus")] + pub index_status: IndexStatus, + /// Whether the index is still being populated. + /// + /// Measured against the service on 2026-08-06 in us-east-1, by seeding 3000 + /// items of 1024 dimensions and then adding the index with `UpdateTable`, so + /// the backfill was slow enough to observe. It took 8.5 minutes. Three + /// distinct states appeared, in this order: + /// + /// | elapsed | `IndexStatus` | `Backfilling` | + /// |---|---|---| + /// | +0.01s | `CREATING` | present, `false` | + /// | +30.8s | `CREATING` | present, `true` | + /// | +511.5s | `ACTIVE` | absent | + /// + /// Two consequences for anyone implementing this. Present does not imply + /// backfilling: the member appears as `false` first, meaning the index exists + /// but its backfill has not started, so a client must read the value rather + /// than test for presence. And the member is removed once the index is + /// `ACTIVE` rather than reported as `false`, which matches the documented GSI + /// behaviour, so `None` here must serialise as an absent member. + /// + /// An earlier probe saw only the absence-when-`ACTIVE` case and wrongly + /// inferred the flag might never be set at all. It was measuring an index over + /// a handful of tiny items, which finished backfilling before the first poll. + #[serde( + rename = "Backfilling", + skip_serializing_if = "Option::is_none", + default + )] + pub backfilling: Option, + #[serde(rename = "IndexSizeBytes")] + pub index_size_bytes: i64, + #[serde(rename = "ItemCount")] + pub item_count: i64, + #[serde(rename = "IndexArn")] + pub index_arn: String, + /// Returned by the service on every vector index description. + #[serde( + rename = "Projection", + skip_serializing_if = "Option::is_none", + default + )] + pub projection: Option, +} + +/// A single change to a table's vector indexes, as carried by +/// `UpdateTable.VectorIndexUpdates`. +/// +/// Exactly one action per element, mirroring `GlobalSecondaryIndexUpdate`. The +/// service accepts a list, so several changes may arrive in one request. +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize, Default)] +pub struct VectorIndexUpdate { + #[serde(rename = "Create", skip_serializing_if = "Option::is_none", default)] + pub create: Option, + #[serde(rename = "Delete", skip_serializing_if = "Option::is_none", default)] + pub delete: Option, +} + +/// Remove a vector index from an existing table. +/// +/// Measured on 2026-08-06: the service accepts a delete of an index that is still +/// backfilling, and of one that is `ACTIVE`. Core therefore imposes no readiness +/// condition on deletion; an earlier instinct to forbid deleting a backfilling +/// index would have been stricter than the service. +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct DeleteVectorIndexAction { + #[serde(rename = "IndexName")] + pub index_name: String, +} + +/// Message the service returns when a vector index is created whose name is taken +/// by an index that has finished building. +/// +/// Measured on 2026-08-06, and paired with +/// [`VECTOR_INDEX_CREATE_IN_USE_PREFIX`]: the error *class* depends on the state +/// of the existing index, so a backend cannot report one message for both. +/// `ACTIVE` gives a `ValidationException` carrying this text. +/// Maximum vector indexes per table. Measured 2026-08-13: five are accepted, +/// six are refused, on both the create and the update path. +pub const MAX_VECTOR_INDEXES_PER_TABLE: usize = 5; + +/// Vector indexes require on-demand billing. Measured 2026-08-13, identical on +/// `CreateTable` with `PROVISIONED` and on an `UpdateTable` that switches a +/// table holding vector indexes to `PROVISIONED`. +pub const VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST: &str = "One or more parameter values were invalid: Vector indexes are only supported for \ + PAY_PER_REQUEST tables"; + +/// Per-table vector index limit exceeded on `CreateTable`. +/// +/// The create and update paths differ in BOTH class and text for this one rule, +/// so they get separate constants rather than one shared message. Measured +/// 2026-08-13: `CreateTable` reports this as a `ValidationException`. +pub const VECTOR_INDEX_COUNT_LIMIT_CREATE: &str = + "One or more parameter values were invalid: VectorIndex count exceeds the per-table limit of 5"; + +/// Per-table vector index limit exceeded on `UpdateTable`. +/// +/// Measured 2026-08-13: reported as a `LimitExceededException`, not a +/// `ValidationException`, and with different wording from the create path. +pub const VECTOR_INDEX_COUNT_LIMIT_UPDATE: &str = + "Subscriber limit exceeded: Number of vector secondary indexes exceeds per-table limit of 5"; + +/// A `SearchSchema` element names an attribute with no `AttributeDefinition`. +/// +/// Measured 2026-08-13, same text on both paths. On `UpdateTable` the +/// definition must be present in THAT request, even when the attribute is +/// already declared on the table. +pub const VECTOR_SEARCH_SCHEMA_UNDECLARED: &str = "One or more parameter values were invalid: One element in SearchSchema is not defined in \ + attribute definitions"; + +/// The vector attribute must NOT appear in `AttributeDefinitions`. +/// +/// The opposite of the rule for key attributes, and the reason a table +/// partition key cannot be used as the vector attribute on `CreateTable`: the +/// key must be declared, so declaring it trips this. Measured 2026-08-13. +#[must_use] +pub fn vector_attribute_conflicting_definition(attribute_name: &str) -> String { + format!( + "One or more parameter values were invalid: Conflicting attribute definition for \ + '{attribute_name}'. An attribute cannot be defined in AttributeDefinitions when used \ + as a VectorAttribute." + ) +} + +/// The vector attribute collides with an existing key attribute. +/// +/// Distinct from [`vector_attribute_conflicting_definition`]: seen on +/// `UpdateTable`, where the key is not re-declared in the request so the +/// conflicting-definition rule cannot fire first. The message embeds both +/// schemas, reporting the vector as type `L` with its dimension count. +/// Measured 2026-08-13. +#[must_use] +pub fn vector_attribute_redefines_key( + attribute_name: &str, + existing_type: &str, + existing_key_type: &str, + dimensions: u32, +) -> String { + format!( + "One or more parameter values were invalid: Attributes cannot be redefined. Please check \ + that your attribute has the same type as previously defined. Existing schema: \ + Schema:[SchemaElement: key{{{attribute_name}:{existing_type}:{existing_key_type}}}] \ + New schema: VectorIndexSchema:[VectorAttribute: key{{{attribute_name}:L:{dimensions}}}]" + ) +} + +pub const VECTOR_INDEX_ALREADY_EXISTS: &str = "Attempting to create an index which already exists"; + +/// Prefix of the message the service returns when the name is taken by an index +/// that is still being created. Measured on 2026-08-06; carried as a +/// `ResourceInUseException`, not a `ValidationException`, and continues +/// " Table: {table} Index: {index}" with two spaces after the full stop. +pub const VECTOR_INDEX_CREATE_IN_USE_PREFIX: &str = + "Attempt to change a resource which is still in use: Index is being created."; + +impl VectorIndexDescription { + /// Reject a description whose reported state the service would never produce. + /// + /// Two rules, both measured rather than assumed (see + /// [`VectorIndexDescription::backfilling`] for the observed lifecycle). + /// + /// `ACTIVE` with a backfill in flight is a contradiction. An index that has + /// not finished being populated cannot answer a search completely, so + /// reporting it ready makes a client's first search silently undercount, which + /// is the failure RFC 236 guards against. `Backfilling: true` was only ever + /// observed alongside `CREATING`. + /// + /// `ACTIVE` carrying the member at all, even `false`, is also rejected. The + /// service removes it on completion rather than reporting `false`, matching the + /// documented GSI behaviour, so a backend that keeps emitting it diverges on + /// the wire from what a client is entitled to expect. + /// + /// A contradictory pair is a backend defect rather than anything the caller + /// did, hence `InternalServerError`. + /// + /// # Errors + /// Returns [`DynamoDbError::InternalServerError`](crate::error::DynamoDbError::InternalServerError) + /// if the index is reported `ACTIVE` with any `backfilling` value present. + pub fn validate_readiness(&self) -> Result<(), crate::error::DynamoDbError> { + if !self.index_status.is_active() { + return Ok(()); + } + match self.backfilling { + Some(true) => Err(crate::error::DynamoDbError::InternalServerError(format!( + "vector index '{}' reported ACTIVE while still \ + backfilling; it cannot serve complete results yet", + self.index_name + ))), + Some(false) => Err(crate::error::DynamoDbError::InternalServerError(format!( + "vector index '{}' reported ACTIVE with a \ + Backfilling member; the service removes it once the index is \ + active rather than reporting false", + self.index_name + ))), + None => Ok(()), + } + } +} + +impl TableDescription { + /// Validate the reported readiness of every vector index on the table. + /// + /// Called on the paths that hand a description to a client, so a backend that + /// reports a contradictory state is caught once, centrally, rather than each + /// response path having to remember. No in-tree backend populates + /// `vector_indexes` yet, so this guards the backend implementations to come + /// rather than anything shipping today. + /// + /// # Errors + /// Propagates the first failure from + /// [`VectorIndexDescription::validate_readiness`]. + pub fn validate_vector_index_readiness(&self) -> Result<(), crate::error::DynamoDbError> { + for index in self.vector_indexes.iter().flatten() { + index.validate_readiness()?; + } + Ok(()) + } +} + /// Full description of a Virtual `DynamoDB` table, returned by `CreateTable`, /// `DeleteTable`, and `DescribeTable`. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +/// Derives [`Default`] deliberately: a storage backend assembles this and must +/// be able to write `..Default::default()` so that adding a field here, for a +/// feature that backend does not implement, does not break its build. Note that +/// `#[non_exhaustive]` would defeat that, since it forbids functional update +/// syntax from other crates entirely. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub struct TableDescription { #[serde(rename = "TableName")] pub table_name: String, @@ -260,6 +661,8 @@ pub struct TableDescription { skip_serializing_if = "Option::is_none" )] pub local_secondary_indexes: Option>, + #[serde(rename = "VectorIndexes", skip_serializing_if = "Option::is_none")] + pub vector_indexes: Option>, #[serde( rename = "StreamSpecification", skip_serializing_if = "Option::is_none" @@ -280,7 +683,7 @@ pub struct TableDescription { } /// `CreateTable` request body. -#[derive(Debug, Clone, PartialEq, Deserialize)] +#[derive(Debug, Clone, PartialEq, Deserialize, Default)] pub struct CreateTableInput { #[serde(rename = "TableName")] pub table_name: String, @@ -288,7 +691,10 @@ pub struct CreateTableInput { pub key_schema: Vec, #[serde(rename = "AttributeDefinitions")] pub attribute_definitions: Vec, - #[serde(rename = "BillingMode")] + // Some clients send the billing/throughput mode under the field name + // TableThroughputMode instead of BillingMode; accept it as an alias so such + // requests are not rejected for a missing billing mode. + #[serde(rename = "BillingMode", alias = "TableThroughputMode")] pub billing_mode: Option, #[serde(rename = "ProvisionedThroughput")] pub provisioned_throughput: Option, @@ -296,6 +702,8 @@ pub struct CreateTableInput { pub global_secondary_indexes: Option>, #[serde(rename = "LocalSecondaryIndexes")] pub local_secondary_indexes: Option>, + #[serde(rename = "VectorIndexes")] + pub vector_indexes: Option>, #[serde(rename = "StreamSpecification")] pub stream_specification: Option, #[serde(rename = "SSESpecification")] @@ -417,7 +825,9 @@ pub struct UpdateGsiAction { pub struct UpdateTableInput { #[serde(rename = "TableName")] pub table_name: String, - #[serde(rename = "BillingMode")] + // Accept TableThroughputMode as an alias for BillingMode (see + // CreateTableInput): some clients send the billing mode under that name. + #[serde(rename = "BillingMode", alias = "TableThroughputMode")] pub billing_mode: Option, #[serde(rename = "ProvisionedThroughput")] pub provisioned_throughput: Option, @@ -433,6 +843,11 @@ pub struct UpdateTableInput { pub table_class: Option, #[serde(rename = "OnDemandThroughput")] pub on_demand_throughput: Option, + /// Vector index changes. `None` and an empty list both mean no change, so an + /// ordinary `UpdateTable` against a backend without vector support is + /// unaffected. + #[serde(rename = "VectorIndexUpdates", default)] + pub vector_index_updates: Option>, } /// `UpdateTable` response body. @@ -568,6 +983,39 @@ pub struct DescribeLimitsOutput { mod tests { use super::*; + #[test] + fn create_table_accepts_table_throughput_mode_alias() { + // A client that sends the billing mode under `TableThroughputMode` + // (instead of `BillingMode`) must populate billing_mode all the same. + let json = r#"{ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "TableThroughputMode": "PAY_PER_REQUEST" + }"#; + let input: CreateTableInput = serde_json::from_str(json).unwrap(); + assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); + } + + #[test] + fn create_table_billing_mode_still_wins_when_present() { + let json = r#"{ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + }"#; + let input: CreateTableInput = serde_json::from_str(json).unwrap(); + assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); + } + + #[test] + fn update_table_accepts_table_throughput_mode_alias() { + let json = r#"{"TableName": "t", "TableThroughputMode": "PROVISIONED"}"#; + let input: UpdateTableInput = serde_json::from_str(json).unwrap(); + assert_eq!(input.billing_mode, Some(BillingMode::Provisioned)); + } + #[test] fn on_demand_throughput_round_trips_json() { let odt = OnDemandThroughput { @@ -632,3 +1080,192 @@ mod tests { assert_eq!(odt.max_write_request_units, Some(5)); } } + +#[cfg(test)] +mod vector_index_readiness_tests { + + /// The enum must round-trip the service's own strings, and an unrecognised + /// value must parse rather than fail: a new status appearing upstream should + /// not turn DescribeTable into an outage. + #[test] + fn index_status_round_trips_the_service_values() { + for (value, expected) in [ + ("\"CREATING\"", IndexStatus::Creating), + ("\"UPDATING\"", IndexStatus::Updating), + ("\"DELETING\"", IndexStatus::Deleting), + ("\"ACTIVE\"", IndexStatus::Active), + ] { + let parsed: IndexStatus = serde_json::from_str(value).expect("parses"); + assert_eq!(parsed, expected, "{value}"); + assert_eq!( + serde_json::to_string(&expected).expect("serialises"), + value, + "must serialise back to the service's spelling" + ); + } + let unknown: IndexStatus = + serde_json::from_str("\"SOMETHING_NEW\"").expect("an unknown status must still parse"); + assert_eq!(unknown, IndexStatus::Unknown); + assert!(!unknown.is_active(), "an unknown status is not serviceable"); + } + use super::*; + use crate::error::DynamoDbError; + + fn description(index_status: IndexStatus, backfilling: Option) -> VectorIndexDescription { + VectorIndexDescription { + index_name: "vidx".to_owned(), + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + dimensions: 4, + search_schema: None, + distance_function: DistanceFunction::Cosine, + index_status, + backfilling, + index_size_bytes: 0, + item_count: 0, + index_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/t/index/vidx".to_owned(), + projection: Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + } + } + + /// The contradiction the invariant exists to catch. + #[test] + fn active_while_backfilling_is_rejected() { + let err = description(IndexStatus::Active, Some(true)) + .validate_readiness() + .expect_err("ACTIVE plus a backfill in flight must not be reportable"); + match err { + DynamoDbError::InternalServerError(m) => { + assert!(m.contains("vidx"), "should name the index: {m}"); + assert!(m.contains("backfilling"), "should say why: {m}"); + } + other => panic!("expected InternalServerError, got {other:?}"), + } + } + + /// ACTIVE must not carry the member at all. The service removes it on + /// completion rather than reporting false, so emitting false diverges on the + /// wire from what a client is entitled to expect. + #[test] + fn active_carrying_the_member_at_all_is_rejected() { + let err = description(IndexStatus::Active, Some(false)) + .validate_readiness() + .expect_err("ACTIVE must not carry a Backfilling member"); + match err { + DynamoDbError::InternalServerError(m) => { + assert!(m.contains("removes it"), "should say why: {m}"); + } + other => panic!("expected InternalServerError, got {other:?}"), + } + } + + /// The three states actually observed, in the order observed, must all be + /// reportable. Measured on 2026-08-06 by adding an index to a table of 3000 + /// items with UpdateTable and polling through the 8.5 minute backfill. + #[test] + fn the_observed_lifecycle_is_reportable() { + for (status, backfilling, when) in [ + ( + IndexStatus::Creating, + Some(false), + "t+0.01s, created, backfill not started", + ), + (IndexStatus::Creating, Some(true), "t+30.8s, backfilling"), + ( + IndexStatus::Active, + None, + "t+511.5s, complete, member removed", + ), + ] { + assert!( + description(status, backfilling) + .validate_readiness() + .is_ok(), + "observed state must be permitted ({when})" + ); + } + } + + /// Non-ACTIVE statuses are unconstrained: the rules are about not claiming + /// readiness, so anything short of ACTIVE passes whatever the flag says. + #[test] + fn non_active_statuses_are_unconstrained() { + for (status, backfilling) in [ + (IndexStatus::Creating, None), + (IndexStatus::Deleting, Some(true)), + (IndexStatus::Deleting, None), + (IndexStatus::Updating, Some(false)), + ] { + assert!( + description(status, backfilling) + .validate_readiness() + .is_ok(), + "{status:?} with backfilling={backfilling:?} should be permitted" + ); + } + } + + /// The table-level check reports the offending index rather than passing + /// because a sibling is fine. + #[test] + fn table_level_check_finds_a_bad_index_among_good_ones() { + // Built the way a backend builds one, which is the pattern the contract + // relies on for opt-out. + let table = TableDescription { + vector_indexes: Some(vec![ + description(IndexStatus::Active, None), + description(IndexStatus::Active, Some(true)), + ]), + ..Default::default() + }; + assert!(table.validate_vector_index_readiness().is_err()); + } + + #[test] + fn a_table_with_no_vector_indexes_passes() { + assert!( + TableDescription::default() + .validate_vector_index_readiness() + .is_ok() + ); + let table = TableDescription { + vector_indexes: Some(Vec::new()), + ..Default::default() + }; + assert!(table.validate_vector_index_readiness().is_ok()); + } + + /// Wire shape. The service omitted `Backfilling` entirely for an ACTIVE index, + /// so `None` must not serialise to `"Backfilling": null`, which a client would + /// see as a different response. + #[test] + fn backfilling_is_omitted_from_the_wire_when_absent() { + let json = + serde_json::to_string(&description(IndexStatus::Active, None)).expect("serialises"); + assert!( + !json.contains("Backfilling"), + "absent must mean omitted, not null: {json}" + ); + let json = serde_json::to_string(&description(IndexStatus::Creating, Some(true))) + .expect("serialises"); + assert!( + json.contains(r#""Backfilling":true"#), + "present must serialise under the service's member name: {json}" + ); + } + + /// A response without the member deserialises, which is the shape the service + /// actually returned. + #[test] + fn a_response_without_backfilling_deserialises() { + let json = + serde_json::to_string(&description(IndexStatus::Active, None)).expect("serialises"); + let round_tripped: VectorIndexDescription = + serde_json::from_str(&json).expect("deserialises without the member"); + assert_eq!(round_tripped.backfilling, None); + } +} diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index 9bf4ea69..621256f3 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -1,6 +1,9 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 pub mod number; +pub mod vector_item; + +pub use vector_item::{MAX_HASH_KEY_SIZE, MAX_INLINE_FILTER_SIZE, validate_vector_write}; use crate::error::{DynamoDbError, ErrorMessageKey, error_message}; use crate::limits::LimitsConfig; @@ -102,10 +105,165 @@ pub fn validate_create_table( validate_lsi_requires_range_key(input)?; validate_unique_index_names(input)?; validate_index_projections(input)?; + validate_vector_indexes(input)?; validate_stream_specification(input)?; Ok(()) } +/// Validate the `VectorIndexes` on a `CreateTable` request. +/// +/// POC scope: each index name must be well-formed, `Dimensions` must be in +/// `1..=4096`, and the vector attribute name must be non-empty. The distance +/// function is enforced by the type system (enum deserialization). +/// Rules that apply to one vector index specification, wherever it arrives from. +/// +/// `CreateTable` and `UpdateTable`'s create action carry the same shape, so they +/// get the same rules: a malformed index is rejected identically whichever path +/// it arrives by, rather than each handler enforcing its own subset. +/// +/// `position` numbers the element within its request list, from 1, because that is +/// how the service numbers it in the message below. +/// Attribute-definition rules for one vector index, applied on both paths. +/// +/// Both are decidable from the request alone, so they belong here rather than +/// in a backend: the vector attribute must NOT be declared in +/// `AttributeDefinitions` (the opposite of the rule for key attributes), and +/// every `SearchSchema` element MUST be. Measured 2026-08-13; on `UpdateTable` +/// the search-schema definition must be present in that request even when the +/// attribute is already declared on the table. +fn validate_vector_index_attribute_definitions( + vi: &crate::types::VectorIndexSpecification, + attribute_definitions: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + let declared = |name: &str| { + attribute_definitions + .iter() + .any(|ad| ad.attribute_name == name) + }; + let vector_attr = &vi.vector_attribute.attribute_name; + if declared(vector_attr) { + return Err(DynamoDbError::ValidationException( + crate::types::vector_attribute_conflicting_definition(vector_attr), + )); + } + for element in vi.search_schema.iter().flatten() { + if !declared(&element.attribute_name) { + return Err(DynamoDbError::ValidationException( + crate::types::VECTOR_SEARCH_SCHEMA_UNDECLARED.to_owned(), + )); + } + } + Ok(()) +} + +fn validate_one_vector_index( + vi: &crate::types::VectorIndexSpecification, + position: usize, + field: &str, +) -> Result<(), DynamoDbError> { + // Projection is required by the service. Reported with the service's own + // message, which numbers the offending list element from 1, not 0. Measured + // on 2026-08-06 by bypassing botocore's client-side check so the request + // reached the service: + // Value null at 'vectorIndexes.1.member.projection' failed to satisfy + // constraint: Member must not be null + if vi.projection.is_none() { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Value null at \ + '{field}.{position}.member.projection' failed to satisfy constraint: \ + Member must not be null" + ))); + } + validate_index_name(&vi.index_name)?; + if vi.dimensions < 1 || vi.dimensions > 4096 { + // Verified against the service 2026-08-05: Dimensions=4097 and 8192 + // both return exactly this message. The lower bound is not observable + // through an SDK because botocore rejects Dimensions=0 client-side, but + // the service text names both bounds, so it is reused here. + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: Number of dimensions must be \ + between 1 and 4096 inclusive." + .to_owned(), + )); + } + if vi.vector_attribute.attribute_name.is_empty() { + return Err(DynamoDbError::ValidationException( + "VectorAttribute.AttributeName must not be empty".to_owned(), + )); + } + Ok(()) +} + +fn validate_vector_indexes(input: &CreateTableInput) -> Result<(), DynamoDbError> { + let Some(vis) = input.vector_indexes.as_ref() else { + return Ok(()); + }; + if vis.is_empty() { + return Ok(()); + } + + // Vector indexes are on-demand only. Measured 2026-08-13. + if input.billing_mode == Some(BillingMode::Provisioned) { + return Err(DynamoDbError::ValidationException( + crate::types::VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST.to_owned(), + )); + } + + // Per-table count limit. This path reports it as a ValidationException; + // UpdateTable reports the same limit as a LimitExceededException with + // different wording, which is why the two are not shared. + if vis.len() > crate::types::MAX_VECTOR_INDEXES_PER_TABLE { + return Err(DynamoDbError::ValidationException( + crate::types::VECTOR_INDEX_COUNT_LIMIT_CREATE.to_owned(), + )); + } + + let attribute_definitions = input.attribute_definitions.as_slice(); + for (position, vi) in vis.iter().enumerate() { + validate_one_vector_index(vi, position + 1, "vectorIndexes")?; + validate_vector_index_attribute_definitions(vi, attribute_definitions)?; + } + Ok(()) +} + +/// Validate the vector index changes on an `UpdateTable` request. +/// +/// Deliberately limited to what core can decide from the request alone. It does +/// not check whether a created name is already taken, or whether a deleted index +/// exists, because the service's error for a name clash changes CLASS with the +/// state of the existing index, `ValidationException` when it is ACTIVE and +/// `ResourceInUseException` while it is still creating, and `TableKeyInfo` does +/// not carry index status. Reporting the wrong class would be worse than leaving +/// it to the layer that knows. Those messages are recorded as +/// [`VECTOR_INDEX_ALREADY_EXISTS`](crate::types::VECTOR_INDEX_ALREADY_EXISTS) and +/// [`VECTOR_INDEX_CREATE_IN_USE_PREFIX`](crate::types::VECTOR_INDEX_CREATE_IN_USE_PREFIX) +/// so both backends produce identical text. +/// +/// # Errors +/// Returns [`DynamoDbError::ValidationException`] if vector search is disabled for +/// the deployment, or if a create action carries a malformed index. +pub fn validate_vector_index_updates( + updates: Option<&Vec>, + attribute_definitions: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + let Some(updates) = updates else { + return Ok(()); + }; + if updates.is_empty() { + return Ok(()); + } + for (position, update) in updates.iter().enumerate() { + if let Some(create) = update.create.as_ref() { + validate_one_vector_index(create, position + 1, "vectorIndexUpdates")?; + validate_vector_index_attribute_definitions(create, attribute_definitions)?; + } + if let Some(delete) = update.delete.as_ref() { + validate_index_name(&delete.index_name)?; + } + } + Ok(()) +} + /// Validate that INCLUDE-projection secondary indexes specify `NonKeyAttributes`. /// /// Real `DynamoDB` rejects a GSI or LSI whose `ProjectionType` is `INCLUDE` @@ -387,6 +545,20 @@ fn validate_attribute_definitions(input: &CreateTableInput) -> Result<(), Dynamo } } } + // Vector-index search-schema attributes are declared in AttributeDefinitions + // but are not part of the base or secondary-index key schema, so count them + // as used to satisfy the definition/key correspondence check. + if let Some(vis) = &input.vector_indexes { + for vi in vis { + if let Some(schema) = &vi.search_schema { + for element in schema { + if !key_attrs.contains(&element.attribute_name.as_str()) { + key_attrs.push(&element.attribute_name); + } + } + } + } + } // Every key attribute must have a definition let def_names: Vec<&str> = input @@ -1410,11 +1582,340 @@ fn validate_unique_index_names(input: &CreateTableInput) -> Result<(), DynamoDbE } } } + if let Some(vis) = &input.vector_indexes { + for vi in vis { + if !names.insert(&vi.index_name) { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Duplicate index name: {}", + vi.index_name + ))); + } + } + } Ok(()) } #[cfg(test)] mod tests { + + /// A create arriving by UpdateTable gets the same rules as one arriving by + /// CreateTable, including the field name in the message path. + #[test] + fn update_table_creates_get_the_same_rules_as_create_table() { + use crate::types::{DeleteVectorIndexAction, VectorIndexUpdate}; + let spec = |projection| VectorIndexSpecification { + index_name: "vidx".to_owned(), + dimensions: 4, + distance_function: DistanceFunction::Cosine, + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + search_schema: None, + projection, + }; + let all = || { + Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }) + }; + + // None and empty are not changes, so an ordinary UpdateTable is unaffected. + assert!(validate_vector_index_updates(None, &[]).is_ok()); + assert!(validate_vector_index_updates(Some(&Vec::new()), &[]).is_ok()); + + // Missing projection is caught here too, under this request's field name. + let updates = vec![crate::types::VectorIndexUpdate { + create: Some(spec(None)), + delete: None, + }]; + let err = validate_vector_index_updates(Some(&updates), &[]) + .expect_err("a malformed create must be rejected on this path as well"); + match err { + DynamoDbError::ValidationException(m) => assert!( + m.contains("vectorIndexUpdates.1.member.projection"), + "should name this request's field and element: {m}" + ), + other => panic!("expected ValidationException, got {other:?}"), + } + + // Dimensions bound applies here too. + let updates = vec![crate::types::VectorIndexUpdate { + create: Some(VectorIndexSpecification { + dimensions: 4097, + ..spec(all()) + }), + delete: None, + }]; + assert!(validate_vector_index_updates(Some(&updates), &[]).is_err()); + + // A well-formed create, and a delete, both pass. + let updates = vec![ + VectorIndexUpdate { + create: Some(spec(all())), + delete: None, + }, + VectorIndexUpdate { + create: None, + delete: Some(DeleteVectorIndexAction { + index_name: "other".to_owned(), + }), + }, + ]; + assert!(validate_vector_index_updates(Some(&updates), &[]).is_ok()); + } + + use crate::types::{DistanceFunction, VectorAttribute, VectorIndexSpecification}; + + /// The service requires `Projection` on every vector index, and numbers the + /// offending element from 1. Measured on 2026-08-06 by bypassing botocore's + /// client-side check so the request reached the service. Asserted exactly, + /// because a client parsing the path would be misled by 0-based numbering. + /// Rules below are pinned to messages measured against the live service on + /// 2026-08-13 (us-east-1). They are parity tests, not tests of our own + /// wording. + + fn vi_spec(name: &str, attr: &str, schema: &[&str]) -> VectorIndexSpecification { + VectorIndexSpecification { + index_name: name.to_owned(), + vector_attribute: VectorAttribute { + attribute_name: attr.to_owned(), + }, + dimensions: 8, + distance_function: DistanceFunction::Cosine, + projection: Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + search_schema: if schema.is_empty() { + None + } else { + Some( + schema + .iter() + .map(|n| crate::types::SearchSchemaElement { + attribute_name: (*n).to_owned(), + element_type: crate::types::SearchSchemaElementType::Hash, + }) + .collect(), + ) + }, + } + } + + fn vector_create_input( + vis: Vec, + defs: Vec, + ) -> CreateTableInput { + let mut input = base_input(vec![make_ks("pk", KeyType::Hash)], defs); + input.vector_indexes = Some(vis); + input + } + + #[test] + fn vector_index_on_provisioned_table_is_rejected() { + let mut input = vector_create_input( + vec![vi_spec("vidx", "emb", &["tenant"])], + vec![ + make_ad("pk", ScalarAttributeType::S), + make_ad("tenant", ScalarAttributeType::S), + ], + ); + input.billing_mode = Some(BillingMode::Provisioned); + let err = validate_vector_indexes(&input).unwrap_err(); + assert_eq!( + err.to_string(), + crate::types::VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST + ); + } + + #[test] + fn five_vector_indexes_are_allowed_and_six_are_not() { + let defs = vec![ + make_ad("pk", ScalarAttributeType::S), + make_ad("tenant", ScalarAttributeType::S), + ]; + let five: Vec<_> = (0..5) + .map(|i| vi_spec(&format!("vidx{i}"), &format!("emb{i}"), &["tenant"])) + .collect(); + validate_vector_indexes(&vector_create_input(five, defs.clone())).unwrap(); + + let six: Vec<_> = (0..6) + .map(|i| vi_spec(&format!("vidx{i}"), &format!("emb{i}"), &["tenant"])) + .collect(); + let err = validate_vector_indexes(&vector_create_input(six, defs)).unwrap_err(); + assert_eq!( + err.to_string(), + crate::types::VECTOR_INDEX_COUNT_LIMIT_CREATE + ); + assert!( + matches!(err, DynamoDbError::ValidationException(_)), + "CreateTable reports the count limit as a ValidationException; \ + UpdateTable uses LimitExceededException with different text" + ); + } + + #[test] + fn vector_attribute_declared_in_attribute_definitions_is_rejected() { + let err = validate_vector_indexes(&vector_create_input( + vec![vi_spec("vidx", "emb", &["tenant"])], + vec![ + make_ad("pk", ScalarAttributeType::S), + make_ad("tenant", ScalarAttributeType::S), + make_ad("emb", ScalarAttributeType::S), + ], + )) + .unwrap_err(); + assert_eq!( + err.to_string(), + crate::types::vector_attribute_conflicting_definition("emb") + ); + } + + /// The table partition key must be declared, so naming it as the vector + /// attribute trips the same rule. This is why CreateTable reports the + /// conflicting-definition message rather than a key-specific one. + #[test] + fn table_partition_key_as_vector_attribute_is_rejected() { + let err = validate_vector_indexes(&vector_create_input( + vec![vi_spec("vidx", "pk", &["tenant"])], + vec![ + make_ad("pk", ScalarAttributeType::S), + make_ad("tenant", ScalarAttributeType::S), + ], + )) + .unwrap_err(); + assert_eq!( + err.to_string(), + crate::types::vector_attribute_conflicting_definition("pk") + ); + } + + #[test] + fn search_schema_attribute_without_a_definition_is_rejected() { + let err = validate_vector_indexes(&vector_create_input( + vec![vi_spec("vidx", "emb", &["undeclared"])], + vec![make_ad("pk", ScalarAttributeType::S)], + )) + .unwrap_err(); + assert_eq!( + err.to_string(), + crate::types::VECTOR_SEARCH_SCHEMA_UNDECLARED + ); + } + + #[test] + fn search_schema_attribute_with_a_definition_is_accepted() { + validate_vector_indexes(&vector_create_input( + vec![vi_spec("vidx", "emb", &["tenant"])], + vec![ + make_ad("pk", ScalarAttributeType::S), + make_ad("tenant", ScalarAttributeType::S), + ], + )) + .unwrap(); + } + + /// The same two attribute-definition rules apply on the update path, against + /// the definitions carried by THAT request. Measured: omitting the + /// definition fails even when the attribute is already on the table. + #[test] + fn update_path_applies_the_attribute_definition_rules() { + let create = vi_spec("vidx", "emb", &["tenant"]); + let updates = vec![crate::types::VectorIndexUpdate { + create: Some(create), + delete: None, + }]; + let err = + validate_vector_index_updates(Some(&updates), &[make_ad("pk", ScalarAttributeType::S)]) + .unwrap_err(); + assert_eq!( + err.to_string(), + crate::types::VECTOR_SEARCH_SCHEMA_UNDECLARED + ); + validate_vector_index_updates( + Some(&updates), + &[ + make_ad("pk", ScalarAttributeType::S), + make_ad("tenant", ScalarAttributeType::S), + ], + ) + .unwrap(); + + let declared = vi_spec("vidx", "emb", &[]); + let updates = vec![crate::types::VectorIndexUpdate { + create: Some(declared), + delete: None, + }]; + let err = validate_vector_index_updates( + Some(&updates), + &[make_ad("emb", ScalarAttributeType::S)], + ) + .unwrap_err(); + assert_eq!( + err.to_string(), + crate::types::vector_attribute_conflicting_definition("emb") + ); + } + + #[test] + fn missing_vector_index_projection_matches_the_service_message() { + fn spec(projection: Option) -> VectorIndexSpecification { + VectorIndexSpecification { + index_name: "vidx".to_owned(), + dimensions: 4, + distance_function: DistanceFunction::Cosine, + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + search_schema: None, + projection, + } + } + let all = || { + Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }) + }; + + // First element omitted: reported as element 1. + let input = CreateTableInput { + table_name: "t".to_owned(), + vector_indexes: Some(vec![spec(None)]), + ..Default::default() + }; + let err = validate_vector_indexes(&input).expect_err("must be rejected"); + assert!( + format!("{err:?}").contains("vectorIndexes.1.member.projection"), + "expected element 1 in the path: {err:?}" + ); + + // Second element omitted: reported as element 2, not 1. + let input = CreateTableInput { + table_name: "t".to_owned(), + vector_indexes: Some(vec![spec(all()), spec(None)]), + ..Default::default() + }; + let err = validate_vector_indexes(&input).expect_err("must be rejected"); + match err { + DynamoDbError::ValidationException(m) => assert_eq!( + m, + "1 validation error detected: Value null at \ + 'vectorIndexes.2.member.projection' failed to satisfy constraint: \ + Member must not be null" + ), + other => panic!("expected ValidationException, got {other:?}"), + } + + // Present on every element: accepted. + let input = CreateTableInput { + table_name: "t".to_owned(), + vector_indexes: Some(vec![spec(all()), spec(all())]), + ..Default::default() + }; + assert!(validate_vector_indexes(&input).is_ok()); + } use super::*; use crate::types::{GsiInput, Projection, ProjectionType}; @@ -1444,6 +1945,7 @@ mod tests { provisioned_throughput: None, global_secondary_indexes: None, local_secondary_indexes: None, + vector_indexes: None, stream_specification: None, sse_specification: None, tags: None, diff --git a/crates/core/src/validation/vector_item.rs b/crates/core/src/validation/vector_item.rs new file mode 100644 index 00000000..c877f244 --- /dev/null +++ b/crates/core/src/validation/vector_item.rs @@ -0,0 +1,395 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Write-path validation for vector-valued and search-schema attributes. +//! +//! When a table has a vector index, an item written to that table must carry a +//! well-formed vector attribute (a list of 32-bit floats of the declared +//! dimension) when present, and any search-schema attribute it carries must +//! match the declared scalar type and stay within the size limits. Missing +//! attributes are allowed: the write simply is not indexed for that field. +//! +//! Pure synchronous Rust: no async, no I/O. + +use crate::error::DynamoDbError; +use crate::types::{ + AttributeDefinition, AttributeValue, Item, ScalarAttributeType, SearchSchemaElementType, + VectorIndexKeyInfo, attribute_value_size, +}; + +/// Maximum byte size of a search-schema partition-key attribute value. +pub const MAX_HASH_KEY_SIZE: usize = 2048; +/// Maximum byte size of a search-schema inline-filter attribute value. +pub const MAX_INLINE_FILTER_SIZE: usize = 10240; + +fn invalid(msg: impl Into) -> DynamoDbError { + DynamoDbError::ValidationException(msg.into()) +} + +/// Validate the vector-relevant attributes of an item being written against the +/// table's vector indexes. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` when a present vector attribute +/// is not a list of the declared dimension of 32-bit floats, or when a present +/// search-schema attribute has the wrong type or exceeds its size limit. +pub fn validate_vector_write( + item: &Item, + vector_indexes: &[VectorIndexKeyInfo], + attribute_definitions: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + for index in vector_indexes { + if let Some(value) = item.get(&index.vector_attribute_name) { + validate_vector_attribute(value, index)?; + } + for element in &index.search_schema { + if let Some(value) = item.get(&element.attribute_name) { + validate_search_schema_attribute( + &element.attribute_name, + element.element_type, + value, + attribute_definitions, + )?; + } + } + } + Ok(()) +} + +/// Validate a single vector-valued attribute against an index definition. +fn validate_vector_attribute( + value: &AttributeValue, + index: &VectorIndexKeyInfo, +) -> Result<(), DynamoDbError> { + let attr = &index.vector_attribute_name; + let index_name = &index.index_name; + let dimensions = index.dimensions as usize; + + let AttributeValue::L(elements) = value else { + return Err(invalid(format!( + "One or more parameter values were invalid: Invalid type for parameter {attr}, \ + Expected: a list of numbers. IndexName: {index_name}" + ))); + }; + + if elements.len() != dimensions { + // Punctuation matches the service exactly, verified 2026-08-05: + // "...were invalid. Invalid size for parameter emb, Expected: 4, + // Actual: 3 IndexName: vidx" + // Note the full stop after "invalid", the comma after the parameter + // name, and the absence of one after the actual count. + return Err(invalid(format!( + "One or more parameter values were invalid. Invalid size for parameter {attr}, \ + Expected: {dimensions}, Actual: {} IndexName: {index_name}", + elements.len() + ))); + } + + for (position, element) in elements.iter().enumerate() { + match element { + AttributeValue::N(number) => { + // A component is in range when it parses to a finite 32-bit + // float. Comparing decimal magnitude against f32::MAX would + // wrongly reject the boundary value, whose shortest decimal + // rounds just above the exact f32 maximum. + let representable = + matches!(number.parse::(), Ok(parsed) if parsed.is_finite()); + if !representable { + let display = number + .parse::() + .map(format_scientific) + .unwrap_or_else(|_| number.clone()); + return Err(invalid(format!( + "Invalid value for parameter {attr}[{position}], Value: {display} is \ + outside valid range [-3.4028235E38, 3.4028235E38]. IndexName: {index_name}" + ))); + } + } + other => { + return Err(invalid(format!( + "One or more parameter values were invalid: Invalid type for parameter \ + {attr}[{position}], Expected: 32-bit floating point number, Actual: {}. \ + IndexName: {index_name}", + attribute_type_token(other) + ))); + } + } + } + + Ok(()) +} + +/// Validate a single search-schema attribute value (type then size). +fn validate_search_schema_attribute( + name: &str, + element_type: SearchSchemaElementType, + value: &AttributeValue, + attribute_definitions: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + if let Some(definition) = attribute_definitions + .iter() + .find(|definition| definition.attribute_name == name) + && !value_matches_scalar_type(value, definition.attribute_type) + { + return Err(invalid(format!( + "One or more parameter values were invalid: SearchSchema attribute '{name}' type \ + mismatch: value does not match the declared attribute type" + ))); + } + + let size = attribute_value_size(value); + match element_type { + SearchSchemaElementType::Hash if size > MAX_HASH_KEY_SIZE => Err(invalid(format!( + "One or more parameter values were invalid: Aggregate size for HASH key attributes \ + exceeds the maximum of {MAX_HASH_KEY_SIZE} bytes" + ))), + SearchSchemaElementType::InlineFilter if size > MAX_INLINE_FILTER_SIZE => { + Err(invalid(format!( + "One or more parameter values were invalid: Size limit exceeded for SearchSchema \ + attribute '{name}': maximum {MAX_INLINE_FILTER_SIZE} bytes" + ))) + } + _ => Ok(()), + } +} + +/// Whether an attribute value matches a scalar key type (`S`, `N`, or `B`). +fn value_matches_scalar_type(value: &AttributeValue, scalar_type: ScalarAttributeType) -> bool { + matches!( + (value, scalar_type), + (AttributeValue::S(_), ScalarAttributeType::S) + | (AttributeValue::N(_), ScalarAttributeType::N) + | (AttributeValue::B(_), ScalarAttributeType::B) + ) +} + +/// DynamoDB type token for an attribute value (used in error messages). +fn attribute_type_token(value: &AttributeValue) -> &'static str { + match value { + AttributeValue::S(_) => "S", + AttributeValue::N(_) => "N", + AttributeValue::B(_) => "B", + AttributeValue::Bool(_) => "BOOL", + AttributeValue::Null => "NULL", + AttributeValue::M(_) => "M", + AttributeValue::L(_) => "L", + AttributeValue::SS(_) => "SS", + AttributeValue::NS(_) => "NS", + AttributeValue::BS(_) => "BS", + } +} + +/// Format a float in upper-case scientific notation with an explicit exponent +/// sign, e.g. `1.3E+40` or `-1.3E+40`. +fn format_scientific(value: f64) -> String { + let formatted = format!("{value:E}"); + if let Some(exponent_pos) = formatted.find('E') { + let (mantissa, exponent) = formatted.split_at(exponent_pos); + let digits = &exponent[1..]; + if digits.starts_with('-') || digits.starts_with('+') { + formatted + } else { + format!("{mantissa}E+{digits}") + } + } else { + formatted + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::SearchSchemaElement; + + fn index() -> VectorIndexKeyInfo { + VectorIndexKeyInfo { + index_name: "ProductIndex".to_owned(), + dimensions: 5, + vector_attribute_name: "ProductEmbedding".to_owned(), + search_schema: vec![ + SearchSchemaElement { + attribute_name: "Country".to_owned(), + element_type: SearchSchemaElementType::Hash, + }, + SearchSchemaElement { + attribute_name: "Category".to_owned(), + element_type: SearchSchemaElementType::InlineFilter, + }, + ], + } + } + + fn defs() -> Vec { + vec![ + AttributeDefinition { + attribute_name: "Country".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "Category".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ] + } + + fn num_vec(values: &[&str]) -> AttributeValue { + AttributeValue::L( + values + .iter() + .map(|v| AttributeValue::N((*v).to_owned())) + .collect(), + ) + } + + fn item_with_vector(value: AttributeValue) -> Item { + let mut item = Item::new(); + item.insert("ProductId".to_owned(), AttributeValue::S("p1".to_owned())); + item.insert("ProductEmbedding".to_owned(), value); + item + } + + fn err(item: &Item) -> String { + match validate_vector_write(item, &[index()], &defs()).unwrap_err() { + DynamoDbError::ValidationException(m) => m, + other => panic!("expected ValidationException, got {other:?}"), + } + } + + #[test] + fn accepts_valid_vector() { + let item = item_with_vector(num_vec(&["0.1", "0.2", "0.3", "0.4", "0.5"])); + validate_vector_write(&item, &[index()], &defs()).unwrap(); + } + + #[test] + fn missing_vector_and_schema_attributes_are_allowed() { + let mut item = Item::new(); + item.insert("ProductId".to_owned(), AttributeValue::S("p1".to_owned())); + validate_vector_write(&item, &[index()], &defs()).unwrap(); + } + + #[test] + fn rejects_too_few_dimensions() { + let message = err(&item_with_vector(num_vec(&["0.1", "0.2", "0.3"]))); + assert!(message.contains("Invalid size for parameter ProductEmbedding")); + assert!(message.contains("Expected: 5, Actual: 3")); + assert!(message.contains("IndexName: ProductIndex")); + } + + /// Exact wording, not fragments. + /// + /// Verified against real DynamoDB on 2026-08-05: a `PutItem` carrying three + /// values against a four-dimension index returned + /// "One or more parameter values were invalid. Invalid size for parameter + /// emb, Expected: 4, Actual: 3 IndexName: vidx". + /// + /// The punctuation is load-bearing: a full stop after "invalid", a comma + /// after the parameter name, and no separator after the actual count. The + /// fragment assertions above cannot catch a regression in any of those, so + /// this asserts the whole string. + #[test] + fn dimension_mismatch_message_matches_the_service_exactly() { + let message = err(&item_with_vector(num_vec(&["0.1", "0.2", "0.3"]))); + assert_eq!( + message, + "One or more parameter values were invalid. Invalid size for parameter \ + ProductEmbedding, Expected: 5, Actual: 3 IndexName: ProductIndex" + ); + } + + #[test] + fn rejects_empty_vector() { + let message = err(&item_with_vector(AttributeValue::L(vec![]))); + assert!(message.contains("Expected: 5, Actual: 0")); + } + + #[test] + fn rejects_non_list_vector() { + let message = err(&item_with_vector(AttributeValue::N("0.1".to_owned()))); + assert!(message.contains("Invalid type for parameter ProductEmbedding")); + } + + #[test] + fn rejects_string_element() { + let value = AttributeValue::L(vec![ + AttributeValue::N("0.1".to_owned()), + AttributeValue::N("0.2".to_owned()), + AttributeValue::S("x".to_owned()), + AttributeValue::N("0.4".to_owned()), + AttributeValue::N("0.5".to_owned()), + ]); + let message = err(&item_with_vector(value)); + assert!(message.contains("Invalid type for parameter ProductEmbedding[2]")); + assert!(message.contains("Expected: 32-bit floating point number, Actual: S")); + } + + #[test] + fn rejects_value_out_of_range() { + let value = AttributeValue::L(vec![ + AttributeValue::N("0.1".to_owned()), + AttributeValue::N("1.3E40".to_owned()), + AttributeValue::N("0.3".to_owned()), + AttributeValue::N("0.4".to_owned()), + AttributeValue::N("0.5".to_owned()), + ]); + let message = err(&item_with_vector(value)); + assert!(message.contains( + "Invalid value for parameter ProductEmbedding[1], Value: 1.3E+40 is outside \ + valid range [-3.4028235E38, 3.4028235E38]. IndexName: ProductIndex" + )); + } + + #[test] + fn rejects_negative_value_out_of_range() { + let value = AttributeValue::L(vec![ + AttributeValue::N("0.1".to_owned()), + AttributeValue::N("-1.3E40".to_owned()), + AttributeValue::N("0.3".to_owned()), + AttributeValue::N("0.4".to_owned()), + AttributeValue::N("0.5".to_owned()), + ]); + assert!(err(&item_with_vector(value)).contains("Value: -1.3E+40 is outside")); + } + + #[test] + fn rejects_partition_key_type_mismatch() { + let mut item = item_with_vector(num_vec(&["0.1", "0.2", "0.3", "0.4", "0.5"])); + item.insert("Country".to_owned(), AttributeValue::N("123".to_owned())); + assert!(err(&item).contains("type mismatch")); + } + + #[test] + fn rejects_partition_key_too_large() { + let mut item = item_with_vector(num_vec(&["0.1", "0.2", "0.3", "0.4", "0.5"])); + item.insert( + "Country".to_owned(), + AttributeValue::S("A".repeat(MAX_HASH_KEY_SIZE + 1)), + ); + assert!(err(&item).contains("Aggregate size for HASH key attributes")); + } + + #[test] + fn rejects_inline_filter_too_large() { + let mut item = item_with_vector(num_vec(&["0.1", "0.2", "0.3", "0.4", "0.5"])); + item.insert( + "Category".to_owned(), + AttributeValue::S("B".repeat(MAX_INLINE_FILTER_SIZE + 1)), + ); + assert!(err(&item).contains("Size limit exceeded for SearchSchema")); + } + + #[test] + fn accepts_f32_max_boundary() { + // The shortest decimal for f32::MAX rounds just above the exact value; + // it must still be accepted as representable. + let item = item_with_vector(num_vec(&["0.1", "0.2", "0.3", "0.4", "3.4028235E38"])); + validate_vector_write(&item, &[index()], &defs()).unwrap(); + } + + #[test] + fn format_scientific_adds_exponent_sign() { + assert_eq!(format_scientific(1.3e40), "1.3E+40"); + assert_eq!(format_scientific(-1.3e40), "-1.3E+40"); + } +} diff --git a/crates/core/src/vector_capacity.rs b/crates/core/src/vector_capacity.rs new file mode 100644 index 00000000..3fdd82d9 --- /dev/null +++ b/crates/core/src/vector_capacity.rs @@ -0,0 +1,386 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Vector-index write metering. +//! +//! Vector indexes bill in their own units, separate from table read and write +//! capacity. Writes replicated into an index are reported as +//! `ConsumedCapacity.VectorIndexes..VectorWriteRequestBytes`. +//! +//! The model here was measured against the live service in us-east-1 on +//! 2026-08-13 across 20 probes, and reproduces every one of them exactly (not +//! approximately). Three blind predictions computed before measuring also +//! matched to the byte. Unlike the search-side figure, which the service +//! reports non-deterministically in one of two modes, the write figure is +//! deterministic: four identical writes reported an identical value at both +//! 1024 and 2048 dimensions. + +use std::collections::BTreeSet; + +use crate::types::{Item, attribute_value_size}; + +/// Byte floor for a single vector-index write. +/// +/// Measured: an 8-dimension index reported exactly 1024 for a small item, +/// where the unfloored model gives 47. +pub const VECTOR_WRITE_FLOOR_BYTES: f64 = 1024.0; + +/// Bytes metered per vector dimension. +/// +/// Derived from (8207 - 4111) / (2048 - 1024) = 4.0, i.e. one f32 per +/// component: the index stores the raw vector, so the wire representation is +/// irrelevant. Confirmed by writing the same vector with 1-character and +/// 20-character numbers, which reported the same figure. +pub const BYTES_PER_DIMENSION: f64 = 4.0; + +/// Which attributes an index projects, for metering purposes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProjectedAttributes<'a> { + /// Every attribute of the item is projected (`ProjectionType: ALL`). + All, + /// Only the table key, the vector attribute, and the search-schema + /// attributes are projected (`ProjectionType: KEYS_ONLY`). + /// + /// Measured: a 2000-byte non-projected attribute did not change the figure + /// under KEYS_ONLY, where it added exactly its own size under ALL. + KeysOnly { + /// Table key attribute names, always projected. + key_attributes: &'a [&'a str], + /// Search-schema attribute names, always projected. + search_schema_attributes: &'a [&'a str], + }, +} + +/// Bytes reported for replicating one item image into one vector index. +/// +/// `image` is the item as it exists in the index after the write, or the +/// deleted image for a delete. `vector_attribute` contributes only its NAME to +/// the byte count, because its payload is metered by the dimension term +/// instead; this is not an approximation, it is what makes the measured +/// arithmetic come out exactly. +/// +/// This is the charge for ONE index entry. A write that moves an entry between +/// search-schema partitions is charged twice; see +/// [`search_schema_partition_moved`]. +#[must_use] +pub fn vector_write_request_bytes( + dimensions: u32, + image: &Item, + vector_attribute: &str, + projected: ProjectedAttributes<'_>, +) -> f64 { + let projected_names: Option> = match projected { + ProjectedAttributes::All => None, + ProjectedAttributes::KeysOnly { + key_attributes, + search_schema_attributes, + } => Some( + key_attributes + .iter() + .chain(search_schema_attributes.iter()) + .copied() + .collect(), + ), + }; + + let mut bytes = 0usize; + for (name, value) in image { + if name == vector_attribute { + // Name only: the vector's payload is the dimension term. + bytes += name.len(); + continue; + } + let included = projected_names + .as_ref() + .is_none_or(|allowed| allowed.contains(name.as_str())); + if included { + bytes += name.len() + attribute_value_size(value); + } + } + + let unfloored = f64::from(dimensions) * BYTES_PER_DIMENSION + bytes as f64; + unfloored.max(VECTOR_WRITE_FLOOR_BYTES) +} + +/// Whether a write changes what the index holds, and so is charged at all. +/// +/// Measured behaviour, which is NOT what the public documentation describes. +/// The docs say "writes that do not change an indexed attribute do not incur +/// vector write capacity"; the service actually charges whenever the PROJECTED +/// entry changes: +/// +/// * setting the vector to a byte-identical value is NOT charged; +/// * changing a non-indexed attribute IS charged under `ALL`, because the +/// projection includes it, and is NOT charged under `KEYS_ONLY`; +/// * an item with no vector attribute is never in the index, so neither +/// writing nor deleting it is charged. +#[must_use] +pub fn projected_entry_changed( + before: Option<&Item>, + after: Option<&Item>, + vector_attribute: &str, + projected: ProjectedAttributes<'_>, +) -> bool { + let in_index = + |image: Option<&Item>| -> bool { image.is_some_and(|i| i.contains_key(vector_attribute)) }; + let (was, is) = (in_index(before), in_index(after)); + if !was && !is { + // Never in the index: nothing is replicated either way. + return false; + } + if was != is { + // Entering or leaving the index always replicates. + return true; + } + // Present both sides: compare only what the index actually holds. + let (Some(b), Some(a)) = (before, after) else { + return true; + }; + match projected { + ProjectedAttributes::All => b != a, + ProjectedAttributes::KeysOnly { + key_attributes, + search_schema_attributes, + } => key_attributes + .iter() + .chain(search_schema_attributes.iter()) + .chain(std::iter::once(&vector_attribute)) + .any(|name| b.get(*name) != a.get(*name)), + } +} + +/// Whether the entry moves between search-schema partitions, which the service +/// charges as a delete plus an insert. +/// +/// Measured: changing the search-schema HASH value on a 1024-dimension index +/// reported 8224, exactly twice the 4112 a single write of that image costs. +#[must_use] +pub fn search_schema_partition_moved( + before: Option<&Item>, + after: Option<&Item>, + search_schema_hash_attribute: Option<&str>, +) -> bool { + let Some(hash_attr) = search_schema_hash_attribute else { + return false; + }; + match (before, after) { + (Some(b), Some(a)) => { + let (bv, av) = (b.get(hash_attr), a.get(hash_attr)); + bv.is_some() && av.is_some() && bv != av + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::AttributeValue; + + fn emb(n: usize) -> AttributeValue { + AttributeValue::L( + (0..n) + .map(|_| AttributeValue::N("0.1".to_owned())) + .collect(), + ) + } + + /// Build the exact item shape used by the live probes. + fn probe_item(pk: &str, dims: usize, blob: Option) -> Item { + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S(pk.to_owned())); + item.insert("tenant".to_owned(), AttributeValue::S("t1".to_owned())); + item.insert("emb".to_owned(), emb(dims)); + if let Some(n) = blob { + item.insert("blob".to_owned(), AttributeValue::S("x".repeat(n))); + } + item + } + + fn all() -> ProjectedAttributes<'static> { + ProjectedAttributes::All + } + + fn keys_only() -> ProjectedAttributes<'static> { + ProjectedAttributes::KeysOnly { + key_attributes: &["pk"], + search_schema_attributes: &["tenant"], + } + } + + // Each case below is a figure MEASURED against the live service on + // 2026-08-13, us-east-1, account 964157134968. They are regression pins on + // real service behaviour, not on our own arithmetic. + + #[test] + fn measured_8_dims_small_item_hits_the_floor() { + let got = vector_write_request_bytes(8, &probe_item("a1", 8, None), "emb", all()); + assert_eq!(got, 1024.0, "measured 1024 (unfloored model gives 47)"); + } + + #[test] + fn measured_8_dims_with_2000_byte_attribute() { + let got = vector_write_request_bytes(8, &probe_item("a2", 8, Some(2000)), "emb", all()); + assert_eq!(got, 2051.0, "measured 2051"); + } + + #[test] + fn measured_1024_dims_small_item() { + let got = vector_write_request_bytes(1024, &probe_item("b1", 1024, None), "emb", all()); + assert_eq!(got, 4111.0, "measured 4111"); + } + + #[test] + fn measured_2048_dims_small_item() { + let got = vector_write_request_bytes(2048, &probe_item("b1", 2048, None), "emb", all()); + assert_eq!(got, 8207.0, "measured 8207"); + } + + #[test] + fn measured_1024_dims_with_2000_byte_attribute() { + let got = + vector_write_request_bytes(1024, &probe_item("b2", 1024, Some(2000)), "emb", all()); + assert_eq!(got, 6115.0, "measured 6115"); + } + + #[test] + fn measured_2048_dims_with_2000_byte_attribute() { + let got = + vector_write_request_bytes(2048, &probe_item("b2", 2048, Some(2000)), "emb", all()); + assert_eq!(got, 10211.0, "measured 10211"); + } + + #[test] + fn measured_delete_image_longer_key() { + let got = vector_write_request_bytes(1024, &probe_item("del1", 1024, None), "emb", all()); + assert_eq!(got, 4113.0, "measured 4113 on delete of pk=del1"); + } + + #[test] + fn measured_delete_image_longer_key_with_blob() { + let got = + vector_write_request_bytes(1024, &probe_item("del2", 1024, Some(2000)), "emb", all()); + assert_eq!(got, 6117.0, "measured 6117"); + } + + /// Under KEYS_ONLY the unprojected 2000-byte attribute must not be counted, + /// giving the same figure as the item without it. Measured 4111 for both. + #[test] + fn measured_keys_only_ignores_unprojected_attribute() { + let with_blob = vector_write_request_bytes( + 1024, + &probe_item("k1", 1024, Some(2000)), + "emb", + keys_only(), + ); + assert_eq!(with_blob, 4111.0, "measured 4111 under KEYS_ONLY"); + let without = + vector_write_request_bytes(1024, &probe_item("k1", 1024, None), "emb", keys_only()); + assert_eq!(with_blob, without, "projection must make these identical"); + } + + /// The same item under ALL does count it, which is what makes the previous + /// test discriminating rather than a tautology. + #[test] + fn projection_all_counts_what_keys_only_ignores() { + let item = probe_item("k1", 1024, Some(2000)); + let a = vector_write_request_bytes(1024, &item, "emb", all()); + let k = vector_write_request_bytes(1024, &item, "emb", keys_only()); + assert_eq!(a, 6115.0, "same shape measured 6115 under ALL"); + assert_eq!(k, 4111.0); + assert!(a > k); + } + + /// Blind prediction from the live run: pk="pred-g" plus three small + /// attributes measured 4127. + #[test] + fn measured_blind_prediction_case() { + let mut item = probe_item("pred-g", 1024, None); + item.insert("a".to_owned(), AttributeValue::S("1".to_owned())); + item.insert("bb".to_owned(), AttributeValue::S("22".to_owned())); + item.insert("ccc".to_owned(), AttributeValue::S("333".to_owned())); + let got = vector_write_request_bytes(1024, &item, "emb", all()); + assert_eq!(got, 4127.0, "predicted and measured 4127"); + } + + #[test] + fn identical_rewrite_is_not_charged() { + let item = probe_item("a1", 8, None); + assert!(!projected_entry_changed( + Some(&item), + Some(&item), + "emb", + all() + )); + } + + #[test] + fn item_without_vector_is_never_charged() { + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S("a3".to_owned())); + assert!(!projected_entry_changed(None, Some(&item), "emb", all())); + assert!(!projected_entry_changed(Some(&item), None, "emb", all())); + } + + #[test] + fn entering_and_leaving_the_index_is_charged() { + let with_vec = probe_item("a1", 8, None); + let mut without = Item::new(); + without.insert("pk".to_owned(), AttributeValue::S("a1".to_owned())); + assert!(projected_entry_changed(None, Some(&with_vec), "emb", all())); + assert!(projected_entry_changed(Some(&with_vec), None, "emb", all())); + assert!(projected_entry_changed( + Some(&without), + Some(&with_vec), + "emb", + all() + )); + } + + /// Measured divergence from the public documentation: a non-indexed + /// attribute change IS charged under ALL and is NOT under KEYS_ONLY. + #[test] + fn non_indexed_attribute_change_follows_the_projection() { + let before = probe_item("mv1", 8, None); + let mut after = before.clone(); + after.insert("label".to_owned(), AttributeValue::S("zz".to_owned())); + assert!( + projected_entry_changed(Some(&before), Some(&after), "emb", all()), + "ALL projects it, so it is charged" + ); + assert!( + !projected_entry_changed(Some(&before), Some(&after), "emb", keys_only()), + "KEYS_ONLY does not project it, so it is not charged" + ); + } + + #[test] + fn search_schema_move_detected_only_on_a_real_change() { + let before = probe_item("mv1", 8, None); + let mut after = before.clone(); + after.insert("tenant".to_owned(), AttributeValue::S("t9".to_owned())); + assert!(search_schema_partition_moved( + Some(&before), + Some(&after), + Some("tenant") + )); + assert!(!search_schema_partition_moved( + Some(&before), + Some(&before), + Some("tenant") + )); + assert!( + !search_schema_partition_moved(Some(&before), Some(&after), None), + "an index with no search-schema HASH cannot move" + ); + } + + /// Measured: the search-schema change at 1024 dimensions reported 8224, + /// exactly twice the 4112 one write of that image costs. + #[test] + fn measured_search_schema_move_is_exactly_double() { + let single = vector_write_request_bytes(1024, &probe_item("mv1", 1024, None), "emb", all()); + assert_eq!(single, 4112.0); + assert_eq!(single * 2.0, 8224.0, "measured 8224 for the move"); + } +} diff --git a/crates/engine/src/batch_write_item.rs b/crates/engine/src/batch_write_item.rs index 1ba44342..ef1efcea 100755 --- a/crates/engine/src/batch_write_item.rs +++ b/crates/engine/src/batch_write_item.rs @@ -127,6 +127,11 @@ pub async fn handle_batch_write_item( validate_item_size(&put.item, ctx.limits.max_item_size_bytes)?; validate_attribute_name_sizes(&put.item, &ctx.limits)?; validate_key_sizes(&put.item, &key_info.key_schema, &ctx.limits)?; + extenddb_core::validation::validate_vector_write( + &put.item, + &key_info.vector_indexes, + &key_info.attribute_definitions, + )?; collect_icm_if_needed( input.return_item_collection_metrics, diff --git a/crates/engine/src/create_table.rs b/crates/engine/src/create_table.rs index c9d00179..5e65f325 100755 --- a/crates/engine/src/create_table.rs +++ b/crates/engine/src/create_table.rs @@ -44,13 +44,32 @@ pub async fn handle_create_table( validate_create_table(&input, &ctx.limits)?; + crate::vector_gate::ensure_create_table_supported( + input.vector_indexes.as_ref(), + ctx.storage.as_vector_search(), + )?; + let table_name = input.table_name.clone(); + // Kept for the post-condition check below, since `input` is moved into the + // backend call. + let requested_vector_indexes = input.vector_indexes.clone(); let table_desc = ctx .storage .create_table(&ctx.account_id, input) .await .map_err(storage_err_to_dynamo)?; + // A declared-capable backend must not silently drop the indexes it was asked + // to create. The capability gate above proves only that it *can*. + crate::vector_gate::ensure_vector_indexes_applied( + requested_vector_indexes.as_ref(), + &table_desc, + )?; + + // Same invariant as the describe path: a newly created index must not be + // reported ready while it is still being populated. + table_desc.validate_vector_index_readiness()?; + // Drop any cached TableKeyInfo (typically a negative-cached "not found" // from a prior describe attempt) so requests against the new table see // it immediately. @@ -100,6 +119,11 @@ pub(crate) fn storage_err_to_dynamo(e: extenddb_storage::error::StorageError) -> tracing::error!(internal_error = %msg, "storage connection error"); DynamoDbError::ServiceUnavailable("Service is temporarily unavailable".to_owned()) } + // Not a fault, so deliberately not logged at error level: the backend + // never claimed the feature. ValidationException because DynamoDB has no + // "unsupported" error class, and the request is invalid against this + // deployment rather than a server failure. + StorageError::Unsupported(msg) => DynamoDbError::ValidationException(msg), StorageError::CatalogVersionMismatch { expected, found } => { tracing::error!("Catalog version mismatch: expected {expected}, found {found}"); DynamoDbError::InternalServerError("Internal server error".to_owned()) diff --git a/crates/engine/src/describe_table.rs b/crates/engine/src/describe_table.rs index 17dccc15..449e8611 100755 --- a/crates/engine/src/describe_table.rs +++ b/crates/engine/src/describe_table.rs @@ -25,6 +25,10 @@ pub async fn handle_describe_table( .await .map_err(storage_err_to_dynamo)?; + // A backend must not tell a client an index is ready while it is still being + // populated; the first search would silently undercount. + table_desc.validate_vector_index_readiness()?; + let output = DescribeTableOutput { table: table_desc }; serialize_output(&output) } diff --git a/crates/engine/src/import_export.rs b/crates/engine/src/import_export.rs index ec3788ce..fd4eddef 100755 --- a/crates/engine/src/import_export.rs +++ b/crates/engine/src/import_export.rs @@ -304,6 +304,7 @@ fn create_table_input_from_params(tcp: &TableCreationParameters) -> CreateTableI provisioned_throughput: tcp.provisioned_throughput.clone(), global_secondary_indexes: tcp.global_secondary_indexes.clone(), local_secondary_indexes: None, + vector_indexes: None, stream_specification: None, sse_specification: None, tags: None, diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 02049abe..3b7c7c01 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -29,6 +29,7 @@ mod put_item; mod query; mod read_helpers; mod scan; +mod search_vectors; pub mod stream_capture; mod streams; mod tagging; @@ -38,6 +39,7 @@ mod transact_write_items; mod ttl; mod update_item; mod update_table; +mod vector_gate; pub use batch_get_item::handle_batch_get_item; pub use batch_write_item::handle_batch_write_item; @@ -53,6 +55,7 @@ pub use list_tables::handle_list_tables; pub use put_item::handle_put_item; pub use query::handle_query; pub use scan::handle_scan; +pub use search_vectors::handle_search_vectors; pub use streams::{ handle_describe_stream, handle_get_records, handle_get_shard_iterator, handle_list_streams, }; @@ -91,6 +94,7 @@ pub fn is_known_operation(operation: &str) -> bool { | "UpdateItem" | "Query" | "Scan" + | "SearchVectors" | "BatchGetItem" | "BatchWriteItem" | "TransactGetItems" @@ -367,6 +371,7 @@ pub async fn dispatch( "UpdateItem" => handle_update_item(body, ctx).await, "Query" => handle_query(body, ctx).await, "Scan" => handle_scan(body, ctx).await, + "SearchVectors" => handle_search_vectors(body, ctx).await, "BatchGetItem" => handle_batch_get_item(body, ctx).await, "BatchWriteItem" => handle_batch_write_item(body, ctx).await, "TransactGetItems" => handle_transact_get_items(body, ctx).await, diff --git a/crates/engine/src/put_item.rs b/crates/engine/src/put_item.rs index f7ecc6b2..81cbaef4 100755 --- a/crates/engine/src/put_item.rs +++ b/crates/engine/src/put_item.rs @@ -160,6 +160,12 @@ pub async fn handle_put_item( &key_info.attribute_definitions, )?; + extenddb_core::validation::validate_vector_write( + &input.item, + &key_info.vector_indexes, + &key_info.attribute_definitions, + )?; + let return_old = input.return_values == ReturnValues::AllOld; let capacity_requested = input.return_consumed_capacity != extenddb_core::types::ReturnConsumedCapacity::None; diff --git a/crates/engine/src/query.rs b/crates/engine/src/query.rs index 624f250b..d6b38426 100755 --- a/crates/engine/src/query.rs +++ b/crates/engine/src/query.rs @@ -60,6 +60,15 @@ pub async fn handle_query( None }; + // A vector index is searched only via the vector search API, never queried. + if let Some(ref idx) = index_info + && idx.index_type == IndexType::Vector + { + return Err(DynamoDbError::ValidationException( + "Query operation not supported on this index type".to_owned(), + )); + } + // ConsistentRead is not supported on GSI queries (tenet 1: fidelity). if input.consistent_read == Some(true) && let Some(ref idx) = index_info @@ -103,6 +112,7 @@ pub async fn handle_query( global_secondary_indexes: key_info.global_secondary_indexes.clone(), local_secondary_indexes: key_info.local_secondary_indexes.clone(), stream_specification: None, // Queries don't capture stream records + vector_indexes: key_info.vector_indexes.clone(), } } else { key_info.clone() diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index c8354bf4..9929005d 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -217,6 +217,15 @@ pub async fn handle_scan( None }; + // A vector index is searched only via the vector search API, never scanned. + if let Some(ref idx) = index_info + && idx.index_type == IndexType::Vector + { + return Err(DynamoDbError::ValidationException( + "Scan operation not supported on this index type".to_owned(), + )); + } + // ConsistentRead is not supported on GSI scans (tenet 1: fidelity). if input.consistent_read == Some(true) && let Some(ref idx) = index_info @@ -304,6 +313,7 @@ pub async fn handle_scan( global_secondary_indexes: key_info.global_secondary_indexes.clone(), local_secondary_indexes: key_info.local_secondary_indexes.clone(), stream_specification: None, // Scans don't capture stream records + vector_indexes: key_info.vector_indexes.clone(), } } else { key_info.clone() diff --git a/crates/engine/src/search_vectors.rs b/crates/engine/src/search_vectors.rs new file mode 100644 index 00000000..c8acb7b3 --- /dev/null +++ b/crates/engine/src/search_vectors.rs @@ -0,0 +1,480 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `SearchVectors` operation handler. +//! +//! Runs a similarity search over a vector index: parses the query vector, an +//! optional single-equality prefilter, and an optional projection, calls the +//! storage layer for the top-k nearest neighbors, and returns each item with +//! its similarity score. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::expression::{ + ExpressionMaps, Projection, validate_conditions_against_search_schema, + validate_search_condition_expression, +}; +use extenddb_core::types::{ + AttributeValue, DescribeTableInput, Item, ReturnConsumedCapacity, SearchSchemaElementType, + item_size_bytes, +}; + +use crate::OperationContext; +use crate::create_table::storage_err_to_dynamo; +use crate::serialize_output; +use crate::{DispatchMetrics, DispatchResult}; + +/// Minimum length of a vector index name. +const MIN_INDEX_NAME_LENGTH: usize = 3; +/// Maximum number of nearest neighbors a single search may request. +const MAX_TOP_K: i64 = 100; +/// Maximum number of elements in a search vector. +const MAX_SEARCH_VECTOR_LENGTH: usize = 4096; +/// `SearchVectors` request body. +#[derive(Debug, Clone, Deserialize)] +struct SearchVectorsInput { + #[serde(rename = "TableName")] + table_name: String, + #[serde(rename = "IndexName")] + index_name: String, + #[serde(rename = "SearchVector")] + search_vector: Vec, + #[serde(rename = "TopK")] + top_k: i64, + #[serde(rename = "SearchConditionExpression")] + search_condition_expression: Option, + #[serde(rename = "ProjectionExpression")] + projection_expression: Option, + #[serde(rename = "ExpressionAttributeNames")] + expression_attribute_names: Option>, + #[serde(rename = "ExpressionAttributeValues")] + expression_attribute_values: Option>, + #[serde(rename = "ReturnConsumedCapacity", default)] + return_consumed_capacity: ReturnConsumedCapacity, +} + +/// A single search result: the item plus its similarity score. +#[derive(Debug, Serialize)] +struct SearchResult { + #[serde(rename = "Item")] + item: Item, + #[serde(rename = "Score")] + score: f64, +} + +/// `SearchVectors` response body. +#[derive(Debug, Serialize)] +struct SearchVectorsOutput { + #[serde(rename = "SearchResults")] + search_results: Vec, + #[serde(rename = "ConsumedCapacity", skip_serializing_if = "Option::is_none")] + consumed_capacity: Option, +} + +/// Consumed capacity for a vector search, reported as `VectorSearchUnits`. +#[derive(Debug, Serialize)] +struct VectorCapacity { + /// Verified against the service 2026-08-05: the response field is + /// `VectorSearchRequestBytes`, a byte count. There is no units field. + #[serde(rename = "VectorSearchRequestBytes")] + vector_search_request_bytes: f64, +} + +/// Handle a `SearchVectors` request. +/// +/// # Errors +/// +/// Returns `DynamoDbError` for validation failures, missing tables/indexes, or +/// storage errors. +pub async fn handle_search_vectors( + body: Value, + ctx: &OperationContext, +) -> Result { + let input: SearchVectorsInput = + serde_json::from_value(body).map_err(crate::deserialize_error)?; + + let vector_search = + crate::vector_gate::ensure_search_supported(ctx.storage.as_vector_search())?; + + // Request-shape validation runs before the table lookup, so a malformed + // request against a missing table reports the validation error rather than + // ResourceNotFound. + + // IndexName length. + if input.index_name.len() < MIN_INDEX_NAME_LENGTH { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Value at 'IndexName' failed to satisfy constraint: \ + Member must have length greater than or equal to {MIN_INDEX_NAME_LENGTH}" + ))); + } + + // SearchVector length and element types. + let query_vector = parse_search_vector(&input.search_vector)?; + + // TopK bounds. The lower bound reports the standard constraint message; the + // upper bound reports the documented range message. + if input.top_k < 1 { + return Err(DynamoDbError::ValidationException( + "1 validation error detected: Value at 'TopK' failed to satisfy constraint: \ + Member must have value greater than or equal to 1" + .to_owned(), + )); + } + if input.top_k > MAX_TOP_K { + return Err(DynamoDbError::ValidationException(format!( + "Provided TopK value '{}' is out of valid range. \ + The value must be between 1 and {MAX_TOP_K} inclusive", + input.top_k + ))); + } + + // Structural validation of the filter expression (schema-independent). + let conditions = match input.search_condition_expression.as_deref() { + Some(expr) => validate_search_condition_expression( + expr, + input.expression_attribute_names.as_ref(), + input.expression_attribute_values.as_ref(), + )?, + None => Vec::new(), + }; + + let key_info = ctx + .table_key_info(&input.table_name) + .await + .map_err(storage_err_to_dynamo)?; + + // Schema-aware validation needs the vector index metadata (dimension and + // search schema) plus the table attribute definitions for type checks. + let table = ctx + .storage + .describe_table( + &ctx.account_id, + DescribeTableInput { + table_name: input.table_name.clone(), + }, + ) + .await + .map_err(storage_err_to_dynamo)?; + let vector_index = table + .vector_indexes + .as_deref() + .unwrap_or(&[]) + .iter() + .find(|vi| vi.index_name == input.index_name) + .ok_or_else(|| { + DynamoDbError::ValidationException(format!( + "The table does not have the specified index: {}", + input.index_name + )) + })?; + + if query_vector.len() != vector_index.dimensions as usize { + return Err(DynamoDbError::ValidationException(format!( + "Input search vector dimension {} does not match vector index dimension {}", + query_vector.len(), + vector_index.dimensions + ))); + } + + if !conditions.is_empty() { + validate_conditions_against_search_schema( + &conditions, + vector_index.search_schema.as_deref(), + &table.attribute_definitions, + )?; + } + + // The index's HASH element scopes the search to one partition; the + // remaining conditions narrow within it. Declaring a HASH element is + // optional, but when the index has one the service requires the search to + // supply it, so validation upstream guarantees it is present here. + let hash_attr = vector_index + .search_schema + .as_deref() + .unwrap_or_default() + .iter() + .find(|e| e.element_type == SearchSchemaElementType::Hash) + .map(|e| e.attribute_name.as_str()); + + let hash_key: Option<(&str, &AttributeValue)> = hash_attr.and_then(|name| { + conditions + .iter() + .find(|c| c.attribute_name == name) + .map(|c| (c.attribute_name.as_str(), &c.value)) + }); + + let filters: Vec<(&str, &AttributeValue)> = conditions + .iter() + .filter(|c| Some(c.attribute_name.as_str()) != hash_attr) + .map(|c| (c.attribute_name.as_str(), &c.value)) + .collect(); + + let search_output = vector_search + .search_vectors(extenddb_storage::VectorSearch { + key_info: &key_info, + index_name: &input.index_name, + query_vector: &query_vector, + top_k: input.top_k, + hash_key, + filters: &filters, + }) + .await + .map_err(storage_err_to_dynamo)?; + let hits = search_output.hits; + + // Bytes read from the index for the returned items, excluding the vector + // component (the stored item already omits the vector attribute). + let non_vector_bytes: usize = hits.iter().map(|h| item_size_bytes(&h.item)).sum(); + + // Compile the projection once, if supplied. + let compiled_projection = if let Some(ref proj_str) = input.projection_expression { + let paths = crate::expression_helpers::parse_projection_expr(proj_str, &ctx.limits)?; + let names = input + .expression_attribute_names + .clone() + .unwrap_or_default() + .into_iter() + .map(|(k, v)| (k.trim_start_matches('#').to_owned(), v)) + .collect(); + let proj_maps = ExpressionMaps::new(names, HashMap::new()); + Some(Projection::compile(&paths, &proj_maps, true)?) + } else { + None + }; + + let search_results: Vec = hits + .into_iter() + .map(|hit| { + let item = match compiled_projection.as_ref() { + Some(proj) => proj.apply(&hit.item), + None => hit.item, + }; + SearchResult { + item, + score: hit.score, + } + }) + .collect(); + + // Kept for the dispatch metric only. `SearchVectorsOutput` deliberately does + // NOT carry a `Count` field: measured against the live service on 2026-08-10, + // the response contains only `SearchResults` and `ConsumedCapacity`, across + // five parameter variations (no projection, ReturnConsumedCapacity=INDEXES, + // a ProjectionExpression, and a TopK larger than the item count). + let count = search_results.len() as i64; + + // The service reports `ConsumedCapacity.VectorSearchRequestBytes`, a byte + // figure, not a unit figure. See `search_request_bytes` for the measured + // model and why exact parity is not achievable. + let request_bytes = search_request_bytes(vector_index.dimensions, non_vector_bytes); + let consumed_capacity = match input.return_consumed_capacity { + ReturnConsumedCapacity::None => None, + _ => Some(VectorCapacity { + vector_search_request_bytes: request_bytes, + }), + }; + + let output = SearchVectorsOutput { + search_results, + consumed_capacity, + }; + + let body = serialize_output(&output)?; + Ok(DispatchResult { + body, + metrics: DispatchMetrics { + read_capacity_units: request_bytes, + returned_item_count: count as u64, + index_name: Some(input.index_name), + ..Default::default() + }, + }) +} + +/// Validate a `SearchVector` and convert it into `f32`. +/// +/// Checks the length bounds (1..=`MAX_SEARCH_VECTOR_LENGTH`) and that every +/// element is a finite number before converting. +fn parse_search_vector(values: &[AttributeValue]) -> Result, DynamoDbError> { + if values.is_empty() { + return Err(DynamoDbError::ValidationException( + "1 validation error detected: Value at 'SearchVector' failed to satisfy constraint: \ + Member must have length greater than or equal to 1" + .to_owned(), + )); + } + if values.len() > MAX_SEARCH_VECTOR_LENGTH { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Value at 'SearchVector' failed to satisfy constraint: \ + Member must have length less than or equal to {MAX_SEARCH_VECTOR_LENGTH}" + ))); + } + let mut out = Vec::with_capacity(values.len()); + for v in values { + match v { + AttributeValue::N(n) => { + let f = n.parse::().map_err(|_| { + DynamoDbError::ValidationException( + "Search vector contains invalid values".to_owned(), + ) + })?; + if !f.is_finite() { + return Err(DynamoDbError::ValidationException( + "Search vector contains invalid values".to_owned(), + )); + } + out.push(f); + } + _ => { + return Err(DynamoDbError::ValidationException( + "Search vector contains invalid values".to_owned(), + )); + } + } + } + Ok(out) +} + +/// Bytes reported as `ConsumedCapacity.VectorSearchRequestBytes` for one search. +/// +/// `returned_non_vector_bytes` is the summed stored size of the items actually +/// returned, taken before any projection is applied. +/// +/// Measured against the service on 2026-08-05 in us-east-1. Three properties +/// hold, and this reproduces all three: +/// +/// * A 1 KiB floor. A 4-dimension index reported exactly 1024 for every TopK +/// from 1 to 100, for 1 to 100 returned results, and with 4 to 204 items in +/// the index. +/// * A per-dimension term that is independent of how many items the index +/// holds, so this is not a scan-cost model: growing a 4096-dimension index +/// from 3 to 12 items moved the figure by 2 bytes, which was the returned +/// key's length rather than the extra items. It is also unaffected by the +/// query vector's wire width, since the same search with 1-character and +/// 21-character numbers (49 KB versus 127 KB of JSON) both reported 75201. +/// The vector is metered per dimension, not per byte sent. +/// * Plus the stored bytes of the returned items, excluding the searched +/// vector. Adding one item with a 2000-byte non-vector attribute raised the +/// figure by exactly 1999. Projection does not reduce it, so a caller must +/// sum the items before projecting, not after. +/// +/// Exact parity is NOT achievable and must not be asserted. The service is not +/// deterministic here: byte-identical requests against an unchanged index return +/// one of two values separated by a fixed `dimensions * 3.111` offset, a ratio of +/// 1.176. Twenty-four samples at 1024 dimensions gave 18067 and 21253; at 2048 +/// they gave 36058 and 42429; the mix varies between runs and persists with 10 +/// items in the index. This reproduces the lower and more frequent mode. +fn search_request_bytes(dimensions: u32, returned_non_vector_bytes: usize) -> f64 { + /// Derived from the lower mode: 18067 at 1024 dimensions and 36058 at 2048, + /// both 17.6 bytes per dimension once the returned item is subtracted. + const BYTES_PER_DIMENSION: f64 = 17.6; + /// Observed floor. A 4-dimension search never reported less than this. + const MIN_SEARCH_BYTES: f64 = 1024.0; + + (BYTES_PER_DIMENSION * f64::from(dimensions) + returned_non_vector_bytes as f64) + .max(MIN_SEARCH_BYTES) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_search_vector_ok() { + let v = vec![ + AttributeValue::N("0.1".to_owned()), + AttributeValue::N("-2".to_owned()), + ]; + assert_eq!(parse_search_vector(&v).unwrap(), vec![0.1f32, -2.0]); + } + + #[test] + fn parse_search_vector_rejects_empty_with_length_message() { + let err = parse_search_vector(&[]).unwrap_err(); + let DynamoDbError::ValidationException(msg) = err else { + panic!("expected ValidationException"); + }; + assert!(msg.contains( + "Value at 'SearchVector' failed to satisfy constraint: \ + Member must have length greater than or equal to 1" + )); + } + + #[test] + fn parse_search_vector_rejects_over_max_length() { + let big: Vec = (0..=MAX_SEARCH_VECTOR_LENGTH) + .map(|i| AttributeValue::N(i.to_string())) + .collect(); + let err = parse_search_vector(&big).unwrap_err(); + let DynamoDbError::ValidationException(msg) = err else { + panic!("expected ValidationException"); + }; + assert!(msg.contains("Member must have length less than or equal to 4096")); + } + + #[test] + fn parse_search_vector_rejects_non_number() { + let err = parse_search_vector(&[AttributeValue::S("x".to_owned())]).unwrap_err(); + let DynamoDbError::ValidationException(msg) = err else { + panic!("expected ValidationException"); + }; + assert_eq!(msg, "Search vector contains invalid values"); + } + + #[test] + fn parse_search_vector_rejects_non_finite() { + assert!(parse_search_vector(&[AttributeValue::N("NaN".to_owned())]).is_err()); + assert!(parse_search_vector(&[AttributeValue::N("inf".to_owned())]).is_err()); + } + + /// A 4-dimension search reported exactly 1024 for every TopK from 1 to 100 + /// and for 4 to 204 items in the index, so the floor dominates at low + /// dimensions rather than anything proportional. + #[test] + fn search_bytes_have_a_one_kib_floor() { + assert!((search_request_bytes(4, 0) - 1024.0).abs() < f64::EPSILON); + assert!((search_request_bytes(1, 100) - 1024.0).abs() < f64::EPSILON); + } + + /// Independent of items scanned, so the only inputs are dimensions and the + /// bytes of the items actually returned. + #[test] + fn search_bytes_scale_per_dimension_above_the_floor() { + let a = search_request_bytes(1024, 0); + let b = search_request_bytes(2048, 0); + assert!(a > 1024.0, "1024 dimensions must clear the floor: {a}"); + // Doubling the dimensions doubles the per-dimension term exactly. + assert!((b - 2.0 * a).abs() < 1.0, "{a} then {b}"); + } + + /// Adding one item with a 2000-byte non-vector attribute raised the service's + /// figure by exactly 1999, so returned bytes pass through one-for-one. + #[test] + fn returned_item_bytes_pass_through_one_for_one() { + let base = search_request_bytes(1024, 0); + assert!((search_request_bytes(1024, 2000) - (base + 2000.0)).abs() < f64::EPSILON); + } + + /// Checks the model against the service's own numbers, deliberately with a + /// tolerance rather than equality: the service returns one of two values for + /// a byte-identical request (18067 or 21253 at 1024 dimensions, 36058 or + /// 42429 at 2048), so asserting equality would encode a coin flip. Roughly + /// 20 bytes of returned item accompanied each observation. + #[test] + fn model_tracks_the_observed_lower_mode() { + for (dimensions, observed) in [(1024_u32, 18067.0_f64), (2048, 36058.0)] { + let modelled = search_request_bytes(dimensions, 20); + let error = (modelled - observed).abs() / observed; + assert!( + error < 0.01, + "{dimensions} dimensions: modelled {modelled} against observed {observed} \ + is {:.2}% out", + error * 100.0 + ); + } + } +} diff --git a/crates/engine/src/transact_write_items.rs b/crates/engine/src/transact_write_items.rs index 7690d0cb..80d088d0 100755 --- a/crates/engine/src/transact_write_items.rs +++ b/crates/engine/src/transact_write_items.rs @@ -18,7 +18,10 @@ use crate::transact_write_helpers::{ }; use crate::{DispatchMetrics, DispatchResult}; use extenddb_core::error::DynamoDbError; -use extenddb_core::types::{TransactWriteItem, TransactWriteItemsInput, TransactWriteItemsOutput}; +use extenddb_core::expression::{Expr, ExpressionMaps, PathElement, UpdateAction}; +use extenddb_core::types::{ + CancellationReason, Item, TransactWriteItem, TransactWriteItemsInput, TransactWriteItemsOutput, +}; use extenddb_core::validation::{ validate_attribute_name_sizes, validate_attribute_values_nesting_depth, validate_item_nesting_depth, validate_item_size, validate_key_not_empty, @@ -141,6 +144,19 @@ pub async fn handle_transact_write_items( } // Build storage operations + // Vector-valued and search-schema validation failures surface as per-item + // cancellation reasons rather than a top-level ValidationException. + if let Some(reasons) = collect_vector_cancellation_reasons(&prepared) { + let codes: Vec = reasons.iter().map(|r| r.code.clone()).collect(); + return Err(DynamoDbError::TransactionCanceledException { + message: format!( + "Transaction cancelled, please refer cancellation reasons for specific reasons [{}]", + codes.join(", ") + ), + cancellation_reasons: reasons, + }); + } + let ops: Vec> = prepared.iter().map(|p| p.to_storage_op()).collect(); @@ -234,6 +250,85 @@ pub async fn handle_transact_write_items( }) } +/// Validate the vector-valued and search-schema attributes of each prepared +/// write operation. Returns per-item cancellation reasons when any operation +/// fails, or `None` when all pass. +fn collect_vector_cancellation_reasons(prepared: &[PreparedOp]) -> Option> { + let mut reasons = Vec::with_capacity(prepared.len()); + let mut any_error = false; + for op in prepared { + let result = match op { + PreparedOp::Put { key_info, item, .. } => { + extenddb_core::validation::validate_vector_write( + item, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + } + PreparedOp::Update { + key_info, + actions, + maps, + .. + } => { + let assigned = assigned_vector_attributes(actions, maps); + extenddb_core::validation::validate_vector_write( + &assigned, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + } + _ => Ok(()), + }; + match result { + Ok(()) => reasons.push(CancellationReason::none()), + Err(DynamoDbError::ValidationException(message)) => { + any_error = true; + reasons.push(CancellationReason::validation_error(message)); + } + Err(other) => { + any_error = true; + reasons.push(CancellationReason::validation_error(other.to_string())); + } + } + } + any_error.then_some(reasons) +} + +/// Collect direct `SET attr = :value` assignments into a partial item for +/// vector validation. +/// +/// This covers the bare-placeholder form ONLY, and deliberately so: it exists +/// to produce the correct per-item `CancellationReason` vector for the common +/// case, which the storage layer cannot reproduce because it aborts at the +/// first failing operation. It is NOT the authoritative vector check and must +/// never be treated as one. Any RHS that is not a bare placeholder +/// (`list_append`, `if_not_exists`, an attribute copy, or a form added later) +/// is invisible here and is caught instead by +/// `expression::apply_update_validated` on the evaluated image inside the +/// transaction, which every backend applies. +fn assigned_vector_attributes(actions: &[UpdateAction], maps: &ExpressionMaps) -> Item { + let mut assigned = Item::new(); + for action in actions { + if let UpdateAction::Set { + path, + value: Expr::Placeholder(placeholder), + } = action + && path.len() == 1 + && let PathElement::Attribute(name) = &path[0] + { + let resolved = name + .strip_prefix('#') + .and_then(|reference| maps.names.get(reference).map(String::as_str)) + .unwrap_or(name.as_str()); + if let Some(value) = maps.values.get(placeholder) { + assigned.insert(resolved.to_owned(), value.clone()); + } + } + } + assigned +} + /// Parse and validate a single `TransactWriteItem`, returning a `PreparedOp`. async fn prepare_write_op( twi: &TransactWriteItem, diff --git a/crates/engine/src/update_item.rs b/crates/engine/src/update_item.rs index 3ec921a0..24d143a8 100755 --- a/crates/engine/src/update_item.rs +++ b/crates/engine/src/update_item.rs @@ -195,6 +195,16 @@ pub async fn handle_update_item( &key_info.attribute_definitions, )?; + // Vector and search-schema attributes are deliberately NOT validated here. + // Their validity is a property of the stored value, and the stored value + // does not exist until the actions have been evaluated against the + // pre-update image, which only the storage layer holds. Validating + // expression forms here instead would silently miss every RHS that is not + // a bare placeholder (`list_append`, `if_not_exists`, attribute copies, and + // anything added later). The authoritative check is + // `expression::apply_update_validated`, which every backend applies to the + // evaluated image and which no expression form can bypass. + // Amazon DynamoDB enforces nesting depth on values that are stored as item // attributes. For UpdateExpression, walk each SET action's RHS to find the // EAV placeholders it references, resolve them against `maps.values`, and diff --git a/crates/engine/src/update_table.rs b/crates/engine/src/update_table.rs index bafb34ca..4fcfd1f9 100755 --- a/crates/engine/src/update_table.rs +++ b/crates/engine/src/update_table.rs @@ -35,10 +35,31 @@ pub async fn handle_update_table( )); } + crate::vector_gate::ensure_update_table_supported( + input.vector_index_updates.as_ref(), + ctx.storage.as_vector_search(), + )?; + // Same per-index rules CreateTable applies, so a malformed index is rejected + // identically whichever path it arrives by. + extenddb_core::validation::validate_vector_index_updates( + input.vector_index_updates.as_ref(), + input.attribute_definitions.as_deref().unwrap_or_default(), + )?; + let has_gsi_updates = input .global_secondary_index_updates .as_ref() .is_some_and(|u| !u.is_empty()); + // Vector index changes count as a specified field. Measured 2026-08-06: the + // service accepts an UpdateTable carrying only VectorIndexUpdates, which is + // how the backfill lifecycle was observed. Omitting it here rejected that + // request as empty, so a vector-capable backend could never have been reached + // by the one operation the contract models a lifecycle for. An empty list is + // not a change, matching how the capability gate treats it. + let has_vector_updates = input + .vector_index_updates + .as_ref() + .is_some_and(|u| !u.is_empty()); // Validate: at least one field must be specified. if input.billing_mode.is_none() @@ -48,6 +69,7 @@ pub async fn handle_update_table( && input.table_class.is_none() && input.on_demand_throughput.is_none() && !has_gsi_updates + && !has_vector_updates { return Err(DynamoDbError::ValidationException( "At least one of BillingMode, ProvisionedThroughput, DeletionProtectionEnabled, StreamSpecification, or GlobalSecondaryIndexUpdates must be specified".to_owned(), @@ -146,6 +168,9 @@ pub async fn handle_update_table( } let table_name = input.table_name.clone(); + // Kept for the post-condition check below, since `input` is moved into the + // backend call. + let vector_index_updates = input.vector_index_updates.clone(); let desc = ctx .storage .update_table(&ctx.account_id, input) @@ -181,6 +206,10 @@ pub async fn handle_update_table( } })?; + // A declared-capable backend must not silently drop the vector index change. + // Checked against the description it returned, so this cannot be opted out of. + crate::vector_gate::ensure_vector_updates_applied(vector_index_updates.as_ref(), &desc)?; + // Drop the cached TableKeyInfo: index changes, stream-spec changes, and // throughput changes all alter what the cached value contains. // diff --git a/crates/engine/src/vector_gate.rs b/crates/engine/src/vector_gate.rs new file mode 100644 index 00000000..655c3ece --- /dev/null +++ b/crates/engine/src/vector_gate.rs @@ -0,0 +1,444 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Backend capability gate for vector operations. +//! +//! The decision lives here, in core, rather than in each handler or each backend. +//! A backend hands over a [`VectorSearchEngine`] or it does not: it needs no code +//! to refuse a feature it has not implemented, and the refusal is identical +//! whichever backend is installed. The gate takes the accessor's result rather +//! than a boolean, so there is no state in which a backend claims support without +//! having provided an implementation. + +use extenddb_core::error::DynamoDbError; +use extenddb_core::types::{ + IndexStatus, TableDescription, VectorIndexDescription, VectorIndexSpecification, + VectorIndexUpdate, +}; +use extenddb_storage::VectorSearchEngine; + +/// Reject a `CreateTable` that asks for vector indexes the backend cannot serve. +/// +/// Without this the request passes shape validation, reaches a backend that does +/// not read `vector_indexes`, and produces a table with no index. The caller is +/// told the index exists and only finds out otherwise on the first search, which +/// is the worst way to learn it: silently, and later. +/// +/// A request carrying no vector indexes, or an empty list, is unaffected, so an +/// ordinary `CreateTable` against a backend without vector support still works. +pub(crate) fn ensure_create_table_supported( + vector_indexes: Option<&Vec>, + vector_search: Option<&dyn VectorSearchEngine>, +) -> Result<(), DynamoDbError> { + let asked_for_vector_indexes = vector_indexes.is_some_and(|v| !v.is_empty()); + if asked_for_vector_indexes && vector_search.is_none() { + return Err(DynamoDbError::ValidationException( + "Vector indexes are not supported by this storage backend".to_owned(), + )); + } + Ok(()) +} + +/// Reject an `UpdateTable` that changes vector indexes on a backend that cannot +/// serve them. +/// +/// `UpdateTable` is the second creation path, and it is the one the service's own +/// backfill lifecycle is observable through, so leaving it ungated would let a +/// caller add an index that a backend silently ignores: the same silent-drop hole +/// `CreateTable` was fixed for. +/// +/// `Delete` is gated too, deliberately. A backend without vector support cannot +/// hold an index to delete, so the honest answer is that the operation is not +/// supported here, rather than letting the request through to be interpreted as a +/// no-op or a not-found. +pub(crate) fn ensure_update_table_supported( + vector_index_updates: Option<&Vec>, + vector_search: Option<&dyn VectorSearchEngine>, +) -> Result<(), DynamoDbError> { + let asked_for_changes = vector_index_updates.is_some_and(|u| !u.is_empty()); + if asked_for_changes && vector_search.is_none() { + return Err(DynamoDbError::ValidationException( + "Vector indexes are not supported by this storage backend".to_owned(), + )); + } + Ok(()) +} + +/// Resolve the backend's vector-search implementation, or reject the request. +/// +/// Returns the engine rather than a unit, so the handler cannot reach a search +/// without passing the gate: there is no separate accessor call it could make +/// instead, and no defaulted method on the backend for it to fall through to. +pub(crate) fn ensure_search_supported( + vector_search: Option<&dyn VectorSearchEngine>, +) -> Result<&dyn VectorSearchEngine, DynamoDbError> { + vector_search.ok_or_else(|| { + DynamoDbError::ValidationException( + "SearchVectors is not supported by this storage backend".to_owned(), + ) + }) +} + +/// Fail an `UpdateTable` whose vector index changes the backend did not apply. +/// +/// A post-condition rather than a pre-condition, and the reason it exists is worth +/// stating plainly: the capability gate above only proves a backend *can* serve +/// vector indexes, not that it acted on this request. A backend that declares the +/// capability but never reads `vector_index_updates` returns 200 with the field +/// silently dropped, and the caller is told the change happened. Measured against +/// the SQLite backend before it implemented this path: a Create returned 200 and +/// created nothing, discoverable only on the first search, and a Delete returned +/// 200 while the index stayed ACTIVE, stayed in `DescribeTable`, and kept +/// returning hits. The delete case is the dangerous one, because deleting an index +/// to stop serving embeddings is a thing people do for reasons that are not +/// performance. +/// +/// Checked against the description the backend itself returned, so no backend can +/// opt out. Deliberately tolerant about *which* post-state is correct, since that +/// is unmeasured: a created index must merely be present, whatever its status, and +/// a deleted one must merely not be present and `ACTIVE`. That catches doing +/// nothing without asserting a lifecycle this contract has not yet observed. +pub(crate) fn ensure_vector_updates_applied( + updates: Option<&Vec>, + description: &TableDescription, +) -> Result<(), DynamoDbError> { + let Some(updates) = updates else { + return Ok(()); + }; + for update in updates { + if let Some(create) = &update.create { + let present = find_index(description, &create.index_name).is_some(); + if !present { + return Err(dropped(&create.index_name, "create")); + } + } + if let Some(delete) = &update.delete { + let still_serving = find_index(description, &delete.index_name) + .is_some_and(|index| index.index_status == IndexStatus::Active); + if still_serving { + return Err(dropped(&delete.index_name, "delete")); + } + } + } + Ok(()) +} + +/// Fail a `CreateTable` whose vector indexes the backend did not create. +/// +/// Same reasoning as [`ensure_vector_updates_applied`], on the other path. No +/// in-tree backend has failed this, but a backend that reads `vector_indexes` on +/// one path and not the other is a likelier mistake than one that ignores both, +/// because the paths are implemented separately. +pub(crate) fn ensure_vector_indexes_applied( + requested: Option<&Vec>, + description: &TableDescription, +) -> Result<(), DynamoDbError> { + for spec in requested.into_iter().flatten() { + if find_index(description, &spec.index_name).is_none() { + return Err(dropped(&spec.index_name, "create")); + } + } + Ok(()) +} + +fn find_index<'a>( + description: &'a TableDescription, + index_name: &str, +) -> Option<&'a VectorIndexDescription> { + description + .vector_indexes + .iter() + .flatten() + .find(|index| index.index_name == index_name) +} + +/// Reported as an internal fault, not a validation error, because it is a bug in +/// the backend rather than anything the caller did wrong. +fn dropped(index_name: &str, verb: &str) -> DynamoDbError { + tracing::error!( + index_name, + operation = verb, + "backend declared vector support but did not apply the vector index change" + ); + DynamoDbError::InternalServerError("Internal server error".to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A description carrying the named vector indexes at the given status. + fn described(indexes: &[(&str, IndexStatus)]) -> TableDescription { + use extenddb_core::types::{DistanceFunction, VectorAttribute}; + TableDescription { + vector_indexes: Some( + indexes + .iter() + .map(|(name, status)| VectorIndexDescription { + index_name: (*name).to_owned(), + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + dimensions: 4, + search_schema: None, + distance_function: DistanceFunction::Cosine, + index_status: *status, + backfilling: None, + index_size_bytes: 0, + item_count: 0, + index_arn: format!("arn:aws:dynamodb:us-east-1:1:table/t/index/{name}"), + projection: None, + }) + .collect(), + ), + ..Default::default() + } + } + + /// The exact behaviour measured against the SQLite backend before it + /// implemented this path: `UpdateTable` returned 200 and created nothing, and + /// the caller only found out on the first search. + #[test] + fn a_create_the_backend_ignored_is_an_error_not_a_success() { + let err = ensure_vector_updates_applied(Some(&vec![create_update()]), &described(&[])) + .expect_err("a dropped create must not pass"); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "a backend bug is not the caller's validation error: {err:?}" + ); + } + + /// The other measured behaviour, and the more dangerous one: `UpdateTable` + /// returned 200 while the index stayed ACTIVE and kept returning hits. + #[test] + fn a_delete_the_backend_ignored_is_an_error_not_a_success() { + let err = ensure_vector_updates_applied( + Some(&vec![delete_update()]), + &described(&[("vidx", IndexStatus::Active)]), + ) + .expect_err("a dropped delete must not pass"); + assert!(matches!(err, DynamoDbError::InternalServerError(_))); + } + + /// Deliberately tolerant about which post-state is correct, since that is + /// unmeasured. A created index need only be present, at any status. + #[test] + fn a_created_index_passes_at_any_status() { + for status in [ + IndexStatus::Creating, + IndexStatus::Active, + IndexStatus::Updating, + ] { + assert!( + ensure_vector_updates_applied( + Some(&vec![create_update()]), + &described(&[("vidx", status)]) + ) + .is_ok(), + "status {status:?} must be accepted" + ); + } + } + + /// A deleted index may be gone or on its way out; only still serving is wrong. + #[test] + fn a_deleted_index_passes_when_absent_or_deleting() { + assert!( + ensure_vector_updates_applied(Some(&vec![delete_update()]), &described(&[])).is_ok() + ); + assert!( + ensure_vector_updates_applied( + Some(&vec![delete_update()]), + &described(&[("vidx", IndexStatus::Deleting)]) + ) + .is_ok() + ); + } + + /// The guard must not fire on the ordinary case, or every `UpdateTable` breaks. + #[test] + fn no_vector_updates_is_always_fine() { + assert!(ensure_vector_updates_applied(None, &described(&[])).is_ok()); + assert!(ensure_vector_updates_applied(Some(&vec![]), &described(&[])).is_ok()); + } + + /// Only the requested index is checked: an unrelated one being absent, or + /// present, says nothing about this request. + #[test] + fn only_the_requested_index_is_checked() { + assert!( + ensure_vector_updates_applied( + Some(&vec![create_update()]), + &described(&[ + ("other", IndexStatus::Active), + ("vidx", IndexStatus::Active) + ]) + ) + .is_ok() + ); + assert!( + ensure_vector_updates_applied( + Some(&vec![delete_update()]), + &described(&[("other", IndexStatus::Active)]) + ) + .is_ok() + ); + } + + /// The same hole on the CreateTable path, which is implemented separately and + /// so can regress independently. + #[test] + fn create_table_indexes_the_backend_ignored_are_an_error() { + assert!(ensure_vector_indexes_applied(Some(&vec![spec()]), &described(&[])).is_err()); + assert!( + ensure_vector_indexes_applied( + Some(&vec![spec()]), + &described(&[("vidx", IndexStatus::Creating)]) + ) + .is_ok() + ); + assert!(ensure_vector_indexes_applied(None, &described(&[])).is_ok()); + } + + /// Stands in for a backend that implements vector search. Defining it is the + /// point: the gate cannot be handed a "supported" value without something + /// that actually implements the trait. + struct Capable; + + impl VectorSearchEngine for Capable { + fn search_vectors( + &self, + _req: extenddb_storage::VectorSearch<'_>, + ) -> extenddb_storage::BoxedFuture<'_, extenddb_storage::VectorSearchResult> { + Box::pin(async { unreachable!("the gate tests never run a search") }) + } + } + + fn capable() -> &'static dyn VectorSearchEngine { + &Capable + } + use extenddb_core::types::{ + DeleteVectorIndexAction, DistanceFunction, Projection, ProjectionType, VectorAttribute, + }; + + fn spec() -> VectorIndexSpecification { + VectorIndexSpecification { + index_name: "vidx".to_owned(), + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + dimensions: 4, + distance_function: DistanceFunction::Cosine, + search_schema: None, + projection: Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + } + } + + fn create_update() -> VectorIndexUpdate { + VectorIndexUpdate { + create: Some(spec()), + delete: None, + } + } + + fn delete_update() -> VectorIndexUpdate { + VectorIndexUpdate { + create: None, + delete: Some(DeleteVectorIndexAction { + index_name: "vidx".to_owned(), + }), + } + } + + fn message(err: &DynamoDbError) -> String { + match err { + DynamoDbError::ValidationException(m) => m.clone(), + other => panic!("expected ValidationException, got {other:?}"), + } + } + + #[test] + fn create_table_without_vector_indexes_is_allowed_on_any_backend() { + // The common case: a backend with no vector support must still serve + // ordinary CreateTable requests. + assert!(ensure_create_table_supported(None, None).is_ok()); + } + + #[test] + fn an_empty_vector_index_list_is_not_a_request_for_vector_indexes() { + let empty: Vec = Vec::new(); + assert!(ensure_create_table_supported(Some(&empty), None).is_ok()); + } + + #[test] + fn create_table_with_vector_indexes_is_rejected_on_an_unsupporting_backend() { + let specs = vec![spec()]; + let err = ensure_create_table_supported(Some(&specs), None) + .expect_err("must be rejected rather than silently dropped"); + assert_eq!( + message(&err), + "Vector indexes are not supported by this storage backend" + ); + } + + #[test] + fn create_table_with_vector_indexes_is_allowed_on_a_supporting_backend() { + let specs = vec![spec()]; + assert!(ensure_create_table_supported(Some(&specs), Some(capable())).is_ok()); + } + + #[test] + fn search_is_rejected_on_an_unsupporting_backend() { + let err = ensure_search_supported(None) + .map(|_| ()) + .expect_err("must be rejected before reaching the backend"); + assert_eq!( + message(&err), + "SearchVectors is not supported by this storage backend" + ); + } + + #[test] + fn search_is_allowed_on_a_supporting_backend() { + assert!(ensure_search_supported(Some(capable())).is_ok()); + assert!(ensure_search_supported(Some(capable())).is_ok()); + } + + #[test] + fn update_table_without_vector_changes_is_allowed_on_any_backend() { + // The common case. An ordinary UpdateTable, adding a GSI or changing + // throughput, must still work on a backend with no vector support. + assert!(ensure_update_table_supported(None, None).is_ok()); + let empty: Vec = Vec::new(); + assert!(ensure_update_table_supported(Some(&empty), None).is_ok()); + } + + #[test] + fn update_table_creating_a_vector_index_is_rejected_on_an_unsupporting_backend() { + let updates = vec![create_update()]; + let err = ensure_update_table_supported(Some(&updates), None) + .expect_err("UpdateTable is the second creation path and must be gated too"); + assert_eq!( + message(&err), + "Vector indexes are not supported by this storage backend" + ); + } + + /// Deleting is gated as well. A backend without support cannot hold an index + /// to delete, so "not supported here" is the honest answer rather than + /// letting it through to look like a no-op or a not-found. + #[test] + fn update_table_deleting_a_vector_index_is_rejected_on_an_unsupporting_backend() { + let updates = vec![delete_update()]; + assert!(ensure_update_table_supported(Some(&updates), None).is_err()); + } + + #[test] + fn update_table_vector_changes_are_allowed_on_a_supporting_backend() { + let updates = vec![create_update(), delete_update()]; + assert!(ensure_update_table_supported(Some(&updates), Some(capable())).is_ok()); + } +} diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index 1febebbd..00fd6743 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -490,6 +490,10 @@ impl BackupEngine for MongoEngine { sse_specification, table_class, on_demand_throughput, + // Fields for features this backend does not implement, vector + // indexes today, take their defaults. Adding one to + // CreateTableInput then does not break this build. + ..Default::default() }; // Create the table with the ACTIVE transition deferred: it enters diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index eb58495d..4a6f4570 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -809,8 +809,14 @@ impl MongoEngine { }; let mut new_item = existing_item; - expression::apply_update(actions, &mut new_item, maps) - .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + expression::apply_update_validated( + actions, + &mut new_item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; // Reject wrong-type or empty index-key attributes on // the resulting item โ€” D-M10, RFC-0003 ยง2.3. Same @@ -2523,7 +2529,14 @@ impl MongoEngine { } } - expression::apply_update(actions, &mut item, maps).map_err(|e| { + expression::apply_update_validated( + actions, + &mut item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| { TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; @@ -2777,8 +2790,14 @@ impl MongoEngine { // Condition allows the upsert. Build the new item // from `key` + apply update actions. let mut new_item = key.clone(); - expression::apply_update(actions, &mut new_item, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; + expression::apply_update_validated( + actions, + &mut new_item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; let new_doc = item_to_document( &new_item, &key_info.key_schema, @@ -2798,8 +2817,14 @@ impl MongoEngine { let existing_item = document_to_item(&existing)?; let mut new_item = existing_item.clone(); - expression::apply_update(actions, &mut new_item, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; + expression::apply_update_validated( + actions, + &mut new_item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; let new_doc = item_to_document( &new_item, diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 8e033ccf..b1c1604f 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -527,6 +527,10 @@ impl MongoEngine { sse_description, table_class_summary, on_demand_throughput: input.on_demand_throughput, + // Fields for features this backend does not implement, vector + // indexes today, take their defaults. Adding one to + // TableDescription then does not break this build. + ..Default::default() }) } @@ -985,19 +989,20 @@ impl MongoEngine { .find(doc! { "_id.table_id": &table_id }) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let mut global_secondary_indexes = Vec::new(); - let mut local_secondary_indexes = Vec::new(); + let mut infos = Vec::new(); while let Some(idx_doc) = idx_cursor .try_next() .await .map_err(|e| StorageError::Internal(e.to_string()))? { - let info = index_info_from_doc(&idx_doc)?; - match info.index_type { - IndexType::Gsi => global_secondary_indexes.push(info), - IndexType::Lsi => local_secondary_indexes.push(info), - } + infos.push(index_info_from_doc(&idx_doc)?); } + // Grouped by core rather than matched here, so a new IndexType variant + // does not break this backend. `index_info_from_doc` already rejects any + // kind this backend cannot have created. + let grouped = extenddb_core::types::partition_indexes(infos); + let global_secondary_indexes = grouped.gsis; + let local_secondary_indexes = grouped.lsis; let has_lsi = !local_secondary_indexes.is_empty(); Ok(TableKeyInfo { @@ -1011,6 +1016,10 @@ impl MongoEngine { global_secondary_indexes, local_secondary_indexes, stream_specification, + // Fields for features this backend does not implement, vector + // indexes today, take their defaults. Adding one to TableKeyInfo + // then does not break this build. + ..Default::default() }) } @@ -1334,6 +1343,10 @@ impl MongoEngine { sse_description, table_class_summary, on_demand_throughput, + // Fields for features this backend does not implement, vector + // indexes today, take their defaults. Adding one to + // TableDescription then does not break this build. + ..Default::default() }) } diff --git a/crates/storage-postgres/src/backup_engine.rs b/crates/storage-postgres/src/backup_engine.rs index ee02a17c..94e4f449 100755 --- a/crates/storage-postgres/src/backup_engine.rs +++ b/crates/storage-postgres/src/backup_engine.rs @@ -448,6 +448,7 @@ impl BackupEngine for PostgresEngine { sse_specification: None, table_class: None, on_demand_throughput: None, + ..Default::default() }; // Create the table with the ACTIVE transition deferred: it enters diff --git a/crates/storage-postgres/src/create_table.rs b/crates/storage-postgres/src/create_table.rs index 9ef2e39d..8a76fe79 100755 --- a/crates/storage-postgres/src/create_table.rs +++ b/crates/storage-postgres/src/create_table.rs @@ -430,6 +430,7 @@ impl PostgresEngine { .as_ref() .map(|tc| serde_json::json!({ "TableClass": tc })), on_demand_throughput: input.on_demand_throughput, + ..Default::default() }) } } diff --git a/crates/storage-postgres/src/data/ddl.rs b/crates/storage-postgres/src/data/ddl.rs index e7211935..e2f62ac6 100755 --- a/crates/storage-postgres/src/data/ddl.rs +++ b/crates/storage-postgres/src/data/ddl.rs @@ -315,6 +315,10 @@ impl PostgresEngine { global_secondary_indexes, local_secondary_indexes, stream_specification, + // Fields for features this backend does not implement, vector + // indexes today, take their defaults. Adding one to TableKeyInfo + // then does not break this build. + ..Default::default() }) } @@ -334,8 +338,7 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let mut gsis = Vec::new(); - let mut lsis = Vec::new(); + let mut infos = Vec::new(); for (index_name, idx_type_str, index_id, ks_json, proj_json) in rows { let index_type = match idx_type_str.as_str() { "GSI" => IndexType::Gsi, @@ -357,12 +360,13 @@ impl PostgresEngine { key_schema, projection, }; - match index_type { - IndexType::Gsi => gsis.push(info), - IndexType::Lsi => lsis.push(info), - } + infos.push(info); } - Ok((gsis, lsis)) + // Grouped by core rather than matched here, so a new IndexType variant + // does not break this backend. The string parse above already rejects + // any kind this backend cannot have created. + let grouped = extenddb_core::types::partition_indexes(infos); + Ok((grouped.gsis, grouped.lsis)) } /// Fetch metadata for a secondary index from the catalog. diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index 5e098046..2b19b668 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -430,9 +430,14 @@ async fn execute_transact_write_op( *return_values_on_ccf, existing.as_ref(), )?; - expression::apply_update(actions, &mut item, maps).map_err(|e| { - TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) - })?; + expression::apply_update_validated( + actions, + &mut item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; // Validate post-update item size validation::validate_item_size(&item, max_item_size_bytes).map_err(|e| { TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index a2c668af..8522eea9 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -117,8 +117,14 @@ impl PostgresEngine { } // Apply update actions - expression::apply_update(actions, &mut item, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; + expression::apply_update_validated( + actions, + &mut item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; // Validate post-update item size (400 KB limit) validation::validate_item_size(&item, self.max_item_size_bytes) diff --git a/crates/storage-postgres/src/table_helpers.rs b/crates/storage-postgres/src/table_helpers.rs index 516f8121..8357f448 100755 --- a/crates/storage-postgres/src/table_helpers.rs +++ b/crates/storage-postgres/src/table_helpers.rs @@ -351,6 +351,10 @@ impl PostgresEngine { on_demand_throughput: row .on_demand_throughput .and_then(|v| serde_json::from_value(v).ok()), + // Fields for features this backend does not implement, vector + // indexes today, take their defaults. Adding one to + // TableDescription then does not break this build. + ..Default::default() }) } } diff --git a/crates/storage-sqlite/src/backup.rs b/crates/storage-sqlite/src/backup.rs index 838d8c62..48b1dc4f 100644 --- a/crates/storage-sqlite/src/backup.rs +++ b/crates/storage-sqlite/src/backup.rs @@ -352,6 +352,7 @@ impl BackupEngine for SqliteEngine { sse_specification: None, table_class: None, on_demand_throughput: None, + ..Default::default() }; let desc = self.create_table(&account_id, create_input).await?; @@ -368,6 +369,7 @@ impl BackupEngine for SqliteEngine { global_secondary_indexes: Vec::new(), local_secondary_indexes: Vec::new(), stream_specification: None, + ..Default::default() }; let items: Vec<(String,)> = diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs index 20c17d30..8c4e2202 100644 --- a/crates/storage-sqlite/src/create_table.rs +++ b/crates/storage-sqlite/src/create_table.rs @@ -398,6 +398,10 @@ impl SqliteEngine { .as_ref() .map(|tc| serde_json::json!({ "TableClass": tc })), on_demand_throughput: input.on_demand_throughput, + // Fields for features this backend does not implement, vector + // indexes today, take their defaults, so adding one to this type + // does not break this build. + ..Default::default() }) } } diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs index d326d022..49c9880a 100644 --- a/crates/storage-sqlite/src/data/ddl.rs +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -241,6 +241,10 @@ impl SqliteEngine { global_secondary_indexes, local_secondary_indexes, stream_specification, + // Fields for features this backend does not implement, vector + // indexes today, take their defaults, so adding one to this type + // does not break this build. + ..Default::default() }) } @@ -259,8 +263,7 @@ impl SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let mut gsis = Vec::new(); - let mut lsis = Vec::new(); + let mut infos = Vec::new(); for (index_name, idx_type_str, index_id, ks_json, proj_json) in rows { let index_type = match idx_type_str.as_str() { "GSI" => IndexType::Gsi, @@ -282,12 +285,13 @@ impl SqliteEngine { key_schema, projection, }; - match index_type { - IndexType::Gsi => gsis.push(info), - IndexType::Lsi => lsis.push(info), - } + infos.push(info); } - Ok((gsis, lsis)) + // Grouped by core rather than matched here, so a new IndexType variant + // does not break this backend. The string parse above already rejects + // any kind this backend cannot have created. + let grouped = extenddb_core::types::partition_indexes(infos); + Ok((grouped.gsis, grouped.lsis)) } /// Fetch `IndexInfo` for a secondary index, validating the table is ACTIVE. diff --git a/crates/storage-sqlite/src/data/transactions.rs b/crates/storage-sqlite/src/data/transactions.rs index f3ae8dec..2e5002bc 100644 --- a/crates/storage-sqlite/src/data/transactions.rs +++ b/crates/storage-sqlite/src/data/transactions.rs @@ -379,9 +379,14 @@ async fn execute_transact_write_op( existing.as_ref(), )?; let mut item = existing.clone().unwrap_or_else(|| (*key).clone()); - expression::apply_update(actions, &mut item, maps).map_err(|e| { - TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) - })?; + expression::apply_update_validated( + actions, + &mut item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; validation::validate_item_size(&item, max_item_size_bytes).map_err(|e| { TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs index 92ac84a5..f3e45368 100644 --- a/crates/storage-sqlite/src/data/update_item.rs +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -64,8 +64,14 @@ impl SqliteEngine { // Start from the existing image, or from the key for a fresh upsert. let mut item = old.clone().unwrap_or_else(|| key.clone()); - expression::apply_update(actions, &mut item, maps) - .map_err(|e| StorageError::Validation(e.to_string()))?; + expression::apply_update_validated( + actions, + &mut item, + maps, + &key_info.vector_indexes, + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; validation::validate_item_size(&item, self.max_item_size_bytes) .map_err(|e| StorageError::Validation(e.to_string()))?; // Secondary-index key validation on the post-update item, matching the diff --git a/crates/storage-sqlite/src/table_helpers.rs b/crates/storage-sqlite/src/table_helpers.rs index c2b9aa72..52b64b85 100644 --- a/crates/storage-sqlite/src/table_helpers.rs +++ b/crates/storage-sqlite/src/table_helpers.rs @@ -240,6 +240,10 @@ impl SqliteEngine { .on_demand_throughput .as_deref() .and_then(|s| serde_json::from_str(s).ok()), + // Fields for features this backend does not implement, vector + // indexes today, take their defaults, so adding one to this type + // does not break this build. + ..Default::default() }) } diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index a2043460..fa7f39e4 100755 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -43,6 +43,11 @@ pub enum StorageError { CatalogNotInitialized, #[error("Connection error: {0}")] Connection(String), + /// The backend does not implement the requested feature. Distinct from + /// `Internal`, which reports a fault: this reports a capability the backend + /// never claimed, so it is not a bug and must not be logged as one. + #[error("Not supported by this storage backend: {0}")] + Unsupported(String), #[error("Internal error: {0}")] Internal(String), } diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 205adf09..46115f41 100755 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -58,12 +58,17 @@ use std::sync::Arc; use futures::future::BoxFuture; +// Re-exported because the storage traits' public signatures return it, so an +// out-of-tree backend must be able to name the type without taking its own +// `futures` dependency and hoping the version matches ours. +pub use futures::future::BoxFuture as BoxedFuture; + use extenddb_core::expression::{Expr, ExpressionMaps, KeyCondition, UpdateAction}; use extenddb_core::types::{ - CreateTableInput, DeleteTableInput, DescribeStreamInput, DescribeTableInput, IndexInfo, Item, - ListTablesInput, ListTablesOutput, StreamDescription, StreamRecord, StreamSummary, - StreamViewType, TableDescription, TableKeyInfo, Tag, TimeToLiveDescription, UpdateTableInput, - UserIdentity, + AttributeValue, CreateTableInput, DeleteTableInput, DescribeStreamInput, DescribeTableInput, + IndexInfo, Item, ListTablesInput, ListTablesOutput, StreamDescription, StreamRecord, + StreamSummary, StreamViewType, TableDescription, TableKeyInfo, Tag, TimeToLiveDescription, + UpdateTableInput, UserIdentity, }; use error::StorageError; @@ -73,6 +78,79 @@ use error::StorageError; pub type ItemPairResult = Result<(Option, Option), StorageError>; /// Result of a query or scan: items plus an optional last-evaluated-key for pagination. pub type QueryResult = Result<(Vec, Option), StorageError>; +/// Result of a vector search: the ranked hits plus the metering the caller needs +/// to populate `ConsumedCapacity`. +pub type VectorSearchResult = Result; + +/// A single vector search result. +#[derive(Debug, Clone)] +pub struct VectorHit { + /// The projected item, honouring the index projection and any + /// `projection_expression` on the request. + pub item: Item, + /// The raw score as the backend computed it. + /// + /// The direction depends on the index's distance function and is **not** + /// uniform: for `Cosine` and `Euclidean` a lower score is more similar and + /// 0.0 means identical, while for `DotProduct` a higher score is more + /// similar. Never normalise these into a single "similarity" number, and + /// never compare two scores without consulting + /// [`VectorSearchOutput::distance_function`]; prefer + /// [`extenddb_core::types::DistanceFunction::ranks_before`]. + pub score: f64, +} + +/// Output of a vector search. +#[derive(Debug, Clone)] +pub struct VectorSearchOutput { + /// Hits ordered most-relevant-first under `distance_function`. + pub hits: Vec, + /// The distance function the index is defined with, so a caller can + /// interpret [`VectorHit::score`] without re-reading the index definition. + pub distance_function: extenddb_core::types::DistanceFunction, +} + +/// Parameters for a vector similarity search. +/// +/// Grouped in a struct so the request surface can evolve without breaking the +/// `DataEngine` trait signature for every backend. +/// +/// Note what is deliberately absent: there is no projection expression. The +/// engine compiles and applies any request projection to the items a backend +/// returns, so a backend serves the index projection and never parses an +/// expression. Keep it that way; two projection implementations would diverge. +pub struct VectorSearch<'a> { + pub key_info: &'a TableKeyInfo, + pub index_name: &'a str, + /// The query vector, already validated to match the index dimensionality. + /// + /// Narrowed to `f32` from the wire representation, which is a list of `N` + /// (arbitrary-precision decimal). Embedding models emit single-precision + /// floats and vector extensions store them that way, so the narrowing is + /// deliberate, but it is lossy for a caller that supplies more precision + /// than `f32` can carry. + pub query_vector: &'a [f32], + /// Maximum hits to return. Validated upstream against the service ceiling. + pub top_k: i64, + /// Equality on the index HASH attribute, scoping the search to a single + /// partition. + /// + /// `None` only when the index declares no HASH element in its search + /// schema, which is permitted. When the index does declare one this is + /// always populated, because the service requires the scope to be supplied + /// on every search against such an index. Backends may therefore treat + /// `Some` as a mandatory predicate rather than a hint. + pub hash_key: Option<(&'a str, &'a AttributeValue)>, + /// Equality filters over the index's inline-filter attributes, combined + /// with logical AND. Empty means no additional filtering. + /// + /// Equality only, deliberately. The wire surface accepts a filter + /// expression, which the engine parses and lowers to these pairs; that is + /// lossless today because only exact-match conditions are supported, and + /// range or function conditions are rejected before reaching a backend. If + /// the service ever admits range conditions this type must widen. + pub filters: &'a [(&'a str, &'a AttributeValue)], +} /// TTL table info: `(account_id, table_name, ttl_attribute)`. pub type TtlTableInfo = (String, String, String); /// Stream records result: records plus an optional next shard iterator. @@ -293,6 +371,28 @@ pub trait DataEngine: Send + Sync { index_name: Option<&str>, ) -> BoxFuture<'_, QueryResult>; + /// The vector-search implementation, if this backend has one. + /// + /// Defaults to `None`, so a backend that has never heard of vector search is + /// correct by omission and needs no vector code at all. Returning `Some` is + /// not a claim, it is the implementation: the returned value must implement + /// [`VectorSearchEngine`], so a backend cannot advertise vector search + /// without providing it. That is the difference between this and a boolean + /// capability flag, which could be set true by a backend that had + /// implemented nothing. + /// + /// The engine calls this before any vector work. A `CreateTable` carrying + /// vector indexes, an `UpdateTable` changing them, or a `SearchVectors` is + /// rejected while this returns `None`, before anything reaches storage. + /// + /// Returning `Some` is a promise about two things this accessor cannot + /// express in the type system: that the backend persists and reports the + /// vector indexes it is given on the table paths, and that it maintains them + /// on writes. + fn as_vector_search(&self) -> Option<&dyn VectorSearchEngine> { + None + } + /// Execute multiple get operations in a single consistent snapshot. /// /// Returns one `Option` per request, in the same order as `ops`. @@ -343,6 +443,24 @@ pub trait DataEngine: Send + Sync { /// /// Methods that operate on table-scoped resources receive `account_id`. /// Tag methods use ARN (which embeds `account_id`) so they don't need it separately. +/// Vector similarity search over a vector index. +/// +/// A separate trait rather than defaulted methods on [`DataEngine`], because not +/// every backend can implement every feature and an optional feature should be +/// impossible to half-declare. There are deliberately **no default bodies**: a +/// backend either implements this trait and hands it over via +/// [`DataEngine::as_vector_search`], or it does not and the engine rejects vector +/// requests before they arrive. +pub trait VectorSearchEngine: Send + Sync { + /// Search a vector index for the nearest vectors to a query vector. + /// + /// # Errors + /// + /// Returns [`StorageError::IndexNotFound`] if the named vector index does not + /// exist on the table, and [`StorageError::Internal`] on query failure. + fn search_vectors(&self, req: VectorSearch<'_>) -> BoxFuture<'_, VectorSearchResult>; +} + pub trait MetadataEngine: Send + Sync { /// Return the TTL configuration for a table. fn describe_ttl( @@ -659,3 +777,155 @@ mod tests { fn _assert_dyn(_: Arc) {} } } + +/// Compile-time and behaviour guard for a backend that does not implement vector +/// search. +/// +/// `MinimalDataEngine` implements only the methods [`DataEngine`] requires. If a +/// vector method loses its default, or a new required method appears, this stops +/// compiling, which is the same breakage an out-of-tree backend crate would hit. +/// That is the property the contract is supposed to have: a backend that has +/// never heard of vector search needs no vector code at all. +/// +/// It also pins the runtime half. The default capability must be `false`, so the +/// engine gates vector requests away before they arrive, and the defaulted +/// `search_vectors` must fail rather than return an empty result set, because a +/// silent empty answer to a search is indistinguishable from a table with no +/// matches. +#[cfg(test)] +mod vector_opt_out_tests { + use super::*; + + struct MinimalDataEngine; + + impl DataEngine for MinimalDataEngine { + fn put_item( + &self, + _key_info: &TableKeyInfo, + _item: Item, + _return_old: bool, + _condition: Option<&Expr>, + _maps: &ExpressionMaps, + _stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(None) }) + } + fn get_item( + &self, + _key_info: &TableKeyInfo, + _key: &Item, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(None) }) + } + fn delete_item( + &self, + _key_info: &TableKeyInfo, + _key: &Item, + _return_old: bool, + _condition: Option<&Expr>, + _maps: &ExpressionMaps, + _stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(None) }) + } + fn update_item( + &self, + _key_info: &TableKeyInfo, + _key: &Item, + _actions: &[UpdateAction], + _return_old: bool, + _return_new: bool, + _condition: Option<&Expr>, + _maps: &ExpressionMaps, + _stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, ItemPairResult> { + Box::pin(async { Ok((None, None)) }) + } + fn query( + &self, + _key_info: &TableKeyInfo, + _key_condition: &KeyCondition, + _maps: &ExpressionMaps, + _forward: bool, + _limit: Option, + _exclusive_start_key: Option<&Item>, + _index_name: Option<&str>, + ) -> BoxFuture<'_, QueryResult> { + Box::pin(async { Ok((Vec::new(), None)) }) + } + fn scan( + &self, + _key_info: &TableKeyInfo, + _limit: Option, + _exclusive_start_key: Option<&Item>, + _segment: Option, + _total_segments: Option, + _index_name: Option<&str>, + ) -> BoxFuture<'_, QueryResult> { + Box::pin(async { Ok((Vec::new(), None)) }) + } + fn transact_get_items( + &self, + _ops: &[TransactGetOp<'_>], + ) -> BoxFuture<'_, Result>, StorageError>> { + Box::pin(async { Ok(Vec::new()) }) + } + fn transact_write_items( + &self, + _ops: &[TransactWriteOp<'_>], + _idempotency: Option>, + ) -> BoxFuture<'_, Result<(), StorageError>> { + Box::pin(async { Ok(()) }) + } + fn cleanup_expired_idempotency_tokens( + &self, + _max_age_seconds: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async { Ok(0) }) + } + } + + /// A backend that writes no vector code hands over nothing, so the engine + /// gates every vector request away before it reaches storage. + #[test] + fn a_backend_that_ignores_vector_search_hands_over_nothing() { + let engine: Box = Box::new(MinimalDataEngine); + assert!(engine.as_vector_search().is_none()); + } + + /// A participating backend, proving the other half of the pattern. The point + /// is enforced at this fixture's definition rather than by an assertion: + /// `Some(self)` only compiles because `VectorCapableEngine` implements + /// `VectorSearchEngine`, so a backend cannot advertise vector search without + /// providing it. That is what the previous boolean capability could not do. + struct VectorCapableEngine; + + impl VectorSearchEngine for VectorCapableEngine { + fn search_vectors(&self, _req: VectorSearch<'_>) -> BoxFuture<'_, VectorSearchResult> { + Box::pin(async { + Ok(VectorSearchOutput { + hits: Vec::new(), + distance_function: extenddb_core::types::DistanceFunction::Cosine, + }) + }) + } + } + + #[tokio::test] + async fn a_participating_backend_hands_over_a_working_implementation() { + let engine = VectorCapableEngine; + let key_info = TableKeyInfo::default(); + let out = engine + .search_vectors(VectorSearch { + key_info: &key_info, + index_name: "vidx", + query_vector: &[0.5, 0.5], + top_k: 10, + hash_key: None, + filters: &[], + }) + .await + .expect("a participating backend answers a search"); + assert!(out.hits.is_empty()); + } +} diff --git a/tests/rust/Cargo.lock b/tests/rust/Cargo.lock index 06882090..27443df0 100644 --- a/tests/rust/Cargo.lock +++ b/tests/rust/Cargo.lock @@ -586,6 +586,7 @@ dependencies = [ "aws-config", "aws-credential-types", "aws-sdk-dynamodb", + "aws-sigv4", "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-smithy-types", diff --git a/tests/rust/Cargo.toml b/tests/rust/Cargo.toml index 0e5d7759..07aeff32 100755 --- a/tests/rust/Cargo.toml +++ b/tests/rust/Cargo.toml @@ -24,6 +24,7 @@ aws-config = { version = "1", default-features = false, features = ["behavior-ve aws-credential-types = "1" aws-smithy-types = "1" aws-smithy-runtime-api = "1" +aws-sigv4 = "1" aws-smithy-http-client = { version = "1", features = ["rustls-ring"] } tokio = { version = "1", features = ["full"] } uuid = { version = "1", features = ["v4"] } diff --git a/tests/rust/src/main.rs b/tests/rust/src/main.rs index 711f08eb..c14d4c69 100755 --- a/tests/rust/src/main.rs +++ b/tests/rust/src/main.rs @@ -114,6 +114,8 @@ mod update_item_number_validation; #[cfg(test)] mod update_table_billing_validation; #[cfg(test)] +mod vector_index_unsupported; +#[cfg(test)] mod wording_parity_validation; fn main() { diff --git a/tests/rust/src/vector_index_unsupported.rs b/tests/rust/src/vector_index_unsupported.rs new file mode 100644 index 00000000..b0a2a073 --- /dev/null +++ b/tests/rust/src/vector_index_unsupported.rs @@ -0,0 +1,487 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Wire-level tests for the vector index surface. +//! +//! These go over real HTTP rather than through the engine in-process, because the +//! contract's purpose is byte-level parity and the parts that only exist on the +//! wire cannot be tested any other way: the HTTP status, the `__type` in the body, +//! and the serialized shape of the response. +//! +//! The AWS SDK is not usable here. `aws-sdk-dynamodb` has no vector types at any +//! published version (checked to 1.119.0), so every request is hand-built and +//! SigV4-signed. That is also closer to what a non-SDK client would send. +//! +//! Scope: the NEGATIVE path only, which is the whole of what the contract +//! guarantees today. No in-tree backend implements vector search, so every vector +//! request must be refused, and refused identically whichever backend is running. +//! The positive path belongs with the first backend that implements it. + +use aws_credential_types::Credentials; +use aws_sigv4::http_request::{sign, SignableBody, SignableRequest, SigningSettings}; +use aws_sigv4::sign::v4; +use aws_smithy_runtime_api::client::identity::Identity; +use std::time::SystemTime; + +use crate::test_base::is_real_dynamodb; + +fn endpoint() -> String { + std::env::var("EXTENDDB_TEST_ENDPOINT") + .unwrap_or_else(|_| "https://dynamodb.us-east-1.amazonaws.com".into()) +} + +fn region() -> String { + std::env::var("AWS_DEFAULT_REGION").unwrap_or_else(|_| "us-east-1".into()) +} + +fn http_client() -> reqwest::Client { + let mut builder = reqwest::Client::builder().danger_accept_invalid_certs(true); + if let Ok(ca_path) = std::env::var("EXTENDDB_CA_CERT") { + if let Ok(pem) = std::fs::read(&ca_path) { + if let Ok(cert) = reqwest::Certificate::from_pem(&pem) { + builder = builder.add_root_certificate(cert); + } + } + } + builder.build().unwrap() +} + +/// A signed request for an operation the SDK cannot model. +/// +/// Signing matters rather than being incidental: authorization runs before +/// dispatch, so an unsigned request never reaches the capability gate and the +/// test would pass for the wrong reason, asserting an auth failure while +/// believing it asserted a vector refusal. +async fn call(target: &str, body: &str) -> (u16, String) { + let access_key = std::env::var("AWS_ACCESS_KEY_ID").expect("AWS_ACCESS_KEY_ID must be set"); + let secret_key = + std::env::var("AWS_SECRET_ACCESS_KEY").expect("AWS_SECRET_ACCESS_KEY must be set"); + let session_token = std::env::var("AWS_SESSION_TOKEN").ok(); + + let creds = Credentials::new( + access_key, + secret_key, + session_token, + None, + "extenddb-integration-tests", + ); + let identity = Identity::from(creds); + let region = region(); + let params = v4::signing_params::Builder::default() + .identity(&identity) + .region(®ion) + .name("dynamodb") + .time(SystemTime::now()) + .settings(SigningSettings::default()) + .build() + .expect("signing params") + .into(); + + let url = endpoint(); + let amz_target = format!("DynamoDB_20120810.{target}"); + let host = url + .split("://") + .nth(1) + .expect("endpoint has a scheme") + .trim_end_matches('/') + .to_owned(); + let headers: Vec<(&str, &str)> = vec![ + ("content-type", "application/x-amz-json-1.0"), + ("x-amz-target", &amz_target), + ("host", &host), + ]; + let signable = SignableRequest::new( + "POST", + &url, + headers.into_iter(), + SignableBody::Bytes(body.as_bytes()), + ) + .expect("signable request"); + + let (instructions, _sig) = sign(signable, ¶ms).expect("sign").into_parts(); + + let mut req = http_client() + .post(&url) + .header("Content-Type", "application/x-amz-json-1.0") + .header("X-Amz-Target", &amz_target) + .body(body.to_owned()); + for (name, value) in instructions.headers() { + req = req.header(name, value); + } + + let resp = req.send().await.expect("request sent"); + let status = resp.status().as_u16(); + let text = resp.text().await.expect("body read"); + (status, text) +} + +/// The exact refusal the contract promises for the table paths. Compared in full +/// rather than by substring, so the wording cannot drift unnoticed: a backend +/// author reads this message and has to reproduce it. +const NOT_SUPPORTED_INDEXES: &str = "Vector indexes are not supported by this storage backend"; +/// The refusal for the read path, which is worded for the operation rather than +/// the feature, matching what a caller of `SearchVectors` would expect to see. +const NOT_SUPPORTED_SEARCH: &str = "SearchVectors is not supported by this storage backend"; + +fn assert_validation_exception(status: u16, body: &str, expected_message: &str) { + assert_eq!(status, 400, "expected HTTP 400, body: {body}"); + let json: serde_json::Value = serde_json::from_str(body).expect("body is JSON"); + let type_field = json + .get("__type") + .and_then(|t| t.as_str()) + .unwrap_or_else(|| panic!("no __type in body: {body}")); + assert!( + type_field.ends_with("ValidationException"), + "expected ValidationException, got {type_field}" + ); + let message = json + .get("message") + .or_else(|| json.get("Message")) + .and_then(|m| m.as_str()) + .unwrap_or_else(|| panic!("no message in body: {body}")); + assert_eq!(message, expected_message); +} + +fn table_name(suffix: &str) -> String { + format!("vec_wire_{}_{}", suffix, uuid::Uuid::new_v4().simple()) +} + +/// Whether the running backend implements vector indexes. +/// +/// Probed by attempting the smallest real vector `CreateTable` and reading the +/// answer, because the capability is not otherwise observable over the wire. +/// Every test here asserts a refusal, so on a backend that *does* implement +/// vector search they must skip rather than fail: the refusals are the contract +/// for non-participating backends only. No in-tree backend implements it today, +/// so this returns false for both, and the mechanism exists for the first one that +/// does. +/// +/// Deliberately distinguishes the refusal from any other failure. Treating "not a +/// 200" as unsupported would make the whole suite skip silently the first time an +/// unrelated error appeared, which is the failure mode a self-skipping suite is +/// most prone to. +async fn backend_supports_vectors() -> bool { + let name = table_name("probe"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "probeidx", + "Dimensions": 4, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + if status == 200 { + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + return true; + } + if text.contains(NOT_SUPPORTED_INDEXES) { + return false; + } + panic!("vector support probe failed for an unrelated reason: {status} {text}"); +} + +/// What this run asserts about the backend's vector capability. +/// +/// Three states on purpose. A suite that adapts to whatever the backend reports +/// never asserts *which* backend is under test, so a backend that silently gained +/// or lost vector support would change what is exercised and still report green. +/// `EXTENDDB_EXPECT_VECTORS` lets a CI job state its expectation: +/// +/// - `1`: the backend must support vectors, so a positive suite failing to run is +/// an error rather than a skip. No such suite exists yet; it arrives with the +/// first participating backend, and the variable is defined here so that suite +/// inherits the guard rather than inventing one. +/// - `0`: the backend must not, so these refusal tests failing to run is an error. +/// - unset: adapt quietly, so a plain local `cargo test` works against either +/// backend without ceremony. +/// +/// An unrecognised value panics rather than being read as one of the two, because +/// a typo that silently meant "unset" would disable the guard it was added for. +pub(crate) fn expect_vectors() -> Option { + match std::env::var("EXTENDDB_EXPECT_VECTORS").ok()?.as_str() { + "1" => Some(true), + "0" => Some(false), + other => panic!("EXTENDDB_EXPECT_VECTORS must be 0 or 1, got {other:?}"), + } +} + +/// Skip guard for every refusal test in this file. +/// +/// If this run expects vector support, the refusal tests are correctly +/// inapplicable and skip. If the run and the backend disagree in either +/// direction, that is a surprise worth failing on rather than skipping past. +/// +/// The expectation is read *before* probing, not inside the assertion. Reading it +/// inside a short-circuiting `&&` meant an invalid value was never validated on a +/// backend without vector support, which is every backend today: the typo guard +/// was dead exactly where it was needed. +async fn skip_if_supported() -> bool { + if is_real_dynamodb() { + return true; + } + let expected = expect_vectors(); + let supported = backend_supports_vectors().await; + assert!( + !(supported && expected == Some(false)), + "EXTENDDB_EXPECT_VECTORS=0 but the backend supports vector indexes, so \ + these refusal tests would skip: either a backend gained vector support \ + unnoticed, or this job sets the wrong expectation" + ); + // The converse matters here too, and not only in a positive suite. Until a + // backend implements vector search there is no positive suite to notice a run + // that claims support the backend does not have, so the claim would pass + // unchecked. + assert!( + !(!supported && expected == Some(true)), + "EXTENDDB_EXPECT_VECTORS=1 but the backend refuses vector indexes: either \ + a backend lost vector support unnoticed, or this job sets the wrong \ + expectation" + ); + supported +} + +/// A `CreateTable` naming vector indexes is refused before the table is created. +/// +/// This is the hole the gate exists to close. Without it the request passes shape +/// validation, reaches a backend that ignores `VectorIndexes`, and produces a +/// table with no index: the caller is told the index exists and only learns +/// otherwise on the first search. +#[tokio::test] +async fn create_table_with_vector_indexes_is_refused() { + if skip_if_supported().await { + return; + } + let name = table_name("create"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 4, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_validation_exception(status, &text, NOT_SUPPORTED_INDEXES); + + // The refusal must also be effective, not merely worded: no table may exist. + let (desc_status, desc_body) = + call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + assert_eq!( + desc_status, 400, + "a refused CreateTable must leave no table behind, got: {desc_body}" + ); + assert!( + desc_body.contains("ResourceNotFoundException"), + "expected ResourceNotFoundException, got: {desc_body}" + ); +} + +/// An ordinary `CreateTable` still succeeds, proving the gate does not +/// over-reject. Without this the suite would pass just as well if the gate +/// refused every table. +/// +/// Guarded on the endpoint only, not on `skip_if_supported`. This assertion holds +/// on every backend, so skipping it once a backend implements vector search would +/// lose coverage for no reason. The same applies to the two tests below it. Only +/// the tests that assert a *refusal* are specific to non-participating backends. +#[tokio::test] +async fn create_table_without_vector_indexes_still_succeeds() { + if is_real_dynamodb() { + return; + } + let name = table_name("control"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "control table must be created, body: {text}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// `UpdateTable` is the second creation path, and the one whose backfill +/// lifecycle the contract models, so leaving it ungated would reopen the same +/// silent-drop hole on a different operation. +#[tokio::test] +async fn update_table_creating_a_vector_index_is_refused() { + if skip_if_supported().await { + return; + } + let name = table_name("update"); + let create = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + let (status, text) = call("CreateTable", &create).await; + assert_eq!(status, 200, "setup table must be created, body: {text}"); + + let body = format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{ + "Create": {{ + "IndexName": "vidx", + "Dimensions": 4, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}} + }} + }}] + }}"# + ); + let (status, text) = call("UpdateTable", &body).await; + assert_validation_exception(status, &text, NOT_SUPPORTED_INDEXES); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Deleting is refused too, deliberately. A backend with no vector support cannot +/// hold an index to delete, so "not supported here" is the honest answer rather +/// than letting the request through to be read as a no-op or a not-found. +#[tokio::test] +async fn update_table_deleting_a_vector_index_is_refused() { + if skip_if_supported().await { + return; + } + let name = table_name("delete"); + let create = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + let (status, text) = call("CreateTable", &create).await; + assert_eq!(status, 200, "setup table must be created, body: {text}"); + + let body = format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{"Delete": {{"IndexName": "vidx"}}}}] + }}"# + ); + let (status, text) = call("UpdateTable", &body).await; + assert_validation_exception(status, &text, NOT_SUPPORTED_INDEXES); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// An `UpdateTable` carrying an empty list is not a request for vector indexes. +/// It must not draw the vector refusal, and because an empty list is also not a +/// change, the correct answer is the ordinary "nothing specified" error. This is +/// the boundary the gate is most likely to get wrong in a future edit, in either +/// direction. +#[tokio::test] +async fn update_table_with_an_empty_vector_list_is_not_a_vector_request() { + if is_real_dynamodb() { + return; + } + let name = table_name("empty"); + let create = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + let (status, text) = call("CreateTable", &create).await; + assert_eq!(status, 200, "setup table must be created, body: {text}"); + + let body = format!(r#"{{"TableName": "{name}", "VectorIndexUpdates": []}}"#); + let (_status, text) = call("UpdateTable", &body).await; + assert!( + !text.contains("not supported by this storage backend"), + "an empty list must not draw the vector refusal: {text}" + ); + assert!( + text.contains("At least one of"), + "an empty list is no change, so the nothing-specified error is expected: {text}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// `SearchVectors` is refused with its own wording. Checked against a table that +/// exists, so the refusal cannot be a disguised ResourceNotFound. +#[tokio::test] +async fn search_vectors_is_refused() { + if skip_if_supported().await { + return; + } + let name = table_name("search"); + let create = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + let (status, text) = call("CreateTable", &create).await; + assert_eq!(status, 200, "setup table must be created, body: {text}"); + + let body = format!( + r#"{{ + "TableName": "{name}", + "IndexName": "vidx", + "SearchVector": [{{"N": "0.1"}}, {{"N": "0.2"}}, {{"N": "0.3"}}, {{"N": "0.4"}}], + "TopK": 5 + }}"# + ); + let (status, text) = call("SearchVectors", &body).await; + assert_validation_exception(status, &text, NOT_SUPPORTED_SEARCH); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// `SearchVectors` is a known operation, so an unauthenticated call must fail on +/// authentication rather than on the operation name. This pins the ordering the +/// other tests depend on: if the operation were unknown, or if auth ran after +/// dispatch, the refusals above would be asserting something else. +#[tokio::test] +async fn search_vectors_is_a_known_operation_and_requires_auth() { + if is_real_dynamodb() { + return; + } + let resp = http_client() + .post(endpoint()) + .header("Content-Type", "application/x-amz-json-1.0") + .header("X-Amz-Target", "DynamoDB_20120810.SearchVectors") + .body(r#"{"TableName": "t", "IndexName": "i", "SearchVector": [{"N": "0.1"}], "TopK": 1}"#) + .send() + .await + .expect("request sent"); + let status = resp.status().as_u16(); + let body = resp.text().await.expect("body read"); + assert!( + !body.contains("UnknownOperationException"), + "SearchVectors must be a recognized operation, got: {body}" + ); + assert_eq!(status, 400, "expected an auth failure, body: {body}"); +}