Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions crates/core/src/types/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,10 @@ pub use table::{
Tag, TagResourceInput, TimeToLiveDescription, TimeToLiveSpecification,
TimeToLiveSpecificationOutput, TimeToLiveStatus, UntagResourceInput, UpdateGsiAction,
UpdateTableInput, UpdateTableOutput, UpdateTimeToLiveInput, UpdateTimeToLiveOutput,
VECTOR_INDEX_ALREADY_EXISTS, VECTOR_INDEX_COUNT_LIMIT_CREATE, VECTOR_INDEX_COUNT_LIMIT_UPDATE,
VECTOR_INDEX_CREATE_IN_USE_PREFIX, VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST,
VECTOR_INDEX_ALREADY_EXISTS, VECTOR_INDEX_BACKFILLING_SCAN_PREFIX,
VECTOR_INDEX_COUNT_LIMIT_CREATE, VECTOR_INDEX_COUNT_LIMIT_UPDATE,
VECTOR_INDEX_CREATE_IN_USE_PREFIX, VECTOR_INDEX_QUERY_NOT_SUPPORTED,
VECTOR_INDEX_REQUIRES_PAY_PER_REQUEST, VECTOR_INDEX_SCAN_NOT_SUPPORTED,
VECTOR_SEARCH_SCHEMA_UNDECLARED, VECTOR_TABLE_REQUIRES_PAY_PER_REQUEST_MODE, VectorAttribute,
VectorIndexDescription, VectorIndexSpecification, VectorIndexUpdate,
vector_attribute_conflicting_definition, vector_attribute_redefines_key,
Expand Down
15 changes: 15 additions & 0 deletions crates/core/src/types/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,21 @@ pub fn vector_index_delete_in_allocation_phase(table_name: &str, index_name: &st
)
}

/// Refusal for a Query naming a vector index. Measured 2026-08-20; note the
/// trailing period, which the Scan variant does not have.
pub const VECTOR_INDEX_QUERY_NOT_SUPPORTED: &str =
"Query operation not supported on this index type.";

/// Refusal for a Scan naming a vector index that is past its backfill.
/// Measured 2026-08-20.
pub const VECTOR_INDEX_SCAN_NOT_SUPPORTED: &str = "Scan operation not supported on this index type";

/// Prefix of the Scan refusal while a vector index is still backfilling;
/// continues with the index name. The service reuses the GSI wording even
/// though the index is a vector index. Measured 2026-08-20.
pub const VECTOR_INDEX_BACKFILLING_SCAN_PREFIX: &str =
"Cannot read from backfilling global secondary index: ";

impl VectorIndexDescription {
/// Reject a description whose reported state the service would never produce.
///
Expand Down
181 changes: 179 additions & 2 deletions crates/engine/src/index_helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@

//! Shared helpers for secondary index operations in the engine layer.

use extenddb_core::error::DynamoDbError;
use extenddb_core::types::{
AttributeDefinition, AttributeValue, IndexInfo, Item, KeySchemaElement, ProjectionType,
ScalarAttributeType,
AttributeDefinition, AttributeValue, DescribeTableInput, IndexInfo, IndexStatus, Item,
KeySchemaElement, ProjectionType, ScalarAttributeType, TableKeyInfo, VectorIndexDescription,
};

use crate::OperationContext;

/// Build the combined key schema for `LastEvaluatedKey` extraction.
///
/// For index queries/scans, the LEK includes both the base table key attributes
Expand Down Expand Up @@ -162,3 +165,177 @@ fn attr_value_matches_scalar(value: &AttributeValue, scalar: ScalarAttributeType
| (AttributeValue::B(_), ScalarAttributeType::B)
)
}

/// How Query and Scan must refuse an index name that is not a row in the
/// `indexes` catalog. A vector index never is one, so both handlers land here
/// for vector indexes and for genuinely absent names alike.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VectorIndexReadRefusal {
/// No vector index carries the name either, or one does but the service
/// reports it as absent (measured 2026-08-20: CREATING before the
/// backfill starts is indistinguishable from a nonexistent index).
NotFound,
/// A vector index mid-backfill. Scan gets the backfilling wording while
/// Query keeps the ordinary type refusal (measured 2026-08-20).
Backfilling,
/// A vector index past its backfill: the operation-specific type refusal.
NotSupported,
}

/// Classify a vector index (or its absence) for a Query/Scan refusal.
///
/// Mirrors the measured lifecycle: ACTIVE and CREATING-while-backfilling are
/// recognized as vector indexes; every other state reads as index-not-found,
/// matching how `SearchVectors` treats a non-ACTIVE index as absent. The
/// backfilling arm requires CREATING so a stale `Backfilling` member on any
/// other status cannot resurrect the backfilling wording.
pub fn classify_vector_index_read(
vector_index: Option<&VectorIndexDescription>,
) -> VectorIndexReadRefusal {
match vector_index {
Some(vi) if vi.index_status.is_active() => VectorIndexReadRefusal::NotSupported,
Some(vi) if vi.index_status == IndexStatus::Creating && vi.backfilling == Some(true) => {
VectorIndexReadRefusal::Backfilling
}
_ => VectorIndexReadRefusal::NotFound,
}
}

/// Resolve an index name that `index_info_by_table_id` reported missing
/// against the table's vector index metadata.
///
/// The `key_info` name pre-filter keeps the dominant case (a mistyped GSI
/// name on a table with no vector indexes) free of the extra round trip; the
/// `describe_table` read runs only when the name matches a known vector
/// index, because the key-info cache carries no lifecycle status and backfill
/// transitions never invalidate it.
///
/// # Errors
/// Propagates storage failures from `describe_table`.
pub async fn classify_unresolved_index_read(
ctx: &OperationContext,
key_info: &TableKeyInfo,
index_name: &str,
) -> Result<VectorIndexReadRefusal, DynamoDbError> {
if !key_info
.vector_indexes
.iter()
.any(|vi| vi.index_name == index_name)
{
return Ok(VectorIndexReadRefusal::NotFound);
}
let table = ctx
.storage
.describe_table(
&ctx.account_id,
DescribeTableInput {
table_name: key_info.table_name.clone(),
},
)
.await
.map_err(crate::create_table::storage_err_to_dynamo)?;
Ok(classify_vector_index_read(
table
.vector_indexes
.as_deref()
.unwrap_or(&[])
.iter()
.find(|vi| vi.index_name == index_name),
))
}

#[cfg(test)]
mod tests {
use super::*;
use extenddb_core::types::{
DistanceFunction, IndexStatus, Projection, VectorAttribute, VectorIndexDescription,
};

fn description(index_status: IndexStatus, backfilling: Option<bool>) -> VectorIndexDescription {
VectorIndexDescription {
index_name: "vidx".to_owned(),
vector_attribute: VectorAttribute {
attribute_name: "emb".to_owned(),
},
dimensions: 4,
search_schema: None,
distance_function: DistanceFunction::Cosine,
index_status,
backfilling,
index_size_bytes: 0,
item_count: 0,
index_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/t/index/vidx".to_owned(),
projection: Some(Projection {
projection_type: ProjectionType::All,
non_key_attributes: None,
}),
}
}

#[test]
fn absent_index_is_not_found() {
assert_eq!(
classify_vector_index_read(None),
VectorIndexReadRefusal::NotFound
);
}

#[test]
fn active_index_gets_the_type_refusal() {
let vi = description(IndexStatus::Active, None);
assert_eq!(
classify_vector_index_read(Some(&vi)),
VectorIndexReadRefusal::NotSupported
);
}

#[test]
fn creating_before_backfill_reads_as_not_found() {
// Measured 2026-08-20: CREATING with Backfilling=false answers with
// the index-not-found message, exactly like a nonexistent index.
let vi = description(IndexStatus::Creating, Some(false));
assert_eq!(
classify_vector_index_read(Some(&vi)),
VectorIndexReadRefusal::NotFound
);
let vi = description(IndexStatus::Creating, None);
assert_eq!(
classify_vector_index_read(Some(&vi)),
VectorIndexReadRefusal::NotFound
);
}

#[test]
fn creating_while_backfilling_is_backfilling() {
let vi = description(IndexStatus::Creating, Some(true));
assert_eq!(
classify_vector_index_read(Some(&vi)),
VectorIndexReadRefusal::Backfilling
);
}

#[test]
fn stale_backfilling_on_a_non_creating_status_stays_not_found() {
// The backfilling wording is tied to CREATING; a stale Backfilling
// member on any other status must not resurrect it.
let vi = description(IndexStatus::Deleting, Some(true));
assert_eq!(
classify_vector_index_read(Some(&vi)),
VectorIndexReadRefusal::NotFound
);
let vi = description(IndexStatus::Updating, Some(true));
assert_eq!(
classify_vector_index_read(Some(&vi)),
VectorIndexReadRefusal::NotFound
);
}

#[test]
fn deleting_reads_as_not_found() {
let vi = description(IndexStatus::Deleting, None);
assert_eq!(
classify_vector_index_read(Some(&vi)),
VectorIndexReadRefusal::NotFound
);
}
}
73 changes: 52 additions & 21 deletions crates/engine/src/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,19 @@ use extenddb_core::error::DynamoDbError;
use extenddb_core::expression::PathElement;
use extenddb_core::expression::{ExpressionKind, ExpressionMaps, Projection};
use extenddb_core::types::{
IndexType, KeyType, ProjectionType, QueryInput, QueryOutput, Select, TableKeyInfo, extract_key,
item_size_bytes,
IndexType, KeyType, ProjectionType, QueryInput, QueryOutput, Select, TableKeyInfo,
VECTOR_INDEX_QUERY_NOT_SUPPORTED, extract_key, item_size_bytes,
};
use extenddb_storage::error::StorageError;

use crate::OperationContext;
use crate::capacity_helpers;
use crate::create_table::storage_err_to_dynamo;
use crate::expression_helpers::{build_expression_maps, parse_optional_filter};
use crate::index_helpers::{combined_lek_key_schema, validate_query_exclusive_start_key};
use crate::index_helpers::{
VectorIndexReadRefusal, classify_unresolved_index_read, combined_lek_key_schema,
validate_query_exclusive_start_key,
};
use crate::legacy_filter::{desugar_filter, desugar_key_conditions};
use crate::read_helpers::apply_post_read;
use crate::serialize_output;
Expand Down Expand Up @@ -75,30 +79,48 @@ pub async fn handle_query(

// GSI/LSI: resolve index metadata if querying a secondary index.
// Uses table_id from pre-fetched key_info to skip redundant table lookup (P118 #4).
let index_info = if let Some(ref idx_name) = input.index_name {
Some(
ctx.storage
// A vector index is not a row in the `indexes` catalog, so a not-found
// result is re-resolved against the vector index metadata before the
// name is treated as absent.
let (index_info, vector_index_named) = match input.index_name {
Some(ref idx_name) => {
match ctx
.storage
.index_info_by_table_id(&key_info.table_id, idx_name)
.await
.map_err(storage_err_to_dynamo)?,
)
} else {
None
{
// Defense in depth: no in-tree backend stores a vector index in
// `indexes`, but if one ever surfaces here it must be refused,
// not sent down the GSI/LSI data path.
Ok(info) if info.index_type == IndexType::Vector => (None, true),
Ok(info) => (Some(info), false),
Err(err @ StorageError::IndexNotFound(_)) => {
match classify_unresolved_index_read(ctx, &key_info, idx_name).await? {
VectorIndexReadRefusal::NotFound => {
return Err(storage_err_to_dynamo(err));
}
// Query refuses a backfilling vector index with the same
// message as an active one (measured 2026-08-20; Scan
// differs). The refusal itself fires further down, after
// the KeyConditionExpression checks.
VectorIndexReadRefusal::Backfilling
| VectorIndexReadRefusal::NotSupported => (None, true),
}
}
Err(err) => return Err(storage_err_to_dynamo(err)),
}
}
None => (None, false),
};

// A vector index is searched only via the vector search API, never queried.
if let Some(ref idx) = index_info
&& idx.index_type == IndexType::Vector
{
return Err(DynamoDbError::ValidationException(
"Query operation not supported on this index type".to_owned(),
));
}

// ConsistentRead is not supported on GSI queries (tenet 1: fidelity).
// Measured 2026-08-20: it fires for a vector index too, with the same
// wording, and before the vector-index refusal.
if input.consistent_read == Some(true)
&& let Some(ref idx) = index_info
&& idx.index_type == IndexType::Gsi
&& (vector_index_named
|| index_info
.as_ref()
.is_some_and(|idx| idx.index_type == IndexType::Gsi))
{
return Err(DynamoDbError::ValidationException(
"Consistent reads are not supported on global secondary indexes".to_owned(),
Expand Down Expand Up @@ -232,6 +254,15 @@ pub async fn handle_query(
));
};

// A vector index is searched only via the vector search API, never
// queried. Measured 2026-08-20: the service refuses after the
// KeyConditionExpression presence and syntax checks, hence below the parse.
if vector_index_named {
return Err(DynamoDbError::ValidationException(
VECTOR_INDEX_QUERY_NOT_SUPPORTED.to_owned(),
));
}

// Use legacy maps for key condition resolution if KeyConditions was used
let effective_maps = if let Some(ref kc_maps) = legacy_kc_maps {
kc_maps
Expand Down
Loading
Loading