Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/dynavec/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from .config import DynavecConfig
from .credentials import AWSCredentials
from .exceptions import (
AccessDeniedError,
ConfigurationError,
ConflictError,
DimensionMismatchError,
Expand All @@ -42,6 +43,7 @@
MissingDependencyError,
NotFoundError,
ProvisioningError,
ValidationError,
)
from .fusion import FitResult, RRFWeightFitter
from .graph import GraphStore
Expand Down Expand Up @@ -212,6 +214,8 @@
"EmbeddingError",
"DimensionMismatchError",
"NotFoundError",
"AccessDeniedError",
"ValidationError",
"ItemTooLargeError",
"ConflictError",
"MissingDependencyError",
Expand Down
8 changes: 8 additions & 0 deletions src/dynavec/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,14 @@ class NotFoundError(DynavecError):
"""Raised when a namespace, index, or document does not exist."""


class AccessDeniedError(DynavecError):
"""Raised when access to an AWS resource is denied."""


class ValidationError(DynavecError):
"""Raised when AWS rejects an invalid request."""


class ItemTooLargeError(DynavecError):
"""Raised when a document would exceed DynamoDB's 400 KB item size limit."""

Expand Down
36 changes: 36 additions & 0 deletions src/dynavec/stores/_s3vectors_errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
"""Shared error translation for S3 Vectors AWS calls."""

from __future__ import annotations

from collections.abc import Iterator, Mapping
from contextlib import contextmanager

from botocore.exceptions import ClientError

from ..exceptions import AccessDeniedError, DynavecError, NotFoundError, ValidationError

_ERROR_TYPES: dict[str, type[DynavecError]] = {
"NotFoundException": NotFoundError,
"AccessDeniedException": AccessDeniedError,
"ValidationException": ValidationError,
}


@contextmanager
def translate_client_errors() -> Iterator[None]:
"""Translate selected permanent errors, leaving retryable errors intact."""
try:
yield
except ClientError as exc:
response = exc.response
error = response.get("Error") if isinstance(response, Mapping) else None
if not isinstance(error, Mapping):
raise
code = error.get("Code")
error_type = _ERROR_TYPES.get(code) if isinstance(code, str) else None
if error_type is None:
raise
message = error.get("Message")
if not isinstance(message, str) or not message:
message = f"S3 Vectors request failed with {code}."
raise error_type(message) from exc
41 changes: 23 additions & 18 deletions src/dynavec/stores/async_s3vectors.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from ..config import DynavecConfig
from ..logging import log_store_event
from ..utils import TokenBucket, async_retry
from ._s3vectors_errors import translate_client_errors
from .s3vectors import _GET_LIMIT, _MAX_TOP_K, _PUT_LIMIT, Metadata, _f32


Expand Down Expand Up @@ -98,11 +99,12 @@ async def _put_batch(
await self._put_limiter.acquire_async()

client = self._require_client()
await client.put_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
vectors=payload,
)
with translate_client_errors():
await client.put_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
vectors=payload,
)

async def put_vectors(
self,
Expand Down Expand Up @@ -283,22 +285,24 @@ async def query_pages(
)

client = self._require_client()
paginator = client.get_paginator("query_vectors")

yielded = 0
buffer: list[dict[str, Any]] = []

page_iterator = paginator.paginate(
PaginationConfig={"MaxItems": top_k},
**kwargs,
).__aiter__()
with translate_client_errors():
paginator = client.get_paginator("query_vectors")
page_iterator = paginator.paginate(
PaginationConfig={"MaxItems": top_k},
**kwargs,
).__aiter__()

while True:
if self._query_limiter is not None:
await self._query_limiter.acquire_async()

try:
page = await page_iterator.__anext__()
with translate_client_errors():
page = await page_iterator.__anext__()
except StopAsyncIteration:
break

Expand Down Expand Up @@ -364,13 +368,14 @@ async def get_vectors(
for start in range(0, len(keys), _GET_LIMIT):
chunk = keys[start : start + _GET_LIMIT]

response = await client.get_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
keys=chunk,
returnData=True,
returnMetadata=return_metadata,
)
with translate_client_errors():
response = await client.get_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
keys=chunk,
returnData=True,
returnMetadata=return_metadata,
)

for vector in response.get("vectors", []):
out[vector["key"]] = {
Expand Down
74 changes: 41 additions & 33 deletions src/dynavec/stores/s3vectors.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from ..config import DynavecConfig
from ..logging import log_store_event
from ..utils import TokenBucket, retry
from ._s3vectors_errors import translate_client_errors

Metadata = dict[str, Any]

Expand Down Expand Up @@ -59,21 +60,23 @@ def __init__(self, config: DynavecConfig, boto_session: Any | None = None) -> No
self._logger = logging.getLogger("dynavec.stores.s3vectors")

def get_index(self) -> dict[str, Any]:
result: dict[str, Any] = self._client.get_index(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
)
with translate_client_errors():
result: dict[str, Any] = self._client.get_index(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
)
return result

@retry()
def _put_batch(self, payload: list[dict[str, Any]]) -> None:
if self._put_limiter is not None:
self._put_limiter.acquire()
self._client.put_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
vectors=payload,
)
with translate_client_errors():
self._client.put_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
vectors=payload,
)


def put_vectors(
Expand Down Expand Up @@ -227,23 +230,25 @@ def query_pages(
return_distance,
)

paginator = self._client.get_paginator("query_vectors")
yielded = 0
buffer: list[dict[str, Any]] = []

page_iterator = iter(
paginator.paginate(
PaginationConfig={"MaxItems": top_k},
**kwargs,
with translate_client_errors():
paginator = self._client.get_paginator("query_vectors")
page_iterator = iter(
paginator.paginate(
PaginationConfig={"MaxItems": top_k},
**kwargs,
)
)
)

while True:
if self._query_limiter is not None:
self._query_limiter.acquire()

try:
page = next(page_iterator)
with translate_client_errors():
page = next(page_iterator)
except StopIteration:
break

Expand Down Expand Up @@ -322,11 +327,12 @@ def list_pages(
if page_size is not None:
pagination_config["PageSize"] = page_size

paginator = self._client.get_paginator("list_vectors")
for page in paginator.paginate(PaginationConfig=pagination_config, **kwargs):
vectors = page.get("vectors", [])
if vectors:
yield vectors
with translate_client_errors():
paginator = self._client.get_paginator("list_vectors")
for page in paginator.paginate(PaginationConfig=pagination_config, **kwargs):
vectors = page.get("vectors", [])
if vectors:
yield vectors

@retry()
def get_vectors(
Expand All @@ -336,13 +342,14 @@ def get_vectors(
out: dict[str, dict[str, Any]] = {}
for start in range(0, len(keys), _GET_LIMIT):
chunk = keys[start : start + _GET_LIMIT]
resp = self._client.get_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
keys=chunk,
returnData=True,
returnMetadata=return_metadata,
)
with translate_client_errors():
resp = self._client.get_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
keys=chunk,
returnData=True,
returnMetadata=return_metadata,
)
for v in resp.get("vectors", []):
out[v["key"]] = {
"vector": v.get("data", {}).get("float32"),
Expand All @@ -365,11 +372,12 @@ def delete_vectors(self, keys: list[str]) -> None:
return
for start in range(0, len(keys), _PUT_LIMIT):
chunk = keys[start : start + _PUT_LIMIT]
self._client.delete_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
keys=chunk,
)
with translate_client_errors():
self._client.delete_vectors(
vectorBucketName=self._config.vector_bucket,
indexName=self._config.index,
keys=chunk,
)
log_store_event(
self._logger,
"s3vectors.delete_vectors",
Expand Down
Loading
Loading