diff --git a/python/packages/qdrant/agent_framework_qdrant/_vector_store.py b/python/packages/qdrant/agent_framework_qdrant/_vector_store.py index 35bb5e62743..2deedbb0eaf 100644 --- a/python/packages/qdrant/agent_framework_qdrant/_vector_store.py +++ b/python/packages/qdrant/agent_framework_qdrant/_vector_store.py @@ -91,6 +91,9 @@ def _create_client( _WRITE_OPTIONS = {"ordering", "shard_key_selector", "timeout"} _BATCH_SIZE = 256 _MAX_EXACT_INTEGER = 2**53 - 1 +_SCALAR_FIELD_TYPES = frozenset({"str", "int", "float", "bool"}) +# Data field types that can never hold an array, so ``is_empty`` alone separates a missing key from a value. +_NON_ARRAY_FIELD_TYPES = _SCALAR_FIELD_TYPES | {"dict"} _CREATE_OPTIONS = { "shard_number", "replication_factor", @@ -221,17 +224,36 @@ def _prepare_false_condition() -> models.HasIdCondition: return models.HasIdCondition(has_id=[]) -def _prepare_presence_condition(name: str) -> models.FieldCondition: - # Unlike is_empty, the server's values_count distinguishes missing from null and []. - return models.FieldCondition(key=name, values_count=models.ValuesCount(gte=0)) +def _prepare_empty_condition(name: str) -> models.IsEmptyCondition: + # is_empty matches a missing key, null, and [] on every supported server version. + return models.IsEmptyCondition(is_empty=models.PayloadField(key=name)) + + +def _prepare_null_condition(name: str) -> models.Filter: + # Since Qdrant 1.19.1, is_null also matches arrays containing null; is_empty restricts it to an explicit null. + return models.Filter( + must=[models.IsNullCondition(is_null=models.PayloadField(key=name)), _prepare_empty_condition(name)] + ) -def _prepare_null_condition(name: str) -> models.IsNullCondition: - return models.IsNullCondition(is_null=models.PayloadField(key=name)) +def _prepare_presence_condition(field: VectorStoreField) -> models.Condition: + name = field.storage_name or field.name + if field.type_ in _NON_ARRAY_FIELD_TYPES: + # Since Qdrant 1.19.0, values_count treats a missing key as zero values. A declared scalar or dict + # cannot hold [] ({} is not empty for Qdrant), so it is present exactly when it is non-empty or an + # explicit null. + return models.Filter( + should=[models.Filter(must_not=[_prepare_empty_condition(name)]), _prepare_null_condition(name)] + ) + # Only values_count separates [] from a missing key, and only on Qdrant servers before 1.19.0. + return models.FieldCondition(key=name, values_count=models.ValuesCount(gte=0)) -def _prepare_non_null_condition(name: str) -> models.Filter: - return models.Filter(must=[_prepare_presence_condition(name)], must_not=[_prepare_null_condition(name)]) +def _prepare_non_null_condition(field: VectorStoreField) -> models.Filter: + name = field.storage_name or field.name + if field.type_ in _NON_ARRAY_FIELD_TYPES: + return models.Filter(must_not=[_prepare_empty_condition(name)]) + return models.Filter(must=[_prepare_presence_condition(field)], must_not=[_prepare_null_condition(name)]) def _prepare_range_operand(value: Any) -> float: @@ -332,6 +354,8 @@ class QdrantCollection(BaseVectorCollection[KeyT, ModelT], BaseVectorSearch[KeyT Portable filters require a server: Qdrant's local emulator has different null/missing and numeric matching behavior across supported SDK versions. Local mode supports unfiltered CRUD and dense search. Ordered retrieval is unsupported. + Qdrant 1.19+ servers cannot distinguish ``[]`` from a missing key, so ``exists``, + ``is_not_null``, and empty ``contains_all`` on collection fields also match points without the key. """ supported_key_types: ClassVar[set[str] | None] = {"int", "str", "UUID"} @@ -610,15 +634,15 @@ def _prepare_filter_condition(self, expression: FilterExpression) -> models.Cond name = field.storage_name or field.name operator, value = expression.operator, expression.value if operator == "exists": - return _prepare_presence_condition(name) + return _prepare_presence_condition(field) if operator == "is_null": return _prepare_null_condition(name) if operator == "is_not_null": - return _prepare_non_null_condition(name) + return _prepare_non_null_condition(field) if operator in {"eq", "ne"}: condition = _prepare_equality_condition(name, value, field) return ( - models.Filter(must=[_prepare_presence_condition(name)], must_not=[condition]) + models.Filter(must=[_prepare_presence_condition(field)], must_not=[condition]) if operator == "ne" else condition ) @@ -638,15 +662,15 @@ def _prepare_filter_condition(self, expression: FilterExpression) -> models.Cond else _prepare_false_condition() ) if operator == "not_in": - return models.Filter(must=[_prepare_non_null_condition(name)], must_not=[condition]) - return models.Filter(must=[_prepare_non_null_condition(name), condition]) + return models.Filter(must=[_prepare_non_null_condition(field)], must_not=[condition]) + return models.Filter(must=[_prepare_non_null_condition(field), condition]) if operator in {"contains", "contains_any", "contains_all"}: if field.type_ not in {"list", "tuple", "set", "Sequence"}: raise NotImplementedError("Qdrant collection membership requires a declared collection field type.") operands: Sequence[Any] = [value] if operator == "contains" else value conditions = [_prepare_equality_condition(name, item, field, element=True) for item in operands] if operator == "contains_all": - return models.Filter(must=[_prepare_non_null_condition(name), *conditions]) + return models.Filter(must=[_prepare_non_null_condition(field), *conditions]) return models.Filter(should=conditions) if conditions else _prepare_false_condition() raise NotImplementedError( f"Qdrant does not support portable filter operator '{operator}'. " diff --git a/python/packages/qdrant/tests/qdrant/test_filters.py b/python/packages/qdrant/tests/qdrant/test_filters.py index 821ab0185d1..486f367429b 100644 --- a/python/packages/qdrant/tests/qdrant/test_filters.py +++ b/python/packages/qdrant/tests/qdrant/test_filters.py @@ -5,7 +5,7 @@ from unittest.mock import AsyncMock import pytest -from agent_framework import Filter, FilterGroup +from agent_framework import Filter, FilterGroup, VectorStoreCollectionDefinition, VectorStoreField from agent_framework._vector_filters import validate_filter from qdrant_client import AsyncQdrantClient @@ -75,6 +75,53 @@ def test_reject_unsafe_numeric_range(collection, value): collection._prepare_filter(Filter("integer", "gt", value)) +def test_scalar_presence_and_null_do_not_depend_on_values_count(collection): + # Qdrant 1.19.0 counts a missing key as zero values; 1.19.1 also matches is_null on arrays containing null. + empty = {"is_empty": {"key": "body"}} + null = {"must": [{"is_null": {"key": "body"}}, empty]} + expected = { + "exists": {"should": [{"must_not": [empty]}, null]}, + "is_null": null, + "is_not_null": {"must_not": [empty]}, + } + for operator, condition in expected.items(): + native = collection._prepare_filter(Filter("text", operator)) + assert native is not None + assert native.model_dump(exclude_none=True) == {"must": [condition]} + native = collection._prepare_filter(Filter("tags", "is_null")) + assert native is not None + assert native.model_dump(exclude_none=True) == { + "must": [{"must": [{"is_null": {"key": "tags"}}, {"is_empty": {"key": "tags"}}]}] + } + + +def test_dict_presence_and_null_do_not_depend_on_values_count(): + # A dict field cannot hold [], so it takes the same is_empty-based path as scalars on Qdrant 1.19+. + client = AsyncMock(spec=AsyncQdrantClient) + client.init_options = {} + collection = QdrantCollection( + dict, + definition=VectorStoreCollectionDefinition([ + VectorStoreField("key", name="id", type_="int"), + VectorStoreField("data", name="meta", type_="dict", storage_name="attributes"), + ]), + collection_name="test", + async_client=client, + ) + empty = {"is_empty": {"key": "attributes"}} + null = {"must": [{"is_null": {"key": "attributes"}}, empty]} + expected = { + "exists": {"should": [{"must_not": [empty]}, null]}, + "is_null": null, + "is_not_null": {"must_not": [empty]}, + } + for operator, condition in expected.items(): + native = collection._prepare_filter(Filter("meta", operator)) + assert native is not None + assert native.model_dump(exclude_none=True) == {"must": [condition]} + assert "values_count" not in str(native.model_dump(exclude_none=True)) + + def test_numeric_equality_preserves_bool_distinction(collection): native = collection._prepare_filter(Filter("integer", "eq", True)) assert native is not None diff --git a/python/packages/qdrant/tests/qdrant/test_server_filters.py b/python/packages/qdrant/tests/qdrant/test_server_filters.py index 761fced9492..e9575716e5c 100644 --- a/python/packages/qdrant/tests/qdrant/test_server_filters.py +++ b/python/packages/qdrant/tests/qdrant/test_server_filters.py @@ -17,7 +17,7 @@ ) from agent_framework._vector_filters import filter_values_equal from agent_framework.exceptions import IntegrationInvalidResponseException -from qdrant_client import models +from qdrant_client import AsyncQdrantClient, models from agent_framework_qdrant import QdrantCollection @@ -119,6 +119,91 @@ async def test_server_native_portable_semantics(payload_collection: QdrantCollec assert [point.id for point in points] == expected +@pytest.mark.parametrize( + ("expression", "expected"), + [ + (Filter("text", "exists"), [1, 2, 3]), + (Filter("text", "is_not_null"), [2, 3]), + (Filter("text", "ne", "a"), [1, 3]), + (Filter("number", "exists"), [1, 2, 3]), + (Filter("number", "ne", 1.0), [1, 3]), + (Filter("integer", "not_in", [1]), [3]), + (Filter("flag", "is_not_null"), [2, 3]), + (Filter("tags", "is_null"), [1]), + (Filter("tags", "contains_all", ["a"]), [2]), + ], +) +async def test_server_presence_ignores_missing_keys_and_null_array_elements( + server_collection: QdrantCollection, expression, expected +): + # Qdrant 1.19.0 counts a missing key as zero values and 1.19.1 matches is_null on arrays containing null. + await server_collection.async_client.upsert( + server_collection.collection_name, + points=[ + models.PointStruct(id=index, vector={}, payload=payload) + for index, payload in enumerate([ + {}, + {"body": None, "price": None, "integer": None, "flag": None, "tags": None}, + {"body": "a", "price": 1.0, "integer": 1, "flag": True, "tags": ["a", None]}, + {"body": "b", "price": 2.0, "integer": 2, "flag": False, "tags": [None]}, + ]) + ], + wait=True, + ) + points, _ = await server_collection.async_client.scroll( + server_collection.collection_name, + scroll_filter=server_collection._prepare_filter(expression), + limit=100, + ) + assert [point.id for point in points] == expected + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + (Filter("meta", "exists"), [1, 2, 3]), + (Filter("meta", "is_null"), [1]), + (Filter("meta", "is_not_null"), [2, 3]), + ], +) +async def test_server_dict_presence_ignores_missing_keys(expression, expected): + # Qdrant 1.19.0 counts a missing key as zero values, so values_count(gte=0) would also match point 0. + definition = VectorStoreCollectionDefinition([ + VectorStoreField("key", name="id", type_="int"), + VectorStoreField("data", name="meta", type_="dict", storage_name="attributes"), + ]) + async with QdrantCollection( + dict, + definition=definition, + collection_name=f"af_qdrant_test_{uuid4().hex}", + async_client=AsyncQdrantClient(url=os.environ["QDRANT_TEST_URL"], check_compatibility=False), + managed_client=True, + ) as collection: + await collection.ensure_collection_exists() + try: + await collection.async_client.upsert( + collection.collection_name, + points=[ + models.PointStruct(id=index, vector={}, payload=payload) + for index, payload in enumerate([ + {}, + {"attributes": None}, + {"attributes": {}}, + {"attributes": {"a": 1}}, + ]) + ], + wait=True, + ) + points, _ = await collection.async_client.scroll( + collection.collection_name, + scroll_filter=collection._prepare_filter(expression), + limit=100, + ) + assert [point.id for point in points] == expected + finally: + await collection.ensure_collection_deleted() + + async def test_filtered_get_search_and_tool_params(server_collection, record): collection = server_collection await collection.upsert(