Python: Fix Qdrant null/presence filters on Qdrant server 1.19

python/packages/qdrant/agent_framework_qdrant/_vector_store.py

@@ -91,6 +91,9 @@ def _create_client(
 _WRITE_OPTIONS = {"ordering", "shard_key_selector", "timeout"}
 _BATCH_SIZE = 256
 _MAX_EXACT_INTEGER = 2**53 - 1
+_SCALAR_FIELD_TYPES = frozenset({"str", "int", "float", "bool"})
+# Data field types that can never hold an array, so ``is_empty`` alone separates a missing key from a value.
+_NON_ARRAY_FIELD_TYPES = _SCALAR_FIELD_TYPES | {"dict"}
 _CREATE_OPTIONS = {
     "shard_number",
     "replication_factor",
@@ -221,17 +224,36 @@ def _prepare_false_condition() -> models.HasIdCondition:
     return models.HasIdCondition(has_id=[])
 
 
-def _prepare_presence_condition(name: str) -> models.FieldCondition:
-    # Unlike is_empty, the server's values_count distinguishes missing from null and [].
-    return models.FieldCondition(key=name, values_count=models.ValuesCount(gte=0))
+def _prepare_empty_condition(name: str) -> models.IsEmptyCondition:
+    # is_empty matches a missing key, null, and [] on every supported server version.
+    return models.IsEmptyCondition(is_empty=models.PayloadField(key=name))
+
+
+def _prepare_null_condition(name: str) -> models.Filter:
+    # Since Qdrant 1.19.1, is_null also matches arrays containing null; is_empty restricts it to an explicit null.
+    return models.Filter(
+        must=[models.IsNullCondition(is_null=models.PayloadField(key=name)), _prepare_empty_condition(name)]
+    )
 
 
-def _prepare_null_condition(name: str) -> models.IsNullCondition:
-    return models.IsNullCondition(is_null=models.PayloadField(key=name))
+def _prepare_presence_condition(field: VectorStoreField) -> models.Condition:
+    name = field.storage_name or field.name
+    if field.type_ in _NON_ARRAY_FIELD_TYPES:
+        # Since Qdrant 1.19.0, values_count treats a missing key as zero values. A declared scalar or dict
+        # cannot hold [] ({} is not empty for Qdrant), so it is present exactly when it is non-empty or an
+        # explicit null.
+        return models.Filter(
+            should=[models.Filter(must_not=[_prepare_empty_condition(name)]), _prepare_null_condition(name)]
+        )
+    # Only values_count separates [] from a missing key, and only on Qdrant servers before 1.19.0.
+    return models.FieldCondition(key=name, values_count=models.ValuesCount(gte=0))
 
 
-def _prepare_non_null_condition(name: str) -> models.Filter:
-    return models.Filter(must=[_prepare_presence_condition(name)], must_not=[_prepare_null_condition(name)])
+def _prepare_non_null_condition(field: VectorStoreField) -> models.Filter:
+    name = field.storage_name or field.name
+    if field.type_ in _NON_ARRAY_FIELD_TYPES:
+        return models.Filter(must_not=[_prepare_empty_condition(name)])
+    return models.Filter(must=[_prepare_presence_condition(field)], must_not=[_prepare_null_condition(name)])
 
 
 def _prepare_range_operand(value: Any) -> float:
@@ -332,6 +354,8 @@ class QdrantCollection(BaseVectorCollection[KeyT, ModelT], BaseVectorSearch[KeyT
     Portable filters require a server: Qdrant's local emulator has different
     null/missing and numeric matching behavior across supported SDK versions.
     Local mode supports unfiltered CRUD and dense search. Ordered retrieval is unsupported.
+    Qdrant 1.19+ servers cannot distinguish ``[]`` from a missing key, so ``exists``,
+    ``is_not_null``, and empty ``contains_all`` on collection fields also match points without the key.
     """
 
     supported_key_types: ClassVar[set[str] | None] = {"int", "str", "UUID"}
@@ -610,15 +634,15 @@ def _prepare_filter_condition(self, expression: FilterExpression) -> models.Cond
         name = field.storage_name or field.name
         operator, value = expression.operator, expression.value
         if operator == "exists":
-            return _prepare_presence_condition(name)
+            return _prepare_presence_condition(field)
         if operator == "is_null":
             return _prepare_null_condition(name)
         if operator == "is_not_null":
-            return _prepare_non_null_condition(name)
+            return _prepare_non_null_condition(field)
         if operator in {"eq", "ne"}:
             condition = _prepare_equality_condition(name, value, field)
             return (
-                models.Filter(must=[_prepare_presence_condition(name)], must_not=[condition])
+                models.Filter(must=[_prepare_presence_condition(field)], must_not=[condition])
                 if operator == "ne"
                 else condition
             )
@@ -638,15 +662,15 @@ def _prepare_filter_condition(self, expression: FilterExpression) -> models.Cond
                 else _prepare_false_condition()
             )
             if operator == "not_in":
-                return models.Filter(must=[_prepare_non_null_condition(name)], must_not=[condition])
-            return models.Filter(must=[_prepare_non_null_condition(name), condition])
+                return models.Filter(must=[_prepare_non_null_condition(field)], must_not=[condition])
+            return models.Filter(must=[_prepare_non_null_condition(field), condition])
         if operator in {"contains", "contains_any", "contains_all"}:
             if field.type_ not in {"list", "tuple", "set", "Sequence"}:
                 raise NotImplementedError("Qdrant collection membership requires a declared collection field type.")
             operands: Sequence[Any] = [value] if operator == "contains" else value
             conditions = [_prepare_equality_condition(name, item, field, element=True) for item in operands]
             if operator == "contains_all":
-                return models.Filter(must=[_prepare_non_null_condition(name), *conditions])
+                return models.Filter(must=[_prepare_non_null_condition(field), *conditions])
             return models.Filter(should=conditions) if conditions else _prepare_false_condition()
         raise NotImplementedError(
             f"Qdrant does not support portable filter operator '{operator}'. "

python/packages/qdrant/tests/qdrant/test_filters.py

@@ -5,7 +5,7 @@
 from unittest.mock import AsyncMock
 
 import pytest
-from agent_framework import Filter, FilterGroup
+from agent_framework import Filter, FilterGroup, VectorStoreCollectionDefinition, VectorStoreField
 from agent_framework._vector_filters import validate_filter
 from qdrant_client import AsyncQdrantClient
 
@@ -75,6 +75,53 @@ def test_reject_unsafe_numeric_range(collection, value):
         collection._prepare_filter(Filter("integer", "gt", value))
 
 
+def test_scalar_presence_and_null_do_not_depend_on_values_count(collection):
+    # Qdrant 1.19.0 counts a missing key as zero values; 1.19.1 also matches is_null on arrays containing null.
+    empty = {"is_empty": {"key": "body"}}
+    null = {"must": [{"is_null": {"key": "body"}}, empty]}
+    expected = {
+        "exists": {"should": [{"must_not": [empty]}, null]},
+        "is_null": null,
+        "is_not_null": {"must_not": [empty]},
+    }
+    for operator, condition in expected.items():
+        native = collection._prepare_filter(Filter("text", operator))
+        assert native is not None
+        assert native.model_dump(exclude_none=True) == {"must": [condition]}
+    native = collection._prepare_filter(Filter("tags", "is_null"))
+    assert native is not None
+    assert native.model_dump(exclude_none=True) == {
+        "must": [{"must": [{"is_null": {"key": "tags"}}, {"is_empty": {"key": "tags"}}]}]
+    }
+
+
+def test_dict_presence_and_null_do_not_depend_on_values_count():
+    # A dict field cannot hold [], so it takes the same is_empty-based path as scalars on Qdrant 1.19+.
+    client = AsyncMock(spec=AsyncQdrantClient)
+    client.init_options = {}
+    collection = QdrantCollection(
+        dict,
+        definition=VectorStoreCollectionDefinition([
+            VectorStoreField("key", name="id", type_="int"),
+            VectorStoreField("data", name="meta", type_="dict", storage_name="attributes"),
+        ]),
+        collection_name="test",
+        async_client=client,
+    )
+    empty = {"is_empty": {"key": "attributes"}}
+    null = {"must": [{"is_null": {"key": "attributes"}}, empty]}
+    expected = {
+        "exists": {"should": [{"must_not": [empty]}, null]},
+        "is_null": null,
+        "is_not_null": {"must_not": [empty]},
+    }
+    for operator, condition in expected.items():
+        native = collection._prepare_filter(Filter("meta", operator))
+        assert native is not None
+        assert native.model_dump(exclude_none=True) == {"must": [condition]}
+        assert "values_count" not in str(native.model_dump(exclude_none=True))
+
+
 def test_numeric_equality_preserves_bool_distinction(collection):
     native = collection._prepare_filter(Filter("integer", "eq", True))
     assert native is not None

python/packages/qdrant/tests/qdrant/test_server_filters.py

@@ -17,7 +17,7 @@
 )
 from agent_framework._vector_filters import filter_values_equal
 from agent_framework.exceptions import IntegrationInvalidResponseException
-from qdrant_client import models
+from qdrant_client import AsyncQdrantClient, models
 
 from agent_framework_qdrant import QdrantCollection
 
@@ -119,6 +119,91 @@ async def test_server_native_portable_semantics(payload_collection: QdrantCollec
     assert [point.id for point in points] == expected
 
 
+@pytest.mark.parametrize(
+    ("expression", "expected"),
+    [
+        (Filter("text", "exists"), [1, 2, 3]),
+        (Filter("text", "is_not_null"), [2, 3]),
+        (Filter("text", "ne", "a"), [1, 3]),
+        (Filter("number", "exists"), [1, 2, 3]),
+        (Filter("number", "ne", 1.0), [1, 3]),
+        (Filter("integer", "not_in", [1]), [3]),
+        (Filter("flag", "is_not_null"), [2, 3]),
+        (Filter("tags", "is_null"), [1]),
+        (Filter("tags", "contains_all", ["a"]), [2]),
+    ],
+)
+async def test_server_presence_ignores_missing_keys_and_null_array_elements(
+    server_collection: QdrantCollection, expression, expected
+):
+    # Qdrant 1.19.0 counts a missing key as zero values and 1.19.1 matches is_null on arrays containing null.
+    await server_collection.async_client.upsert(
+        server_collection.collection_name,
+        points=[
+            models.PointStruct(id=index, vector={}, payload=payload)
+            for index, payload in enumerate([
+                {},
+                {"body": None, "price": None, "integer": None, "flag": None, "tags": None},
+                {"body": "a", "price": 1.0, "integer": 1, "flag": True, "tags": ["a", None]},
+                {"body": "b", "price": 2.0, "integer": 2, "flag": False, "tags": [None]},
+            ])
+        ],
+        wait=True,
+    )
+    points, _ = await server_collection.async_client.scroll(
+        server_collection.collection_name,
+        scroll_filter=server_collection._prepare_filter(expression),
+        limit=100,
+    )
+    assert [point.id for point in points] == expected
+
+
+@pytest.mark.parametrize(
+    ("expression", "expected"),
+    [
+        (Filter("meta", "exists"), [1, 2, 3]),
+        (Filter("meta", "is_null"), [1]),
+        (Filter("meta", "is_not_null"), [2, 3]),
+    ],
+)
+async def test_server_dict_presence_ignores_missing_keys(expression, expected):
+    # Qdrant 1.19.0 counts a missing key as zero values, so values_count(gte=0) would also match point 0.
+    definition = VectorStoreCollectionDefinition([
+        VectorStoreField("key", name="id", type_="int"),
+        VectorStoreField("data", name="meta", type_="dict", storage_name="attributes"),
+    ])
+    async with QdrantCollection(
+        dict,
+        definition=definition,
+        collection_name=f"af_qdrant_test_{uuid4().hex}",
+        async_client=AsyncQdrantClient(url=os.environ["QDRANT_TEST_URL"], check_compatibility=False),
+        managed_client=True,
+    ) as collection:
+        await collection.ensure_collection_exists()
+        try:
+            await collection.async_client.upsert(
+                collection.collection_name,
+                points=[
+                    models.PointStruct(id=index, vector={}, payload=payload)
+                    for index, payload in enumerate([
+                        {},
+                        {"attributes": None},
+                        {"attributes": {}},
+                        {"attributes": {"a": 1}},
+                    ])
+                ],
+                wait=True,
+            )
+            points, _ = await collection.async_client.scroll(
+                collection.collection_name,
+                scroll_filter=collection._prepare_filter(expression),
+                limit=100,
+            )
+            assert [point.id for point in points] == expected
+        finally:
+            await collection.ensure_collection_deleted()
+
+
 async def test_filtered_get_search_and_tool_params(server_collection, record):
     collection = server_collection
     await collection.upsert(