From d121cfe0e10a4abf7dc18e0826c3a80d250f58a8 Mon Sep 17 00:00:00 2001 From: Nanduu24 Date: Tue, 13 Oct 2026 10:01:30 -0500 Subject: [PATCH 1/7] Python: add Milvus vector store (agent-framework-milvus) Adds a new agent-framework-milvus package providing MilvusStore, MilvusCollection and MilvusSettings on the BaseVectorStore / BaseVectorCollection interfaces, so Milvus is available as a vector store alongside the Postgres, Qdrant, Redis, MongoDB and other connectors. - The data model definition drives the collection schema: the key field becomes the primary key (INT64/VARCHAR), data fields become typed scalar fields, and vector fields become FLOAT_VECTOR fields, each with an index. - Distance functions map to Milvus metrics (COSINE/IP/L2) and index kinds to Milvus index types (HNSW/FLAT/IVF_FLAT/DISKANN/AUTOINDEX). - Portable filters (eq, ne, gt, gte, lt, lte, between, in, not_in and and/or/not groups) are translated to Milvus boolean expressions; connector-side score thresholds respect each metric's comparison direction. - Upsert, get (by key or filter), delete, vector search and collection lifecycle, with the sync MilvusClient wrapped in asyncio.to_thread. - README, a runnable sample, and unit tests (filters, settings, collection) plus integration tests gated behind MILVUS_TEST_URI. Registered in the workspace. --- python/packages/milvus/LICENSE | 21 + python/packages/milvus/README.md | 59 ++ .../milvus/agent_framework_milvus/__init__.py | 14 + .../agent_framework_milvus/_vector_store.py | 660 ++++++++++++++++++ .../milvus/agent_framework_milvus/py.typed | 0 python/packages/milvus/pyproject.toml | 67 ++ python/packages/milvus/samples/README.md | 12 + .../packages/milvus/samples/milvus_vectors.py | 78 +++ .../packages/milvus/tests/milvus/conftest.py | 94 +++ .../milvus/tests/milvus/test_collection.py | 159 +++++ .../milvus/tests/milvus/test_filters.py | 56 ++ .../milvus/tests/milvus/test_settings.py | 58 ++ python/pyproject.toml | 1 + python/uv.lock | 146 ++++ 14 files changed, 1425 insertions(+) create mode 100644 python/packages/milvus/LICENSE create mode 100644 python/packages/milvus/README.md create mode 100644 python/packages/milvus/agent_framework_milvus/__init__.py create mode 100644 python/packages/milvus/agent_framework_milvus/_vector_store.py create mode 100644 python/packages/milvus/agent_framework_milvus/py.typed create mode 100644 python/packages/milvus/pyproject.toml create mode 100644 python/packages/milvus/samples/README.md create mode 100644 python/packages/milvus/samples/milvus_vectors.py create mode 100644 python/packages/milvus/tests/milvus/conftest.py create mode 100644 python/packages/milvus/tests/milvus/test_collection.py create mode 100644 python/packages/milvus/tests/milvus/test_filters.py create mode 100644 python/packages/milvus/tests/milvus/test_settings.py diff --git a/python/packages/milvus/LICENSE b/python/packages/milvus/LICENSE new file mode 100644 index 00000000000..96a394c43c3 --- /dev/null +++ b/python/packages/milvus/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/python/packages/milvus/README.md b/python/packages/milvus/README.md new file mode 100644 index 00000000000..1767cda28c3 --- /dev/null +++ b/python/packages/milvus/README.md @@ -0,0 +1,59 @@ +# Milvus for Microsoft Agent Framework + +`agent-framework-milvus` provides a [Milvus](https://milvus.io/) vector store for the +Microsoft Agent Framework, implementing the framework's `BaseVectorStore` / +`BaseVectorCollection` interfaces. It works alongside the Postgres, Qdrant, Redis and +other vector-store connectors. + +## Installation + +```bash +pip install agent-framework-milvus +``` + +## Usage + +```python +from agent_framework import VectorStoreCollectionDefinition, VectorStoreField +from agent_framework_milvus import MilvusStore + +definition = VectorStoreCollectionDefinition([ + VectorStoreField("key", name="id", type_="int"), + VectorStoreField("data", name="text", type_="str"), + VectorStoreField("vector", name="embedding", dimensions=1536, distance_function="cosine_similarity"), +]) + +# Connects to a Milvus server; set MILVUS_URI or pass uri=... +# A local ".db" path uses Milvus Lite. +async with MilvusStore(uri="http://localhost:19530") as store: + collection = store.get_collection(dict, definition=definition, collection_name="docs") + await collection.ensure_collection_exists() + await collection.upsert({"id": 1, "text": "hello", "embedding": [...]}) + results = await collection.search(vector=[...], top=5) + async for hit in results: + print(hit.record, hit.score) +``` + +## Configuration + +Settings are resolved from explicit arguments, then an optional `.env` file, then +environment variables prefixed with `MILVUS_`: + +| Setting | Environment variable | Notes | +| ---------- | -------------------- | -------------------------------------------------- | +| `uri` | `MILVUS_URI` | Server URI or a local `.db` path (Milvus Lite). | +| `token` | `MILVUS_TOKEN` | Access token (API key or `user:password`). | +| `user` | `MILVUS_USER` | Username. | +| `password` | `MILVUS_PASSWORD` | Password. | +| `db_name` | `MILVUS_DB_NAME` | Database name. | + +## Notes + +- The key field becomes the Milvus primary key (`INT64` for `int` keys, `VARCHAR` + otherwise). Data fields map to typed scalar fields and vector fields to + `FLOAT_VECTOR` fields, each with its own index. +- Distance functions map to Milvus metrics (`cosine_similarity`/`cosine_distance` → + `COSINE`, `dot_prod` → `IP`, `euclidean_distance` → `L2`) and index kinds to Milvus + index types (`hnsw`, `flat`, `ivf_flat`, `disk_ann`, and `default` → `AUTOINDEX`). +- Portable filters support `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `between`, `in`, + `not_in`, and `and`/`or`/`not` groups, translated to Milvus boolean expressions. diff --git a/python/packages/milvus/agent_framework_milvus/__init__.py b/python/packages/milvus/agent_framework_milvus/__init__.py new file mode 100644 index 00000000000..b59f31b72e9 --- /dev/null +++ b/python/packages/milvus/agent_framework_milvus/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Milvus vector stores for Microsoft Agent Framework.""" + +import importlib.metadata + +from ._vector_store import MilvusCollection, MilvusSettings, MilvusStore + +try: + __version__ = importlib.metadata.version(__name__) +except importlib.metadata.PackageNotFoundError: + __version__ = "0.0.0" + +__all__ = ["MilvusCollection", "MilvusSettings", "MilvusStore", "__version__"] diff --git a/python/packages/milvus/agent_framework_milvus/_vector_store.py b/python/packages/milvus/agent_framework_milvus/_vector_store.py new file mode 100644 index 00000000000..5a145b6ae61 --- /dev/null +++ b/python/packages/milvus/agent_framework_milvus/_vector_store.py @@ -0,0 +1,660 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Async Milvus dense-vector collections and stores for Microsoft Agent Framework.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Mapping, Sequence +from typing import Any, ClassVar, Final, Generic, cast + +from agent_framework import ( + BaseVectorCollection, + BaseVectorSearch, + BaseVectorStore, + FilterGroup, + SearchResults, + SearchType, + SecretString, + VectorStoreCollectionDefinition, + load_settings, +) +from agent_framework._telemetry import FeatureIndex, mark_feature_used +from agent_framework._vector_filters import FilterExpression +from agent_framework._vectors import EmbeddingClient, Vector +from agent_framework.exceptions import IntegrationException, IntegrationInvalidResponseException +from pymilvus import DataType, MilvusClient +from typing_extensions import TypedDict, TypeVar + +KeyT = TypeVar("KeyT", default=Any) +ModelT = TypeVar("ModelT", default=Any) + +DEFAULT_URI: Final[str] = "http://localhost:19530" +DEFAULT_VARCHAR_LENGTH: Final[int] = 65535 + +# Milvus metric types, see https://milvus.io/docs/metric.md +_METRICS: Final[dict[str, str]] = { + "DEFAULT": "COSINE", + "cosine_similarity": "COSINE", + "cosine_distance": "COSINE", + "dot_prod": "IP", + "euclidean_distance": "L2", + "euclidean_squared_distance": "L2", +} +# Metrics where a larger score is a closer match; used to interpret score_threshold. +_HIGHER_IS_BETTER: Final[set[str]] = {"COSINE", "IP"} +# Milvus index types, see https://milvus.io/docs/index.md +_INDEX_KINDS: Final[dict[str, str]] = { + "default": "AUTOINDEX", + "hnsw": "HNSW", + "flat": "FLAT", + "ivf_flat": "IVF_FLAT", + "disk_ann": "DISKANN", +} +# Data-model scalar types mapped to Milvus field types. +_SCALAR_TYPES: Final[dict[str, DataType]] = { + "str": DataType.VARCHAR, + "int": DataType.INT64, + "float": DataType.DOUBLE, + "bool": DataType.BOOL, + "list": DataType.JSON, + "tuple": DataType.JSON, + "set": DataType.JSON, + "Sequence": DataType.JSON, + "dict": DataType.JSON, +} + +_READ_OPTIONS: Final[set[str]] = {"timeout"} +_WRITE_OPTIONS: Final[set[str]] = {"timeout"} +_CREATE_OPTIONS: Final[set[str]] = {"timeout", "consistency_level"} +_SEARCH_OPTIONS: Final[set[str]] = {"timeout", "search_params"} + + +class MilvusSettings(TypedDict, total=False): + """Connection settings loaded through Agent Framework's ``load_settings``. + + Explicit constructor arguments take precedence over an explicitly selected + ``.env`` file, then environment variables prefixed with ``MILVUS_``. + + Keys: + uri: Server URI from ``MILVUS_URI`` (e.g. ``http://localhost:19530``), or a local + ``.db`` path for Milvus Lite. When unset, defaults to ``http://localhost:19530``. + token: Optional access token from ``MILVUS_TOKEN``, stored as an AF ``SecretString``. + user: Optional username from ``MILVUS_USER``. + password: Optional password from ``MILVUS_PASSWORD``, stored as an AF ``SecretString``. + db_name: Optional database name from ``MILVUS_DB_NAME``. + """ + + uri: str | None + token: SecretString | None + user: str | None + password: SecretString | None + db_name: str | None + + +def _validate_operation_options(options: Mapping[str, Any] | None, allowed: set[str]) -> dict[str, Any]: + result = dict(options or {}) + unknown = result.keys() - allowed + if unknown: + raise ValueError(f"Unsupported Milvus operation option(s): {', '.join(sorted(unknown))}.") + return result + + +def _create_client( + *, + uri: str | None, + token: str | SecretString | None, + user: str | None, + password: str | SecretString | None, + db_name: str | None, + env_file_path: str | None, + env_file_encoding: str | None, +) -> MilvusClient: + settings = load_settings( + MilvusSettings, + env_prefix="MILVUS_", + uri=uri, + token=token, + user=user, + password=password, + db_name=db_name, + env_file_path=env_file_path, + env_file_encoding=env_file_encoding, + ) + resolved_token = settings.get("token") + resolved_password = settings.get("password") + client_kwargs: dict[str, Any] = {"uri": settings.get("uri") or DEFAULT_URI} + if resolved_token is not None: + client_kwargs["token"] = resolved_token.get_secret_value() + if settings.get("user"): + client_kwargs["user"] = settings.get("user") + if resolved_password is not None: + client_kwargs["password"] = resolved_password.get_secret_value() + if settings.get("db_name"): + client_kwargs["db_name"] = settings.get("db_name") + try: + return MilvusClient(**client_kwargs) + except Exception as exc: # pragma: no cover - surfaced as an integration error + raise IntegrationException("Failed to create the Milvus client.") from exc + + +def _format_scalar(value: Any) -> str: + """Format a Python scalar as a Milvus boolean-expression literal.""" + if isinstance(value, bool): + return "true" if value else "false" + if value is None: + return "null" + if isinstance(value, str): + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + if isinstance(value, (int, float)): + return repr(value) + raise NotImplementedError(f"Milvus filters support only scalar operands, not {type(value).__name__}.") + + +def _format_list(values: Any) -> str: + return "[" + ", ".join(_format_scalar(item) for item in values) + "]" + + +class MilvusCollection(BaseVectorCollection[KeyT, ModelT], BaseVectorSearch[KeyT, ModelT], Generic[KeyT, ModelT]): + """Store and search records with the official Milvus client. + + The data model definition drives the collection schema: the key field becomes the + primary key (``INT64``/``VARCHAR``), data fields become typed scalar fields, and + vector fields become ``FLOAT_VECTOR`` fields, each with its own index. The default + metric is cosine similarity (larger is better); scores retain native Milvus units. + + The synchronous ``MilvusClient`` is used under ``asyncio.to_thread`` so the connector + presents the framework's async interface while remaining compatible across the + supported ``pymilvus`` range. Supplied clients remain caller-owned unless managed. + """ + + supported_key_types: ClassVar[set[str] | None] = {"int", "str"} + supported_vector_types: ClassVar[set[str] | None] = {"float", "float32", "float64"} + supported_search_types: ClassVar[set[SearchType]] = {"vector"} + + def __init__( + self, + record_type: type[ModelT], + *, + definition: VectorStoreCollectionDefinition | None = None, + collection_name: str | None = None, + embedding_generator: EmbeddingClient | None = None, + client: MilvusClient | None = None, + uri: str | None = None, + token: str | SecretString | None = None, + user: str | None = None, + password: str | SecretString | None = None, + db_name: str | None = None, + managed_client: bool | None = None, + env_file_path: str | None = None, + env_file_encoding: str | None = None, + ) -> None: + """Initialize a collection. + + Args: + record_type: Registered application model, or dict with an explicit definition. + definition: Explicit dictionary schema. + collection_name: Collection name, overriding the model name. + embedding_generator: Default local embedding client. + client: Existing client, borrowed by default. Use this for Milvus Lite or advanced configuration. + uri: Server URI override; otherwise loaded from MilvusSettings, then defaults to localhost. + token: Access token override as a string or SecretString; otherwise loaded from MilvusSettings. + user: Username override; otherwise loaded from MilvusSettings. + password: Password override as a string or SecretString; otherwise loaded from MilvusSettings. + db_name: Database name override; otherwise loaded from MilvusSettings. + managed_client: Whether to close the client. Defaults to True only for connector-created clients. + env_file_path: Optional .env file to load before process environment variables. + env_file_encoding: Encoding for the .env file; defaults to UTF-8. + """ + if client is not None and any( + value is not None for value in (uri, token, user, password, db_name, env_file_path, env_file_encoding) + ): + raise ValueError("Pass client or connection settings, not both.") + super().__init__( + record_type, + definition=definition, + collection_name=collection_name, + embedding_generator=embedding_generator, + managed_client=client is None if managed_client is None else managed_client, + ) + self.client = ( + client + if client is not None + else _create_client( + uri=uri, + token=token, + user=user, + password=password, + db_name=db_name, + env_file_path=env_file_path, + env_file_encoding=env_file_encoding, + ) + ) + self._closed = False + # pymilvus ships only partial type information, so operational calls go through an + # Any-typed handle to keep the strict type checker clean; the public ``client`` stays typed. + self._client: Any = self.client + + @property + def _key_storage_name(self) -> str: + return self.definition.key_field_storage_name + + def _validate_data_model(self) -> None: + super()._validate_data_model() + if self.definition.key_field.is_auto_generated: + raise NotImplementedError("Milvus requires application-provided primary keys; it does not generate keys.") + for field in self.definition.vector_fields: + if not isinstance(field.dimensions, int) or isinstance(field.dimensions, bool): + raise TypeError("Milvus vector dimensions must be an integer.") + if field.distance_function not in _METRICS: + raise NotImplementedError(f"Milvus distance function '{field.distance_function}' is not supported.") + if field.index_kind not in _INDEX_KINDS: + raise NotImplementedError(f"Milvus index kind '{field.index_kind}' is not supported.") + for field in self.definition.data_fields: + if (field.type_ or "str") not in _SCALAR_TYPES: + raise NotImplementedError(f"Milvus data field type '{field.type_}' is not supported.") + + def _output_fields(self, include_vectors: bool) -> list[str]: + fields = [field.storage_name or field.name for field in self.definition.data_fields] + fields.append(self._key_storage_name) + if include_vectors: + fields.extend(field.storage_name or field.name for field in self.definition.vector_fields) + return fields + + async def collection_exists(self, *, operation_options: Mapping[str, Any] | None = None) -> bool: + """Check whether this collection exists.""" + options = _validate_operation_options(operation_options, _READ_OPTIONS) + mark_feature_used(FeatureIndex.CORE_VECTOR_STORES) + return await asyncio.to_thread(self._client.has_collection, self.collection_name, **options) + + def _build_schema_and_index(self) -> tuple[Any, Any]: + schema = cast("Any", MilvusClient).create_schema(auto_id=False, enable_dynamic_field=False) + key_field = self.definition.key_field + if key_field.type_ == "int": + schema.add_field(field_name=self._key_storage_name, datatype=DataType.INT64, is_primary=True) + else: + schema.add_field( + field_name=self._key_storage_name, + datatype=DataType.VARCHAR, + is_primary=True, + max_length=DEFAULT_VARCHAR_LENGTH, + ) + for field in self.definition.data_fields: + datatype = _SCALAR_TYPES[field.type_ or "str"] + if datatype == DataType.VARCHAR: + schema.add_field( + field_name=field.storage_name or field.name, datatype=datatype, max_length=DEFAULT_VARCHAR_LENGTH + ) + else: + schema.add_field(field_name=field.storage_name or field.name, datatype=datatype) + index_params = self._client.prepare_index_params() + for field in self.definition.vector_fields: + schema.add_field( + field_name=field.storage_name or field.name, + datatype=DataType.FLOAT_VECTOR, + dim=cast(int, field.dimensions), + ) + index_params.add_index( + field_name=field.storage_name or field.name, + index_type=_INDEX_KINDS[cast(str, field.index_kind)], + metric_type=_METRICS[cast(str, field.distance_function)], + ) + return schema, index_params + + async def ensure_collection_exists(self, *, operation_options: Mapping[str, Any] | None = None) -> None: + """Create the collection and its vector indexes when it does not exist.""" + options = _validate_operation_options(operation_options, _CREATE_OPTIONS) + if await self.collection_exists(): + return + schema, index_params = self._build_schema_and_index() + await asyncio.to_thread( + self._client.create_collection, + collection_name=self.collection_name, + schema=schema, + index_params=index_params, + **options, + ) + + async def ensure_collection_deleted(self, *, operation_options: Mapping[str, Any] | None = None) -> None: + """Delete this collection if it exists.""" + options = _validate_operation_options(operation_options, _WRITE_OPTIONS) + if await self.collection_exists(): + await asyncio.to_thread(self._client.drop_collection, self.collection_name, **options) + + async def aclose(self) -> None: + """Close an owned client once; borrowed clients are left open.""" + if self.managed_client and not self._closed: + await asyncio.to_thread(self._client.close) + self._closed = True + + async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + """Close the owned client on context exit.""" + await self.aclose() + + def _serialize_dicts_to_store_models( + self, + records: Sequence[dict[str, Any]], + *, + context: Mapping[str, Any] | None = None, + ) -> Sequence[dict[str, Any]]: + models: list[dict[str, Any]] = [] + for record in records: + model: dict[str, Any] = {self._key_storage_name: record[self._key_storage_name]} + for field in self.definition.data_fields: + name = field.storage_name or field.name + model[name] = record[name] + for field in self.definition.vector_fields: + name = field.storage_name or field.name + value = record.get(name) + if value is not None: + model[name] = [float(component) for component in value] + models.append(model) + return models + + def _deserialize_store_models_to_dicts( + self, + records: Sequence[Any], + *, + context: Mapping[str, Any] | None = None, + ) -> Sequence[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for row in records: + if not isinstance(row, Mapping): + raise IntegrationInvalidResponseException("Milvus responses must contain mapping rows.") + record: dict[str, Any] = {} + if self._key_storage_name in row: + record[self._key_storage_name] = row[self._key_storage_name] + for field in self.definition.data_fields: + name = field.storage_name or field.name + if name in row: + record[name] = row[name] + for field in self.definition.vector_fields: + name = field.storage_name or field.name + if name in row: + record[name] = row[name] + result.append(record) + return result + + async def _inner_upsert( + self, + records: Sequence[Any], + *, + operation_options: Mapping[str, Any] | None = None, + ) -> Sequence[KeyT]: + options = _validate_operation_options(operation_options, _WRITE_OPTIONS) + if not records: + return [] + await asyncio.to_thread( + self._client.upsert, collection_name=self.collection_name, data=list(records), **options + ) + return [cast(KeyT, record[self._key_storage_name]) for record in records] + + async def _inner_get( + self, + *, + keys: Sequence[KeyT] | None = None, + filter: FilterExpression | None = None, + top: int = 10, + skip: int = 0, + order_by: Mapping[str, bool] | None = None, + include_vectors: bool = False, + operation_options: Mapping[str, Any] | None = None, + ) -> Sequence[Any] | None: + options = _validate_operation_options(operation_options, _READ_OPTIONS) + if order_by: + raise NotImplementedError("Milvus ordered retrieval is not supported. Omit order_by.") + output_fields = self._output_fields(include_vectors) + if keys is not None: + if skip: + raise ValueError("Milvus key retrieval cannot be combined with skip.") + if not keys: + return [] + return await asyncio.to_thread( + self._client.get, + collection_name=self.collection_name, + ids=list(keys), + output_fields=output_fields, + **options, + ) + return await asyncio.to_thread( + self._client.query, + collection_name=self.collection_name, + filter=self._prepare_filter(filter), + output_fields=output_fields, + limit=top, + offset=skip, + **options, + ) + + async def _inner_delete( + self, + keys: Sequence[KeyT], + *, + operation_options: Mapping[str, Any] | None = None, + ) -> None: + options = _validate_operation_options(operation_options, _WRITE_OPTIONS) + if not keys: + return + await asyncio.to_thread(self._client.delete, collection_name=self.collection_name, ids=list(keys), **options) + + async def _inner_search( + self, + *, + search_type: SearchType, + filter: FilterExpression | None = None, + values: Any | None = None, + vector: Vector | None = None, + top: int = 3, + skip: int = 0, + include_vectors: bool = False, + vector_property_name: str | None = None, + additional_property_name: str | None = None, + score_threshold: float | None = None, + operation_options: Mapping[str, Any] | None = None, + ) -> SearchResults[dict[str, Any]]: + options = _validate_operation_options(operation_options, _SEARCH_OPTIONS) + if additional_property_name is not None: + raise NotImplementedError("Milvus keyword-hybrid search is not supported by this connector.") + if vector is None: + raise ValueError("Milvus search requires a dense vector or a configured embedding generator.") + field = self.definition.try_get_vector_field(vector_property_name) + if field is None: + raise ValueError("Milvus search requires a declared vector field.") + metric = _METRICS[cast(str, field.distance_function)] + search_params: dict[str, Any] = {"metric_type": metric} + search_params.update(options.pop("search_params", {}) or {}) + if skip: + search_params["offset"] = skip + hits = await asyncio.to_thread( + self._client.search, + collection_name=self.collection_name, + data=[[float(component) for component in vector]], + anns_field=field.storage_name or field.name, + limit=top, + filter=self._prepare_filter(filter), + output_fields=self._output_fields(include_vectors), + search_params=search_params, + **options, + ) + results = list(hits[0]) if hits else [] + if score_threshold is not None: + # Milvus has no portable score cutoff across metrics, so enforce it connector-side. + # For similarity metrics (COSINE/IP) a larger score is closer; for L2 a smaller score is closer. + keep_higher = metric in _HIGHER_IS_BETTER + results = [ + hit for hit in results if self._passes_threshold(hit.get("distance", 0.0), score_threshold, keep_higher) + ] + return SearchResults(results) + + @staticmethod + def _passes_threshold(distance: float, threshold: float, keep_higher: bool) -> bool: + return distance >= threshold if keep_higher else distance <= threshold + + def _get_record_from_result(self, result: Any) -> dict[str, Any]: + if not isinstance(result, Mapping): + raise IntegrationInvalidResponseException("Expected a Milvus search hit mapping.") + hit = cast("Mapping[str, Any]", result) + record: dict[str, Any] = dict(hit.get("entity") or {}) + if "id" in hit: + record.setdefault(self._key_storage_name, hit["id"]) + return record + + def _get_score_from_result(self, result: Any) -> float | None: + if not isinstance(result, Mapping): + raise IntegrationInvalidResponseException("Expected a Milvus search hit mapping.") + distance = cast("Mapping[str, Any]", result).get("distance") + return float(distance) if distance is not None else None + + def _prepare_filter(self, filter: FilterExpression | None) -> str: + if filter is None: + return "" + return self._prepare_filter_condition(filter) + + def _prepare_filter_condition(self, expression: FilterExpression) -> str: + if isinstance(expression, FilterGroup): + children = [self._prepare_filter_condition(child) for child in expression.filters] + if expression.operator == "and": + return "(" + " and ".join(children) + ")" + if expression.operator == "or": + return "(" + " or ".join(children) + ")" + return f"(not {children[0]})" + + field = self.definition.try_get_field(expression.field_name) + if field is None or "." in expression.field_name: + raise NotImplementedError("Milvus filters support declared top-level fields only.") + if field.field_type == "vector": + raise NotImplementedError("Milvus filters do not operate on vector fields.") + name = field.storage_name or field.name + operator, value = expression.operator, expression.value + if operator == "eq": + return f"({name} == {_format_scalar(value)})" + if operator == "ne": + return f"({name} != {_format_scalar(value)})" + if operator == "gt": + return f"({name} > {_format_scalar(value)})" + if operator == "gte": + return f"({name} >= {_format_scalar(value)})" + if operator == "lt": + return f"({name} < {_format_scalar(value)})" + if operator == "lte": + return f"({name} <= {_format_scalar(value)})" + if operator == "between": + return f"({name} >= {_format_scalar(value[0])} and {name} <= {_format_scalar(value[1])})" + if operator == "in": + return f"({name} in {_format_list(value)})" + if operator == "not_in": + return f"({name} not in {_format_list(value)})" + raise NotImplementedError(f"Milvus does not support portable filter operator '{operator}'.") + + +class MilvusStore(BaseVectorStore): + """Create Milvus collection clients sharing one connection.""" + + def __init__( + self, + *, + client: MilvusClient | None = None, + uri: str | None = None, + token: str | SecretString | None = None, + user: str | None = None, + password: str | SecretString | None = None, + db_name: str | None = None, + embedding_generator: EmbeddingClient | None = None, + managed_client: bool | None = None, + env_file_path: str | None = None, + env_file_encoding: str | None = None, + ) -> None: + """Initialize a store. + + Args: + client: Existing client; borrowed unless managed_client=True. + uri: Server URI override; otherwise loaded from MilvusSettings, then defaults to localhost. + token: Access token override as a string or SecretString; otherwise loaded from MilvusSettings. + user: Username override; otherwise loaded from MilvusSettings. + password: Password override as a string or SecretString; otherwise loaded from MilvusSettings. + db_name: Database name override; otherwise loaded from MilvusSettings. + embedding_generator: Default embedding client inherited by collections. + managed_client: Whether to close the client. Defaults to True for connector-created clients. + env_file_path: Optional .env file to load before process environment variables. + env_file_encoding: Encoding for the .env file; defaults to UTF-8. + """ + if client is not None and any( + value is not None for value in (uri, token, user, password, db_name, env_file_path, env_file_encoding) + ): + raise ValueError("Pass client or connection settings, not both.") + super().__init__( + embedding_generator=embedding_generator, + managed_client=client is None if managed_client is None else managed_client, + ) + self.client = ( + client + if client is not None + else _create_client( + uri=uri, + token=token, + user=user, + password=password, + db_name=db_name, + env_file_path=env_file_path, + env_file_encoding=env_file_encoding, + ) + ) + self._closed = False + # pymilvus ships only partial type information, so operational calls go through an + # Any-typed handle to keep the strict type checker clean; the public ``client`` stays typed. + self._client: Any = self.client + + def get_collection( + self, + record_type: type[ModelT], + *, + definition: VectorStoreCollectionDefinition | None = None, + collection_name: str | None = None, + embedding_generator: EmbeddingClient | None = None, + ) -> MilvusCollection[Any, ModelT]: + """Create a collection that borrows this store's client.""" + return MilvusCollection( + record_type, + definition=definition, + collection_name=collection_name, + embedding_generator=embedding_generator if embedding_generator is not None else self.embedding_generator, + client=self.client, + managed_client=False, + ) + + async def list_collection_names(self, *, operation_options: Mapping[str, Any] | None = None) -> Sequence[str]: + """List collection names on the server.""" + options = _validate_operation_options(operation_options, _READ_OPTIONS) + mark_feature_used(FeatureIndex.CORE_VECTOR_STORES) + return await asyncio.to_thread(self._client.list_collections, **options) + + async def collection_exists( + self, + collection_name: str, + *, + operation_options: Mapping[str, Any] | None = None, + ) -> bool: + """Check collection existence without downloading the collection list.""" + options = _validate_operation_options(operation_options, _READ_OPTIONS) + mark_feature_used(FeatureIndex.CORE_VECTOR_STORES) + return await asyncio.to_thread(self._client.has_collection, collection_name, **options) + + async def _inner_ensure_collection_deleted( + self, + collection_name: str, + *, + operation_options: Mapping[str, Any] | None = None, + ) -> None: + options = _validate_operation_options(operation_options, _WRITE_OPTIONS) + await asyncio.to_thread(self._client.drop_collection, collection_name, **options) + + async def aclose(self) -> None: + """Close an owned client once, leaving borrowed clients open.""" + if self.managed_client and not self._closed: + await asyncio.to_thread(self._client.close) + self._closed = True + + async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + """Close the owned client on context exit.""" + await self.aclose() diff --git a/python/packages/milvus/agent_framework_milvus/py.typed b/python/packages/milvus/agent_framework_milvus/py.typed new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/packages/milvus/pyproject.toml b/python/packages/milvus/pyproject.toml new file mode 100644 index 00000000000..ab7462dbf40 --- /dev/null +++ b/python/packages/milvus/pyproject.toml @@ -0,0 +1,67 @@ +[project] +name = "agent-framework-milvus" +description = "Milvus vector stores for Microsoft Agent Framework." +authors = [{ name = "Microsoft", email = "af-support@microsoft.com" }] +readme = "README.md" +requires-python = ">=3.10" +version = "1.0.0a261002" +license-files = ["LICENSE"] +urls.homepage = "https://aka.ms/agent-framework" +urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" +urls.issues = "https://github.com/microsoft/agent-framework/issues" +classifiers = [ + "License :: OSI Approved :: MIT License", + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Typing :: Typed", +] +dependencies = [ + "agent-framework-core>=1.19.0,<2", + "pymilvus>=2.5,<3", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra -q -r fEX" +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" +timeout = 120 +markers = [ + "integration: marks tests as integration tests that require external services", +] + +[tool.ruff] +extend = "../../pyproject.toml" + +[tool.ruff.lint.per-file-ignores] +"samples/**" = ["D", "INP", "commented-out-code", "RUF", "S", "print", "CPY"] +"tests/**" = ["D", "INP", "TD", "commented-out-code", "RUF", "S"] + +[tool.coverage.run] +omit = ["**/__init__.py"] + +[tool.pyright] +extends = "../../pyproject.toml" +include = ["agent_framework_milvus"] + +[tool.poe] +executor.type = "uv" +include = "../../shared_tasks.toml" + +[tool.poe.tasks.test] +help = "Run Milvus unit tests." +cmd = 'pytest -m "not integration" --cov=agent_framework_milvus --cov-report=term-missing:skip-covered tests' + +[tool.poe.tasks.test-integration] +help = "Run tests against the Milvus server at MILVUS_TEST_URI." +cmd = 'pytest -m integration tests' + +[build-system] +requires = ["flit-core>=3.11,<5.0"] +build-backend = "flit_core.buildapi" diff --git a/python/packages/milvus/samples/README.md b/python/packages/milvus/samples/README.md new file mode 100644 index 00000000000..79b917a3203 --- /dev/null +++ b/python/packages/milvus/samples/README.md @@ -0,0 +1,12 @@ +# Milvus samples + +`milvus_vectors.py` demonstrates batch CRUD, dense-vector search, portable filtering +translated to Milvus boolean expressions, and selective vector retrieval using +deterministic vectors. It creates and deletes its own unique collection. + +Set `MILVUS_URI` to a disposable Milvus 2.5+ server (e.g. `http://localhost:19530`) or +to a local `.db` path to use Milvus Lite, then follow the +[package README](../README.md#configuration). No embedding service is needed. +Optional `MILVUS_TOKEN`, `MILVUS_USER`, `MILVUS_PASSWORD` and `MILVUS_DB_NAME` are +supported by the connector's `MilvusSettings`, which also supports explicitly selected +`.env` files. diff --git a/python/packages/milvus/samples/milvus_vectors.py b/python/packages/milvus/samples/milvus_vectors.py new file mode 100644 index 00000000000..d8cfc418dbf --- /dev/null +++ b/python/packages/milvus/samples/milvus_vectors.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Store and search dense vectors in a disposable Milvus collection. + +Run from python/: + export MILVUS_URI=http://localhost:19530 # or a local ".db" path for Milvus Lite + uv run --package agent-framework-milvus python packages/milvus/samples/milvus_vectors.py + +Use a development server or a local Milvus Lite file, not production. No embedding +model, inference service, or OpenAI credentials are used: the vectors are deliberately +small and deterministic. +""" + +from __future__ import annotations + +import asyncio +import os +from dataclasses import dataclass +from typing import Annotated +from uuid import uuid4 + +from agent_framework import Filter, VectorStoreField, vectorstoremodel + +from agent_framework_milvus import MilvusStore + + +@vectorstoremodel +@dataclass +class Document: + id: Annotated[int, VectorStoreField("key")] + title: Annotated[str, VectorStoreField("data", storage_name="document_title")] + category: Annotated[str, VectorStoreField("data")] + embedding: Annotated[ + list[float] | None, + VectorStoreField("vector", dimensions=3, distance_function="cosine_similarity"), + ] = None + + +async def main() -> None: + # 1. The store owns its client. Child collections borrow it. + async with MilvusStore(uri=os.environ.get("MILVUS_URI", "http://localhost:19530")) as store: + collection = store.get_collection(Document, collection_name=f"af_milvus_sample_{uuid4().hex}") + await collection.ensure_collection_exists() + try: + # 2. Preserve supplied vectors instead of generating embeddings. + await collection.upsert( + [ + Document(1, "Billing guide", "reference", [1.0, 0.0, 0.0]), + Document(2, "Image guide", "reference", [0.0, 1.0, 0.0]), + Document(3, "Unrelated", "notes", [0.0, 0.0, 1.0]), + ], + generate_vectors=False, + ) + + # 3. Vector search with a portable filter translated to a Milvus expression. + results = await collection.search( + vector=[1.0, 0.0, 0.0], + filter=Filter("category", "eq", "reference"), + top=2, + ) + async for result in results: + print(f"{result['record'].title}: {result['score']:.2f}") + + # 4. Vectors are excluded from retrieval unless requested. + documents = await collection.get([1], include_vectors=True) + print(f"Embedding: {documents[0].embedding}") + finally: + # Delete only this sample's unique collection, including on errors. + await collection.ensure_collection_deleted() + + +if __name__ == "__main__": + asyncio.run(main()) + +# Expected output (scores are cosine similarity, larger is closer): +# Billing guide: 1.00 +# Image guide: 0.00 +# Embedding: [1.0, 0.0, 0.0] diff --git a/python/packages/milvus/tests/milvus/conftest.py b/python/packages/milvus/tests/milvus/conftest.py new file mode 100644 index 00000000000..076b81ae8f1 --- /dev/null +++ b/python/packages/milvus/tests/milvus/conftest.py @@ -0,0 +1,94 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import os +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest +from agent_framework import VectorStoreCollectionDefinition, VectorStoreField +from pymilvus import MilvusClient + +from agent_framework_milvus import MilvusCollection, MilvusStore + +SERVER_MARKS = [ + pytest.mark.integration, + pytest.mark.skipif(not os.getenv("MILVUS_TEST_URI"), reason="Set MILVUS_TEST_URI to a disposable Milvus server."), +] + + +@pytest.fixture(autouse=True) +def clear_connection_environment(monkeypatch): + for name in ("MILVUS_URI", "MILVUS_TOKEN", "MILVUS_USER", "MILVUS_PASSWORD", "MILVUS_DB_NAME"): + monkeypatch.delenv(name, raising=False) + + +@pytest.fixture +def definition(): + return VectorStoreCollectionDefinition([ + VectorStoreField("key", name="id", type_="int", storage_name="point_key"), + VectorStoreField("data", name="text", type_="str", storage_name="body"), + VectorStoreField("data", name="number", type_="float", storage_name="price"), + VectorStoreField("data", name="flag", type_="bool"), + VectorStoreField( + "vector", + name="embedding", + storage_name="dense", + dimensions=3, + distance_function="cosine_similarity", + ), + ]) + + +@pytest.fixture +def record(): + def make(id=1, **values): + return { + "id": id, + "text": "hello", + "number": float(id), + "flag": True, + "embedding": [1.0, 0.0, 0.0], + } | values + + return make + + +@pytest.fixture +def mock_client(): + client = MagicMock(spec=MilvusClient) + client.has_collection.return_value = True + client.list_collections.return_value = ["docs"] + client.get.return_value = [{"point_key": 1, "body": "hello", "price": 1.0, "flag": True}] + client.query.return_value = [{"point_key": 1, "body": "hello", "price": 1.0, "flag": True}] + client.search.return_value = [ + [ + {"id": 1, "distance": 0.95, "entity": {"body": "hello", "price": 1.0, "flag": True}}, + {"id": 2, "distance": 0.10, "entity": {"body": "world", "price": 2.0, "flag": False}}, + ] + ] + return client + + +@pytest.fixture +def collection(definition, mock_client): + return MilvusCollection(dict, definition=definition, collection_name="docs", client=mock_client) + + +@pytest.fixture +def store(mock_client): + return MilvusStore(client=mock_client) + + +@pytest.fixture +async def server_collection(definition): + if not os.getenv("MILVUS_TEST_URI"): + pytest.skip("Set MILVUS_TEST_URI to a disposable Milvus server.") + async with MilvusStore(uri=os.environ["MILVUS_TEST_URI"]) as store: + collection = store.get_collection(dict, definition=definition, collection_name=f"af_milvus_test_{uuid4().hex}") + await collection.ensure_collection_exists() + try: + yield collection + finally: + await collection.ensure_collection_deleted() diff --git a/python/packages/milvus/tests/milvus/test_collection.py b/python/packages/milvus/tests/milvus/test_collection.py new file mode 100644 index 00000000000..a33c079178c --- /dev/null +++ b/python/packages/milvus/tests/milvus/test_collection.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import pytest +from agent_framework import Filter, VectorStoreCollectionDefinition, VectorStoreField +from pymilvus import DataType + +from agent_framework_milvus import MilvusCollection, MilvusStore + + +async def test_upsert_returns_keys(collection, mock_client, record): + keys = await collection.upsert([record(1), record(2)], generate_vectors=False) + assert keys == [1, 2] + mock_client.upsert.assert_called_once() + assert mock_client.upsert.call_args.kwargs["collection_name"] == "docs" + + +async def test_upsert_empty(collection): + assert await collection.upsert([], generate_vectors=False) == [] + + +async def test_get_by_keys_maps_storage_to_model(collection, mock_client): + records = await collection.get([1]) + assert records[0]["id"] == 1 + assert records[0]["text"] == "hello" + assert records[0]["number"] == 1.0 + mock_client.get.assert_called_once() + assert mock_client.get.call_args.kwargs["ids"] == [1] + + +async def test_get_without_keys_uses_query(collection, mock_client): + records = await collection.get(top=5) + assert records[0]["id"] == 1 + mock_client.query.assert_called_once() + + +async def test_get_empty_keys(collection): + assert await collection.get([]) == [] + + +async def test_get_rejects_order_by(collection): + with pytest.raises(NotImplementedError): + await collection.get(order_by={"number": False}, top=1) + + +async def test_delete(collection, mock_client): + await collection.delete([1, 2]) + mock_client.delete.assert_called_once() + assert mock_client.delete.call_args.kwargs["ids"] == [1, 2] + + +async def test_collection_exists(collection, mock_client): + assert await collection.collection_exists() is True + mock_client.has_collection.assert_called_with("docs") + + +async def test_ensure_collection_exists_creates_when_absent(collection, mock_client): + mock_client.has_collection.return_value = False + await collection.ensure_collection_exists() + mock_client.create_collection.assert_called_once() + assert mock_client.create_collection.call_args.kwargs["collection_name"] == "docs" + + +async def test_ensure_collection_exists_skips_when_present(collection, mock_client): + mock_client.has_collection.return_value = True + await collection.ensure_collection_exists() + mock_client.create_collection.assert_not_called() + + +async def test_ensure_collection_deleted(collection, mock_client): + mock_client.has_collection.return_value = True + await collection.ensure_collection_deleted() + mock_client.drop_collection.assert_called_once_with("docs") + + +async def test_search_returns_records_and_scores(collection, mock_client): + results = [item async for item in await collection.search(vector=[1.0, 0.0, 0.0], top=2)] + assert [item["record"]["id"] for item in results] == [1, 2] + assert [item["score"] for item in results] == [0.95, 0.10] + assert mock_client.search.call_args.kwargs["anns_field"] == "dense" + assert mock_client.search.call_args.kwargs["search_params"]["metric_type"] == "COSINE" + + +async def test_search_applies_filter(collection, mock_client): + _ = [item async for item in await collection.search(vector=[1.0, 0.0, 0.0], filter=Filter("flag", "eq", True))] + assert mock_client.search.call_args.kwargs["filter"] == "(flag == true)" + + +async def test_search_score_threshold_cosine(collection): + results = [item async for item in await collection.search(vector=[1.0, 0.0, 0.0], score_threshold=0.5)] + assert [item["record"]["id"] for item in results] == [1] + + +async def test_search_requires_query(collection): + with pytest.raises(ValueError, match="values or a precomputed vector"): + await collection.search(top=1) + + +def test_serialize_round_trip(collection): + # The handler serializes a model to a storage-keyed dict before calling this method. + storage_record = {"point_key": 7, "body": "hello", "price": 7.0, "flag": True, "dense": [1.0, 0.0, 0.0]} + stored = collection._serialize_dicts_to_store_models([storage_record]) + assert stored[0] == storage_record + back = collection._deserialize_store_models_to_dicts(stored) + assert back[0]["point_key"] == 7 + assert back[0]["dense"] == [1.0, 0.0, 0.0] + + +def test_build_schema_has_primary_and_vector(collection): + schema, _ = collection._build_schema_and_index() + field_types = {field.name: field.dtype for field in schema.fields} + assert field_types["point_key"] == DataType.INT64 + assert field_types["dense"] == DataType.FLOAT_VECTOR + assert next(field for field in schema.fields if field.name == "point_key").is_primary + + +def test_auto_generated_key_rejected(mock_client): + definition = VectorStoreCollectionDefinition([ + VectorStoreField("key", name="id", type_="int", is_auto_generated=True), + VectorStoreField("vector", name="v", dimensions=3, distance_function="cosine_similarity"), + ]) + with pytest.raises(NotImplementedError, match="application-provided primary keys"): + MilvusCollection(dict, definition=definition, collection_name="docs", client=mock_client) + + +def test_unsupported_distance_rejected(mock_client): + definition = VectorStoreCollectionDefinition([ + VectorStoreField("key", name="id", type_="int"), + VectorStoreField("vector", name="v", dimensions=3, distance_function="manhattan"), + ]) + with pytest.raises(NotImplementedError, match="distance function"): + MilvusCollection(dict, definition=definition, collection_name="docs", client=mock_client) + + +def test_client_and_settings_conflict(mock_client): + with pytest.raises(ValueError, match="not both"): + MilvusStore(client=mock_client, uri="http://localhost:19530") + + +async def test_store_list_collection_names(store, mock_client): + assert await store.list_collection_names() == ["docs"] + + +async def test_store_get_collection_borrows_client(store, definition): + collection = store.get_collection(dict, definition=definition, collection_name="docs") + assert collection.client is store.client + assert collection.managed_client is False + + +async def test_store_ensure_collection_deleted(store, mock_client): + mock_client.has_collection.return_value = True + await store.ensure_collection_deleted("docs") + mock_client.drop_collection.assert_called_once() + + +async def test_injected_client_not_closed(collection, mock_client): + await collection.aclose() + mock_client.close.assert_not_called() diff --git a/python/packages/milvus/tests/milvus/test_filters.py b/python/packages/milvus/tests/milvus/test_filters.py new file mode 100644 index 00000000000..541d6fee5f6 --- /dev/null +++ b/python/packages/milvus/tests/milvus/test_filters.py @@ -0,0 +1,56 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +import pytest +from agent_framework import Filter, FilterGroup + + +def test_equality_and_comparisons(collection): + assert collection._prepare_filter(Filter("text", "eq", "a")) == '(body == "a")' + assert collection._prepare_filter(Filter("text", "ne", "a")) == '(body != "a")' + assert collection._prepare_filter(Filter("number", "gt", 1)) == "(price > 1)" + assert collection._prepare_filter(Filter("number", "gte", 1.5)) == "(price >= 1.5)" + assert collection._prepare_filter(Filter("number", "lt", 2)) == "(price < 2)" + assert collection._prepare_filter(Filter("number", "lte", 2)) == "(price <= 2)" + + +def test_between_in_not_in(collection): + assert collection._prepare_filter(Filter("number", "between", [1, 3])) == "(price >= 1 and price <= 3)" + assert collection._prepare_filter(Filter("id", "in", [1, 2])) == "(point_key in [1, 2])" + assert collection._prepare_filter(Filter("text", "not_in", ["a", "b"])) == '(body not in ["a", "b"])' + + +def test_bool_literals(collection): + assert collection._prepare_filter(Filter("flag", "eq", True)) == "(flag == true)" + assert collection._prepare_filter(Filter("flag", "eq", False)) == "(flag == false)" + + +def test_and_or_not_groups(collection): + expr = FilterGroup("and", [Filter("text", "eq", "a"), Filter("number", "gt", 1)]) + assert collection._prepare_filter(expr) == '((body == "a") and (price > 1))' + + expr = FilterGroup("or", [Filter("text", "eq", "a"), Filter("number", "gt", 1)]) + assert collection._prepare_filter(expr) == '((body == "a") or (price > 1))' + + expr = FilterGroup("not", [Filter("text", "eq", "a")]) + assert collection._prepare_filter(expr) == '(not (body == "a"))' + + +def test_none_filter_is_empty(collection): + assert collection._prepare_filter(None) == "" + + +def test_unsupported_operator(collection): + with pytest.raises(NotImplementedError): + collection._prepare_filter(Filter("text", "contains", "a")) + + +def test_unknown_field(collection): + with pytest.raises(NotImplementedError): + collection._prepare_filter(Filter("missing", "eq", "a")) + + +def test_vector_field_rejected(collection): + with pytest.raises(NotImplementedError): + collection._prepare_filter(Filter("embedding", "eq", "a")) diff --git a/python/packages/milvus/tests/milvus/test_settings.py b/python/packages/milvus/tests/milvus/test_settings.py new file mode 100644 index 00000000000..cdac102f6d2 --- /dev/null +++ b/python/packages/milvus/tests/milvus/test_settings.py @@ -0,0 +1,58 @@ +# Copyright (c) Microsoft. All rights reserved. + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest +from pymilvus import MilvusClient + +from agent_framework_milvus import MilvusStore + +_CLIENT_PATH = "agent_framework_milvus._vector_store.MilvusClient" + + +def test_default_uri_when_unset(): + with patch(_CLIENT_PATH) as client_cls: + client_cls.return_value = MagicMock(spec=MilvusClient) + MilvusStore() + assert client_cls.call_args.kwargs["uri"] == "http://localhost:19530" + + +def test_uri_from_environment(monkeypatch): + monkeypatch.setenv("MILVUS_URI", "http://milvus.internal:19530") + with patch(_CLIENT_PATH) as client_cls: + client_cls.return_value = MagicMock(spec=MilvusClient) + MilvusStore() + assert client_cls.call_args.kwargs["uri"] == "http://milvus.internal:19530" + + +def test_explicit_uri_overrides_environment(monkeypatch): + monkeypatch.setenv("MILVUS_URI", "http://env:19530") + with patch(_CLIENT_PATH) as client_cls: + client_cls.return_value = MagicMock(spec=MilvusClient) + MilvusStore(uri="http://explicit:19530") + assert client_cls.call_args.kwargs["uri"] == "http://explicit:19530" + + +def test_token_and_db_forwarded(): + with patch(_CLIENT_PATH) as client_cls: + client_cls.return_value = MagicMock(spec=MilvusClient) + MilvusStore(uri="http://localhost:19530", token="secret-token", db_name="analytics") + assert client_cls.call_args.kwargs["token"] == "secret-token" + assert client_cls.call_args.kwargs["db_name"] == "analytics" + + +def test_token_from_environment(monkeypatch): + monkeypatch.setenv("MILVUS_TOKEN", "env-token") + with patch(_CLIENT_PATH) as client_cls: + client_cls.return_value = MagicMock(spec=MilvusClient) + MilvusStore(uri="http://localhost:19530") + assert client_cls.call_args.kwargs["token"] == "env-token" + + +def test_client_creation_failure_raises(): + from agent_framework.exceptions import IntegrationException + + with patch(_CLIENT_PATH, side_effect=RuntimeError("boom")), pytest.raises(IntegrationException): + MilvusStore(uri="http://localhost:19530") diff --git a/python/pyproject.toml b/python/pyproject.toml index 4c154ea4143..6d55122d24e 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -99,6 +99,7 @@ agent-framework-hosting-responses = { workspace = true } agent-framework-hosting-telegram = { workspace = true } agent-framework-hyperlight = { workspace = true } agent-framework-mem0 = { workspace = true } +agent-framework-milvus = { workspace = true } agent-framework-mistral = { workspace = true } agent-framework-mongodb = { workspace = true } agent-framework-monty = { workspace = true } diff --git a/python/uv.lock b/python/uv.lock index 13726f1d35c..d062c4bc491 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -52,6 +52,7 @@ members = [ "agent-framework-hosting-telegram", "agent-framework-hyperlight", "agent-framework-mem0", + "agent-framework-milvus", "agent-framework-mistral", "agent-framework-mongodb", "agent-framework-monty", @@ -848,6 +849,21 @@ requires-dist = [ { name = "mem0ai", specifier = ">=2.0.0,<3" }, ] +[[package]] +name = "agent-framework-milvus" +version = "1.0.0a261002" +source = { editable = "packages/milvus" } +dependencies = [ + { name = "agent-framework-core" }, + { name = "pymilvus" }, +] + +[package.metadata] +requires-dist = [ + { name = "agent-framework-core", editable = "packages/core" }, + { name = "pymilvus", specifier = ">=2.5,<3" }, +] + [[package]] name = "agent-framework-mistral" version = "1.0.0b260918" @@ -1631,6 +1647,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/39/3a0db1ba57531062cda616dae68b69faef8114939414a2a291b3aac7617b/botocore-1.43.107-py3-none-any.whl", hash = "sha256:23cbe854e815dbaccf097f7fd32b461c9e1d2ed7e0c7dcc5658218704509d840", size = 16002666, upload-time = "2026-10-01T19:31:14.463Z" }, ] +[[package]] +name = "cachetools" +version = "7.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/44/71476a5812da1ddf2c9a3efd31ae76d01480a1cf03ed13ac28aa8f2402e4/cachetools-7.2.1.tar.gz", hash = "sha256:b1a7537025c06abf96fcc1443e496af9a3fb95e774e70e1f0af226f73f7f2dcc", size = 41357, upload-time = "2026-10-05T18:40:06.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/c9/2a61d784caf0d869a3326728c57c7203f50cc53f3cca2ee76bf924769eb4/cachetools-7.2.1-py3-none-any.whl", hash = "sha256:63aa53dfe7473c10cccdd5a01dedf76ef2c4b73a58840d9396e7d0752cbdac3b", size = 17006, upload-time = "2026-10-05T18:40:04.827Z" }, +] + [[package]] name = "certifi" version = "2026.7.22" @@ -4036,6 +4061,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/6c/d8a02ffb24876b5f51fbd781f479fc6525a518553a4196bd0433dae9ff8e/orderedmultidict-1.0.2-py2.py3-none-any.whl", hash = "sha256:ab5044c1dca4226ae4c28524cfc5cc4c939f0b49e978efa46a6ad6468049f79b", size = 11897, upload-time = "2025-11-18T08:00:41.44Z" }, ] +[[package]] +name = "orjson" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/72/380b97dc45bd162d23afe5194721ef678d9eac7cfaa549fe2873f7f0a518/orjson-3.13.0.tar.gz", hash = "sha256:d1de5eb04485110c5da4c657e49168995d55e076b1ce60f1a042e254f4186c4f", size = 2732604, upload-time = "2026-10-07T14:09:25.719Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/a3/0be3b115907fea61ed340639fb0e1562cd18969bad5b3f486f808197aaff/orjson-3.13.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:948bad47f2e2e43527f14248364a0e5dee26dd3184691010ec4a1ebeb0fd6771", size = 223146, upload-time = "2026-10-07T14:08:06.474Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f7/665935edb16163f8b764182e29a30cf056947a66893ed032191e5f01eb3d/orjson-3.13.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:1807c2fa49d393c7ee95fd1ef1b39cbb24aa3ccd81f30b84503ba59407666960", size = 123546, upload-time = "2026-10-07T14:08:08.324Z" }, + { url = "https://files.pythonhosted.org/packages/67/ec/e7cde480c0e212594d17ba2b2bd210c002052e9147fc1a1aeafaabe722fb/orjson-3.13.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:637dbca1fccffe83780e806fbc0f17427c0c59bf822528eb0acc8f0aa9f19acb", size = 113290, upload-time = "2026-10-07T14:08:09.816Z" }, + { url = "https://files.pythonhosted.org/packages/36/59/4455fb11a297af73611dfc437f0f89456220227ed1cb1544a5a0ee9d6c03/orjson-3.13.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:554948becd1110123ef9f6a6e1310fd92b2d07d2cbac6dbf65df3de75702e736", size = 130342, upload-time = "2026-10-07T14:08:11.253Z" }, + { url = "https://files.pythonhosted.org/packages/ca/80/0eec5fbde2e52407646b4cb3118f63175bdcee1e2390c2759dc96e0bc62a/orjson-3.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9d9a101bd8dbfad112170f009cd155e52bb8c936468821a0d03cbb96c0e426", size = 129138, upload-time = "2026-10-07T14:08:12.814Z" }, + { url = "https://files.pythonhosted.org/packages/cd/cc/c0874f13819ae346d69ca00d074d464710b494abd4442bdebf75ac404a98/orjson-3.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89bcf2d4bc6c9a7e1763c8cf534f38712e66b76a0fefda7fb7785462f0d635e4", size = 130518, upload-time = "2026-10-07T14:08:14.392Z" }, + { url = "https://files.pythonhosted.org/packages/25/ab/140dd9adff84bf64b862c4fcfe2d055af6014d5ba03a075f95c9addb2ec7/orjson-3.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a79cdc4934fe81f593072c94e13da3095e9d41c2deef8f6ff2901794ca1c5042", size = 134924, upload-time = "2026-10-07T14:08:16.09Z" }, + { url = "https://files.pythonhosted.org/packages/08/0a/e8f6deb032b1d98a39043cf99b863d8b9e842e2ffc2d2067d2e2a88c18e4/orjson-3.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:50a5202ba388b3850ba24437951727d3aa6d79a21964a30ae8dc6a059a5fd34c", size = 126704, upload-time = "2026-10-07T14:08:17.439Z" }, + { url = "https://files.pythonhosted.org/packages/af/cf/be64b99ff75f7983488390d4ef5df72115119770eed295691c0a715d492a/orjson-3.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0377d6962fa431c93ecd78fdea771bb62ec545b24ee0c5d4e32acf2260af259", size = 121287, upload-time = "2026-10-07T14:08:18.843Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/1b8ca186baf3420f12db1f2819fcc5f2cae69e4cf051168501726a64c0fa/orjson-3.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:1d84820b2ec4ac975cba482214032de5b0dbdd17046170c98e642ef9c4a4ee4b", size = 126314, upload-time = "2026-10-07T14:08:20.452Z" }, + { url = "https://files.pythonhosted.org/packages/98/17/ed65f84ed5ed6a1e06eb628611b4172e7480fc4ad92594856751a6363cac/orjson-3.13.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:fb8644dc6d705e1269ed2842bf4dbe2b4e50d670de503bf79d5cef3a5148a4c7", size = 223063, upload-time = "2026-10-07T14:08:21.979Z" }, + { url = "https://files.pythonhosted.org/packages/6f/4d/9332eb96d2e379384be0f211f543835eebc81f460c9403b84abe1294c431/orjson-3.13.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:6ff2a2c67f35202f7d823753d38ad371a9b7fc297567cdfff4420e763cb9f6f8", size = 123364, upload-time = "2026-10-07T14:08:24.026Z" }, + { url = "https://files.pythonhosted.org/packages/b4/06/558456b7da27e974a8c9ea09117b07119f6fa131cd62b8b9ecad9eea94e1/orjson-3.13.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:65c4e0e106ccc7265b488385659117a6805c37d042f737558ecd68aa0c67ad8f", size = 113199, upload-time = "2026-10-07T14:08:25.476Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f2/1187a9c09965620348262ec0f406868f6d7c234b2e9b5ee51020bdde5748/orjson-3.13.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:fbbad6b9b1da43f25c1f5b20cd5a268e028a2fc95d5a8d1ade6059973bc71584", size = 130329, upload-time = "2026-10-07T14:08:26.877Z" }, + { url = "https://files.pythonhosted.org/packages/46/07/5d1a151bc11600434fe799e73abfc6a4d463d02e149a20e47c59d3a985ae/orjson-3.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae1d895cf7bbfd50ef34bb63bb727b14514f259f3e3f8dd010783bd38e864c6e", size = 129072, upload-time = "2026-10-07T14:08:28.355Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8c/bb07c368abbf4021c4cd01c12edb526e00090f7f750ff1b88da6e6b6c7a6/orjson-3.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bceadfd314bd238f584fc229a4bbaf0e573597e7a026dec5429fbf29fd66c641", size = 130612, upload-time = "2026-10-07T14:08:30.041Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8d/4b66d19619ed344ac000ffea7c006477d0061d580646e736ef0e203759e8/orjson-3.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b74c30e56346aad067937d766846ee74c231d1d18aad3f324e9b9261de3b2d5e", size = 134632, upload-time = "2026-10-07T14:08:31.474Z" }, + { url = "https://files.pythonhosted.org/packages/ea/88/f8221f6593e37eb26ec4706e185b9ac6f38ff0c8f7bad5459844031ffd2d/orjson-3.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4329c19b8a25693f60a77b867c9d2a3ab637b20e36f5b7bea7f5acb492b44b15", size = 126807, upload-time = "2026-10-07T14:08:32.914Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/a1ca7321eeafd7d72e174cdc388cc96301f41516d863e7b1f64f0a1735be/orjson-3.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:b571236d8393edcd3236e07423f762bfcf571f852aad667a3bce9e7b755e0790", size = 121538, upload-time = "2026-10-07T14:08:34.325Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a0/1f19b4779c910104370932fceb9ed436b47ac077f297db74008062525c04/orjson-3.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:8594956a75223f657e1e68c568c0eeb3dd145f02cd6b78a47fd9a8095dbc4eae", size = 126259, upload-time = "2026-10-07T14:08:35.765Z" }, + { url = "https://files.pythonhosted.org/packages/a9/56/f8ad2546150168858c16915c452b00eecb79597597524d1ad6ae14ad4eab/orjson-3.13.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:64e8f345048d988c8b68d3882e5d41028fca1219a9939b32e4a77be34c8ae8e3", size = 222892, upload-time = "2026-10-07T14:08:37.495Z" }, + { url = "https://files.pythonhosted.org/packages/1f/19/725d23160b2471a3f27026c55bb79af34687652d8be8f5f583cee5dcd42f/orjson-3.13.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ded33b972cffdaf4ca0ac917338ab61d2bb10d68987dbcae641c313fbfdbf499", size = 123319, upload-time = "2026-10-07T14:08:38.989Z" }, + { url = "https://files.pythonhosted.org/packages/ac/08/e5d81a00b22c73dfcb60d80da3bd92d5a7684346593536565f184dbae3c9/orjson-3.13.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:45e34deb3437509f4ec9888dd9ee5dc426cfe21be10f1eb4ea3a9e4d33034f9e", size = 113196, upload-time = "2026-10-07T14:08:40.383Z" }, + { url = "https://files.pythonhosted.org/packages/67/78/fda6117c69a43e470b1e9dff38dd8c5f0bc6fd8a47e4d4561ab023039335/orjson-3.13.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9825b954155b345c4759f24e5f8d652b9aec2261bb5d4e1abe06bba0a1200535", size = 130245, upload-time = "2026-10-07T14:08:41.878Z" }, + { url = "https://files.pythonhosted.org/packages/6d/31/d0cfebd456defb234414795ae7599696bf124843dfe077d0c9ece0c93554/orjson-3.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b081f0e7b600ff24513dec4ca75507fa05e904607847e386e8310d5b7b96b6c7", size = 128981, upload-time = "2026-10-07T14:08:43.716Z" }, + { url = "https://files.pythonhosted.org/packages/45/46/f8d83189ff5b7b2ff225a58c5908618cc4e86afe09e65d17a30ac68c9da4/orjson-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbed5f4c4b88d94bcc36115f4c3bb3aa25da1563a5c3328aa3acebce2b083040", size = 130370, upload-time = "2026-10-07T14:08:45.132Z" }, + { url = "https://files.pythonhosted.org/packages/e6/6a/d6344c305003ea826b3fa0482645a897a3cd6d477ed74e1fe15d3322cb23/orjson-3.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e9b61676116f755126b90e740a9cff36b91562f47ec330056cc88cc3b9f02f4b", size = 134595, upload-time = "2026-10-07T14:08:46.63Z" }, + { url = "https://files.pythonhosted.org/packages/9f/52/d73fa44f88d53e02d10de1cf77c16ed13204ff5bca47e1692da6b406619c/orjson-3.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3ef75ed7e81dae34a3649f82df52cd85f9ac839a7d6ec78ab355b33b3b27ef7f", size = 126513, upload-time = "2026-10-07T14:08:48.111Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f8/bcfc50b4ab851c4f9c0ee62f52bf3b28f0bcd0d9fe08e0ad98d4585148db/orjson-3.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:4ee06e53b998c71ce3eb93b86222912fdd9dcced685ac64d4525d36fac338ea4", size = 121371, upload-time = "2026-10-07T14:08:49.549Z" }, + { url = "https://files.pythonhosted.org/packages/7b/7a/d6927845712ec2b1e89263cd12d7203531db185dbad67f914226f2fca156/orjson-3.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:89efecad02515df7f318d0613b5dfd6d2a1acd323a2b8294712789a715945525", size = 126134, upload-time = "2026-10-07T14:08:51.118Z" }, + { url = "https://files.pythonhosted.org/packages/f0/10/98b5a3cdc086abf78d8cd20bb0cba124485d4b6a745722197bd209d967a5/orjson-3.13.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a7bfc7db961c7d96cb75889dc6a1e4ae1e91d87ee61da564f582bd742b8dfeef", size = 222889, upload-time = "2026-10-07T14:08:52.673Z" }, + { url = "https://files.pythonhosted.org/packages/22/7c/7728c5280ab5202f4891ff4b0b96e2e1dbd5520dfee53edf083c54409a64/orjson-3.13.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:91d933e668ff0ffe164d7c2daec36beba6d1ce7fadb71538fbe142a71f8a1e6e", size = 123312, upload-time = "2026-10-07T14:08:54.25Z" }, + { url = "https://files.pythonhosted.org/packages/a9/a5/d9a44321e6f66c0f64b45be587395f87ad94cb447bce7d92286f6b97d46a/orjson-3.13.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:6c8bfe728b81b0fd58a3c7f3f9c5a113f87f2992c9948e0f28707aafd737c0bc", size = 113146, upload-time = "2026-10-07T14:08:55.803Z" }, + { url = "https://files.pythonhosted.org/packages/80/da/d95c80d413f288feb471e16d82e5c1512d2439728e3bac917d058c31f098/orjson-3.13.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e8e05549f3b30f9d8a8e28c5aba11cc2a4b90b90961ec685ca58444b0815fc09", size = 130348, upload-time = "2026-10-07T14:08:57.31Z" }, + { url = "https://files.pythonhosted.org/packages/04/0f/36fdfb32ad1852997bac00e3ce52c7888d8a1094ba9dcdcbb22fcc6b953a/orjson-3.13.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c749ab3ac30b5ab1ffb7677f8b92eacfdfdc5260210baa398f845bc3714c05d8", size = 128971, upload-time = "2026-10-07T14:08:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/25/de/a82acf93bdcca0c79ccff25ef0c6868d24ccbc2e72f21fae39c8cabce4f1/orjson-3.13.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58a9619d88f8818d9ab6b39d70d203789457ba13c1ed5d274f33ce9ae7e81a36", size = 130359, upload-time = "2026-10-07T14:09:00.412Z" }, + { url = "https://files.pythonhosted.org/packages/71/ca/2bc4f7697cb9f6897bf61aca11803df096a5d971bf69ef5538b243bb1fa8/orjson-3.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2715c4808d1571029ed18fd07a82140bf3ba7def0dc89f8d015c416e3649bf87", size = 134583, upload-time = "2026-10-07T14:09:02.047Z" }, + { url = "https://files.pythonhosted.org/packages/23/b3/12b1af9b87ff9fa0aaf4e5724c87672b30bb5de76f275f7fac64e8219c1b/orjson-3.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08bf722f923d2100bc5e5a5dcf72c656db557049c1bea26582fdd5dd9d5395a1", size = 126500, upload-time = "2026-10-07T14:09:03.863Z" }, + { url = "https://files.pythonhosted.org/packages/ad/ea/cf257fc8a7f4b18f5677c22b3a9673a1b51d4b7161f25177ed389b76560e/orjson-3.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:6adcaa85d79977659a448b4123a88eb33511a11ed2db243535ad7ea88a6668e0", size = 121378, upload-time = "2026-10-07T14:09:05.375Z" }, + { url = "https://files.pythonhosted.org/packages/05/0a/9f4643f849e9918eab11983b83928af3aac14bedb04002e28e885ee1936f/orjson-3.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:83705c12b4afde10c62a5dd3fe6fdb21b7900bd0dcd5af1c85612ae94d0ee590", size = 126123, upload-time = "2026-10-07T14:09:07.085Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -4045,6 +4118,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "pandas" +version = "3.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/17/d7b106e05bfa642e8694451e7d3d759c6a241c5386a5d962e4f66c047e06/pandas-3.0.6.tar.gz", hash = "sha256:66b07ef7315a31bfe1089cd3d71a7de781c9dca986762d0b4fe7c0ef17465d10", size = 4667686, upload-time = "2026-09-17T23:23:18.345Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/48/88e8d250d28efa8163294f6809a71683c7ee67f63ac7c33021c0503b3547/pandas-3.0.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:085e3786ae6b2e82b406266bce36690f72b9dc1421903ba9296b2981a9fcf586", size = 10391798, upload-time = "2026-09-17T23:20:20.96Z" }, + { url = "https://files.pythonhosted.org/packages/55/a6/39db5d41f3eb5d7846626312cb30e0cea48e988fc47a3725698bc931ad3c/pandas-3.0.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d7564d86a94c2eb8ab290b07f63ddaae5c032fa53897c29a2ff2197d43aee8af", size = 10023718, upload-time = "2026-09-17T23:20:25.094Z" }, + { url = "https://files.pythonhosted.org/packages/54/b7/707e966129f77ee8a39d41a41bbd989b790b3fef581c6e26b821315cba6b/pandas-3.0.6-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e7c0afdcaf6661d795fcefc2f647ddd1136f62cdc153fba177c685d97a87808", size = 10612800, upload-time = "2026-09-17T23:20:27.99Z" }, + { url = "https://files.pythonhosted.org/packages/63/be/dfb6cc9329d0bbe76dadda8a1113d026bb996ec76c509221368dc68349f4/pandas-3.0.6-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47121f9571503f724c9b93e297ab6254ac99c77adf5e9ed085ea419fd585c258", size = 11108900, upload-time = "2026-09-17T23:20:30.65Z" }, + { url = "https://files.pythonhosted.org/packages/1a/6f/3d58f15bbe972f3d7bfa13ee06d731785a76fe8d232c92b2655a8de14127/pandas-3.0.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:994a79608263fe1c14cc48ffa7300e2b834b7d1cb406ffe96a08828cb0cdd79b", size = 11630226, upload-time = "2026-09-17T23:20:33.582Z" }, + { url = "https://files.pythonhosted.org/packages/58/54/9b494de4a3dd92fc6eb19187db1b21afb50fdc21962f32ea1aca9f270cb2/pandas-3.0.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a3a22e07fe75347eaacc75b0e85297947af4fba6b4aae23916bd8b6828d0bba3", size = 12146942, upload-time = "2026-09-17T23:20:36.792Z" }, + { url = "https://files.pythonhosted.org/packages/d3/dc/d2df02854aec5d47659acfb2be352eecc691845b2f86e99c84f1010a8671/pandas-3.0.6-cp311-cp311-win_amd64.whl", hash = "sha256:2e5fa32ff162dfdbc280157d664f44d23049ae414725af9676df339c501d82cd", size = 9859246, upload-time = "2026-09-17T23:20:43.976Z" }, + { url = "https://files.pythonhosted.org/packages/79/1e/2a30df0d7dede5c195300a1820b0bc21cea3aa24e4c8b6c4431565ecc79a/pandas-3.0.6-cp311-cp311-win_arm64.whl", hash = "sha256:5e75072773c1b2f7cb63faa3a6f562aede11f3976f68ed34cb538bc091a28171", size = 9106923, upload-time = "2026-09-17T23:20:46.611Z" }, + { url = "https://files.pythonhosted.org/packages/77/4c/597d588c055d4373cff19cbbc32d4dd046c7be8fadee957585b5ba9e5b24/pandas-3.0.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7dac2d65e9087e8e7b5a45fe15c4920911a221df061ab629943ce016489145c7", size = 10397411, upload-time = "2026-09-17T23:20:49.465Z" }, + { url = "https://files.pythonhosted.org/packages/18/8f/48907c7c707b61a8e5018c32e1a3f70623209bfb59020a2f6196159d3aa7/pandas-3.0.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9dab635a549e58a053c7b0fa054dc0bd7be22f0ed9a720f4a85d5fb993276172", size = 10051279, upload-time = "2026-09-17T23:20:52.409Z" }, + { url = "https://files.pythonhosted.org/packages/67/fa/613d867c3d9554a61bafdec6f79565c8a3e73235feb52cc4a72ad2e0fa6a/pandas-3.0.6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3dccb584123b399c07562ac4d62543e90ede49ddf8ce3c13ffc64cbe828c281", size = 10303429, upload-time = "2026-09-17T23:20:55.597Z" }, + { url = "https://files.pythonhosted.org/packages/cb/67/0c0f18e38d7f2d2af8c24b3315bc4046e73bbdd4a5540405506671ad0c0d/pandas-3.0.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0704044b676496b8350e023b09f174a26772456c974a2b11c36bebb558c9490d", size = 10788193, upload-time = "2026-09-17T23:20:58.617Z" }, + { url = "https://files.pythonhosted.org/packages/39/53/1b57f3162501fe36687e4870e1b918a6458ca7386b173af75663ace95857/pandas-3.0.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e7c1905ef02c3d6d43d9dbd5b6ccb4da4870a0b0c821bbc103fbdb6f3ad2707b", size = 11323501, upload-time = "2026-09-17T23:21:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d2/b1182e8d39100369d7f13f4a125a3fb6b096fef112c46d0566c25781ff68/pandas-3.0.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:569e114072b24fc4970c12e2b4bab252671668a40b324318903380cab0254c0c", size = 11838173, upload-time = "2026-09-17T23:21:04.85Z" }, + { url = "https://files.pythonhosted.org/packages/c7/33/5b717af24d2f27995e51e0875a269dddd373045216e34cf62e3aa764eaa1/pandas-3.0.6-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:2a8fc94be2ee5f1d86f97aacd8cc566f81680b6498e76f3007421bb5d98151bf", size = 7118499, upload-time = "2026-09-17T23:21:07.661Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/14b3b17cd75cef4a1ee1af4234eb41cc1afe4103b98c2d80e8916abfd42b/pandas-3.0.6-cp312-cp312-win_amd64.whl", hash = "sha256:3ef908d28590b3f42d7070e7ad8f9b34b442b260b7f3c1afb57e0040c58cdb1b", size = 9661439, upload-time = "2026-09-17T23:21:10.959Z" }, + { url = "https://files.pythonhosted.org/packages/3b/11/3d580a604a1e35d69f6676847bd12db7d14344bc677b717f4413e79c5d0d/pandas-3.0.6-cp312-cp312-win_arm64.whl", hash = "sha256:f4e7c52eb108d752e7592268108fd3e98efd76d83a3125cdd06c621c2e44359b", size = 8959806, upload-time = "2026-09-17T23:21:13.851Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/143605a1f6443ad50ebda78a31e5a3a10147fec2590e931584aaa5ff0a09/pandas-3.0.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ae8073aed8e21d1a7fe263dcdc6840743549722a6738198a0a46000fa9476f2", size = 10418900, upload-time = "2026-09-17T23:21:16.594Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ca/87f8548f73d452aab35e4a90f8b39ae303295e0f2ef0b4055c44d6b3f1be/pandas-3.0.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:60d81f9e1799b36f3739e7fff44d1fbb2e8fd5a271b3863e03de9715fccda0fa", size = 10064785, upload-time = "2026-09-17T23:21:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/43/1a/d951442e5607c6e3b2462eff8f420797d428aa74b87c6ecfe4f48553626e/pandas-3.0.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:097090508a1dd335013d39106fc10b20f4fd4a171638e47b77d55798ed9dab6c", size = 10245290, upload-time = "2026-09-17T23:21:22.797Z" }, + { url = "https://files.pythonhosted.org/packages/50/fa/96d50e1e6cd0b08b5e2b7c838f65ae644940f75a124063380b5ef73b6866/pandas-3.0.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e92d9fa834c7d877130027cddc0cad8dcff97c1f6cca26bd6310f847228b658", size = 10757657, upload-time = "2026-09-17T23:21:25.673Z" }, + { url = "https://files.pythonhosted.org/packages/7b/12/f82d13a2cb703e1a8acee7e01fdc2b898d9cd0c00f07d1dfce63af43e350/pandas-3.0.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b27c8d890e4aa2171437ae2a39de1d215e674158e4865c4023a8b31c932513b2", size = 11249114, upload-time = "2026-09-17T23:21:28.898Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ce/8aef2e561a2f2c8b38c913c67373c65ba6748174e763d27c80271b24bd17/pandas-3.0.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8029ec0f1f89e4f985929ce1f6626dabf3140d61a4e9c1215afdab34eaf9a5d", size = 11820511, upload-time = "2026-09-17T23:21:32.11Z" }, + { url = "https://files.pythonhosted.org/packages/c0/bd/63cb67e6903ef6d9c2871916dbcbc09d254da0fe8b870cf62e16b21945f2/pandas-3.0.6-cp313-cp313-win_amd64.whl", hash = "sha256:f3ce8a6968045481e91a3990e797e348ce13db45ee164a7095bbc824e26c09dd", size = 9638092, upload-time = "2026-09-17T23:21:34.883Z" }, + { url = "https://files.pythonhosted.org/packages/75/2e/e7b35b712edb068d382ddc8b2bea8a04974100515ba2daa22b478b265842/pandas-3.0.6-cp313-cp313-win_arm64.whl", hash = "sha256:cc39303913e2ea129915670de5d1c9fbd647f543bb72e5543bac8baa94e9e42f", size = 8952032, upload-time = "2026-09-17T23:21:37.729Z" }, + { url = "https://files.pythonhosted.org/packages/75/55/1a8875395b05ccd572cbca0b9255dcd2db6e6508e632a558c1a6884b39ad/pandas-3.0.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ee913a91669056c1de1a6b733fbfeab711de9e54e3bee2dfa5fe79d9457247d1", size = 10492213, upload-time = "2026-09-17T23:21:40.746Z" }, + { url = "https://files.pythonhosted.org/packages/35/61/47ae13476995cc8a40cd609e93e7cf11f273d8692925c2903cb6d38aa0d1/pandas-3.0.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ff51a4459ed036e93d1eb1bb5e6e7b28685d3cb6b7c12b91c05b31024e234729", size = 10156618, upload-time = "2026-09-17T23:21:44.142Z" }, + { url = "https://files.pythonhosted.org/packages/bc/f2/cc5f2adb8d6e86a85d9fb5128f8cf205a61189336f70d1f7faf0d1b53ec9/pandas-3.0.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:654aae059295dbba6ecd2328ca12712a2cf1676214c8699f1c29213f7ccf9c34", size = 10375479, upload-time = "2026-09-17T23:21:47.159Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ba/ffdcb19be4ff6bfe7d969e7cef2c567c633df5a3a1cc1053394ad053bca8/pandas-3.0.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f51d7f651c8054c5e82a69265c98082e795d1442df7ca6edc3a545d61214b1", size = 10783244, upload-time = "2026-09-17T23:21:50.367Z" }, + { url = "https://files.pythonhosted.org/packages/77/5b/e150075b2c6eb69fae896f2d9239bc6ed07db97735971d53d66de6553460/pandas-3.0.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:22172a92e7ee678ec0140c7af4fc9366b55413834a1cd86af78b3caa0b0574de", size = 11382223, upload-time = "2026-09-17T23:21:53.355Z" }, + { url = "https://files.pythonhosted.org/packages/d6/8a/b441c587dc7355bf6e1f68a91b4f76a6c29740f0c23be3acc5d4ebbeea6d/pandas-3.0.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:583be68728a31d0d750d5b8d9e00f02b153df0d4655f858bde93cb84cfc4227c", size = 11854881, upload-time = "2026-09-17T23:21:56.342Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e9/f43410fada510b43fec09993c08f552086c3d247d3ee801a678f3cb10ea5/pandas-3.0.6-cp314-cp314-win_amd64.whl", hash = "sha256:77ccbe5057aece6fc172b9b77f19c04335af6882bc2e10c8f3ee4e6bfb3da553", size = 9791672, upload-time = "2026-09-17T23:21:59.332Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9e/db14c059c21f9baa1907d436f8bf30e0c76c6288225c5e8b79a08ba8b2c5/pandas-3.0.6-cp314-cp314-win_arm64.whl", hash = "sha256:fb625f426b375bcc96e3a04c5d5d266cd7be6ae5d6866e0e703382ab5164068c", size = 9121246, upload-time = "2026-09-17T23:22:02.123Z" }, + { url = "https://files.pythonhosted.org/packages/67/ba/bad0f8dac020ab38a8637fddab01a57a82da7a496a6e6f19590aad53ab62/pandas-3.0.6-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9e492cd4bdba6778de4fe0df7f4590c012161ebcf9902dce01b01dc683105514", size = 10920612, upload-time = "2026-09-17T23:22:05.404Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a9/b500982e9aac6d52a58da4ad3f11e14168a315b06906b3f397c427878065/pandas-3.0.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d7dcd21238cbb4828ff148481ba01cac8946dc5121457b5aeba28636f8f99a60", size = 10570213, upload-time = "2026-09-17T23:22:08.44Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fa/e6ecd0073c98be8f840ac3125b955272835d7d9fd69f5944383b164deb5e/pandas-3.0.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ff482fa91fa2bafd92e8fe66ce3645c851824310f295c1f0a2f96e928fc4541", size = 10252356, upload-time = "2026-09-17T23:22:11.302Z" }, + { url = "https://files.pythonhosted.org/packages/04/f5/001e230a7a7803590d9275a1a3f7e1bb605e3a495cfe5e8d3a532090621b/pandas-3.0.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:db7ec631f26223beee8e5c9e0b8f23c24d8197bbd1d982421d4e3188bea51965", size = 10655952, upload-time = "2026-09-17T23:22:14.283Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/bab587148a3852c96aae26c4b5f9e04ce2221801ad94e166b4fbf969ede0/pandas-3.0.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd75ed0c840f709fc2ae26ddd9534ac77ca1a48ac0cce521a74acaa85f3340a7", size = 11274177, upload-time = "2026-09-17T23:22:17.352Z" }, + { url = "https://files.pythonhosted.org/packages/f3/32/74b48d87df2b80892d713c149abfe36d5db4de41b4eccb042a2bc07dafc1/pandas-3.0.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ef738d71d1059245b6bb03e312be06d8b3821326a83486c1ad03b9aba3710e44", size = 11719041, upload-time = "2026-09-17T23:22:20.227Z" }, + { url = "https://files.pythonhosted.org/packages/f6/c6/d64b72d64d7eb0fad9fe424d34138e45dee70ddbea1360dcd0adf30e28f6/pandas-3.0.6-cp314-cp314t-win_amd64.whl", hash = "sha256:429d9df32731ab01383ed98f2baa7a60368090d1a94fc06019a12062510e8630", size = 10191388, upload-time = "2026-09-17T23:22:23.524Z" }, + { url = "https://files.pythonhosted.org/packages/7a/30/5e5b2ccabeca73ae2b03fc82bca3eabb7466cf43737f05ac08d591665d47/pandas-3.0.6-cp314-cp314t-win_arm64.whl", hash = "sha256:a4dbd4dc65cbe645b92b8785d0f96dd7311010dc6606cf620e51b07b8788a12a", size = 9387796, upload-time = "2026-09-17T23:22:26.64Z" }, +] + [[package]] name = "pastel" version = "0.2.1" @@ -4673,6 +4801,24 @@ crypto = [ { name = "cryptography" }, ] +[[package]] +name = "pymilvus" +version = "2.6.17" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "grpcio" }, + { name = "orjson" }, + { name = "pandas" }, + { name = "protobuf" }, + { name = "python-dotenv" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/22/7dad24082f7efa4bf5ff29f0d9241cec6a8c8ebeb08ad126ca7ca4793da7/pymilvus-2.6.17.tar.gz", hash = "sha256:d9f849ef177f242febc4c02ece88feea46a1a9a1d0c67688b600e7cce3b76f43", size = 306908, upload-time = "2026-07-17T06:23:32.592Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/a6/9e2824138ffe1b2932c24f836638dabff5c6fbb5a9afcc1ba5ac5cdc39d8/pymilvus-2.6.17-py3-none-any.whl", hash = "sha256:583c680bea60b5944762ae81dd577169efe3b895b579d78e60bab05c087c4f89", size = 342148, upload-time = "2026-07-17T06:23:31.188Z" }, +] + [[package]] name = "pymongo" version = "4.18.2" From 73796abec85265934f86374cc06a934d74e6ac40 Mon Sep 17 00:00:00 2001 From: Nanduu24 Date: Tue, 13 Oct 2026 12:53:49 -0500 Subject: [PATCH 2/7] Python: address review feedback on the Milvus vector store - ensure_collection_exists: handle concurrent creation (treat an already-created collection as success instead of failing the ensure operation). - Filter string literals are serialized with json.dumps so quotes, backslashes and control characters cannot break out of the Milvus expression. - Reject is_indexed / is_full_text_indexed on data fields until scalar/full-text index creation is implemented, rather than silently ignoring them. - Validate score_threshold as a finite, non-boolean number before searching. - Qualify the Milvus Lite (local .db) docs: it requires the pymilvus[milvus_lite] extra; fix the README usage snippet (sequence upsert, generate_vectors, subscript SearchResponse fields). - Add live integration tests exercising server_collection (CRUD, search, filters), selected by -m integration. - Add a Milvus PR unit-test job and path filter so regressions are caught on PRs. - Register the package in PACKAGE_STATUS.md (alpha / no). --- .github/workflows/python-merge-tests.yml | 26 ++++++++ python/PACKAGE_STATUS.md | 1 + python/packages/milvus/README.md | 22 ++++--- .../agent_framework_milvus/_vector_store.py | 47 +++++++++++---- .../packages/milvus/samples/milvus_vectors.py | 7 ++- .../milvus/tests/milvus/test_collection.py | 31 ++++++++++ .../milvus/tests/milvus/test_filters.py | 5 ++ .../milvus/tests/milvus/test_integration.py | 59 +++++++++++++++++++ 8 files changed, 178 insertions(+), 20 deletions(-) create mode 100644 python/packages/milvus/tests/milvus/test_integration.py diff --git a/.github/workflows/python-merge-tests.yml b/.github/workflows/python-merge-tests.yml index 7872a72bae5..b3dc4c5b70d 100644 --- a/.github/workflows/python-merge-tests.yml +++ b/.github/workflows/python-merge-tests.yml @@ -41,6 +41,7 @@ jobs: cosmosChanged: ${{ steps.filter.outputs.cosmos }} duckdbChanged: ${{ steps.filter.outputs.duckdb }} qdrantChanged: ${{ steps.filter.outputs.qdrant }} + milvusChanged: ${{ steps.filter.outputs.milvus }} mongodbChanged: ${{ steps.filter.outputs.mongodb }} redisChanged: ${{ steps.filter.outputs.redis }} githubCopilotChanged: ${{ steps.filter.outputs.github_copilot }} @@ -98,6 +99,9 @@ jobs: - 'python/packages/qdrant/**' - '.github/workflows/python-merge-tests.yml' - '.github/workflows/python-integration-tests.yml' + milvus: + - 'python/packages/milvus/**' + - '.github/workflows/python-merge-tests.yml' mongodb: - 'python/packages/mongodb/**' - '.github/workflows/python-merge-tests.yml' @@ -144,6 +148,27 @@ jobs: - name: Test DuckDB package run: uv run poe test -P duckdb + python-tests-milvus: + name: Python Tests - Milvus + needs: paths-filter + if: > + github.event_name == 'pull_request' && + (needs.paths-filter.outputs.milvusChanged == 'true' || + needs.paths-filter.outputs.coreChanged == 'true') + runs-on: ubuntu-latest + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up python and install the project + uses: ./.github/actions/python-setup + with: + python-version: ${{ env.UV_PYTHON }} + os: ${{ runner.os }} + - name: Test Milvus package + run: uv run poe test -P milvus + python-tests-qdrant: name: Python Integration Tests - Qdrant needs: paths-filter @@ -980,6 +1005,7 @@ jobs: needs: [ python-tests-duckdb, + python-tests-milvus, python-tests-unit, python-tests-openai, python-tests-azure-openai, diff --git a/python/PACKAGE_STATUS.md b/python/PACKAGE_STATUS.md index dd68cb9bff2..fe06482c49a 100644 --- a/python/PACKAGE_STATUS.md +++ b/python/PACKAGE_STATUS.md @@ -52,6 +52,7 @@ column records whether `pip install agent-framework` installs the workspace pack | `agent-framework-hyperlight` | `python/packages/hyperlight` | `beta` | `yes` | | `agent-framework-lab` | `python/packages/lab` | `beta` | `no` | | `agent-framework-mem0` | `python/packages/mem0` | `beta` | `yes` | +| `agent-framework-milvus` | `python/packages/milvus` | `alpha` | `no` | | `agent-framework-mistral` | `python/packages/mistral` | `beta` | `yes` | | `agent-framework-mongodb` | `python/packages/mongodb` | `alpha` | `no` | | `agent-framework-monty` | `python/packages/monty` | `beta` | `yes` | diff --git a/python/packages/milvus/README.md b/python/packages/milvus/README.md index 1767cda28c3..aefc36d509d 100644 --- a/python/packages/milvus/README.md +++ b/python/packages/milvus/README.md @@ -11,6 +11,13 @@ other vector-store connectors. pip install agent-framework-milvus ``` +This connects to a Milvus server. To use a local `.db` file (Milvus Lite) instead, also +install the Milvus Lite extra, which is published for Linux and macOS only: + +```bash +pip install "pymilvus[milvus_lite]" +``` + ## Usage ```python @@ -20,18 +27,19 @@ from agent_framework_milvus import MilvusStore definition = VectorStoreCollectionDefinition([ VectorStoreField("key", name="id", type_="int"), VectorStoreField("data", name="text", type_="str"), - VectorStoreField("vector", name="embedding", dimensions=1536, distance_function="cosine_similarity"), + VectorStoreField("vector", name="embedding", dimensions=3, distance_function="cosine_similarity"), ]) # Connects to a Milvus server; set MILVUS_URI or pass uri=... -# A local ".db" path uses Milvus Lite. +# A local ".db" path uses Milvus Lite (requires the pymilvus[milvus_lite] extra). async with MilvusStore(uri="http://localhost:19530") as store: collection = store.get_collection(dict, definition=definition, collection_name="docs") await collection.ensure_collection_exists() - await collection.upsert({"id": 1, "text": "hello", "embedding": [...]}) - results = await collection.search(vector=[...], top=5) - async for hit in results: - print(hit.record, hit.score) + # upsert takes a sequence; pass generate_vectors=False when vectors are supplied. + await collection.upsert([{"id": 1, "text": "hello", "embedding": [0.1, 0.2, 0.3]}], generate_vectors=False) + results = await collection.search(vector=[0.1, 0.2, 0.3], top=5) + async for hit in results: # each hit is a SearchResponse mapping + print(hit["record"], hit["score"]) ``` ## Configuration @@ -41,7 +49,7 @@ environment variables prefixed with `MILVUS_`: | Setting | Environment variable | Notes | | ---------- | -------------------- | -------------------------------------------------- | -| `uri` | `MILVUS_URI` | Server URI or a local `.db` path (Milvus Lite). | +| `uri` | `MILVUS_URI` | Server URI, or a local `.db` path (Milvus Lite extra). | | `token` | `MILVUS_TOKEN` | Access token (API key or `user:password`). | | `user` | `MILVUS_USER` | Username. | | `password` | `MILVUS_PASSWORD` | Password. | diff --git a/python/packages/milvus/agent_framework_milvus/_vector_store.py b/python/packages/milvus/agent_framework_milvus/_vector_store.py index 5a145b6ae61..d7984d48243 100644 --- a/python/packages/milvus/agent_framework_milvus/_vector_store.py +++ b/python/packages/milvus/agent_framework_milvus/_vector_store.py @@ -5,6 +5,8 @@ from __future__ import annotations import asyncio +import json +import math from collections.abc import Mapping, Sequence from typing import Any, ClassVar, Final, Generic, cast @@ -78,7 +80,8 @@ class MilvusSettings(TypedDict, total=False): Keys: uri: Server URI from ``MILVUS_URI`` (e.g. ``http://localhost:19530``), or a local - ``.db`` path for Milvus Lite. When unset, defaults to ``http://localhost:19530``. + ``.db`` path for Milvus Lite (requires the ``pymilvus[milvus_lite]`` extra). + When unset, defaults to ``http://localhost:19530``. token: Optional access token from ``MILVUS_TOKEN``, stored as an AF ``SecretString``. user: Optional username from ``MILVUS_USER``. password: Optional password from ``MILVUS_PASSWORD``, stored as an AF ``SecretString``. @@ -145,8 +148,9 @@ def _format_scalar(value: Any) -> str: if value is None: return "null" if isinstance(value, str): - escaped = value.replace("\\", "\\\\").replace('"', '\\"') - return f'"{escaped}"' + # json.dumps escapes quotes, backslashes and control characters (newlines, tabs, …), + # producing a valid double-quoted Milvus string literal for any stored value. + return json.dumps(value) if isinstance(value, (int, float)): return repr(value) raise NotImplementedError(f"Milvus filters support only scalar operands, not {type(value).__name__}.") @@ -254,6 +258,16 @@ def _validate_data_model(self) -> None: for field in self.definition.data_fields: if (field.type_ or "str") not in _SCALAR_TYPES: raise NotImplementedError(f"Milvus data field type '{field.type_}' is not supported.") + # Scalar and full-text indexing are not created by this connector yet; rejecting the + # flags avoids silently reporting success for an index that was never built. + if field.is_indexed: + raise NotImplementedError( + f"Scalar indexing (is_indexed) is not supported by this Milvus connector; field '{field.name}'." + ) + if field.is_full_text_indexed: + raise NotImplementedError( + f"Full-text indexing is not supported by this Milvus connector; field '{field.name}'." + ) def _output_fields(self, include_vectors: bool) -> list[str]: fields = [field.storage_name or field.name for field in self.definition.data_fields] @@ -308,13 +322,20 @@ async def ensure_collection_exists(self, *, operation_options: Mapping[str, Any] if await self.collection_exists(): return schema, index_params = self._build_schema_and_index() - await asyncio.to_thread( - self._client.create_collection, - collection_name=self.collection_name, - schema=schema, - index_params=index_params, - **options, - ) + try: + await asyncio.to_thread( + self._client.create_collection, + collection_name=self.collection_name, + schema=schema, + index_params=index_params, + **options, + ) + except Exception: + # The check-then-create sequence is not atomic: a concurrent worker may have + # created the collection between the existence check and this call. Treat an + # already-created collection as success (ensure semantics); otherwise re-raise. + if not await self.collection_exists(): + raise async def ensure_collection_deleted(self, *, operation_options: Mapping[str, Any] | None = None) -> None: """Delete this collection if it exists.""" @@ -461,6 +482,12 @@ async def _inner_search( field = self.definition.try_get_vector_field(vector_property_name) if field is None: raise ValueError("Milvus search requires a declared vector field.") + if score_threshold is not None: + if isinstance(score_threshold, bool) or not isinstance(score_threshold, (int, float)): + raise ValueError("Milvus score_threshold must be a finite number.") + score_threshold = float(score_threshold) + if not math.isfinite(score_threshold): + raise ValueError("Milvus score_threshold must be a finite number.") metric = _METRICS[cast(str, field.distance_function)] search_params: dict[str, Any] = {"metric_type": metric} search_params.update(options.pop("search_params", {}) or {}) diff --git a/python/packages/milvus/samples/milvus_vectors.py b/python/packages/milvus/samples/milvus_vectors.py index d8cfc418dbf..0f06fd6452b 100644 --- a/python/packages/milvus/samples/milvus_vectors.py +++ b/python/packages/milvus/samples/milvus_vectors.py @@ -6,9 +6,10 @@ export MILVUS_URI=http://localhost:19530 # or a local ".db" path for Milvus Lite uv run --package agent-framework-milvus python packages/milvus/samples/milvus_vectors.py -Use a development server or a local Milvus Lite file, not production. No embedding -model, inference service, or OpenAI credentials are used: the vectors are deliberately -small and deterministic. +Use a development server or a local Milvus Lite file, not production. A local ".db" path +requires the Milvus Lite extra: pip install "pymilvus[milvus_lite]" (Linux/macOS only). +No embedding model, inference service, or OpenAI credentials are used: the vectors are +deliberately small and deterministic. """ from __future__ import annotations diff --git a/python/packages/milvus/tests/milvus/test_collection.py b/python/packages/milvus/tests/milvus/test_collection.py index a33c079178c..aaaf5a832d4 100644 --- a/python/packages/milvus/tests/milvus/test_collection.py +++ b/python/packages/milvus/tests/milvus/test_collection.py @@ -97,6 +97,27 @@ async def test_search_requires_query(collection): await collection.search(top=1) +@pytest.mark.parametrize("bad", [True, float("nan"), float("inf")]) +async def test_search_rejects_invalid_score_threshold(collection, bad): + with pytest.raises(ValueError, match="score_threshold must be a finite number"): + await collection.search(vector=[1.0, 0.0, 0.0], score_threshold=bad) + + +async def test_ensure_collection_exists_swallows_concurrent_create(collection, mock_client): + # Absent at the check, then another worker wins the race: create fails but the + # collection now exists, so ensure semantics should treat it as success. + mock_client.has_collection.side_effect = [False, True] + mock_client.create_collection.side_effect = RuntimeError("collection already exists") + await collection.ensure_collection_exists() # must not raise + + +async def test_ensure_collection_exists_reraises_real_create_error(collection, mock_client): + mock_client.has_collection.side_effect = [False, False] + mock_client.create_collection.side_effect = RuntimeError("boom") + with pytest.raises(RuntimeError, match="boom"): + await collection.ensure_collection_exists() + + def test_serialize_round_trip(collection): # The handler serializes a model to a storage-keyed dict before calling this method. storage_record = {"point_key": 7, "body": "hello", "price": 7.0, "flag": True, "dense": [1.0, 0.0, 0.0]} @@ -124,6 +145,16 @@ def test_auto_generated_key_rejected(mock_client): MilvusCollection(dict, definition=definition, collection_name="docs", client=mock_client) +def test_is_indexed_rejected(mock_client): + definition = VectorStoreCollectionDefinition([ + VectorStoreField("key", name="id", type_="int"), + VectorStoreField("data", name="text", type_="str", is_indexed=True), + VectorStoreField("vector", name="v", dimensions=3, distance_function="cosine_similarity"), + ]) + with pytest.raises(NotImplementedError, match="is_indexed"): + MilvusCollection(dict, definition=definition, collection_name="docs", client=mock_client) + + def test_unsupported_distance_rejected(mock_client): definition = VectorStoreCollectionDefinition([ VectorStoreField("key", name="id", type_="int"), diff --git a/python/packages/milvus/tests/milvus/test_filters.py b/python/packages/milvus/tests/milvus/test_filters.py index 541d6fee5f6..4f490083881 100644 --- a/python/packages/milvus/tests/milvus/test_filters.py +++ b/python/packages/milvus/tests/milvus/test_filters.py @@ -26,6 +26,11 @@ def test_bool_literals(collection): assert collection._prepare_filter(Filter("flag", "eq", False)) == "(flag == false)" +def test_string_control_characters_are_escaped(collection): + # Newlines/quotes/backslashes must not break out of the string literal. + assert collection._prepare_filter(Filter("text", "eq", 'a"b\nc')) == '(body == "a\\"b\\nc")' + + def test_and_or_not_groups(collection): expr = FilterGroup("and", [Filter("text", "eq", "a"), Filter("number", "gt", 1)]) assert collection._prepare_filter(expr) == '((body == "a") and (price > 1))' diff --git a/python/packages/milvus/tests/milvus/test_integration.py b/python/packages/milvus/tests/milvus/test_integration.py new file mode 100644 index 00000000000..d0078d7372c --- /dev/null +++ b/python/packages/milvus/tests/milvus/test_integration.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Live integration tests against a real Milvus server. + +These run only when MILVUS_TEST_URI is set (see the ``server_collection`` fixture) and are +selected with ``pytest -m integration`` / ``poe test-integration``. +""" + +from __future__ import annotations + +import pytest +from agent_framework import Filter + +pytestmark = pytest.mark.integration + + +async def test_upsert_get_delete(server_collection, record): + keys = await server_collection.upsert([record(1), record(2)], generate_vectors=False) + assert sorted(keys) == [1, 2] + + loaded = await server_collection.get([1, 2]) + assert {item["id"] for item in loaded} == {1, 2} + + await server_collection.delete([1]) + assert await server_collection.get([1]) == [] + + +async def test_vector_search_ranks_closest(server_collection, record): + await server_collection.upsert( + [ + record(1, embedding=[1.0, 0.0, 0.0]), + record(2, embedding=[0.0, 1.0, 0.0]), + record(3, embedding=[0.0, 0.0, 1.0]), + ], + generate_vectors=False, + ) + results = [item async for item in await server_collection.search(vector=[1.0, 0.0, 0.0], top=3)] + assert results[0]["record"]["id"] == 1 + assert results[0]["score"] is not None + + +async def test_filtered_search(server_collection, record): + await server_collection.upsert( + [record(1, number=1.0), record(2, number=2.0), record(3, number=3.0)], + generate_vectors=False, + ) + results = [ + item + async for item in await server_collection.search( + vector=[1.0, 0.0, 0.0], top=10, filter=Filter("number", "gte", 2.0) + ) + ] + assert {item["record"]["id"] for item in results} == {2, 3} + + +async def test_get_by_filter(server_collection, record): + await server_collection.upsert([record(1), record(2, flag=False)], generate_vectors=False) + matches = await server_collection.get(filter=Filter("flag", "eq", False), top=10) + assert [item["id"] for item in matches] == [2] From 78c5046bd22cd06b5201b084ce0a2ce61fce2a06 Mon Sep 17 00:00:00 2001 From: Nanduu24 Date: Wed, 14 Oct 2026 06:20:50 -0500 Subject: [PATCH 3/7] Python: validate existing Milvus collections and filter operand types Addresses review feedback on #9346: - ensure_collection_exists now validates an existing collection's fields, vector dimensions and metric types against the data model definition (and does the same after an already-exists race) instead of accepting a stale or unrelated schema. - Portable filter operands are validated against the declared field type: a boolean operand on a numeric field (or any type mismatch) is rejected with a clear error rather than emitting an invalid comparison such as 'price == true'; ordered comparisons require a numeric field. - Added unit tests for schema mismatch (dimension, metric) and operand-type rejection. --- .../agent_framework_milvus/_vector_store.py | 111 +++++++++++++++--- .../packages/milvus/tests/milvus/conftest.py | 14 ++- .../milvus/tests/milvus/test_collection.py | 25 +++- .../milvus/tests/milvus/test_filters.py | 21 ++++ 4 files changed, 151 insertions(+), 20 deletions(-) diff --git a/python/packages/milvus/agent_framework_milvus/_vector_store.py b/python/packages/milvus/agent_framework_milvus/_vector_store.py index d7984d48243..bb187ae58dc 100644 --- a/python/packages/milvus/agent_framework_milvus/_vector_store.py +++ b/python/packages/milvus/agent_framework_milvus/_vector_store.py @@ -19,6 +19,7 @@ SearchType, SecretString, VectorStoreCollectionDefinition, + VectorStoreField, load_settings, ) from agent_framework._telemetry import FeatureIndex, mark_feature_used @@ -160,6 +161,35 @@ def _format_list(values: Any) -> str: return "[" + ", ".join(_format_scalar(item) for item in values) + "]" +def _check_scalar_operand(field: VectorStoreField, value: Any) -> None: + """Reject operands whose Python type does not match the field's declared type. + + Booleans are kept distinct from numbers so a filter like ``eq`` on a numeric field with a + boolean operand is rejected rather than emitting an invalid, type-mismatched comparison. + """ + kind = field.type_ or "str" + if kind == "bool": + if not isinstance(value, bool): + raise ValueError(f"Filter on boolean field '{field.name}' requires a boolean operand.") + elif kind == "str": + if not isinstance(value, str): + raise ValueError(f"Filter on string field '{field.name}' requires a string operand.") + elif kind in ("int", "float"): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"Filter on numeric field '{field.name}' requires a non-boolean numeric operand.") + else: + raise NotImplementedError( + f"Milvus filters do not support equality or membership on '{kind}' field '{field.name}'." + ) + + +def _check_numeric_field(field: VectorStoreField) -> None: + if (field.type_ or "str") not in ("int", "float"): + raise NotImplementedError( + f"Milvus ordered comparisons require a declared int or float field; '{field.name}' is '{field.type_}'." + ) + + class MilvusCollection(BaseVectorCollection[KeyT, ModelT], BaseVectorSearch[KeyT, ModelT], Generic[KeyT, ModelT]): """Store and search records with the official Milvus client. @@ -317,9 +347,15 @@ def _build_schema_and_index(self) -> tuple[Any, Any]: return schema, index_params async def ensure_collection_exists(self, *, operation_options: Mapping[str, Any] | None = None) -> None: - """Create the collection and its vector indexes when it does not exist.""" + """Create the collection and its vector indexes, or validate an existing one. + + When a collection with this name already exists, its schema and vector indexes are + validated against the data model definition so a stale or unrelated collection is not + silently accepted; a mismatch raises ``ValueError``. + """ options = _validate_operation_options(operation_options, _CREATE_OPTIONS) if await self.collection_exists(): + await self._validate_existing_collection() return schema, index_params = self._build_schema_and_index() try: @@ -333,9 +369,48 @@ async def ensure_collection_exists(self, *, operation_options: Mapping[str, Any] except Exception: # The check-then-create sequence is not atomic: a concurrent worker may have # created the collection between the existence check and this call. Treat an - # already-created collection as success (ensure semantics); otherwise re-raise. + # already-created collection as success (ensure semantics) after validating that + # its schema matches; otherwise re-raise the original error. if not await self.collection_exists(): raise + await self._validate_existing_collection() + + async def _validate_existing_collection(self) -> None: + """Validate an existing collection's fields, vector dimensions and metrics.""" + info = cast( + "Mapping[str, Any]", await asyncio.to_thread(self._client.describe_collection, self.collection_name) + ) + raw_fields = cast("Sequence[Mapping[str, Any]]", info.get("fields") or []) + fields: dict[str, Mapping[str, Any]] = {str(field["name"]): field for field in raw_fields} + key = fields.get(self._key_storage_name) + if key is None or not key.get("is_primary"): + raise ValueError( + f"Existing Milvus collection '{self.collection_name}' has no primary key " + f"'{self._key_storage_name}' matching the data model." + ) + for field in self.definition.data_fields: + if (field.storage_name or field.name) not in fields: + raise ValueError( + f"Existing Milvus collection '{self.collection_name}' is missing field " + f"'{field.storage_name or field.name}'." + ) + for field in self.definition.vector_fields: + name = field.storage_name or field.name + existing = fields.get(name) + if existing is None: + raise ValueError( + f"Existing Milvus collection '{self.collection_name}' is missing vector field '{name}'." + ) + dimensions = cast("Mapping[str, Any]", existing.get("params") or {}).get("dim") + if dimensions is not None and int(dimensions) != int(cast(int, field.dimensions)): + raise ValueError( + f"Existing Milvus vector field '{name}' has dimension {dimensions}, expected {field.dimensions}." + ) + index = await asyncio.to_thread(self._client.describe_index, self.collection_name, name) + metric = cast("Mapping[str, Any]", index).get("metric_type") if isinstance(index, Mapping) else None + expected = _METRICS[cast(str, field.distance_function)] + if metric is not None and metric != expected: + raise ValueError(f"Existing Milvus vector field '{name}' uses metric {metric}, expected {expected}.") async def ensure_collection_deleted(self, *, operation_options: Mapping[str, Any] | None = None) -> None: """Delete this collection if it exists.""" @@ -554,24 +629,24 @@ def _prepare_filter_condition(self, expression: FilterExpression) -> str: raise NotImplementedError("Milvus filters do not operate on vector fields.") name = field.storage_name or field.name operator, value = expression.operator, expression.value - if operator == "eq": - return f"({name} == {_format_scalar(value)})" - if operator == "ne": - return f"({name} != {_format_scalar(value)})" - if operator == "gt": - return f"({name} > {_format_scalar(value)})" - if operator == "gte": - return f"({name} >= {_format_scalar(value)})" - if operator == "lt": - return f"({name} < {_format_scalar(value)})" - if operator == "lte": - return f"({name} <= {_format_scalar(value)})" + if operator in ("eq", "ne"): + _check_scalar_operand(field, value) + return f"({name} {'==' if operator == 'eq' else '!='} {_format_scalar(value)})" + if operator in ("gt", "gte", "lt", "lte"): + _check_numeric_field(field) + _check_scalar_operand(field, value) + symbol = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[operator] + return f"({name} {symbol} {_format_scalar(value)})" if operator == "between": + _check_numeric_field(field) + _check_scalar_operand(field, value[0]) + _check_scalar_operand(field, value[1]) return f"({name} >= {_format_scalar(value[0])} and {name} <= {_format_scalar(value[1])})" - if operator == "in": - return f"({name} in {_format_list(value)})" - if operator == "not_in": - return f"({name} not in {_format_list(value)})" + if operator in ("in", "not_in"): + for item in value: + _check_scalar_operand(field, item) + keyword = "in" if operator == "in" else "not in" + return f"({name} {keyword} {_format_list(value)})" raise NotImplementedError(f"Milvus does not support portable filter operator '{operator}'.") diff --git a/python/packages/milvus/tests/milvus/conftest.py b/python/packages/milvus/tests/milvus/conftest.py index 076b81ae8f1..711113bc8dc 100644 --- a/python/packages/milvus/tests/milvus/conftest.py +++ b/python/packages/milvus/tests/milvus/conftest.py @@ -8,7 +8,7 @@ import pytest from agent_framework import VectorStoreCollectionDefinition, VectorStoreField -from pymilvus import MilvusClient +from pymilvus import DataType, MilvusClient from agent_framework_milvus import MilvusCollection, MilvusStore @@ -68,6 +68,18 @@ def mock_client(): {"id": 2, "distance": 0.10, "entity": {"body": "world", "price": 2.0, "flag": False}}, ] ] + # Matches the `definition` fixture so ensure_collection_exists validates cleanly when present. + client.describe_collection.return_value = { + "collection_name": "docs", + "fields": [ + {"name": "point_key", "type": DataType.INT64, "is_primary": True, "params": {}}, + {"name": "body", "type": DataType.VARCHAR, "params": {"max_length": 65535}}, + {"name": "price", "type": DataType.DOUBLE, "params": {}}, + {"name": "flag", "type": DataType.BOOL, "params": {}}, + {"name": "dense", "type": DataType.FLOAT_VECTOR, "params": {"dim": 3}}, + ], + } + client.describe_index.return_value = {"metric_type": "COSINE"} return client diff --git a/python/packages/milvus/tests/milvus/test_collection.py b/python/packages/milvus/tests/milvus/test_collection.py index aaaf5a832d4..69a540a8316 100644 --- a/python/packages/milvus/tests/milvus/test_collection.py +++ b/python/packages/milvus/tests/milvus/test_collection.py @@ -62,10 +62,33 @@ async def test_ensure_collection_exists_creates_when_absent(collection, mock_cli assert mock_client.create_collection.call_args.kwargs["collection_name"] == "docs" -async def test_ensure_collection_exists_skips_when_present(collection, mock_client): +async def test_ensure_collection_exists_validates_when_present(collection, mock_client): mock_client.has_collection.return_value = True await collection.ensure_collection_exists() mock_client.create_collection.assert_not_called() + mock_client.describe_collection.assert_called_once() + + +async def test_ensure_collection_exists_rejects_dimension_mismatch(collection, mock_client): + mock_client.has_collection.return_value = True + mock_client.describe_collection.return_value = { + "fields": [ + {"name": "point_key", "type": DataType.INT64, "is_primary": True, "params": {}}, + {"name": "body", "type": DataType.VARCHAR, "params": {"max_length": 65535}}, + {"name": "price", "type": DataType.DOUBLE, "params": {}}, + {"name": "flag", "type": DataType.BOOL, "params": {}}, + {"name": "dense", "type": DataType.FLOAT_VECTOR, "params": {"dim": 99}}, + ] + } + with pytest.raises(ValueError, match="dimension"): + await collection.ensure_collection_exists() + + +async def test_ensure_collection_exists_rejects_metric_mismatch(collection, mock_client): + mock_client.has_collection.return_value = True + mock_client.describe_index.return_value = {"metric_type": "L2"} + with pytest.raises(ValueError, match="metric"): + await collection.ensure_collection_exists() async def test_ensure_collection_deleted(collection, mock_client): diff --git a/python/packages/milvus/tests/milvus/test_filters.py b/python/packages/milvus/tests/milvus/test_filters.py index 4f490083881..eec611b3bc9 100644 --- a/python/packages/milvus/tests/milvus/test_filters.py +++ b/python/packages/milvus/tests/milvus/test_filters.py @@ -51,6 +51,27 @@ def test_unsupported_operator(collection): collection._prepare_filter(Filter("text", "contains", "a")) +def test_boolean_operand_on_numeric_field_rejected(collection): + # Filter("number", "eq", True) must not emit `price == true`. + with pytest.raises(ValueError, match="numeric field"): + collection._prepare_filter(Filter("number", "eq", True)) + + +def test_type_mismatched_operand_rejected(collection): + with pytest.raises(ValueError, match="string field"): + collection._prepare_filter(Filter("text", "eq", 5)) + + +def test_membership_element_type_checked(collection): + with pytest.raises(ValueError, match="numeric field"): + collection._prepare_filter(Filter("number", "in", [1, True])) + + +def test_ordered_comparison_requires_numeric_field(collection): + with pytest.raises(NotImplementedError, match="ordered comparisons"): + collection._prepare_filter(Filter("text", "gt", "a")) + + def test_unknown_field(collection): with pytest.raises(NotImplementedError): collection._prepare_filter(Filter("missing", "eq", "a")) From b2e1bd98c9e552eb1469feeaabb8f5908a5f1309 Mon Sep 17 00:00:00 2001 From: Nanduu24 Date: Wed, 14 Oct 2026 06:25:58 -0500 Subject: [PATCH 4/7] Python: only swallow genuine already-exists errors in ensure_collection_exists Per review on #9346: create_collection builds the collection and then separately creates indexes and loads it. Treating any exception as success when the collection exists afterwards could mask an index or load failure, leaving an unindexed/unloaded collection. Now only a genuine already-exists/concurrent-create MilvusException is swallowed (then the existing collection is validated); all other create/index/load failures are re-raised. Added tests for the index/load-failure and non-Milvus cases. --- .../agent_framework_milvus/_vector_store.py | 23 +++++++++++++------ .../milvus/tests/milvus/test_collection.py | 23 ++++++++++++++----- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/python/packages/milvus/agent_framework_milvus/_vector_store.py b/python/packages/milvus/agent_framework_milvus/_vector_store.py index bb187ae58dc..069edb4bb80 100644 --- a/python/packages/milvus/agent_framework_milvus/_vector_store.py +++ b/python/packages/milvus/agent_framework_milvus/_vector_store.py @@ -27,6 +27,7 @@ from agent_framework._vectors import EmbeddingClient, Vector from agent_framework.exceptions import IntegrationException, IntegrationInvalidResponseException from pymilvus import DataType, MilvusClient +from pymilvus.exceptions import MilvusException from typing_extensions import TypedDict, TypeVar KeyT = TypeVar("KeyT", default=Any) @@ -142,6 +143,12 @@ def _create_client( raise IntegrationException("Failed to create the Milvus client.") from exc +def _is_already_exists_error(exc: MilvusException) -> bool: + """Whether a Milvus error indicates the collection already exists (a concurrent create).""" + message = getattr(exc, "message", None) or str(exc) + return "already exist" in message.lower() + + def _format_scalar(value: Any) -> str: """Format a Python scalar as a Milvus boolean-expression literal.""" if isinstance(value, bool): @@ -366,14 +373,16 @@ async def ensure_collection_exists(self, *, operation_options: Mapping[str, Any] index_params=index_params, **options, ) - except Exception: - # The check-then-create sequence is not atomic: a concurrent worker may have - # created the collection between the existence check and this call. Treat an - # already-created collection as success (ensure semantics) after validating that - # its schema matches; otherwise re-raise the original error. - if not await self.collection_exists(): + return + except MilvusException as exc: + # create_collection creates the collection and then separately builds indexes and + # loads it. Only swallow a genuine already-exists/concurrent-create error; re-raise + # every other failure (including index/load failures that leave the collection + # existing but incomplete) so it is not masked as success. + if not _is_already_exists_error(exc): raise - await self._validate_existing_collection() + # A concurrent creator won the race; validate the existing collection matches. + await self._validate_existing_collection() async def _validate_existing_collection(self) -> None: """Validate an existing collection's fields, vector dimensions and metrics.""" diff --git a/python/packages/milvus/tests/milvus/test_collection.py b/python/packages/milvus/tests/milvus/test_collection.py index 69a540a8316..a67b5ed95af 100644 --- a/python/packages/milvus/tests/milvus/test_collection.py +++ b/python/packages/milvus/tests/milvus/test_collection.py @@ -5,6 +5,7 @@ import pytest from agent_framework import Filter, VectorStoreCollectionDefinition, VectorStoreField from pymilvus import DataType +from pymilvus.exceptions import MilvusException from agent_framework_milvus import MilvusCollection, MilvusStore @@ -127,15 +128,25 @@ async def test_search_rejects_invalid_score_threshold(collection, bad): async def test_ensure_collection_exists_swallows_concurrent_create(collection, mock_client): - # Absent at the check, then another worker wins the race: create fails but the - # collection now exists, so ensure semantics should treat it as success. - mock_client.has_collection.side_effect = [False, True] - mock_client.create_collection.side_effect = RuntimeError("collection already exists") + # Another worker wins the race: create raises a genuine already-exists error, so ensure + # semantics treat it as success (after validating the existing schema matches). + mock_client.has_collection.return_value = False + mock_client.create_collection.side_effect = MilvusException(message="collection already exist") await collection.ensure_collection_exists() # must not raise + mock_client.describe_collection.assert_called_once() -async def test_ensure_collection_exists_reraises_real_create_error(collection, mock_client): - mock_client.has_collection.side_effect = [False, False] +async def test_ensure_collection_exists_reraises_index_or_load_failure(collection, mock_client): + # create_collection builds the collection then indexes/loads it; a later-step failure must + # NOT be masked as success even though the collection now exists. + mock_client.has_collection.return_value = False + mock_client.create_collection.side_effect = MilvusException(message="create index failed") + with pytest.raises(MilvusException, match="create index failed"): + await collection.ensure_collection_exists() + + +async def test_ensure_collection_exists_reraises_non_milvus_error(collection, mock_client): + mock_client.has_collection.return_value = False mock_client.create_collection.side_effect = RuntimeError("boom") with pytest.raises(RuntimeError, match="boom"): await collection.ensure_collection_exists() From 6061318352fc579f767dffb0070d02ff8dfff14f Mon Sep 17 00:00:00 2001 From: Nanduu24 Date: Wed, 14 Oct 2026 11:08:00 -0500 Subject: [PATCH 5/7] Python: use the native async Milvus client and simplify client handling Addresses review feedback on #9346: - Switch from the sync MilvusClient wrapped in asyncio.to_thread to the native AsyncMilvusClient, awaiting operations directly. - Let the native Milvus client-creation error propagate instead of wrapping it. - Replace the brittle already-exists message match: on a create failure, validate the existing collection and re-raise the original error if it does not match, so only a genuine concurrent-create with a complete, matching collection is accepted. - Tests use MILVUS_URI (not a separate MILVUS_TEST_URI) to gate integration tests; the env-clearing fixture is skipped for integration tests. - Require pymilvus>=2.5.3 (AsyncMilvusClient). --- .../agent_framework_milvus/_vector_store.py | 95 ++++++++----------- python/packages/milvus/pyproject.toml | 3 +- .../packages/milvus/tests/milvus/conftest.py | 17 ++-- .../milvus/tests/milvus/test_collection.py | 9 +- .../milvus/tests/milvus/test_settings.py | 21 ++-- python/uv.lock | 2 +- 6 files changed, 67 insertions(+), 80 deletions(-) diff --git a/python/packages/milvus/agent_framework_milvus/_vector_store.py b/python/packages/milvus/agent_framework_milvus/_vector_store.py index 069edb4bb80..47318e3c855 100644 --- a/python/packages/milvus/agent_framework_milvus/_vector_store.py +++ b/python/packages/milvus/agent_framework_milvus/_vector_store.py @@ -4,7 +4,6 @@ from __future__ import annotations -import asyncio import json import math from collections.abc import Mapping, Sequence @@ -25,8 +24,8 @@ from agent_framework._telemetry import FeatureIndex, mark_feature_used from agent_framework._vector_filters import FilterExpression from agent_framework._vectors import EmbeddingClient, Vector -from agent_framework.exceptions import IntegrationException, IntegrationInvalidResponseException -from pymilvus import DataType, MilvusClient +from agent_framework.exceptions import IntegrationInvalidResponseException +from pymilvus import AsyncMilvusClient, DataType from pymilvus.exceptions import MilvusException from typing_extensions import TypedDict, TypeVar @@ -114,7 +113,7 @@ def _create_client( db_name: str | None, env_file_path: str | None, env_file_encoding: str | None, -) -> MilvusClient: +) -> AsyncMilvusClient: settings = load_settings( MilvusSettings, env_prefix="MILVUS_", @@ -128,6 +127,8 @@ def _create_client( ) resolved_token = settings.get("token") resolved_password = settings.get("password") + # load_settings includes optional keys as ``None`` when unset, so ``or`` (not a dict default) + # is needed to fall back to the default URI. client_kwargs: dict[str, Any] = {"uri": settings.get("uri") or DEFAULT_URI} if resolved_token is not None: client_kwargs["token"] = resolved_token.get_secret_value() @@ -137,16 +138,7 @@ def _create_client( client_kwargs["password"] = resolved_password.get_secret_value() if settings.get("db_name"): client_kwargs["db_name"] = settings.get("db_name") - try: - return MilvusClient(**client_kwargs) - except Exception as exc: # pragma: no cover - surfaced as an integration error - raise IntegrationException("Failed to create the Milvus client.") from exc - - -def _is_already_exists_error(exc: MilvusException) -> bool: - """Whether a Milvus error indicates the collection already exists (a concurrent create).""" - message = getattr(exc, "message", None) or str(exc) - return "already exist" in message.lower() + return AsyncMilvusClient(**client_kwargs) def _format_scalar(value: Any) -> str: @@ -198,16 +190,15 @@ def _check_numeric_field(field: VectorStoreField) -> None: class MilvusCollection(BaseVectorCollection[KeyT, ModelT], BaseVectorSearch[KeyT, ModelT], Generic[KeyT, ModelT]): - """Store and search records with the official Milvus client. + """Store and search records with the official async Milvus client. The data model definition drives the collection schema: the key field becomes the primary key (``INT64``/``VARCHAR``), data fields become typed scalar fields, and vector fields become ``FLOAT_VECTOR`` fields, each with its own index. The default metric is cosine similarity (larger is better); scores retain native Milvus units. - The synchronous ``MilvusClient`` is used under ``asyncio.to_thread`` so the connector - presents the framework's async interface while remaining compatible across the - supported ``pymilvus`` range. Supplied clients remain caller-owned unless managed. + Operations use the official ``AsyncMilvusClient``. Supplied clients remain caller-owned + unless explicitly managed. """ supported_key_types: ClassVar[set[str] | None] = {"int", "str"} @@ -221,7 +212,7 @@ def __init__( definition: VectorStoreCollectionDefinition | None = None, collection_name: str | None = None, embedding_generator: EmbeddingClient | None = None, - client: MilvusClient | None = None, + client: AsyncMilvusClient | None = None, uri: str | None = None, token: str | SecretString | None = None, user: str | None = None, @@ -238,7 +229,7 @@ def __init__( definition: Explicit dictionary schema. collection_name: Collection name, overriding the model name. embedding_generator: Default local embedding client. - client: Existing client, borrowed by default. Use this for Milvus Lite or advanced configuration. + client: Existing async client, borrowed by default. Use this for advanced configuration. uri: Server URI override; otherwise loaded from MilvusSettings, then defaults to localhost. token: Access token override as a string or SecretString; otherwise loaded from MilvusSettings. user: Username override; otherwise loaded from MilvusSettings. @@ -317,10 +308,10 @@ async def collection_exists(self, *, operation_options: Mapping[str, Any] | None """Check whether this collection exists.""" options = _validate_operation_options(operation_options, _READ_OPTIONS) mark_feature_used(FeatureIndex.CORE_VECTOR_STORES) - return await asyncio.to_thread(self._client.has_collection, self.collection_name, **options) + return await self._client.has_collection(self.collection_name, **options) def _build_schema_and_index(self) -> tuple[Any, Any]: - schema = cast("Any", MilvusClient).create_schema(auto_id=False, enable_dynamic_field=False) + schema = cast("Any", AsyncMilvusClient).create_schema(auto_id=False, enable_dynamic_field=False) key_field = self.definition.key_field if key_field.type_ == "int": schema.add_field(field_name=self._key_storage_name, datatype=DataType.INT64, is_primary=True) @@ -366,29 +357,26 @@ async def ensure_collection_exists(self, *, operation_options: Mapping[str, Any] return schema, index_params = self._build_schema_and_index() try: - await asyncio.to_thread( - self._client.create_collection, + await self._client.create_collection( collection_name=self.collection_name, schema=schema, index_params=index_params, **options, ) - return except MilvusException as exc: - # create_collection creates the collection and then separately builds indexes and - # loads it. Only swallow a genuine already-exists/concurrent-create error; re-raise - # every other failure (including index/load failures that leave the collection - # existing but incomplete) so it is not masked as success. - if not _is_already_exists_error(exc): - raise - # A concurrent creator won the race; validate the existing collection matches. - await self._validate_existing_collection() + # create_collection creates the collection and then separately builds indexes and loads + # it, and the check-then-create sequence is not atomic. A concurrent creator may have + # produced a complete, matching collection in the race window; accept that only if the + # existing collection validates. Any other failure (including an index/load failure that + # leaves this collection incomplete) fails validation, so re-raise the original error. + try: + await self._validate_existing_collection() + except Exception: + raise exc from None async def _validate_existing_collection(self) -> None: """Validate an existing collection's fields, vector dimensions and metrics.""" - info = cast( - "Mapping[str, Any]", await asyncio.to_thread(self._client.describe_collection, self.collection_name) - ) + info = cast("Mapping[str, Any]", await self._client.describe_collection(self.collection_name)) raw_fields = cast("Sequence[Mapping[str, Any]]", info.get("fields") or []) fields: dict[str, Mapping[str, Any]] = {str(field["name"]): field for field in raw_fields} key = fields.get(self._key_storage_name) @@ -415,7 +403,7 @@ async def _validate_existing_collection(self) -> None: raise ValueError( f"Existing Milvus vector field '{name}' has dimension {dimensions}, expected {field.dimensions}." ) - index = await asyncio.to_thread(self._client.describe_index, self.collection_name, name) + index = await self._client.describe_index(self.collection_name, name) metric = cast("Mapping[str, Any]", index).get("metric_type") if isinstance(index, Mapping) else None expected = _METRICS[cast(str, field.distance_function)] if metric is not None and metric != expected: @@ -425,12 +413,12 @@ async def ensure_collection_deleted(self, *, operation_options: Mapping[str, Any """Delete this collection if it exists.""" options = _validate_operation_options(operation_options, _WRITE_OPTIONS) if await self.collection_exists(): - await asyncio.to_thread(self._client.drop_collection, self.collection_name, **options) + await self._client.drop_collection(self.collection_name, **options) async def aclose(self) -> None: """Close an owned client once; borrowed clients are left open.""" if self.managed_client and not self._closed: - await asyncio.to_thread(self._client.close) + await self._client.close() self._closed = True async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: @@ -490,9 +478,7 @@ async def _inner_upsert( options = _validate_operation_options(operation_options, _WRITE_OPTIONS) if not records: return [] - await asyncio.to_thread( - self._client.upsert, collection_name=self.collection_name, data=list(records), **options - ) + await self._client.upsert(collection_name=self.collection_name, data=list(records), **options) return [cast(KeyT, record[self._key_storage_name]) for record in records] async def _inner_get( @@ -515,15 +501,13 @@ async def _inner_get( raise ValueError("Milvus key retrieval cannot be combined with skip.") if not keys: return [] - return await asyncio.to_thread( - self._client.get, + return await self._client.get( collection_name=self.collection_name, ids=list(keys), output_fields=output_fields, **options, ) - return await asyncio.to_thread( - self._client.query, + return await self._client.query( collection_name=self.collection_name, filter=self._prepare_filter(filter), output_fields=output_fields, @@ -541,7 +525,7 @@ async def _inner_delete( options = _validate_operation_options(operation_options, _WRITE_OPTIONS) if not keys: return - await asyncio.to_thread(self._client.delete, collection_name=self.collection_name, ids=list(keys), **options) + await self._client.delete(collection_name=self.collection_name, ids=list(keys), **options) async def _inner_search( self, @@ -577,8 +561,7 @@ async def _inner_search( search_params.update(options.pop("search_params", {}) or {}) if skip: search_params["offset"] = skip - hits = await asyncio.to_thread( - self._client.search, + hits = await self._client.search( collection_name=self.collection_name, data=[[float(component) for component in vector]], anns_field=field.storage_name or field.name, @@ -660,12 +643,12 @@ def _prepare_filter_condition(self, expression: FilterExpression) -> str: class MilvusStore(BaseVectorStore): - """Create Milvus collection clients sharing one connection.""" + """Create Milvus collection clients sharing one async connection.""" def __init__( self, *, - client: MilvusClient | None = None, + client: AsyncMilvusClient | None = None, uri: str | None = None, token: str | SecretString | None = None, user: str | None = None, @@ -679,7 +662,7 @@ def __init__( """Initialize a store. Args: - client: Existing client; borrowed unless managed_client=True. + client: Existing async client; borrowed unless managed_client=True. uri: Server URI override; otherwise loaded from MilvusSettings, then defaults to localhost. token: Access token override as a string or SecretString; otherwise loaded from MilvusSettings. user: Username override; otherwise loaded from MilvusSettings. @@ -738,7 +721,7 @@ async def list_collection_names(self, *, operation_options: Mapping[str, Any] | """List collection names on the server.""" options = _validate_operation_options(operation_options, _READ_OPTIONS) mark_feature_used(FeatureIndex.CORE_VECTOR_STORES) - return await asyncio.to_thread(self._client.list_collections, **options) + return await self._client.list_collections(**options) async def collection_exists( self, @@ -749,7 +732,7 @@ async def collection_exists( """Check collection existence without downloading the collection list.""" options = _validate_operation_options(operation_options, _READ_OPTIONS) mark_feature_used(FeatureIndex.CORE_VECTOR_STORES) - return await asyncio.to_thread(self._client.has_collection, collection_name, **options) + return await self._client.has_collection(collection_name, **options) async def _inner_ensure_collection_deleted( self, @@ -758,12 +741,12 @@ async def _inner_ensure_collection_deleted( operation_options: Mapping[str, Any] | None = None, ) -> None: options = _validate_operation_options(operation_options, _WRITE_OPTIONS) - await asyncio.to_thread(self._client.drop_collection, collection_name, **options) + await self._client.drop_collection(collection_name, **options) async def aclose(self) -> None: """Close an owned client once, leaving borrowed clients open.""" if self.managed_client and not self._closed: - await asyncio.to_thread(self._client.close) + await self._client.close() self._closed = True async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: diff --git a/python/packages/milvus/pyproject.toml b/python/packages/milvus/pyproject.toml index ab7462dbf40..82c21391ce5 100644 --- a/python/packages/milvus/pyproject.toml +++ b/python/packages/milvus/pyproject.toml @@ -23,7 +23,8 @@ classifiers = [ ] dependencies = [ "agent-framework-core>=1.19.0,<2", - "pymilvus>=2.5,<3", + # 2.5.3+ provides AsyncMilvusClient. + "pymilvus>=2.5.3,<3", ] [tool.pytest.ini_options] diff --git a/python/packages/milvus/tests/milvus/conftest.py b/python/packages/milvus/tests/milvus/conftest.py index 711113bc8dc..7216bb57310 100644 --- a/python/packages/milvus/tests/milvus/conftest.py +++ b/python/packages/milvus/tests/milvus/conftest.py @@ -8,18 +8,21 @@ import pytest from agent_framework import VectorStoreCollectionDefinition, VectorStoreField -from pymilvus import DataType, MilvusClient +from pymilvus import AsyncMilvusClient, DataType from agent_framework_milvus import MilvusCollection, MilvusStore SERVER_MARKS = [ pytest.mark.integration, - pytest.mark.skipif(not os.getenv("MILVUS_TEST_URI"), reason="Set MILVUS_TEST_URI to a disposable Milvus server."), + pytest.mark.skipif(not os.getenv("MILVUS_URI"), reason="Set MILVUS_URI to a disposable Milvus server."), ] @pytest.fixture(autouse=True) -def clear_connection_environment(monkeypatch): +def clear_connection_environment(request, monkeypatch): + # Integration tests connect to a real server via MILVUS_URI, so leave the environment intact. + if request.node.get_closest_marker("integration"): + return for name in ("MILVUS_URI", "MILVUS_TOKEN", "MILVUS_USER", "MILVUS_PASSWORD", "MILVUS_DB_NAME"): monkeypatch.delenv(name, raising=False) @@ -57,7 +60,7 @@ def make(id=1, **values): @pytest.fixture def mock_client(): - client = MagicMock(spec=MilvusClient) + client = MagicMock(spec=AsyncMilvusClient) client.has_collection.return_value = True client.list_collections.return_value = ["docs"] client.get.return_value = [{"point_key": 1, "body": "hello", "price": 1.0, "flag": True}] @@ -95,9 +98,9 @@ def store(mock_client): @pytest.fixture async def server_collection(definition): - if not os.getenv("MILVUS_TEST_URI"): - pytest.skip("Set MILVUS_TEST_URI to a disposable Milvus server.") - async with MilvusStore(uri=os.environ["MILVUS_TEST_URI"]) as store: + if not os.getenv("MILVUS_URI"): + pytest.skip("Set MILVUS_URI to a disposable Milvus server.") + async with MilvusStore(uri=os.environ["MILVUS_URI"]) as store: collection = store.get_collection(dict, definition=definition, collection_name=f"af_milvus_test_{uuid4().hex}") await collection.ensure_collection_exists() try: diff --git a/python/packages/milvus/tests/milvus/test_collection.py b/python/packages/milvus/tests/milvus/test_collection.py index a67b5ed95af..0f8d39380b1 100644 --- a/python/packages/milvus/tests/milvus/test_collection.py +++ b/python/packages/milvus/tests/milvus/test_collection.py @@ -128,8 +128,8 @@ async def test_search_rejects_invalid_score_threshold(collection, bad): async def test_ensure_collection_exists_swallows_concurrent_create(collection, mock_client): - # Another worker wins the race: create raises a genuine already-exists error, so ensure - # semantics treat it as success (after validating the existing schema matches). + # A concurrent creator produced a complete, matching collection in the race window: create + # raises, but the existing collection validates, so ensure semantics treat it as success. mock_client.has_collection.return_value = False mock_client.create_collection.side_effect = MilvusException(message="collection already exist") await collection.ensure_collection_exists() # must not raise @@ -137,10 +137,11 @@ async def test_ensure_collection_exists_swallows_concurrent_create(collection, m async def test_ensure_collection_exists_reraises_index_or_load_failure(collection, mock_client): - # create_collection builds the collection then indexes/loads it; a later-step failure must - # NOT be masked as success even though the collection now exists. + # create_collection builds the collection then indexes/loads it; if the index step fails the + # collection exists but has no index (describe_index errors), so the original error re-raises. mock_client.has_collection.return_value = False mock_client.create_collection.side_effect = MilvusException(message="create index failed") + mock_client.describe_index.side_effect = MilvusException(message="index not found") with pytest.raises(MilvusException, match="create index failed"): await collection.ensure_collection_exists() diff --git a/python/packages/milvus/tests/milvus/test_settings.py b/python/packages/milvus/tests/milvus/test_settings.py index cdac102f6d2..e84b7b6cec1 100644 --- a/python/packages/milvus/tests/milvus/test_settings.py +++ b/python/packages/milvus/tests/milvus/test_settings.py @@ -5,16 +5,16 @@ from unittest.mock import MagicMock, patch import pytest -from pymilvus import MilvusClient +from pymilvus import AsyncMilvusClient from agent_framework_milvus import MilvusStore -_CLIENT_PATH = "agent_framework_milvus._vector_store.MilvusClient" +_CLIENT_PATH = "agent_framework_milvus._vector_store.AsyncMilvusClient" def test_default_uri_when_unset(): with patch(_CLIENT_PATH) as client_cls: - client_cls.return_value = MagicMock(spec=MilvusClient) + client_cls.return_value = MagicMock(spec=AsyncMilvusClient) MilvusStore() assert client_cls.call_args.kwargs["uri"] == "http://localhost:19530" @@ -22,7 +22,7 @@ def test_default_uri_when_unset(): def test_uri_from_environment(monkeypatch): monkeypatch.setenv("MILVUS_URI", "http://milvus.internal:19530") with patch(_CLIENT_PATH) as client_cls: - client_cls.return_value = MagicMock(spec=MilvusClient) + client_cls.return_value = MagicMock(spec=AsyncMilvusClient) MilvusStore() assert client_cls.call_args.kwargs["uri"] == "http://milvus.internal:19530" @@ -30,14 +30,14 @@ def test_uri_from_environment(monkeypatch): def test_explicit_uri_overrides_environment(monkeypatch): monkeypatch.setenv("MILVUS_URI", "http://env:19530") with patch(_CLIENT_PATH) as client_cls: - client_cls.return_value = MagicMock(spec=MilvusClient) + client_cls.return_value = MagicMock(spec=AsyncMilvusClient) MilvusStore(uri="http://explicit:19530") assert client_cls.call_args.kwargs["uri"] == "http://explicit:19530" def test_token_and_db_forwarded(): with patch(_CLIENT_PATH) as client_cls: - client_cls.return_value = MagicMock(spec=MilvusClient) + client_cls.return_value = MagicMock(spec=AsyncMilvusClient) MilvusStore(uri="http://localhost:19530", token="secret-token", db_name="analytics") assert client_cls.call_args.kwargs["token"] == "secret-token" assert client_cls.call_args.kwargs["db_name"] == "analytics" @@ -46,13 +46,12 @@ def test_token_and_db_forwarded(): def test_token_from_environment(monkeypatch): monkeypatch.setenv("MILVUS_TOKEN", "env-token") with patch(_CLIENT_PATH) as client_cls: - client_cls.return_value = MagicMock(spec=MilvusClient) + client_cls.return_value = MagicMock(spec=AsyncMilvusClient) MilvusStore(uri="http://localhost:19530") assert client_cls.call_args.kwargs["token"] == "env-token" -def test_client_creation_failure_raises(): - from agent_framework.exceptions import IntegrationException - - with patch(_CLIENT_PATH, side_effect=RuntimeError("boom")), pytest.raises(IntegrationException): +def test_client_creation_error_propagates(): + # The native client error is surfaced directly rather than wrapped. + with patch(_CLIENT_PATH, side_effect=RuntimeError("boom")), pytest.raises(RuntimeError, match="boom"): MilvusStore(uri="http://localhost:19530") diff --git a/python/uv.lock b/python/uv.lock index 900419a13b6..7a010003312 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -873,7 +873,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "agent-framework-core", editable = "packages/core" }, - { name = "pymilvus", specifier = ">=2.5,<3" }, + { name = "pymilvus", specifier = ">=2.5.3,<3" }, ] [[package]] From 117ab0e09dd8304c113723d3acfe126c956d608c Mon Sep 17 00:00:00 2001 From: Nanduu24 Date: Wed, 14 Oct 2026 12:20:09 -0500 Subject: [PATCH 6/7] Python: declare Python 3.15 support for the Milvus package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified pymilvus 2.5.3+ and its dependencies (grpcio, numpy, …) resolve, install and import on Python 3.15, matching the framework's new 3.15 support. --- python/packages/milvus/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/packages/milvus/pyproject.toml b/python/packages/milvus/pyproject.toml index 82c21391ce5..9f4e2cd90ff 100644 --- a/python/packages/milvus/pyproject.toml +++ b/python/packages/milvus/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Typing :: Typed", ] dependencies = [ From 0a8f6d5754bd711979e6b864ec40cce4371eac41 Mon Sep 17 00:00:00 2001 From: Nanduu24 Date: Thu, 22 Oct 2026 10:39:08 -0500 Subject: [PATCH 7/7] Python: strengthen Milvus schema/vector validation and fix Lite docs Addresses review feedback on #9346: - Existing-collection validation now also checks the primary-key type, every data/vector field DataType, and the (non-AUTOINDEX) vector index kind, in addition to dimensions and metric, so a same-named but incompatible collection is rejected. Added key/data/vector type and index-kind mismatch tests. - Added a shared dense-vector validator used for both stored and query vectors: rejects non-numeric sequences (incl. strings like "123"), booleans, and NaN/infinity or out-of-float32-range values, instead of coercing with float(). Added regression tests. - Removed the now-unused ty: ignore[unresolved-import] on the unrelated optimized_data_formats sample (pymilvus pulls pandas into the workspace, so ty resolves the import), fixing the PR-caused Samples/Typing CI failures. - Clarified docs: Milvus Lite is selected via the uri= argument with a local .db path; the MILVUS_URI env var must be a server address (pymilvus reserves it). --- python/packages/milvus/README.md | 8 +- .../agent_framework_milvus/_vector_store.py | 78 ++++++++++++++--- python/packages/milvus/samples/README.md | 5 +- .../packages/milvus/samples/milvus_vectors.py | 11 +-- .../milvus/tests/milvus/test_collection.py | 84 +++++++++++++++++++ .../vector_stores/optimized_data_formats.py | 2 +- 6 files changed, 165 insertions(+), 23 deletions(-) diff --git a/python/packages/milvus/README.md b/python/packages/milvus/README.md index aefc36d509d..430d7b6ec1a 100644 --- a/python/packages/milvus/README.md +++ b/python/packages/milvus/README.md @@ -30,8 +30,10 @@ definition = VectorStoreCollectionDefinition([ VectorStoreField("vector", name="embedding", dimensions=3, distance_function="cosine_similarity"), ]) -# Connects to a Milvus server; set MILVUS_URI or pass uri=... -# A local ".db" path uses Milvus Lite (requires the pymilvus[milvus_lite] extra). +# Connects to a Milvus server; set the MILVUS_URI env var or pass uri=... +# For Milvus Lite, pass a local ".db" path as the uri argument, e.g. +# MilvusStore(uri="milvus.db") (requires the pymilvus[milvus_lite] extra). The +# MILVUS_URI env var must be a server address, not a file path. async with MilvusStore(uri="http://localhost:19530") as store: collection = store.get_collection(dict, definition=definition, collection_name="docs") await collection.ensure_collection_exists() @@ -49,7 +51,7 @@ environment variables prefixed with `MILVUS_`: | Setting | Environment variable | Notes | | ---------- | -------------------- | -------------------------------------------------- | -| `uri` | `MILVUS_URI` | Server URI, or a local `.db` path (Milvus Lite extra). | +| `uri` | `MILVUS_URI` | Server URI (e.g. `http://localhost:19530`). For Milvus Lite, pass a local `.db` path as the `uri=` argument instead — the `MILVUS_URI` env var must be a server address. | | `token` | `MILVUS_TOKEN` | Access token (API key or `user:password`). | | `user` | `MILVUS_USER` | Username. | | `password` | `MILVUS_PASSWORD` | Password. | diff --git a/python/packages/milvus/agent_framework_milvus/_vector_store.py b/python/packages/milvus/agent_framework_milvus/_vector_store.py index 47318e3c855..ce4446954c5 100644 --- a/python/packages/milvus/agent_framework_milvus/_vector_store.py +++ b/python/packages/milvus/agent_framework_milvus/_vector_store.py @@ -189,6 +189,31 @@ def _check_numeric_field(field: VectorStoreField) -> None: ) +# Maximum magnitude representable as float32; Milvus dense vectors are float32. +_FLOAT32_MAX: Final[float] = 3.4028234663852886e38 + + +def _validate_dense_vector(value: Any, name: str) -> list[float]: + """Validate and materialize a dense float32 vector. + + Core only checks dimensions, so the connector validates the element values here. Rejects + non-sequences and strings/bytes (so a query like ``"123"`` is not read as ``[1, 2, 3]``), + booleans, non-numbers, and NaN/infinity or out-of-float32-range values. Used for both stored + vectors and query vectors so the two paths share identical semantics. + """ + if not isinstance(value, Sequence) or isinstance(value, (str, bytes, bytearray)): + raise TypeError(f"Milvus vector '{name}' must be a dense numeric sequence, not {type(value).__name__}.") + result: list[float] = [] + for item in cast("Sequence[Any]", value): + if isinstance(item, bool) or not isinstance(item, (int, float)): + raise TypeError(f"Milvus vector '{name}' must contain numbers, not {type(item).__name__}.") + number = float(item) + if not math.isfinite(number) or abs(number) > _FLOAT32_MAX: + raise ValueError(f"Milvus vector '{name}' must contain finite float32 values.") + result.append(number) + return result + + class MilvusCollection(BaseVectorCollection[KeyT, ModelT], BaseVectorSearch[KeyT, ModelT], Generic[KeyT, ModelT]): """Store and search records with the official async Milvus client. @@ -374,8 +399,21 @@ async def ensure_collection_exists(self, *, operation_options: Mapping[str, Any] except Exception: raise exc from None + @staticmethod + def _check_field_type(existing: Mapping[str, Any], expected: DataType, name: str) -> None: + actual = existing.get("type") + if actual is not None and DataType(actual) != expected: + raise ValueError( + f"Existing Milvus field '{name}' has type {DataType(actual).name}, expected {expected.name}." + ) + async def _validate_existing_collection(self) -> None: - """Validate an existing collection's fields, vector dimensions and metrics.""" + """Validate an existing collection against the data model. + + Checks the primary key (presence, type), every data field's scalar type, and each vector + field's type, dimensions, metric and (non-auto) index kind, so a same-named but incompatible + collection is rejected up front instead of failing later CRUD/search calls. + """ info = cast("Mapping[str, Any]", await self._client.describe_collection(self.collection_name)) raw_fields = cast("Sequence[Mapping[str, Any]]", info.get("fields") or []) fields: dict[str, Mapping[str, Any]] = {str(field["name"]): field for field in raw_fields} @@ -385,12 +423,17 @@ async def _validate_existing_collection(self) -> None: f"Existing Milvus collection '{self.collection_name}' has no primary key " f"'{self._key_storage_name}' matching the data model." ) + self._check_field_type( + key, + DataType.INT64 if self.definition.key_field.type_ == "int" else DataType.VARCHAR, + self._key_storage_name, + ) for field in self.definition.data_fields: - if (field.storage_name or field.name) not in fields: - raise ValueError( - f"Existing Milvus collection '{self.collection_name}' is missing field " - f"'{field.storage_name or field.name}'." - ) + name = field.storage_name or field.name + existing = fields.get(name) + if existing is None: + raise ValueError(f"Existing Milvus collection '{self.collection_name}' is missing field '{name}'.") + self._check_field_type(existing, _SCALAR_TYPES[field.type_ or "str"], name) for field in self.definition.vector_fields: name = field.storage_name or field.name existing = fields.get(name) @@ -398,16 +441,27 @@ async def _validate_existing_collection(self) -> None: raise ValueError( f"Existing Milvus collection '{self.collection_name}' is missing vector field '{name}'." ) + self._check_field_type(existing, DataType.FLOAT_VECTOR, name) dimensions = cast("Mapping[str, Any]", existing.get("params") or {}).get("dim") if dimensions is not None and int(dimensions) != int(cast(int, field.dimensions)): raise ValueError( f"Existing Milvus vector field '{name}' has dimension {dimensions}, expected {field.dimensions}." ) index = await self._client.describe_index(self.collection_name, name) - metric = cast("Mapping[str, Any]", index).get("metric_type") if isinstance(index, Mapping) else None - expected = _METRICS[cast(str, field.distance_function)] - if metric is not None and metric != expected: - raise ValueError(f"Existing Milvus vector field '{name}' uses metric {metric}, expected {expected}.") + index_info: Mapping[str, Any] = cast("Mapping[str, Any]", index) if isinstance(index, Mapping) else {} + metric = index_info.get("metric_type") + expected_metric = _METRICS[cast(str, field.distance_function)] + if metric is not None and metric != expected_metric: + raise ValueError( + f"Existing Milvus vector field '{name}' uses metric {metric}, expected {expected_metric}." + ) + index_type = index_info.get("index_type") + expected_index = _INDEX_KINDS[cast(str, field.index_kind)] + # AUTOINDEX lets the server pick a concrete type, so only enforce an explicit index kind. + if expected_index != "AUTOINDEX" and index_type is not None and index_type != expected_index: + raise ValueError( + f"Existing Milvus vector field '{name}' uses index {index_type}, expected {expected_index}." + ) async def ensure_collection_deleted(self, *, operation_options: Mapping[str, Any] | None = None) -> None: """Delete this collection if it exists.""" @@ -441,7 +495,7 @@ def _serialize_dicts_to_store_models( name = field.storage_name or field.name value = record.get(name) if value is not None: - model[name] = [float(component) for component in value] + model[name] = _validate_dense_vector(value, name) models.append(model) return models @@ -563,7 +617,7 @@ async def _inner_search( search_params["offset"] = skip hits = await self._client.search( collection_name=self.collection_name, - data=[[float(component) for component in vector]], + data=[_validate_dense_vector(vector, field.storage_name or field.name)], anns_field=field.storage_name or field.name, limit=top, filter=self._prepare_filter(filter), diff --git a/python/packages/milvus/samples/README.md b/python/packages/milvus/samples/README.md index 79b917a3203..bfe09e7e3be 100644 --- a/python/packages/milvus/samples/README.md +++ b/python/packages/milvus/samples/README.md @@ -4,8 +4,9 @@ translated to Milvus boolean expressions, and selective vector retrieval using deterministic vectors. It creates and deletes its own unique collection. -Set `MILVUS_URI` to a disposable Milvus 2.5+ server (e.g. `http://localhost:19530`) or -to a local `.db` path to use Milvus Lite, then follow the +Set `MILVUS_URI` to a disposable Milvus 2.5+ server (e.g. `http://localhost:19530`); for +Milvus Lite, edit the sample to pass a local `.db` path as the `uri=` argument instead +(the `MILVUS_URI` env var must be a server address). Then follow the [package README](../README.md#configuration). No embedding service is needed. Optional `MILVUS_TOKEN`, `MILVUS_USER`, `MILVUS_PASSWORD` and `MILVUS_DB_NAME` are supported by the connector's `MilvusSettings`, which also supports explicitly selected diff --git a/python/packages/milvus/samples/milvus_vectors.py b/python/packages/milvus/samples/milvus_vectors.py index 0f06fd6452b..9cf6d5d907b 100644 --- a/python/packages/milvus/samples/milvus_vectors.py +++ b/python/packages/milvus/samples/milvus_vectors.py @@ -3,13 +3,14 @@ """Store and search dense vectors in a disposable Milvus collection. Run from python/: - export MILVUS_URI=http://localhost:19530 # or a local ".db" path for Milvus Lite + export MILVUS_URI=http://localhost:19530 # a server address (not a file path) uv run --package agent-framework-milvus python packages/milvus/samples/milvus_vectors.py -Use a development server or a local Milvus Lite file, not production. A local ".db" path -requires the Milvus Lite extra: pip install "pymilvus[milvus_lite]" (Linux/macOS only). -No embedding model, inference service, or OpenAI credentials are used: the vectors are -deliberately small and deterministic. +Use a development server, not production. To run against Milvus Lite instead, install the +extra (pip install "pymilvus[milvus_lite]", Linux/macOS only) and pass a local ".db" path as +the uri argument, e.g. MilvusStore(uri="milvus.db") — the MILVUS_URI env var must be a server +address. No embedding model, inference service, or OpenAI credentials are used: the vectors +are deliberately small and deterministic. """ from __future__ import annotations diff --git a/python/packages/milvus/tests/milvus/test_collection.py b/python/packages/milvus/tests/milvus/test_collection.py index 0f8d39380b1..0a9f8702ed4 100644 --- a/python/packages/milvus/tests/milvus/test_collection.py +++ b/python/packages/milvus/tests/milvus/test_collection.py @@ -92,6 +92,90 @@ async def test_ensure_collection_exists_rejects_metric_mismatch(collection, mock await collection.ensure_collection_exists() +def _describe(point_key_type, price_type=DataType.DOUBLE, dense_type=DataType.FLOAT_VECTOR, dim=3): + return { + "fields": [ + {"name": "point_key", "type": point_key_type, "is_primary": True, "params": {"max_length": 65535}}, + {"name": "body", "type": DataType.VARCHAR, "params": {"max_length": 65535}}, + {"name": "price", "type": price_type, "params": {}}, + {"name": "flag", "type": DataType.BOOL, "params": {}}, + {"name": "dense", "type": dense_type, "params": {"dim": dim}}, + ] + } + + +async def test_validate_rejects_key_type_mismatch(collection, mock_client): + mock_client.has_collection.return_value = True + mock_client.describe_collection.return_value = _describe(DataType.VARCHAR) # model key is int -> INT64 + with pytest.raises(ValueError, match="type"): + await collection.ensure_collection_exists() + + +async def test_validate_rejects_data_type_mismatch(collection, mock_client): + mock_client.has_collection.return_value = True + mock_client.describe_collection.return_value = _describe(DataType.INT64, price_type=DataType.VARCHAR) + with pytest.raises(ValueError, match="type"): + await collection.ensure_collection_exists() + + +async def test_validate_rejects_vector_type_mismatch(collection, mock_client): + mock_client.has_collection.return_value = True + mock_client.describe_collection.return_value = _describe(DataType.INT64, dense_type=DataType.INT64) + with pytest.raises(ValueError, match="type"): + await collection.ensure_collection_exists() + + +async def test_validate_rejects_index_kind_mismatch(mock_client): + definition = VectorStoreCollectionDefinition([ + VectorStoreField("key", name="id", type_="int", storage_name="point_key"), + VectorStoreField( + "vector", + name="embedding", + storage_name="dense", + dimensions=3, + distance_function="cosine_similarity", + index_kind="hnsw", + ), + ]) + col = MilvusCollection(dict, definition=definition, collection_name="docs", client=mock_client) + mock_client.has_collection.return_value = True + mock_client.describe_collection.return_value = { + "fields": [ + {"name": "point_key", "type": DataType.INT64, "is_primary": True, "params": {}}, + {"name": "dense", "type": DataType.FLOAT_VECTOR, "params": {"dim": 3}}, + ] + } + mock_client.describe_index.return_value = {"metric_type": "COSINE", "index_type": "FLAT"} + with pytest.raises(ValueError, match="index"): + await col.ensure_collection_exists() + + +async def test_upsert_rejects_boolean_vector_element(collection): + with pytest.raises(TypeError, match="must contain numbers"): + await collection.upsert( + [{"id": 1, "text": "x", "number": 1.0, "flag": True, "embedding": [1.0, True, 3.0]}], + generate_vectors=False, + ) + + +async def test_upsert_rejects_non_finite_vector(collection): + with pytest.raises(ValueError, match="finite float32"): + await collection.upsert( + [{"id": 1, "text": "x", "number": 1.0, "flag": True, "embedding": [1.0, float("nan"), 3.0]}], + generate_vectors=False, + ) + + +async def test_search_rejects_string_vector(collection): + with pytest.raises(TypeError, match="dense numeric sequence"): + await collection.search(vector="123") + + +async def test_search_rejects_non_finite_vector(collection): + with pytest.raises(ValueError, match="finite float32"): + await collection.search(vector=[1.0, float("inf"), 3.0]) + + async def test_ensure_collection_deleted(collection, mock_client): mock_client.has_collection.return_value = True await collection.ensure_collection_deleted() diff --git a/python/samples/02-agents/vector_stores/optimized_data_formats.py b/python/samples/02-agents/vector_stores/optimized_data_formats.py index 95d3beda060..d58a5bc8a37 100644 --- a/python/samples/02-agents/vector_stores/optimized_data_formats.py +++ b/python/samples/02-agents/vector_stores/optimized_data_formats.py @@ -20,7 +20,7 @@ from typing import Annotated, Any, cast import numpy as np -import pandas as pd # pyright: ignore[reportMissingImports] # pyrefly: ignore[missing-import] # ty: ignore[unresolved-import] +import pandas as pd # pyright: ignore[reportMissingImports] # pyrefly: ignore[missing-import] from agent_framework import ( VectorStoreCollectionDefinition, VectorStoreField,