Merge branch '47-relevance-score-and-c-api-renames' into 'dev'

Resolve "Bubble up relevance_score / rename to OBXVectorDistanceType"

Closes #47

See merge request objectbox/objectbox-python!32

Author: dan-obx

example/vectorsearch-cities/model.py

@@ -11,12 +11,13 @@ class City:
     location = Property(np.ndarray, type=PropertyType.floatVector, id=3, uid=1003, index=HnswIndex(
         id=3, uid=10001,
         dimensions=2,
-        distance_type=HnswDistanceType.EUCLIDEAN
+        distance_type=VectorDistanceType.EUCLIDEAN
     ))
 
+
 def get_objectbox_model():
     m = Model()
     m.entity(City, last_property_id=IdUid(3, 1003))
     m.last_entity_id = IdUid(1, 1)
-    m.last_index_id = IdUid(3,10001)
+    m.last_index_id = IdUid(3, 10001)
     return m

objectbox/c.py

@@ -79,7 +79,7 @@ def shlib_name(library: str) -> str:
 OBXPutPaddingMode = ctypes.c_int
 OBXOrderFlags = ctypes.c_int
 OBXHnswFlags = ctypes.c_int
-OBXHnswDistanceType = ctypes.c_int
+OBXVectorDistanceType = ctypes.c_int
 OBXValidateOnOpenPagesFlags = ctypes.c_int
 OBXValidateOnOpenKvFlags = ctypes.c_int
 OBXBackupRestoreFlags = ctypes.c_int
@@ -360,6 +360,12 @@ def c_array_pointer(py_list: Union[List[Any], np.ndarray], c_type):
     return ctypes.cast(c_array(py_list, c_type), ctypes.POINTER(c_type))
 
 
+# OBX_C_API float obx_vector_distance_float32(OBXVectorDistanceType type, const float* vector1, const float* vector2, size_t dimension);
+obx_vector_distance_float32 = c_fn("obx_vector_distance_float32", ctypes.c_float, [OBXVectorDistanceType, ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_float), ctypes.c_size_t])
+
+# OBX_C_API float obx_vector_distance_to_relevance(OBXVectorDistanceType type, float distance);
+obx_vector_distance_to_relevance = c_fn("obx_vector_distance_to_relevance", ctypes.c_float, [OBXVectorDistanceType, ctypes.c_float])
+
 # OBX_model* (void);
 obx_model = c_fn('obx_model', OBX_model_p, [])
 
@@ -393,9 +399,8 @@ def c_array_pointer(py_list: Union[List[Any], np.ndarray], c_type):
 obx_model_property_index_hnsw_flags = \
     c_fn_rc('obx_model_property_index_hnsw_flags', [OBX_model_p, OBXHnswFlags])
 
-# obx_err obx_model_property_index_hnsw_distance_type(OBX_model* model, OBXHnswDistanceType value)
-obx_model_property_index_hnsw_distance_type = \
-    c_fn_rc('obx_model_property_index_hnsw_distance_type', [OBX_model_p, OBXHnswDistanceType])
+# obx_err obx_model_property_index_hnsw_distance_type(OBX_model* model, OBXVectorDistanceType value)
+obx_model_property_index_hnsw_distance_type = c_fn_rc('obx_model_property_index_hnsw_distance_type', [OBX_model_p, OBXVectorDistanceType])
 
 # obx_err obx_model_property_index_hnsw_reparation_backlink_probability(OBX_model* model, float value)
 obx_model_property_index_hnsw_reparation_backlink_probability = \
@@ -980,11 +985,11 @@ def c_array_pointer(py_list: Union[List[Any], np.ndarray], c_type):
 OBXHnswFlags_VECTOR_CACHE_SIMD_PADDING_OFF = 4
 OBXHnswFlags_REPARATION_LIMIT_CANDIDATES = 8
 
-OBXHnswDistanceType_UNKNOWN = 0
-OBXHnswDistanceType_EUCLIDEAN = 1
-OBXHnswDistanceType_COSINE = 2
-OBXHnswDistanceType_DOT_PRODUCT = 3
-OBXHnswDistanceType_DOT_PRODUCT_NON_NORMALIZED = 10
+OBXVectorDistanceType_UNKNOWN = 0
+OBXVectorDistanceType_EUCLIDEAN = 1
+OBXVectorDistanceType_COSINE = 2
+OBXVectorDistanceType_DOT_PRODUCT = 3
+OBXVectorDistanceType_DOT_PRODUCT_NON_NORMALIZED = 10
 
 OBXPutPaddingMode_PaddingAutomatic = 1
 OBXPutPaddingMode_PaddingAllowedByBuffer = 2

objectbox/model/properties.py

@@ -94,26 +94,26 @@ class HnswFlags(IntEnum):
     REPARATION_LIMIT_CANDIDATES = 8
 
 
-class HnswDistanceType(IntEnum):
-    UNKNOWN = OBXHnswDistanceType_UNKNOWN
-    EUCLIDEAN = OBXHnswDistanceType_EUCLIDEAN
-    COSINE = OBXHnswDistanceType_COSINE
-    DOT_PRODUCT = OBXHnswDistanceType_DOT_PRODUCT
-    DOT_PRODUCT_NON_NORMALIZED = OBXHnswDistanceType_DOT_PRODUCT_NON_NORMALIZED
-
-HnswDistanceType.UNKNOWN.__doc__ = "Not a real type, just best practice (e.g. forward compatibility)"
-HnswDistanceType.EUCLIDEAN.__doc__ = "The default; typically 'euclidean squared' internally."
-HnswDistanceType.COSINE.__doc__ = """
+class VectorDistanceType(IntEnum):
+    UNKNOWN = OBXVectorDistanceType_UNKNOWN
+    EUCLIDEAN = OBXVectorDistanceType_EUCLIDEAN
+    COSINE = OBXVectorDistanceType_COSINE
+    DOT_PRODUCT = OBXVectorDistanceType_DOT_PRODUCT
+    DOT_PRODUCT_NON_NORMALIZED = OBXVectorDistanceType_DOT_PRODUCT_NON_NORMALIZED
+
+VectorDistanceType.UNKNOWN.__doc__ = "Not a real type, just best practice (e.g. forward compatibility)"
+VectorDistanceType.EUCLIDEAN.__doc__ = "The default; typically 'euclidean squared' internally."
+VectorDistanceType.COSINE.__doc__ = """
 Cosine similarity compares two vectors irrespective of their magnitude (compares the angle of two vectors).
 Often used for document or semantic similarity.
 Value range: 0.0 - 2.0 (0.0: same direction, 1.0: orthogonal, 2.0: opposite direction)
 """
-HnswDistanceType.DOT_PRODUCT.__doc__ = """
+VectorDistanceType.DOT_PRODUCT.__doc__ = """
 For normalized vectors (vector length == 1.0), the dot product is equivalent to the cosine similarity.
 Because of this, the dot product is often preferred as it performs better.
 Value range (normalized vectors): 0.0 - 2.0 (0.0: same direction, 1.0: orthogonal, 2.0: opposite direction)
 """
-HnswDistanceType.DOT_PRODUCT_NON_NORMALIZED.__doc__ = """
+VectorDistanceType.DOT_PRODUCT_NON_NORMALIZED.__doc__ = """
 A custom dot product similarity measure that does not require the vectors to be normalized.
 Note: this is no replacement for cosine similarity (like DotProduct for normalized vectors is).
 The non-linear conversion provides a high precision over the entire float range (for the raw dot product).
@@ -130,7 +130,7 @@ class HnswIndex:
     neighbors_per_node: Optional[int] = None
     indexing_search_count: Optional[int] = None
     flags: HnswFlags = HnswFlags.NONE
-    distance_type: HnswDistanceType = HnswDistanceType.EUCLIDEAN
+    distance_type: VectorDistanceType = VectorDistanceType.EUCLIDEAN
     reparation_backlink_probability: Optional[float] = None
     vector_cache_hint_size_kb: Optional[float] = None
 

objectbox/query_builder.py

@@ -2,10 +2,11 @@
 import numpy as np
 from typing import *
 
+from objectbox.c import *
 from objectbox.model.properties import Property
 from objectbox.objectbox import ObjectBox
 from objectbox.query import Query
-from objectbox.c import *
+from objectbox.utils import check_float_vector
 
 
 class QueryBuilder:
@@ -108,10 +109,11 @@ def between_2ints(self, prop: Union[int, str, Property], value_a: int, value_b:
         cond = obx_qb_between_2ints(self._c_builder, prop_id, value_a, value_b)
         return cond
 
-    def nearest_neighbors_f32(self, prop: Union[int, str, Property], query_vector: Union[np.ndarray, List[float]],
+    def nearest_neighbors_f32(self,
+                              prop: Union[int, str, Property],
+                              query_vector: Union[np.ndarray, List[float]],
                               element_count: int):
-        if isinstance(query_vector, np.ndarray) and query_vector.dtype != np.float32:
-            raise Exception(f"query_vector dtype is expected to be np.float32, got: {query_vector.dtype}")
+        check_float_vector(query_vector, "query_vector")
         prop_id = self._entity.get_property_id(prop)
         c_query_vector = c_array(query_vector, ctypes.c_float)
         cond = obx_qb_nearest_neighbors_f32(self._c_builder, prop_id, c_query_vector, element_count)

objectbox/utils.py

@@ -0,0 +1,30 @@
+import numpy as np
+
+from objectbox.c import *
+from objectbox.model.properties import VectorDistanceType
+
+
+def check_float_vector(vector: Union[np.ndarray, List[float]], vector_name: str):
+    """ Checks that the given vector is a float vector (either np.ndarray or Python's list). """
+    if isinstance(vector, np.ndarray) and vector.dtype != np.float32:
+        raise Exception(f"{vector_name} dtype is expected to be np.float32, got: {vector.dtype}")
+    elif isinstance(vector, list) and len(vector) > 0 and (type(vector[0]) is not float):
+        raise Exception(f"{vector_name} is expected to be a float list, got vector[0]: {type(vector[0])}")
+
+
+def vector_distance_f32(distance_type: VectorDistanceType,
+                        vector1: Union[np.ndarray, List[float]],
+                        vector2: Union[np.ndarray, List[float]],
+                        dimension: int) -> float:
+    """ Utility function to calculate the distance of two vectors. """
+    check_float_vector(vector1, "vector1")
+    check_float_vector(vector2, "vector2")
+    return obx_vector_distance_float32(distance_type,
+                                       c_array(vector1, ctypes.c_float),
+                                       c_array(vector2, ctypes.c_float),
+                                       dimension)
+
+
+def vector_distance_to_relevance(distance_type: VectorDistanceType, distance: float) -> float:
+    """ Converts the given distance to a relevance score in range [0.0, 1.0], according to its type. """
+    return obx_vector_distance_to_relevance(distance_type, distance)

tests/model.py

@@ -58,23 +58,22 @@ class VectorEntity:
     id = Id(id=1, uid=4001)
     name = Property(str, type=PropertyType.string, id=2, uid=4002)
     vector_euclidean = Property(np.ndarray, type=PropertyType.floatVector, id=3, uid=4003,
-                      index=HnswIndex(
-                          id=3, uid=40001,
-                          dimensions=2, distance_type=HnswDistanceType.EUCLIDEAN)
-                      )
+                                index=HnswIndex(
+                                    id=3, uid=40001,
+                                    dimensions=2, distance_type=VectorDistanceType.EUCLIDEAN)
+                                )
     vector_cosine = Property(np.ndarray, type=PropertyType.floatVector, id=4, uid=4004,
-                      index=HnswIndex(
-                          id=4, uid=40002,
-                          dimensions=2, distance_type=HnswDistanceType.COSINE)
-                      )
+                             index=HnswIndex(
+                                 id=4, uid=40002,
+                                 dimensions=2, distance_type=VectorDistanceType.COSINE)
+                             )
     vector_dot_product = Property(np.ndarray, type=PropertyType.floatVector, id=5, uid=4005,
-                      index=HnswIndex(
-                          id=5, uid=40003,
-                          dimensions=2, distance_type=HnswDistanceType.DOT_PRODUCT)
-                      )
-    #vector_dot_product_non_normalized = Property(np.ndarray, type=PropertyType.floatVector, id=6, uid=4006,
+                                  index=HnswIndex(
+                                      id=5, uid=40003,
+                                      dimensions=2, distance_type=VectorDistanceType.DOT_PRODUCT)
+                                  )
+    # vector_dot_product_non_normalized = Property(np.ndarray, type=PropertyType.floatVector, id=6, uid=4006,
     #                  index=HnswIndex(
     #                      id=6, uid=40004,
-    #                      dimensions=2, distance_type=HnswDistanceType.DOT_PRODUCT_NON_NORMALIZED)
+    #                      dimensions=2, distance_type=VectorDistanceType.DOT_PRODUCT_NON_NORMALIZED)
     #                  )
-    

tests/test_hnsw.py

@@ -15,7 +15,12 @@ def _find_expected_nn(points: np.ndarray, query: np.ndarray, n: int):
     return np.argsort(d)[:n]
 
 
-def _test_random_points(num_points: int, num_query_points: int, seed: Optional[int] = None, distance_type: HnswDistanceType = HnswDistanceType.EUCLIDEAN, min_score: float = 0.5):
+def _test_random_points(
+        num_points: int,
+        num_query_points: int,
+        seed: Optional[int] = None,
+        distance_type: VectorDistanceType = VectorDistanceType.EUCLIDEAN,
+        min_score: float = 0.5):
     """ Generates random points in a 2d plane; checks the queried NN against the expected. """
 
     vector_field_name = "vector_"+distance_type.name.lower()
@@ -76,7 +81,7 @@ def _test_random_points(num_points: int, num_query_points: int, seed: Optional[i
 def test_random_points():
 
     min_score = 0.5
-    distance_type = HnswDistanceType.EUCLIDEAN  
+    distance_type = VectorDistanceType.EUCLIDEAN
     _test_random_points(num_points=100, num_query_points=10, seed=10, distance_type=distance_type, min_score=min_score)
     _test_random_points(num_points=100, num_query_points=10, seed=11, distance_type=distance_type, min_score=min_score)
     _test_random_points(num_points=100, num_query_points=10, seed=12, distance_type=distance_type, min_score=min_score)
@@ -86,8 +91,9 @@ def test_random_points():
 
     # TODO: Cosine and Dot Product may result in 0 score
 
-def _test_combined_nn_search(distance_type: HnswDistanceType = HnswDistanceType.EUCLIDEAN):
-    
+
+def _test_combined_nn_search(distance_type: VectorDistanceType = VectorDistanceType.EUCLIDEAN):
+
     db = create_test_objectbox()
 
     box = objectbox.Box(db, VectorEntity)
@@ -175,6 +181,6 @@ def _test_combined_nn_search(distance_type: HnswDistanceType = HnswDistanceType.
 
 def test_combined_nn_search():
     """ Tests NN search combined with regular query conditions, offset and limit. """
-    distance_type = HnswDistanceType.EUCLIDEAN
+    distance_type = VectorDistanceType.EUCLIDEAN
     _test_combined_nn_search(distance_type)
     # TODO: Cosine, DotProduct  diverges see below

tests/test_utils.py

@@ -0,0 +1,18 @@
+import pytest
+
+from objectbox.utils import *
+
+
+def test_vector_distance_f32():
+    """ Tests distance values between two vectors. """
+
+    a = np.array([3.4, 2.9, -10, 1.0], dtype=np.float32)
+    b = np.array([56., -1.2, 22, 2.0], dtype=np.float32)
+
+    a_norm = a / np.linalg.norm(a)
+    b_norm = b / np.linalg.norm(b)
+
+    assert vector_distance_f32(VectorDistanceType.EUCLIDEAN, a, b, 4) == pytest.approx(np.dot(b - a, b - a))
+    assert vector_distance_f32(VectorDistanceType.COSINE, a, b, 4) == pytest.approx(1.0469311)
+    assert vector_distance_f32(VectorDistanceType.DOT_PRODUCT, a_norm, b_norm, 4) == pytest.approx(1.0469311)
+    assert vector_distance_f32(VectorDistanceType.DOT_PRODUCT_NON_NORMALIZED, a, b, 4) == pytest.approx(1.519307)