introduce Box with basic CRUD, implement entity binary marshalling

Author: Ivan Dlugos

objectbox/__init__.py

@@ -1,7 +1,13 @@
+from objectbox.box import Box
 from objectbox.builder import Builder
 from objectbox.model import Model
+from objectbox.objectbox import ObjectBox
+from objectbox.c import NotFoundException
 
 __all__ = [
+    'Box',
     'Builder',
     'Model',
+    'ObjectBox',
+    'NotFoundException',
 ]

objectbox/box.py

@@ -0,0 +1,54 @@
+from objectbox.model.entity import _Entity
+from objectbox.objectbox import ObjectBox
+from objectbox.c import *
+
+
+class Box:
+    def __init__(self, ob: ObjectBox, entity: _Entity):
+        if not isinstance(entity, _Entity):
+            raise Exception("Given type is not an Entity")
+
+        self._ob = ob
+        self._entity = entity
+        self._c_box = obx_box(ob._c_store, entity.id)
+
+    def is_empty(self) -> bool:
+        is_empty = ctypes.c_bool()
+        obx_box_is_empty(self._c_box, ctypes.byref(is_empty))
+        return bool(is_empty.value)
+
+    def count(self, limit: int = 0) -> int:
+        count = ctypes.c_uint64()
+        obx_box_count(self._c_box, limit, ctypes.byref(count))
+        return int(count.value)
+
+    def put(self, object) -> int:
+        id = object_id = self._entity.get_object_id(object)
+
+        if not id:
+            id = obx_box_id_for_put(self._c_box, 0)
+
+        data = self._entity.marshal(object, id)
+        obx_box_put(self._c_box, id, bytes(data), len(data), OBXPutMode_PUT)
+
+        if id != object_id:
+            self._entity.set_object_id(object, id)
+        return id
+
+    def get(self, id: int):
+        c_data = ctypes.c_void_p()
+        c_size = ctypes.c_size_t()
+        obx_box_get(self._c_box, id, ctypes.byref(c_data), ctypes.byref(c_size))
+
+        # TODO verify which of the following two approaches is better.
+
+        # slice the data from the pointer
+        # data = ctypes.cast(c_data, ctypes.POINTER(ctypes.c_char))[:c_size.value]
+
+        # create a memory view
+        data = memoryview(ctypes.cast(c_data, ctypes.POINTER(ctypes.c_ubyte * c_size.value))[0]).tobytes()
+
+        return self._entity.unmarshal(data)
+
+    def remove(self, id: int):
+        obx_box_remove(self._c_box, id)

objectbox/c.py

@@ -43,7 +43,7 @@ class OBX_store_options(ctypes.Structure):
     ]
 
     def p(self) -> 'ctypes.POINTER(OBX_store_options)':
-        return ctypes.pointer(self)
+        return ctypes.byref(self)
 
 
 OBX_store_options_p = ctypes.POINTER(OBX_store_options)
@@ -118,7 +118,7 @@ class OBX_query(ctypes.Structure):
 C.obx_last_error_code.restype = obx_err
 
 
-class CError(Exception):
+class CoreException(Exception):
     codes = {
         0: "SUCCESS",
         404: "NOT_FOUND",
@@ -150,19 +150,25 @@ class CError(Exception):
     def __init__(self, code):
         self.code = code
         self.message = py_str(C.obx_last_error_message())
-        super(CError, self).__init__("%d (%s) - %s" % (code, self.codes[code], self.message))
+        super(CoreException, self).__init__("%d (%s) - %s" % (code, self.codes[code], self.message))
 
 
-# check obx_err and raise an error
+class NotFoundException(CoreException):
+    pass
+
+
+# assert the the returned obx_err is empty
 def check_obx_err(code: obx_err, func, args):
-    if code != 0:
-        raise CError(code)
+    if code == 404:
+        raise NotFoundException(code)
+    elif code != 0:
+        raise CoreException(code)
 
 
-# check if the returned pointer is null and raise an error
-def check_ptr_result(result, func, args):
+# assert that the returned pointer/int is non-empty
+def check_result(result, func, args):
     if not result:
-        raise CError(C.obx_last_error_code())
+        raise CoreException(C.obx_last_error_code())
     return result
 
 
@@ -173,7 +179,7 @@ def fn(name: str, restype: type, argtypes):
     if restype is obx_err:
         func.errcheck = check_obx_err
     elif restype is not None:
-        func.errcheck = check_ptr_result
+        func.errcheck = check_result
         func.restype = restype
 
     func.argtypes = argtypes
@@ -220,6 +226,28 @@ def c_str(string: str) -> ctypes.c_char_p:
 # obx_err (OBX_store* store);
 obx_store_close = fn('obx_store_close', obx_err, [OBX_store_p])
 
+# OBX_box* (OBX_store* store, obx_schema_id entity_id);
+obx_box = fn('obx_box', OBX_box_p, [OBX_store_p, obx_schema_id])
+
+# obx_err (OBX_box* box, obx_id id, void** data, size_t* size);
+obx_box_get = fn('obx_box_get', obx_err,
+                 [OBX_box_p, obx_id, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_size_t)])
+
+# obx_id (OBX_box* box, obx_id id_or_zero);
+obx_box_id_for_put = fn('obx_box_id_for_put', obx_id, [OBX_box_p, obx_id])
+
+# obx_err (OBX_box* box, obx_id id, const void* data, size_t size, OBXPutMode mode);
+obx_box_put = fn('obx_box_put', obx_err, [OBX_box_p, obx_id, ctypes.c_void_p, ctypes.c_size_t, OBXPutMode])
+
+# obx_err (OBX_box* box, obx_id id);
+obx_box_remove = fn('obx_box_remove', obx_err, [OBX_box_p, obx_id])
+
+# obx_err (OBX_box* box, bool* out_is_empty);
+obx_box_is_empty = fn('obx_box_is_empty', obx_err, [OBX_box_p, ctypes.POINTER(ctypes.c_bool)])
+
+# obx_err obx_box_count(OBX_box* box, uint64_t limit, uint64_t* out_count);
+obx_box_count = fn('obx_box_count', obx_err, [OBX_box_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_uint64)])
+
 OBXPropertyType_Bool = 1
 OBXPropertyType_Byte = 2
 OBXPropertyType_Short = 3
@@ -249,38 +277,38 @@ def c_str(string: str) -> ctypes.c_char_p:
 OBXPropertyFlags_INDEX_HASH64 = 4096
 OBXPropertyFlags_UNSIGNED = 8192
 
-OBXDebugFlags_LOG_TRANSACTIONS_READ = 1,
-OBXDebugFlags_LOG_TRANSACTIONS_WRITE = 2,
-OBXDebugFlags_LOG_QUERIES = 4,
-OBXDebugFlags_LOG_QUERY_PARAMETERS = 8,
-OBXDebugFlags_LOG_ASYNC_QUEUE = 16,
+OBXDebugFlags_LOG_TRANSACTIONS_READ = 1
+OBXDebugFlags_LOG_TRANSACTIONS_WRITE = 2
+OBXDebugFlags_LOG_QUERIES = 4
+OBXDebugFlags_LOG_QUERY_PARAMETERS = 8
+OBXDebugFlags_LOG_ASYNC_QUEUE = 16
 
 # Standard put ("insert or update")
-OBXPutMode_PUT = 1,
+OBXPutMode_PUT = 1
 
 # Put succeeds only if the entity does not exist yet.
-OBXPutMode_INSERT = 2,
+OBXPutMode_INSERT = 2
 
 # Put succeeds only if the entity already exist.
-OBXPutMode_UPDATE = 3,
+OBXPutMode_UPDATE = 3
 
 # The given ID (non-zero) is guaranteed to be new; don't use unless you know exactly what you are doing!
 # This is primarily used internally. Wrong usage leads to inconsistent data (e.g. index data not updated)!
 OBXPutMode_PUT_ID_GUARANTEED_TO_BE_NEW = 4
 
 # Reverts the order from ascending (default) to descending.
-OBXOrderFlags_DESCENDING = 1,
+OBXOrderFlags_DESCENDING = 1
 
 # Makes upper case letters (e.g. "Z") be sorted before lower case letters (e.g. "a").
 # If not specified, the default is case insensitive for ASCII characters.
-OBXOrderFlags_CASE_SENSITIVE = 2,
+OBXOrderFlags_CASE_SENSITIVE = 2
 
 # For scalars only: changes the comparison to unsigned (default is signed).
-OBXOrderFlags_UNSIGNED = 4,
+OBXOrderFlags_UNSIGNED = 4
 
 # null values will be put last.
 # If not specified, by default null values will be put first.
-OBXOrderFlags_NULLS_LAST = 8,
+OBXOrderFlags_NULLS_LAST = 8
 
 # null values should be treated equal to zero (scalars only).
-OBXOrderFlags_NULLS_ZERO = 16,
+OBXOrderFlags_NULLS_ZERO = 16

objectbox/model/__init__.py

@@ -6,5 +6,6 @@
     'Model',
     'Entity',
     'Id',
-    'IdUid'
+    'IdUid',
+    'Property'
 ]

objectbox/model/entity.py

@@ -1,42 +1,136 @@
-from typing import List, Dict
-
+import flatbuffers
+from typing import List
+from objectbox.c import *
 from objectbox.model.properties import Property
 
 
-# entity decorator
+# _Entity class holds model information as well as conversions between python objects and FlatBuffers (ObjectBox data)
 class _Entity(object):
     def __init__(self, cls, id: int, uid: int):
         # currently, ID and UID are mandatory and are not fetched from the model.json
         if id <= 0:
-            raise ValueError("invalid or no 'id; given in the @Entity annotation")
+            raise Exception("invalid or no 'id; given in the @Entity annotation")
 
         if uid <= 0:
-            raise ValueError("invalid or no 'uid' given in the @Entity annotation")
+            raise Exception("invalid or no 'uid' given in the @Entity annotation")
 
         self.cls = cls
         self.name: str = cls.__name__
         self.id = id
         self.uid = uid
 
+        self.last_property_id: 'IdUid' = None  # set in model.entity()
+
         self.properties: List[Property] = list()
-        self.fillProperties()
+        self.offset_properties: List[Property] = list()
+        self.id_property: Property = None
+        self.fill_properties()
 
     def __call__(self):
         return self.cls()
 
-    def fillProperties(self):
+    def fill_properties(self):
         # TODO allow subclassing and support entities with __slots__ defined
         variables = dict(vars(self.cls))
 
         # filter only subclasses of Property
         variables = {k: v for k, v in variables.items() if issubclass(type(v), Property)}
 
-        for k, v in variables.items():
-            v._name = k
-            self.properties.append(v)
+        for k, prop in variables.items():
+            prop._name = k
+            self.properties.append(prop)
+
+            if prop._is_id:
+                if self.id_property:
+                    raise Exception("duplicate ID property: '%s' and '%s'" % (self.id_property._name, prop._name))
+                self.id_property = prop
+
+            if prop._fb_type == flatbuffers.number_types.UOffsetTFlags:
+                assert prop._ob_type in [OBXPropertyType_String, OBXPropertyType_ByteVector], \
+                    "programming error - invalid type OB & FB type combination"
+                self.offset_properties.append(prop)
+
+        if not self.id_property:
+            raise Exception("ID property is not defined")
+        elif self.id_property._ob_type != OBXPropertyType_Long:
+            raise Exception("ID property must be an int")
+
+    def get_value(self, object, prop: Property):
+        # in case value is not overwritten on the object, it's the Property object itself (= as defined in the Class)
+        val = getattr(object, prop._name)
+        if val == prop:
+            return prop._py_type()  # default (empty) value for the given type
+        return val
+
+    def get_object_id(self, object) -> int:
+        return self.get_value(object, self.id_property)
+
+    def set_object_id(self, object, id: int):
+        setattr(object, self.id_property._name, id)
+
+    def marshal(self, object, id: int) -> bytearray:
+        builder = flatbuffers.Builder(256)
+
+        # prepare some properties that need to be built in FB before starting the main object
+        offsets = {}
+        for prop in self.offset_properties:
+            val = self.get_value(object, prop)
+            if prop._ob_type == OBXPropertyType_String:
+                offsets[prop._id] = builder.CreateString(val.encode('utf-8'))
+            elif prop._ob_type == OBXPropertyType_ByteVector:
+                offsets[prop._id] = builder.CreateByteVector(val)
+            else:
+                assert False, "programming error - invalid type OB & FB type combination"
+
+        # start the FlatBuffers object with the largest number of properties that were ever present in the Entity
+        builder.StartObject(self.last_property_id.id)
+
+        # add properties to the FB object
+        for prop in self.properties:
+            if prop._id in offsets:
+                val = offsets[prop._id]
+                if val:
+                    builder.PrependUOffsetTRelative(val)
+            else:
+                val = id if prop == self.id_property else self.get_value(object, prop)
+                builder.Prepend(prop._fb_type, val)
+
+            builder.Slot(prop._fb_slot)
+
+        builder.Finish(builder.EndObject())
+        return builder.Output()
+
+    def unmarshal(self, data: bytes):
+        pos = flatbuffers.encode.Get(flatbuffers.packer.uoffset, data, 0)
+        table = flatbuffers.Table(data, pos)
+
+        # initialize an empty object
+        obj = self.cls()
+
+        # fill it with the data read from FlatBuffers
+        for prop in self.properties:
+            o = table.Offset(prop._fb_v_offset)
+            val = None
+            if not o:
+                val = prop._py_type()  # use default (empty) value if not present in the object
+            elif prop._ob_type == OBXPropertyType_String:
+                val = table.String(o + table.Pos).decode('utf-8')
+            elif prop._ob_type == OBXPropertyType_ByteVector:
+                # access the FB byte vector information
+                start = table.Vector(o)
+                size = table.VectorLen(o)
+
+                # slice the vector as a requested type
+                # TODO test this (immutability, memory safe, etc)
+                val = prop._py_type(table.Bytes[start:start+size])
+            else:
+                val = table.Get(prop._fb_type, o + table.Pos)
+
+            setattr(obj, prop._name, val)
+        return obj
 
 
-# wrap _Entity to allow @Entity(id=, uid=), i.e. no class argument
+# entity decorator - wrap _Entity to allow @Entity(id=, uid=), i.e. no class argument
 def Entity(cls=None, id: int = 0, uid: int = 0):
     if cls:
         return _Entity(cls, id, uid)

objectbox/model/model.py

@@ -25,9 +25,11 @@ def __init__(self):
 
     def entity(self, entity: _Entity, last_property_id: IdUid):
         if not isinstance(entity, _Entity):
-            raise ValueError("Given type is not an Entity. Are you passing an instance instead of a type or did you "
+            raise Exception("Given type is not an Entity. Are you passing an instance instead of a type or did you "
                              "forget the '@Entity' annotation?")
 
+        entity.last_property_id = last_property_id
+
         obx_model_entity(self._c_model, c_str(entity.name), entity.id, entity.uid)
 
         for v in entity.properties: