Add date_value_to_int() to parse date types, unmarshall as UTC #29

When using float as a date type, always interpret values as seconds.

Author: greenrobot

objectbox/model/entity.py

@@ -18,11 +18,12 @@
 from typing import Generic
 import numpy as np
 from math import floor
-from datetime import datetime
+from datetime import datetime, timezone
 from objectbox.c import *
 from objectbox.model.properties import Property
 import threading
 
+
 # _Entity class holds model information as well as conversions between python objects and FlatBuffers (ObjectBox data)
 class _Entity(object):
     def __init__(self, cls, id: int, uid: int):
@@ -47,7 +48,7 @@ def __init__(self, cls, id: int, uid: int):
         self.id_property = None
         self.fill_properties()
         self._tl = threading.local()
-        
+
     def __call__(self, **properties):
         """ The constructor of the user Entity class. """
         object_ = self.cls()
@@ -122,9 +123,9 @@ def get_value(self, object, prop: Property):
             if (val == np.array(prop)).all():
                 return np.array([])
         elif val == prop:
-            if prop._py_type == datetime:
-                return datetime.fromtimestamp(0)
-            if prop._ob_type == OBXPropertyType_Flex:
+            if prop._ob_type == OBXPropertyType_Date or prop._ob_type == OBXPropertyType_DateNano:
+                return 0.0  # For marshalling, prefer float over datetime
+            elif prop._ob_type == OBXPropertyType_Flex:
                 return None
             else:
                 return prop._py_type()  # default (empty) value for the given type
@@ -136,6 +137,19 @@ def get_object_id(self, object) -> int:
     def set_object_id(self, object, id: int):
         setattr(object, self.id_property._name, id)
 
+    @staticmethod
+    def date_value_to_int(value, multiplier: int) -> int:
+        if isinstance(value, datetime):
+            return round(value.timestamp() * multiplier)  # timestamp returns seconds
+        elif isinstance(value, float):
+            return round(value * multiplier)  # floats typically represent seconds
+        elif isinstance(value, int):  # Interpret ints as-is (without the multiplier); e.g. milliseconds or nanoseconds
+            return value
+        else:
+            raise TypeError(
+                f"Unsupported Python datetime type: {type(value)}. Please use datetime, float (seconds based) or "
+                f"int (milliseconds for Date, nanoseconds for DateNano).")
+
     def marshal(self, object, id: int) -> bytearray:
         if not hasattr(self._tl, "builder"):
             self._tl.builder = flatbuffers.Builder(256)
@@ -186,13 +200,9 @@ def marshal(self, object, id: int) -> bytearray:
             else:
                 val = id if prop == self.id_property else self.get_value(object, prop)
                 if prop._ob_type == OBXPropertyType_Date:
-                    if prop._py_type == datetime:
-                        val = val.timestamp() * 1000  # timestamp returns seconds, convert to milliseconds
-                    val = floor(val)  # use floor to allow for float types
+                    val = self.date_value_to_int(val, 1000)  # convert to milliseconds
                 elif prop._ob_type == OBXPropertyType_DateNano:
-                    if prop._py_type == datetime:
-                        val = val.timestamp() * 1000000000  # convert to nanoseconds
-                    val = floor(val)  # use floor to allow for float types
+                    val = self.date_value_to_int(val, 1000000000)  # convert to nanoseconds
                 builder.Prepend(prop._fb_type, val)
 
             builder.Slot(prop._fb_slot)
@@ -211,42 +221,49 @@ def unmarshal(self, data: bytes):
         for prop in self.properties:
             o = table.Offset(prop._fb_v_offset)
             val = None
+            ob_type = prop._ob_type
             if not o:
                 val = prop._py_type()  # use default (empty) value if not present in the object
-            elif prop._ob_type == OBXPropertyType_String:
+            elif ob_type == OBXPropertyType_String:
                 val = table.String(o + table.Pos).decode('utf-8')
-            elif prop._ob_type == OBXPropertyType_BoolVector:
+            elif ob_type == OBXPropertyType_BoolVector:
                 val = table.GetVectorAsNumpy(flatbuffers.number_types.BoolFlags, o)
-            elif prop._ob_type == OBXPropertyType_ByteVector:
+            elif ob_type == OBXPropertyType_ByteVector:
                 # access the FB byte vector information
                 start = table.Vector(o)
                 size = table.VectorLen(o)
                 # slice the vector as a requested type
-                val = prop._py_type(table.Bytes[start:start+size])
-            elif prop._ob_type == OBXPropertyType_ShortVector:
+                val = prop._py_type(table.Bytes[start:start + size])
+            elif ob_type == OBXPropertyType_ShortVector:
                 val = table.GetVectorAsNumpy(flatbuffers.number_types.Int16Flags, o)
-            elif prop._ob_type == OBXPropertyType_CharVector:
+            elif ob_type == OBXPropertyType_CharVector:
                 val = table.GetVectorAsNumpy(flatbuffers.number_types.Int16Flags, o)
-            elif prop._ob_type == OBXPropertyType_Date and prop._py_type == datetime:
-                table_val = table.Get(prop._fb_type, o + table.Pos)
-                val = datetime.fromtimestamp(table_val/1000) if table_val != 0 else datetime.fromtimestamp(0)  # default timestamp
-            elif prop._ob_type == OBXPropertyType_DateNano and prop._py_type == datetime:
-                table_val = table.Get(prop._fb_type, o + table.Pos)
-                val = datetime.fromtimestamp(table_val/1000000000) if table_val != 0 else datetime.fromtimestamp(0)  # default timestamp
-            elif prop._ob_type == OBXPropertyType_IntVector:
+            elif ob_type == OBXPropertyType_Date:
+                val = table.Get(prop._fb_type, o + table.Pos)  # int
+                if prop._py_type == datetime:
+                    val = datetime.fromtimestamp(val / 1000.0, tz=timezone.utc)
+                elif prop._py_type == float:
+                    val = val / 1000.0
+            elif ob_type == OBXPropertyType_DateNano and prop._py_type == datetime:
+                val = table.Get(prop._fb_type, o + table.Pos)  # int
+                if prop._py_type == datetime:
+                    val = datetime.fromtimestamp(val / 1000000000.0, tz=timezone.utc)
+                elif prop._py_type == float:
+                    val = val / 1000000000.0
+            elif ob_type == OBXPropertyType_IntVector:
                 val = table.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
-            elif prop._ob_type == OBXPropertyType_LongVector:
+            elif ob_type == OBXPropertyType_LongVector:
                 val = table.GetVectorAsNumpy(flatbuffers.number_types.Int64Flags, o)
-            elif prop._ob_type == OBXPropertyType_FloatVector:
+            elif ob_type == OBXPropertyType_FloatVector:
                 val = table.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
-            elif prop._ob_type == OBXPropertyType_DoubleVector:
+            elif ob_type == OBXPropertyType_DoubleVector:
                 val = table.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
-            elif prop._ob_type == OBXPropertyType_Flex:
+            elif ob_type == OBXPropertyType_Flex:
                 # access the FB byte vector information
                 start = table.Vector(o)
                 size = table.VectorLen(o)
                 # slice the vector as bytes
-                buf = table.Bytes[start:start+size]
+                buf = table.Bytes[start:start + size]
                 val = flatbuffers.flexbuffers.Loads(buf)
             else:
                 val = table.Get(prop._fb_type, o + table.Pos)
@@ -258,6 +275,8 @@ def unmarshal(self, data: bytes):
 
 def Entity(id: int = 0, uid: int = 0) -> Callable[[Type], _Entity]:
     """ Entity decorator that wraps _Entity to allow @Entity(id=, uid=); i.e. no class arguments. """
+
     def wrapper(class_):
         return _Entity(class_, id, uid)
+
     return wrapper

tests/common.py

@@ -1,3 +1,5 @@
+from datetime import timezone
+
 import objectbox
 import os
 from os import path
@@ -9,17 +11,20 @@
 
 test_dir = 'testdata'
 
+
 def create_default_model() -> objectbox.Model:
     model = objectbox.Model()
     model.entity(TestEntity, last_property_id=IdUid(27, 1027))
     model.last_entity_id = IdUid(2, 2)
     model.last_index_id = IdUid(2, 10002)
     return model
 
+
 def load_empty_test_default_store(db_name: str = test_dir) -> objectbox.Store:
     model = create_default_model()
     return objectbox.Store(model=model, directory=db_name)
 
+
 def load_empty_test_datetime_store(name: str = "") -> objectbox.Store:
     model = objectbox.Model()
     model.entity(TestEntityDatetime, last_property_id=IdUid(4, 2004))
@@ -61,8 +66,10 @@ def create_test_store(db_name: Optional[str] = None, clear_db: bool = True) -> o
 
 
 def assert_equal_prop(actual, expected, default):
-    assert actual == expected or (isinstance(
-        expected, objectbox.model.Property) and actual == default)
+    if isinstance(expected, objectbox.model.Property):
+        assert (actual == default)
+    else:
+        assert (actual == expected)
 
 
 def assert_equal_prop_vector(actual, expected, default):
@@ -72,8 +79,10 @@ def assert_equal_prop_vector(actual, expected, default):
 
 # compare approx values
 def assert_equal_prop_approx(actual, expected, default):
-    assert pytest.approx(actual) == expected or (isinstance(
-        expected, objectbox.model.Property) and actual == default)
+    if isinstance(expected, objectbox.model.Property):
+        assert (actual == default)
+    else:
+        assert (pytest.approx(actual) == expected)
 
 
 def assert_equal(actual: TestEntity, expected: TestEntity):
@@ -100,6 +109,6 @@ def assert_equal(actual: TestEntity, expected: TestEntity):
     assert_equal_prop_approx(actual.longs_list, expected.longs_list, [])
     assert_equal_prop_approx(actual.floats_list, expected.floats_list, [])
     assert_equal_prop_approx(actual.doubles_list, expected.doubles_list, [])
-    assert_equal_prop_approx(actual.date, expected.date, 0)
+    assert_equal_prop_approx(actual.date, expected.date, datetime.fromtimestamp(0, timezone.utc))
     assert_equal_prop(actual.date_nano, expected.date_nano, 0)
     assert_equal_prop(actual.flex, expected.flex, None)

tests/model.py

@@ -31,16 +31,16 @@ class TestEntity:
     longs_list = Int64List(id=22, uid=1022)
     floats_list = Float32List(id=23, uid=1023)
     doubles_list = Float64List(id=24, uid=1024)
-    date = Date(py_type=int, id=25, uid=1025)
-    date_nano = DateNano(py_type=int, id=26, uid=1026)
+    date = Date(id=25, uid=1025)
+    date_nano = DateNano(int, id=26, uid=1026)
     flex = Flex(id=27, uid=1027)
     transient = ""  # not "Property" so it's not stored
 
 
 @Entity(id=2, uid=2)
 class TestEntityDatetime:
     id = Id(id=1, uid=2001)
-    date = Date(id=2, uid=2002)
+    date = Date(float, id=2, uid=2002)
     date_nano = DateNano(id=3, uid=2003)
 
 

tests/test_box.py

@@ -3,7 +3,7 @@
 from tests.model import TestEntity, TestEntityDatetime, TestEntityFlex
 from tests.common import *
 import numpy as np
-from datetime import datetime
+from datetime import datetime, timezone
 import time
 from math import floor
 
@@ -47,8 +47,8 @@ def test_box_basics():
     object.longs_list = [4568, 8714, 1234, 5678, 9012240941]
     object.floats_list = [0.11, 1.22, 2.33, 3.44, 4.5595]
     object.doubles_list = [99.1999, 88.2888, 77.3777, 66.4666, 55.6597555]
-    object.date = time.time() * 1000  # milliseconds since UNIX epoch
-    object.date_nano = time.time_ns()  # nanoseconds since UNIX epoch
+    object.date = time.time()  # seconds since UNIX epoch (float)
+    object.date_nano = time.time_ns()  # nanoseconds since UNIX epoch (int)
     object.flex = dict(a=1, b=2, c=3)
     object.transient = "abcd"
 
@@ -60,20 +60,23 @@ def test_box_basics():
     assert box.count() == 2
 
     # read
+    # wrap date so it can be compared (is read as datetime)
+    object.date = datetime.fromtimestamp(round(object.date * 1000) / 1000, tz=timezone.utc)
     read = box.get(object.id)
     assert_equal(read, object)
     assert read.transient != object.transient  # !=
 
     # update
     object.str = "bar"
-    object.date = floor(time.time_ns() / 1000000)  # check that date can also be int
-    object.date_nano = float(time.time() * 1000000000)  # check that date_nano can also be float
+    object.date = floor(time.time_ns() / 1000000)  # check that date can also be an int
+    object.date_nano = time.time()  # check that date_nano can also be a float
     id = box.put(object)
     assert id == 5
 
     # read again
     read = box.get(object.id)
-    assert_equal(read, object)
+    assert (floor(read.date.timestamp() * 1000) == object.date)
+    assert (read.date_nano == floor(object.date_nano * 1000000000))
 
     # remove
     box.remove(object)
@@ -107,7 +110,8 @@ def test_box_bulk():
     assert objects[2].id == 4
     assert objects[3].id == 1
 
-    assert_equal(box.get(objects[0].id), objects[0])
+    read = box.get(objects[0].id)
+    assert_equal(read, objects[0])
     assert_equal(box.get(objects[1].id), objects[1])
     assert_equal(box.get(objects[2].id), objects[2])
     assert_equal(box.get(objects[3].id), objects[3])
@@ -155,7 +159,9 @@ def test_datetime():
 
     # read
     read = box.get(object.id)
-    assert pytest.approx(read.date.timestamp()) == object.date.timestamp()
+    assert type(read.date) == float
+    assert type(read.date_nano) == datetime
+    assert pytest.approx(read.date) == object.date.timestamp()
 
     # update
     object.str = "bar"
@@ -166,7 +172,7 @@ def test_datetime():
 
     # read again
     read = box.get(object.id)
-    assert pytest.approx(read.date.timestamp()) == object.date.timestamp()
+    assert pytest.approx(read.date) == object.date.timestamp()
 
     # remove
     success = box.remove(object)

tests/test_utils.py

@@ -1,8 +1,48 @@
+from datetime import timezone, datetime, timedelta
+
 import pytest
 
+from objectbox.model.entity import _Entity
 from objectbox.utils import *
 
 
+def test_date_value_to_int__basics():
+    assert _Entity.date_value_to_int(1234, 1000) == 1234
+    assert _Entity.date_value_to_int(1234, 1000000000) == 1234
+    assert _Entity.date_value_to_int(1234.0, 1000) == 1234000  # milliseconds
+    assert _Entity.date_value_to_int(1234.0, 1000000000) == 1234000000000  # nanoseconds
+    assert _Entity.date_value_to_int(datetime.fromtimestamp(1234), 1000) == 1234000  # milliseconds
+
+
+def test_date_value_to_int__timezone():
+    # create datetime object for may 1st, 2000
+    dt_utc = datetime(year=2000, month=5, day=1, hour=12, minute=30, second=45, microsecond=123456, tzinfo=timezone.utc)
+    dt_plus2 = datetime(year=2000, month=5, day=1, hour=14, minute=30, second=45, microsecond=123456,
+                        tzinfo=timezone(offset=timedelta(hours=2)))
+
+    # Demonstrate Python's semantic
+    assert dt_utc == dt_plus2
+    assert dt_utc.timestamp() == dt_plus2.timestamp()
+
+    # Actual test
+    expected: int = 957184245123
+    assert _Entity.date_value_to_int(dt_utc, 1000) == expected
+    assert _Entity.date_value_to_int(dt_plus2, 1000) == expected
+
+
+def test_date_value_to_int__naive():
+    dt_naive = datetime(year=2000, month=5, day=1, hour=12, minute=30, second=45, microsecond=123456)
+    local_tz = datetime.now().astimezone().tzinfo
+    dt_local = datetime(year=2000, month=5, day=1, hour=12, minute=30, second=45, microsecond=123456, tzinfo=local_tz)
+
+    # Demonstrate Python's semantic
+    assert dt_naive.astimezone(timezone.utc) == dt_local  # naive lacks the TZ, so we can't compare directly
+    assert dt_naive.timestamp() == dt_local.timestamp()
+
+    # Actual test
+    assert _Entity.date_value_to_int(dt_naive, 1000) == _Entity.date_value_to_int(dt_local, 1000)
+
+
 def test_vector_distance_f32():
     """ Tests distance values between two vectors. """