Preserve DateTimeOffset's offset across BSON round-trip

src/BLite.Bson/BsonSpanReader.cs

@@ -209,6 +209,17 @@ public int ReadInt32()
         return value;
     }
 
+    /// <summary>Not a standalone BSON element type - used only for the offset-in-minutes trailer in <see cref="ReadDateTimeOffset(BsonType)"/>.</summary>
+    private short ReadInt16()
+    {
+        if (Remaining < 2)
+            throw new InvalidOperationException("Not enough bytes to read Int16");
+
+        var value = BinaryPrimitives.ReadInt16LittleEndian(_buffer.Slice(_position, 2));
+        _position += 2;
+        return value;
+    }
+
     public long ReadInt64()
     {
         if (Remaining < 8)
@@ -323,14 +334,38 @@ public DateTime ReadDateTime()
     }
 
     /// <summary>
-    /// Reads a BSON DateTime as DateTimeOffset (UTC milliseconds since Unix epoch)
+    /// Reads a legacy BSON DateTime field (8-byte UTC millisecond timestamp) as a DateTimeOffset.
+    /// There is no offset on the wire in this format, so the result always has Offset=0 - use
+    /// <see cref="ReadDateTimeOffset(BsonType)"/> when the wire type is known, so a value written
+    /// by <c>WriteDateTimeOffset</c> (tagged <see cref="BsonType.DateTimeOffset"/>) comes back with
+    /// its original offset instead.
     /// </summary>
     public DateTimeOffset ReadDateTimeOffset()
     {
         var milliseconds = ReadInt64();
         return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds);
     }
 
+    /// <summary>
+    /// Reads a DateTimeOffset field, coercing a legacy <see cref="BsonType.DateTime"/> wire value
+    /// (8 bytes, no offset - always predates this type, see <see cref="BsonType.DateTimeOffset"/>)
+    /// the same way <see cref="ReadDateTimeOffset()"/> does, or decoding the full 10-byte
+    /// <see cref="BsonType.DateTimeOffset"/> format (UTC milliseconds + signed offset-in-minutes)
+    /// when the offset was actually preserved on write.
+    /// </summary>
+    public DateTimeOffset ReadDateTimeOffset(BsonType bsonType)
+    {
+        if (bsonType != BsonType.DateTimeOffset)
+        {
+            return ReadDateTimeOffset();
+        }
+
+        var milliseconds = ReadInt64();
+        var offsetMinutes = ReadInt16();
+        var instant = DateTimeOffset.FromUnixTimeMilliseconds(milliseconds);
+        return instant.ToOffset(TimeSpan.FromMinutes(offsetMinutes));
+    }
+
     /// <summary>
     /// Reads a TimeSpan from BSON Int64 (ticks)
     /// </summary>
@@ -433,6 +468,9 @@ public void SkipValue(BsonType type)
             case BsonType.Timestamp:
                 _position += 8;
                 break;
+            case BsonType.DateTimeOffset:
+                _position += 10; // 8-byte UTC millisecond timestamp + 2-byte offset-in-minutes
+                break;
             case BsonType.Decimal128:
                 _position += 16;
                 break;

src/BLite.Bson/BsonSpanWriter.cs

@@ -255,12 +255,22 @@ public void WriteDateTime(string name, DateTime value)
         _position += 8;
     }
 
+    /// <summary>
+    /// Writes a UTC instant plus its original offset (10 bytes: 8-byte UTC millisecond timestamp,
+    /// same layout as <see cref="WriteDateTime"/>, followed by a 2-byte signed offset in minutes).
+    /// Tagged <see cref="BsonType.DateTimeOffset"/>, not <see cref="BsonType.DateTime"/>, so
+    /// <see cref="BsonSpanReader.SkipValue"/> and typed readers know an extra 2 bytes follow and
+    /// can reconstruct the original wall-clock value instead of always normalising to UTC.
+    /// </summary>
     public void WriteDateTimeOffset(string name, DateTimeOffset value)
     {
-        WriteElementHeader(BsonType.DateTime, name);
+        WriteElementHeader(BsonType.DateTimeOffset, name);
         var milliseconds = value.ToUnixTimeMilliseconds();
         BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), milliseconds);
         _position += 8;
+        var offsetMinutes = (short)value.Offset.TotalMinutes;
+        BinaryPrimitives.WriteInt16LittleEndian(_buffer.Slice(_position, 2), offsetMinutes);
+        _position += 2;
     }
 
     public void WriteTimeSpan(string name, TimeSpan value)
@@ -448,12 +458,16 @@ public void WriteArrayDateTime(int index, DateTime value)
         _position += 8;
     }
 
+    /// <summary>Array counterpart of <see cref="WriteDateTimeOffset(string, DateTimeOffset)"/> - see its remarks.</summary>
     public void WriteArrayDateTimeOffset(int index, DateTimeOffset value)
     {
-        WriteArrayElementHeader(BsonType.DateTime, index);
+        WriteArrayElementHeader(BsonType.DateTimeOffset, index);
         var milliseconds = value.ToUnixTimeMilliseconds();
         BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), milliseconds);
         _position += 8;
+        var offsetMinutes = (short)value.Offset.TotalMinutes;
+        BinaryPrimitives.WriteInt16LittleEndian(_buffer.Slice(_position, 2), offsetMinutes);
+        _position += 2;
     }
 
     public void WriteArrayTimeSpan(int index, TimeSpan value)

src/BLite.Bson/BsonType.cs

@@ -25,6 +25,17 @@ public enum BsonType : byte
     Timestamp = 0x11,
     Int64 = 0x12,
     Decimal128 = 0x13,
+
+    /// <summary>
+    /// BLite extension, not part of the BSON spec (0x14 is unassigned there). Distinct from
+    /// <see cref="DateTime"/> because it carries a UTC-offset alongside the instant - see
+    /// <see cref="BsonValue.FromDateTimeOffset"/> / <c>BsonSpanWriter.WriteDateTimeOffset</c>.
+    /// A <see cref="DateTimeOffset"/> value written before this type existed is still tagged
+    /// <see cref="DateTime"/> on disk (offset already lost at write time, not recoverable) -
+    /// readers must keep handling that tag for <see cref="DateTimeOffset"/> fields too.
+    /// </summary>
+    DateTimeOffset = 0x14,
+
     MinKey = 0xFF,
     MaxKey = 0x7F
 }

src/BLite.Bson/BsonValue.cs

@@ -43,7 +43,15 @@ private BsonValue(BsonType type, double numericValue = 0, object? refValue = nul
     public static BsonValue FromBoolean(bool value) => new(BsonType.Boolean, value ? 1 : 0);
     public static BsonValue FromObjectId(ObjectId value) => new(BsonType.ObjectId, refValue: value);
     public static BsonValue FromDateTime(DateTime value) => new(BsonType.DateTime, BitConverter.Int64BitsToDouble(new DateTimeOffset(value.ToUniversalTime()).ToUnixTimeMilliseconds()));
-    public static BsonValue FromDateTimeOffset(DateTimeOffset value) => new(BsonType.DateTime, BitConverter.Int64BitsToDouble(value.ToUnixTimeMilliseconds()));
+
+    /// <summary>
+    /// Tagged <see cref="BsonType.DateTimeOffset"/>, not <see cref="BsonType.DateTime"/>, so the
+    /// offset survives a round-trip through <see cref="WriteTo"/>/<see cref="ReadFrom"/> instead of
+    /// being silently normalised to UTC. The offset itself rides in <c>_refValue</c> (boxed
+    /// <see cref="short"/> minutes) since <c>_numericValue</c> already holds the packed instant.
+    /// </summary>
+    public static BsonValue FromDateTimeOffset(DateTimeOffset value) =>
+        new(BsonType.DateTimeOffset, BitConverter.Int64BitsToDouble(value.ToUnixTimeMilliseconds()), (short)value.Offset.TotalMinutes);
     public static BsonValue FromGuid(Guid value) => new(BsonType.String, refValue: value.ToString());
     public static BsonValue FromBinary(byte[] value) => new(BsonType.Binary, refValue: value ?? throw new ArgumentNullException(nameof(value)));
     public static BsonValue FromDocument(BsonDocument value) => new(BsonType.Document, refValue: value ?? throw new ArgumentNullException(nameof(value)));
@@ -110,13 +118,29 @@ private BsonValue(BsonType type, double numericValue = 0, object? refValue = nul
         ? oid
         : throw new InvalidOperationException($"BsonValue is {_type}, not ObjectId");
 
-    public DateTime AsDateTime => _type == BsonType.DateTime
-        ? DateTimeOffset.FromUnixTimeMilliseconds(BitConverter.DoubleToInt64Bits(_numericValue)).UtcDateTime
-        : throw new InvalidOperationException($"BsonValue is {_type}, not DateTime");
+    public DateTime AsDateTime => _type switch
+    {
+        BsonType.DateTime or BsonType.DateTimeOffset =>
+            DateTimeOffset.FromUnixTimeMilliseconds(BitConverter.DoubleToInt64Bits(_numericValue)).UtcDateTime,
+        _ => throw new InvalidOperationException($"BsonValue is {_type}, not DateTime")
+    };
 
-    public DateTimeOffset AsDateTimeOffset => _type == BsonType.DateTime
-        ? DateTimeOffset.FromUnixTimeMilliseconds(BitConverter.DoubleToInt64Bits(_numericValue))
-        : throw new InvalidOperationException($"BsonValue is {_type}, not DateTime");
+    /// <summary>
+    /// For <see cref="BsonType.DateTime"/> (a value written before offsets were tracked, or a plain
+    /// <see cref="DateTime"/> field) the offset is unknown and comes back as 0 (UTC). For
+    /// <see cref="BsonType.DateTimeOffset"/> the original offset (boxed in <c>_refValue</c>) is
+    /// restored via <see cref="DateTimeOffset.ToOffset"/>, which re-expresses the same instant -
+    /// it does not shift it.
+    /// </summary>
+    public DateTimeOffset AsDateTimeOffset => _type switch
+    {
+        BsonType.DateTimeOffset =>
+            DateTimeOffset.FromUnixTimeMilliseconds(BitConverter.DoubleToInt64Bits(_numericValue))
+                .ToOffset(TimeSpan.FromMinutes((short)_refValue!)),
+        BsonType.DateTime =>
+            DateTimeOffset.FromUnixTimeMilliseconds(BitConverter.DoubleToInt64Bits(_numericValue)),
+        _ => throw new InvalidOperationException($"BsonValue is {_type}, not DateTime")
+    };
 
     public byte[] AsBinary => _type == BsonType.Binary && _refValue is byte[] b
         ? b
@@ -149,6 +173,7 @@ private BsonValue(BsonType type, double numericValue = 0, object? refValue = nul
     public bool IsDouble => Type == BsonType.Double;
     public bool IsBoolean => Type == BsonType.Boolean;
     public bool IsDateTime => Type == BsonType.DateTime;
+    public bool IsDateTimeOffset => Type == BsonType.DateTimeOffset;
     public bool IsDecimal => Type == BsonType.Decimal128;
     public bool IsBinary => Type == BsonType.Binary;
     public bool IsObjectId => Type == BsonType.ObjectId;
@@ -196,6 +221,9 @@ public void WriteTo(ref BsonSpanWriter writer, string fieldName)
                 var dt = DateTimeOffset.FromUnixTimeMilliseconds(BitConverter.DoubleToInt64Bits(_numericValue)).UtcDateTime;
                 writer.WriteDateTime(fieldName, dt);
                 break;
+            case BsonType.DateTimeOffset:
+                writer.WriteDateTimeOffset(fieldName, AsDateTimeOffset);
+                break;
             case BsonType.Binary:
                 writer.WriteBinary(fieldName, (byte[])_refValue!);
                 break;
@@ -256,6 +284,9 @@ public void WriteToArray(ref BsonSpanWriter writer, int index)
             case BsonType.DateTime:
                 writer.WriteArrayDateTime(index, DateTimeOffset.FromUnixTimeMilliseconds(BitConverter.DoubleToInt64Bits(_numericValue)).UtcDateTime);
                 break;
+            case BsonType.DateTimeOffset:
+                writer.WriteArrayDateTimeOffset(index, AsDateTimeOffset);
+                break;
             case BsonType.Null:
                 writer.WriteArrayNull(index);
                 break;
@@ -292,7 +323,13 @@ public static BsonValue ReadFrom(ref BsonSpanReader reader, BsonType type)
             BsonType.String => FromString(reader.ReadString()),
             BsonType.Boolean => FromBoolean(reader.ReadBoolean()),
             BsonType.ObjectId => FromObjectId(reader.ReadObjectId()),
-            BsonType.DateTime => FromDateTimeOffset(reader.ReadDateTimeOffset()),
+            // Must stay tagged BsonType.DateTime, matching the wire tag just read - not
+            // FromDateTimeOffset, which now tags BsonType.DateTimeOffset. Using it here would flip
+            // the in-memory type for every plain DateTime field (and every legacy DateTimeOffset
+            // value still under the old tag), so a later WriteTo would silently re-emit it in the
+            // new 10-byte layout even though nothing about its offset was actually recovered.
+            BsonType.DateTime => FromDateTime(reader.ReadDateTime()),
+            BsonType.DateTimeOffset => FromDateTimeOffset(reader.ReadDateTimeOffset(BsonType.DateTimeOffset)),
             BsonType.Null => Null,
             BsonType.Binary => FromBinary(reader.ReadBinary(out _).ToArray()),
             BsonType.Array => ReadArray(ref reader),
@@ -335,6 +372,11 @@ public bool Equals(BsonValue other)
         {
             BsonType.Int32 or BsonType.Int64 or BsonType.Double or BsonType.Boolean or BsonType.DateTime
                 => _numericValue == other._numericValue,
+            // Unlike DateTimeOffset.Equals (instant-only), the offset is treated as significant here:
+            // it's the whole reason this type exists, so two values landing on the same instant but
+            // written with a different offset are NOT the same BsonValue. Keeps this consistent with
+            // GetHashCode, which already combines both _numericValue and _refValue (the boxed offset).
+            BsonType.DateTimeOffset => _numericValue == other._numericValue && (short)_refValue! == (short)other._refValue!,
             BsonType.String => string.Equals((string?)_refValue, (string?)other._refValue, StringComparison.Ordinal),
             BsonType.ObjectId => Equals(_refValue, other._refValue),
             BsonType.Null => true,
@@ -361,6 +403,7 @@ public override string ToString()
             BsonType.Boolean => (_numericValue != 0).ToString(),
             BsonType.ObjectId => _refValue?.ToString() ?? "(null)",
             BsonType.DateTime => AsDateTime.ToString("O"),
+            BsonType.DateTimeOffset => AsDateTimeOffset.ToString("O"),
             BsonType.Null => "null",
             _ => $"({_type})"
         };

src/BLite.Core/Collections/BsonSchemaGenerator.cs

@@ -115,7 +115,8 @@ private static (BsonType type, BsonSchema? nested, BsonType? itemType) GetBsonTy
         if (type == typeof(bool)) return (BsonType.Boolean, null, null);
         if (type == typeof(double)) return (BsonType.Double, null, null);
         if (type == typeof(decimal)) return (BsonType.Decimal128, null, null);
-        if (type == typeof(DateTime) || type == typeof(DateTimeOffset)) return (BsonType.DateTime, null, null);
+        if (type == typeof(DateTime)) return (BsonType.DateTime, null, null);
+        if (type == typeof(DateTimeOffset)) return (BsonType.DateTimeOffset, null, null);
         if (type == typeof(Guid)) return (BsonType.Binary, null, null); // Guid is usually Binary subtype
         if (type == typeof(byte[])) return (BsonType.Binary, null, null);
 

src/BLite.Core/Collections/DocumentCollection.cs

@@ -408,7 +408,7 @@ private long ExtractTimestampTicks(byte[] bsonBytes, string fieldName)
                 var value = BsonValue.ReadFrom(ref reader, seekType);
                 return value.Type switch
                 {
-                    BsonType.DateTime => value.AsDateTime.Ticks,
+                    BsonType.DateTime or BsonType.DateTimeOffset => value.AsDateTime.Ticks,
                     BsonType.Int64    => value.AsInt64,
                     _                 => 0
                 };
@@ -429,7 +429,7 @@ private long ExtractTimestampTicks(byte[] bsonBytes, string fieldName)
                 var val = BsonValue.ReadFrom(ref reader, type);
                 return val.Type switch
                 {
-                    BsonType.DateTime => val.AsDateTime.Ticks,
+                    BsonType.DateTime or BsonType.DateTimeOffset => val.AsDateTime.Ticks,
                     BsonType.Int64    => val.AsInt64,
                     _                 => 0
                 };

src/BLite.Core/DynamicCollection.cs

@@ -423,7 +423,7 @@ private async Task ApplyRetentionPolicyCoreAsync(CancellationToken ct)
                 {
                     ticks = tsVal.Type switch
                     {
-                        BsonType.DateTime => tsVal.AsDateTime.Ticks,
+                        BsonType.DateTime or BsonType.DateTimeOffset => tsVal.AsDateTime.Ticks,
                         BsonType.Int64    => tsVal.AsInt64,
                         _                 => 0
                     };
@@ -2001,8 +2001,10 @@ private void SnapshotFsiForTransaction(ITransaction transaction, uint pageId)
             BsonType.ObjectId => new IndexKey(value.AsObjectId),
             BsonType.Double   => new IndexKey(BitConverter.GetBytes(value.AsDouble)),
             // DateTime is stored as BitConverter.Int64BitsToDouble(unixMs); key on the raw long
-            // to get the same ordering used by ToIndexObject in BlqlFilter.
-            BsonType.DateTime => new IndexKey(value.AsDateTimeOffset.ToUnixTimeMilliseconds()),
+            // to get the same ordering used by ToIndexObject in BlqlFilter. Deliberately keyed by
+            // instant, not offset, for DateTimeOffset too - the offset must not perturb range-scan
+            // ordering, only how the value is displayed once read back.
+            BsonType.DateTime or BsonType.DateTimeOffset => new IndexKey(value.AsDateTimeOffset.ToUnixTimeMilliseconds()),
             _ => null // Can't index this type as BTree key
         };
     }

src/BLite.Core/Query/Blql/BlqlFilter.cs

@@ -384,7 +384,8 @@ internal override bool TryGetIndexCandidate(out IndexScanCandidate candidate)
             BsonType.ObjectId => v.AsObjectId,
             // DateTime is stored as BitConverter.Int64BitsToDouble(unixMs); extract as long so
             // CreateIndexKeyFromObject can create a comparable IndexKey(long) for BTree range scans.
-            BsonType.DateTime => v.AsDateTimeOffset.ToUnixTimeMilliseconds(),
+            // Keyed by instant for DateTimeOffset too, matching DynamicCollection.ToIndexObject.
+            BsonType.DateTime or BsonType.DateTimeOffset => v.AsDateTimeOffset.ToUnixTimeMilliseconds(),
             _                 => null
         };
     }

src/BLite.Core/Query/Blql/BsonValueComparer.cs

@@ -26,14 +26,22 @@ public static int Compare(BsonValue a, BsonValue b)
         if (IsNumeric(a.Type) && IsNumeric(b.Type))
             return ToDouble(a).CompareTo(ToDouble(b));
 
+        // DateTime/DateTimeOffset: compare by instant across the two tags too, same as the numeric
+        // family above. A collection can hold both during the transition after upgrading BLite - a
+        // DateTimeOffset value written before this type existed stays tagged DateTime forever - so
+        // this must not fall through to TypeOrder below, which would sort DateTimeOffset-tagged rows
+        // as unrelated to DateTime-tagged ones instead of interleaving them by actual time.
+        if (IsDateLike(a.Type) && IsDateLike(b.Type))
+            return a.AsDateTimeOffset.CompareTo(b.AsDateTimeOffset);
+
         // Same type comparisons
         if (a.Type == b.Type)
         {
             return a.Type switch
             {
                 BsonType.String    => string.Compare(a.AsString, b.AsString, StringComparison.Ordinal),
                 BsonType.Boolean   => a.AsBoolean.CompareTo(b.AsBoolean),
-                BsonType.DateTime  => a.AsDateTimeOffset.CompareTo(b.AsDateTimeOffset),
+                BsonType.DateTime or BsonType.DateTimeOffset => a.AsDateTimeOffset.CompareTo(b.AsDateTimeOffset),
                 BsonType.ObjectId  => CompareObjectIds(a.AsObjectId, b.AsObjectId),
                 BsonType.Binary    => CompareBytes(a.AsBinary, b.AsBinary),
                 _                  => 0
@@ -47,6 +55,9 @@ public static int Compare(BsonValue a, BsonValue b)
     private static bool IsNumeric(BsonType t) =>
         t is BsonType.Int32 or BsonType.Int64 or BsonType.Double or BsonType.Decimal128;
 
+    private static bool IsDateLike(BsonType t) =>
+        t is BsonType.DateTime or BsonType.DateTimeOffset;
+
     private static double ToDouble(BsonValue v) => v.Type switch
     {
         BsonType.Int32     => v.AsInt32,
@@ -67,6 +78,7 @@ private static bool IsNumeric(BsonType t) =>
         BsonType.String    => 3,
         BsonType.ObjectId  => 4,
         BsonType.DateTime  => 5,
+        BsonType.DateTimeOffset => 5,
         BsonType.Binary    => 6,
         BsonType.Document  => 7,
         BsonType.Array     => 8,