Improve createdAt column handling

Author: steida

.changeset/clear-mails-float.md

@@ -0,0 +1,19 @@
+---
+"@evolu/common": patch
+---
+
+Refined system (formerly "default") createdAt column handling
+
+### Summary
+
+- `createdAt` is now derived exclusively from the CRDT `Timestamp`. It is injected automatically only on first insert. You can no longer provide `createdAt` in `upsert` mutation – doing so was an anti‑pattern and is now validated against.
+- Introduced `isInsert` flag to `DbChange` to distinguish initial row creation from subsequent updates; this drives automatic `createdAt` population.
+- Added `ValidDbChangeValues` type to reject system columns (`createdAt`, `updatedAt`, `id`) while allowing `isDeleted`.
+- Clock storage changed from sortable string (`TimestampString`) to compact binary (`blob`) representation for space efficiency and fewer conversions.
+- Removed `timestampToTimestampString` / `timestampStringToTimestamp`; added `timestampToDateIso` for converting CRDT timestamps to ISO dates.
+- Schema validation wording updated: "default column" -> "system column" for clarity.
+- Internal protocol encoding updated (tests reflect new binary clock and flag ordering); snapshots adjusted accordingly.
+
+### Notes
+
+- This change reduces payload size (e.g. from 113 to 97).

apps/relay/.gitignore

@@ -10,7 +10,6 @@
 
 db.sqlite
 package-lock.json
-evolu-relay-0.db
 
 # Persistent data directory contents (but keep the directory)
 data/*

apps/web/src/lib/llms.ts

@@ -68,7 +68,7 @@ export const excludePaths = [
   "comparison",
   "conventions",
   "dependency-injection",
-  "evolu-relay",
+  "relay",
   "faq",
   // Add other paths to exclude as needed
 ];

examples/vue-vite-pwa/src/App.vue

@@ -7,7 +7,6 @@ import {
   sqliteTrue,
   createEvolu,
   createFormatTypeError,
-  getOrThrow,
   id,
   kysely,
   maxLength,

packages/common/src/Evolu/Db.ts

@@ -63,11 +63,11 @@ import {
 } from "./Sync.js";
 import {
   Timestamp,
+  TimestampBytes,
+  timestampBytesToTimestamp,
   TimestampConfig,
   TimestampError,
-  TimestampString,
-  timestampStringToTimestamp,
-  timestampToTimestampString,
+  timestampToTimestampBytes,
 } from "./Timestamp.js";
 
 export interface DbConfig extends ConsoleConfig, TimestampConfig {
@@ -387,7 +387,7 @@ const createDbWorkerDeps =
         // }
 
         const configResult = sqlite.exec<{
-          clock: TimestampString;
+          clock: TimestampBytes;
           appOwnerId: OwnerId;
           appOwnerEncryptionKey: OwnerEncryptionKey;
           appOwnerWriteKey: OwnerWriteKey;
@@ -415,7 +415,7 @@ const createDbWorkerDeps =
         };
 
         clock = createClock({ ...platformDeps, sqlite })(
-          timestampStringToTimestamp(config.clock),
+          timestampBytesToTimestamp(config.clock),
         );
       } else {
         appOwner =
@@ -507,7 +507,7 @@ const initializeDb =
 
       sql`
         create table evolu_config (
-          "clock" text not null,
+          "clock" blob not null,
           "appOwnerId" text not null,
           "appOwnerEncryptionKey" blob not null,
           "appOwnerWriteKey" blob not null,
@@ -527,7 +527,7 @@ const initializeDb =
           )
         values
           (
-            ${timestampToTimestampString(initialClock)},
+            ${timestampToTimestampBytes(initialClock)},
             ${initialAppOwner.id},
             ${initialAppOwner.encryptionKey},
             ${initialAppOwner.writeKey},

packages/common/src/Evolu/Evolu.ts

@@ -4,7 +4,7 @@ import {
   isNonEmptyArray,
   isNonEmptyReadonlyArray,
 } from "../Array.js";
-import { assert, assertNonEmptyReadonlyArray } from "../Assert.js";
+import { assertNonEmptyReadonlyArray } from "../Assert.js";
 import { createCallbacks } from "../Callbacks.js";
 import { ConsoleDep } from "../Console.js";
 import { RandomBytesDep, SymmetricCryptoDecryptError } from "../Crypto.js";
@@ -48,7 +48,6 @@ import {
 } from "./Query.js";
 import {
   CreateQuery,
-  DefaultColumns,
   EvoluSchema,
   evoluSchemaToDbSchema,
   IndexesConfig,
@@ -59,6 +58,7 @@ import {
   MutationKind,
   MutationMapping,
   MutationOptions,
+  SystemColumns,
   updateable,
   upsertable,
   ValidateSchema,
@@ -239,7 +239,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> extends Disposable {
    *
    * Evolu does not use SQL for mutations to ensure data can be safely and
    * predictably merged without conflicts. Explicit mutations also allow Evolu
-   * to automatically add and update {@link DefaultColumns}.
+   * to automatically add and update {@link SystemColumns}.
    *
    * ### Example
    *
@@ -284,7 +284,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> extends Disposable {
    *
    * Evolu does not use SQL for mutations to ensure data can be safely and
    * predictably merged without conflicts. Explicit mutations also allow Evolu
-   * to automatically add and update {@link DefaultColumns}.
+   * to automatically add and update {@link SystemColumns}.
    *
    * ### Example
    *
@@ -337,7 +337,7 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> extends Disposable {
    *
    * Evolu does not use SQL for mutations to ensure data can be safely and
    * predictably merged without conflicts. Explicit mutations also allow Evolu
-   * to automatically add and update {@link DefaultColumns}.
+   * to automatically add and update {@link SystemColumns}.
    *
    * ### Example
    *
@@ -408,7 +408,12 @@ export interface Evolu<S extends EvoluSchema = EvoluSchema> extends Disposable {
    */
   readonly reloadApp: () => void;
 
-  /** Export SQLite database file as Uint8Array. */
+  /**
+   * Export SQLite database file as Uint8Array.
+   *
+   * In the future, it will be possible to import a database and export/import
+   * history for 1:1 migrations across owners.
+   */
   readonly exportDatabase: () => Promise<Uint8Array<ArrayBuffer>>;
 
   /**
@@ -716,21 +721,17 @@ const createEvoluInstance =
             const values = { ...result.value };
             delete values.id;
 
-            if (kind === "insert" || kind === "upsert") {
-              // Only set createdAt if not provided by user
-              if (!("createdAt" in values)) {
-                values.createdAt = new Date(deps.time.now()).toISOString();
-              }
-            }
-
-            const dbChange = { table, id, values };
-            assert(
-              DbChange.is(dbChange),
-              `Failed to create DbChange for table "${dbChange.table}"`,
-            );
+            const dbChange = DbChange.orThrow({
+              table,
+              id,
+              values,
+              isInsert: kind === "insert" || kind === "upsert",
+            });
 
-            const mutationChange = { ...dbChange, ownerId: options?.ownerId };
-            mutateMicrotaskQueue.push([mutationChange, options?.onComplete]);
+            mutateMicrotaskQueue.push([
+              { ...dbChange, ownerId: options?.ownerId },
+              options?.onComplete,
+            ]);
           }
 
           if (mutateMicrotaskQueue.length === 1) {

packages/common/src/Evolu/Protocol.ts

@@ -171,6 +171,8 @@
  *   `applyProtocolMessage` function with conditional arguments to reduce code
  *   duplication.
  * - ProtocolQuotaError should return storedBytes and actual quota.
+ * - Replace try-catch with Result + new Error (to preserve stacktraces). Measure
+ *   Result overhead, it should be super small.
  */
 
 import { Packr } from "msgpackr";
@@ -186,8 +188,8 @@ import {
   utf8ToBytes,
 } from "../Buffer.js";
 import {
+  createPadmePadding,
   EncryptionKey,
-  padmePaddingLength,
   RandomBytesDep,
   SymmetricCryptoDecryptError,
   SymmetricCryptoDep,
@@ -436,7 +438,7 @@ export interface ProtocolSyncError extends BaseOwnerError {
 export interface ProtocolTimestampMismatchError {
   readonly type: "ProtocolTimestampMismatchError";
   readonly expected: Timestamp;
-  readonly embedded: Timestamp;
+  readonly timestamp: Timestamp;
 }
 
 /**
@@ -909,8 +911,6 @@ export const applyProtocolMessageAsClient =
       | ProtocolQuotaError
     >
   > => {
-    // try-catch instead of Result for performance and stacktraces
-    // DEV: Measure it again, I think we should use Result with new Error.
     try {
       const input = createBuffer(inputMessage);
       const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
@@ -1057,8 +1057,6 @@ export const applyProtocolMessageAsRelay =
   ): Promise<
     Result<ApplyProtocolMessageAsRelayResult, ProtocolInvalidDataError>
   > => {
-    // try-catch instead of Result for performance and stacktraces
-    // DEV: Measure it again, I think we should use Result with new Error.
     try {
       const input = createBuffer(inputMessage);
       const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
@@ -1664,6 +1662,52 @@ export const decodeNumber = (buffer: Buffer): number => {
   return numberResult.value;
 };
 
+/**
+ * Encodes an array of boolean flags into a single byte.
+ *
+ * Each element in the array corresponds to a bit (0-7). Array can have 0-8
+ * elements.
+ *
+ * ### Example
+ *
+ * ```ts
+ * encodeFlags(buffer, [true, false, true]); // Encodes bits 0, 1, 2
+ * ```
+ */
+export const encodeFlags = (
+  buffer: Buffer,
+  flags: ReadonlyArray<boolean>,
+): void => {
+  let byte = 0;
+  for (let i = 0; i < flags.length && i < 8; i++) {
+    if (flags[i]) {
+      byte |= 1 << i;
+    }
+  }
+  buffer.extend([byte]);
+};
+
+/**
+ * Decodes a byte into an array of boolean flags.
+ *
+ * ### Example
+ *
+ * ```ts
+ * const flags = decodeFlags(buffer, 3); // Decode 3 flags
+ * ```
+ */
+export const decodeFlags = (
+  buffer: Buffer,
+  count: PositiveInt,
+): ReadonlyArray<boolean> => {
+  const byte = buffer.shift();
+  const flags: Array<boolean> = [];
+  for (let i = 0; i < count && i < 8; i++) {
+    flags.push((byte & (1 << i)) !== 0);
+  }
+  return flags;
+};
+
 /**
  * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
  * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
@@ -1675,36 +1719,29 @@ export const decodeNumber = (buffer: Buffer): number => {
 export const encodeAndEncryptDbChange =
   (deps: SymmetricCryptoDep) =>
   (message: CrdtMessage, key: EncryptionKey): EncryptedDbChange => {
-    const change = message.change;
     const buffer = createBuffer();
 
-    // Encode protocol version first for backward compatibility
     encodeNonNegativeInt(buffer, protocolVersion);
 
-    // Encode the timestamp (after version) for tamper verification
-    const timestampBytes = timestampToTimestampBytes(message.timestamp);
-    buffer.extend(timestampBytes);
+    // Encode the timestamp to prevent tampering (e.g., a malicious relay
+    // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
+    buffer.extend(timestampToTimestampBytes(message.timestamp));
 
-    encodeString(buffer, change.table);
+    encodeFlags(buffer, [message.change.isInsert]);
 
-    buffer.extend(idToIdBytes(change.id));
+    encodeString(buffer, message.change.table);
+    buffer.extend(idToIdBytes(message.change.id));
 
-    const entries = objectToEntries(change.values).map(
-      ([column, value]): [string, SqliteValue] => {
-        return [column, value];
-      },
-    );
+    const entries = objectToEntries(message.change.values);
 
     encodeLength(buffer, entries);
-
     for (const [column, value] of entries) {
       encodeString(buffer, column);
       encodeSqliteValue(buffer, value);
     }
 
-    const paddingLength = padmePaddingLength(buffer.getLength());
-    // Add zero bytes as PADMÉ padding - these will be ignored during decoding.
-    buffer.extend(new Uint8Array(paddingLength));
+    // Add PADMÉ padding (ignored during decoding)
+    buffer.extend(createPadmePadding(buffer.getLength()));
 
     const { nonce, ciphertext } = deps.symmetricCrypto.encrypt(
       buffer.unwrap(),
@@ -1735,13 +1772,11 @@ export const decryptAndDecodeDbChange =
     | ProtocolInvalidDataError
     | ProtocolTimestampMismatchError
   > => {
-    // try-catch instead of Result for performance and stacktraces
     try {
       const buffer = createBuffer(message.change);
-      const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
 
-      const ciphertextLength = decodeLength(buffer);
-      const ciphertext = buffer.shiftN(ciphertextLength);
+      const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
+      const ciphertext = buffer.shiftN(decodeLength(buffer));
 
       const plaintextBytes = deps.symmetricCrypto.decrypt(
         ciphertext,
@@ -1753,24 +1788,24 @@ export const decryptAndDecodeDbChange =
       buffer.reset();
       buffer.extend(plaintextBytes.value);
 
-      // Decode version (for future compatibility, no validation needed for now)
+      // Decode version (for future compatibility, not need yet)
       decodeNonNegativeInt(buffer);
 
-      // Decode and verify the embedded timestamp
-      const embeddedTimestampBytes = buffer.shiftN(timestampBytesLength);
-      const embeddedTimestamp = timestampBytesToTimestamp(
-        embeddedTimestampBytes as TimestampBytes,
+      const timestamp = timestampBytesToTimestamp(
+        buffer.shiftN(timestampBytesLength) as TimestampBytes,
       );
 
-      // Verify timestamp integrity
-      if (!eqTimestamp(embeddedTimestamp, message.timestamp)) {
+      if (!eqTimestamp(timestamp, message.timestamp)) {
         return err<ProtocolTimestampMismatchError>({
           type: "ProtocolTimestampMismatchError",
           expected: message.timestamp,
-          embedded: embeddedTimestamp,
+          timestamp,
         });
       }
 
+      const flags = decodeFlags(buffer, PositiveInt.orThrow(1));
+      const isInsert = flags[0];
+
       const table = decodeString(buffer);
       const id = decodeId(buffer);
 
@@ -1783,7 +1818,7 @@ export const decryptAndDecodeDbChange =
         values[column] = value;
       }
 
-      const dbChange = { table, id, values };
+      const dbChange = DbChange.orThrow({ table, id, values, isInsert });
 
       return ok(dbChange);
     } catch (error) {