fix(datagrid): save NULL when a date or number cell is cleared

CHANGELOG.md

@@ -41,6 +41,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Smaller updates: about 6 MB from any of the last five releases and 21 MB from older ones, down from 34 MB.
 - Several statements run together on PostgreSQL from iPhone and iPad commit one at a time, and a failure keeps the ones before it. (#3399)
 - MySQL dumps write TIMESTAMP values in UTC and turn off foreign key checks while they load, as mysqldump does.
+- Set Value > Empty and Set EMPTY offered only on columns that can hold an empty value, not on dates, numbers or booleans.
+- Clearing a date, number or boolean cell in a NOT NULL column keeps the stored value instead of saving an empty string.
+- Clearing a Redis key's TTL in the grid removes its expiry.
 
 ### Removed
 
@@ -120,6 +123,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - GitHub Copilot language server pinned to a reviewed version and checksum instead of the latest npm release.
 - One unreadable setting, or one unknown AI provider, resetting its whole settings group to defaults.
+- Clearing a date, number or boolean cell saving an empty string, stored as `0000-00-00 00:00:00` on MySQL, instead of NULL.
+- Pasting a copied NULL cell writing the text NULL instead of NULL.
+- Inspector's boolean picker showing false for a value set to NULL in the grid.
+- Clearing an Elasticsearch long, short or byte field sending an empty string the mapping refuses.
+- Empty line in pasted cells moving the values below it up a row, or cutting off the columns after it.
 
 ### Security
 

Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift

@@ -361,7 +361,8 @@ extension ElasticsearchPluginDriver {
             columnTypeNames: typeNames,
             rows: rows,
             rowsAffected: 0,
-            executionTime: Date().timeIntervalSince(startTime)
+            executionTime: Date().timeIntervalSince(startTime),
+            columnMeta: ElasticsearchTypeClassification.columnMeta(columns: columns, typeNames: typeNames)
         )
     }
 

Plugins/ElasticsearchDriverPlugin/ElasticsearchTypeClassification.swift

@@ -0,0 +1,44 @@
+//
+//  ElasticsearchTypeClassification.swift
+//  ElasticsearchDriverPlugin
+//
+
+import Foundation
+import TableProPluginKit
+
+enum ElasticsearchTypeClassification {
+    /// The SQL name the app classifies a mapping type by, where the mapping name is not one it
+    /// reads. Unhinted, `long`, `short` and `date_nanos` classify as text: a cleared cell was then
+    /// written as `""`, which the field's mapping refuses, and the grid sorted the numbers as text.
+    static func classificationName(for mappingType: String) -> String? {
+        switch mappingType {
+        case "long", "unsigned_long": return "BIGINT"
+        case "short": return "SMALLINT"
+        case "byte": return "TINYINT"
+        case "token_count": return "INTEGER"
+        case "half_float", "scaled_float": return "FLOAT"
+        case "date_nanos": return "TIMESTAMP"
+        default: return nil
+        }
+    }
+
+    /// The result's columns as `fetchColumns` describes them, carrying the hints above.
+    static func columnMeta(columns: [String], typeNames: [String]) -> [PluginColumnInfo] {
+        zip(columns, typeNames).map { column, typeName in
+            let isID = column == ElasticsearchMappingFlattener.idColumn
+            return PluginColumnInfo(
+                name: column,
+                dataType: typeName,
+                isNullable: !isID,
+                isPrimaryKey: isID,
+                generationExpression: nil,
+                generationKind: nil,
+                ddlSpelling: nil,
+                ddlDefault: nil,
+                ddlGenerationExpression: nil,
+                ddlCollation: nil,
+                classificationTypeName: classificationName(for: typeName)
+            )
+        }
+    }
+}

TablePro/Core/Services/ColumnType.swift

@@ -2,8 +2,8 @@
 //  ColumnType.swift
 //  TablePro
 //
-//  Column type metadata for type-aware formatting and display.
-//  Driver-specific type mapping lives in each plugin; this enum is display-only.
+//  Column type metadata for type-aware formatting, display and editing.
+//  Driver-specific type mapping lives in each plugin.
 //
 
 import Foundation
@@ -163,6 +163,42 @@ enum ColumnType: Equatable, Sendable {
         }
     }
 
+    /// Whether `''` is a value of this type, so an emptied cell can be written as one.
+    ///
+    /// A date, number, boolean, JSON document, geometry or array has no empty value. Sent `''`,
+    /// MySQL without strict mode stores its zero value (`0000-00-00 00:00:00`, `0`), SQL Server
+    /// stores `1900-01-01` or `0`, SQLite keeps the text, and a strict MySQL or PostgreSQL refuses
+    /// the save. An empty SET is the empty set, and an enum has an empty value only when one of
+    /// its labels is the empty string.
+    var hasEmptyValue: Bool {
+        switch self {
+        case .text(let rawType):
+            return !Self.nonCharacterTextTypes.contains(Self.baseTypeName(rawType))
+        case .set, .blob:
+            return true
+        case .enumType(_, let values):
+            return values?.contains("") ?? false
+        case .date, .timestamp, .datetime, .integer, .decimal, .boolean, .json, .spatial, .array:
+            return false
+        }
+    }
+
+    /// Types the classifier files under `.text`, either by name or because nothing else matched,
+    /// that hold no empty string either.
+    private static let nonCharacterTextTypes: Set<String> = [
+        "YEAR", "UUID", "TIMEUUID", "UNIQUEIDENTIFIER",
+        "INET", "CIDR", "MACADDR", "MACADDR8", "IPV4", "IPV6", "INTERVAL"
+    ]
+
+    /// `Nullable(UUID)`, `year(4)` and `INTERVAL DAY TO SECOND` all name their type in the first word.
+    private static func baseTypeName(_ rawType: String?) -> String {
+        guard var name = rawType?.uppercased() else { return "" }
+        while let wrapper = ["NULLABLE(", "LOWCARDINALITY("].first(where: { name.hasPrefix($0) }) {
+            name.removeFirst(wrapper.count)
+        }
+        return String(name.prefix { $0.isLetter || $0.isNumber || $0 == "_" })
+    }
+
     /// The element type of an array column, if this is one
     var arrayElement: ColumnType? {
         switch self {

TablePro/Core/Services/Infrastructure/ClipboardService.swift

@@ -33,13 +33,26 @@ extension GridRowsClipboardPayload {
     }
 }
 
+/// A cell block the grid copied, each field as the grid held it.
+///
+/// The text alone loses two things a paste needs: a NULL copies as the text `NULL`, the same text a
+/// stored `'NULL'` copies as, and a value holding a tab or a line break splits into extra fields.
+struct GridCellsClipboardPayload: Codable, Equatable {
+    /// The text the copy wrote. A paste reads the fields only while the clipboard still holds it.
+    let text: String
+    /// The fields by line, in the order the copy wrote them, nil for a NULL.
+    let fields: [[String?]]
+}
+
 protocol ClipboardProvider {
     func readText() -> String?
     func readGridRows() -> GridRowsClipboardPayload?
+    func readGridCells() -> GridCellsClipboardPayload?
     func writeText(_ text: String)
     func writeCsv(_ csv: String)
     func writeImage(_ image: NSImage)
     func writeRows(tsv: String, html: String?, gridRows: GridRowsClipboardPayload)
+    func writeCells(_ cells: GridCellsClipboardPayload)
     var hasText: Bool { get }
     var hasGridRows: Bool { get }
 }
@@ -56,6 +69,7 @@ struct NSPasteboardClipboardProvider: ClipboardProvider {
     private static let tsvType = NSPasteboard.PasteboardType("public.utf8-tab-separated-values-text")
     private static let csvType = NSPasteboard.PasteboardType("public.comma-separated-values-text")
     private static let gridRowsType = NSPasteboard.PasteboardType("com.TablePro.gridRows")
+    private static let gridCellsType = NSPasteboard.PasteboardType("com.TablePro.gridCells")
 
     /// The convention clipboard managers watch for to keep an item out of their history.
     private static let concealedType = NSPasteboard.PasteboardType("org.nspasteboard.ConcealedType")
@@ -72,6 +86,11 @@ struct NSPasteboardClipboardProvider: ClipboardProvider {
         return try? JSONDecoder().decode(GridRowsClipboardPayload.self, from: data)
     }
 
+    func readGridCells() -> GridCellsClipboardPayload? {
+        guard let data = NSPasteboard.general.data(forType: Self.gridCellsType) else { return nil }
+        return try? JSONDecoder().decode(GridCellsClipboardPayload.self, from: data)
+    }
+
     func writeText(_ text: String) {
         let pb = NSPasteboard.general
         pb.clearContents()
@@ -116,6 +135,13 @@ struct NSPasteboardClipboardProvider: ClipboardProvider {
         }
     }
 
+    func writeCells(_ cells: GridCellsClipboardPayload) {
+        writeText(cells.text)
+        if let data = try? JSONEncoder().encode(cells) {
+            NSPasteboard.general.setData(data, forType: Self.gridCellsType)
+        }
+    }
+
     var hasText: Bool {
         PasteboardTextReader.hasText()
     }

TablePro/Core/Services/Query/RowOperationsManager.swift

@@ -320,6 +320,14 @@ final class RowOperationsManager {
             return PasteRowsResult(pastedRows: [], delta: .none)
         }
 
+        if parser == nil, let cells = clipboardProvider.readGridCells(), cells.text == clipboardText {
+            let parsedRows = Self.rows(fromCopiedCells: cells, schema: schema, tableRows: tableRows)
+            return insertParsedRows(
+                keepsMissingFields ? parsedRows.map(Self.nullsAsMissingFields) : parsedRows,
+                into: &tableRows
+            )
+        }
+
         let rowParser = parser ?? Self.detectParser(for: clipboardText)
         let parseResult = rowParser.parse(clipboardText, schema: schema)
 
@@ -366,6 +374,26 @@ final class RowOperationsManager {
         }
     }
 
+    /// Cells the grid copied, as wide as the table, become rows field by field, so a value holding a
+    /// tab or a line break stays one value and a NULL stays NULL. An empty field is read for its
+    /// column, as a typed one is.
+    private static func rows(
+        fromCopiedCells cells: GridCellsClipboardPayload,
+        schema: TableSchema,
+        tableRows: TableRows
+    ) -> [ParsedRow] {
+        cells.fields.enumerated().map { index, fields in
+            var values: [PluginCellValue] = (0..<schema.columnCount).map { column in
+                guard column < fields.count, let text = fields[column] else { return .null }
+                return .entered(text, columnType: tableRows.editingColumnType(at: column))
+            }
+            if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count {
+                values[pkIndex] = .text("__DEFAULT__")
+            }
+            return ParsedRow(values: values, sourceLineNumber: index + 1)
+        }
+    }
+
     /// Text says nothing about which fields a row lacks, so on an engine that tells a missing field
     /// from NULL a pasted NULL leaves the field out, which is what a new row does with it too.
     private static func nullsAsMissingFields(_ row: ParsedRow) -> ParsedRow {

TablePro/Extensions/PluginCellValue+EnteredText.swift

@@ -0,0 +1,20 @@
+//
+//  PluginCellValue+EnteredText.swift
+//  TablePro
+//
+
+import Foundation
+import TableProPluginKit
+
+internal extension PluginCellValue {
+    /// What text entered for a column means as a value.
+    ///
+    /// Emptying a cell whose type has no empty value asks for no value, which is NULL. Writing `''`
+    /// there turned a cleared `deleted_at` into `0000-00-00 00:00:00` on MySQL, so the row still
+    /// read as deleted. A text column keeps `''`, which is a real value there.
+    static func entered(_ text: String?, columnType: ColumnType?) -> PluginCellValue {
+        guard let text else { return .null }
+        guard text.isEmpty, let columnType, !columnType.hasEmptyValue else { return .text(text) }
+        return .null
+    }
+}

TablePro/Models/Query/TableRows.swift

@@ -94,6 +94,30 @@ struct TableRows: Sendable {
         return !ColumnDefaultLiteral.isNull(defaultValue)
     }
 
+    /// The column's type with the labels its catalog declared, which a result set alone never carries.
+    func editingColumnType(at index: Int) -> ColumnType? {
+        guard columnTypes.indices.contains(index) else { return nil }
+        let columnType = columnTypes[index]
+        guard columns.indices.contains(index),
+              let labels = columnEnumValues[columns[index]], !labels.isEmpty else { return columnType }
+        return columnType.withAllowedValues(labels)
+    }
+
+    /// What text entered for a column stages, or nil when the column holds neither it nor NULL.
+    ///
+    /// An emptied date in a NOT NULL column has no value to write. MySQL turns NULL there into the
+    /// current time when `explicit_defaults_for_timestamp` is off, and `''` into the zero date, so
+    /// the edit is refused rather than staged.
+    func enteredValue(_ text: String?, forColumn index: Int) -> PluginCellValue? {
+        let value = PluginCellValue.entered(text, columnType: editingColumnType(at: index))
+        guard value.isNull, refusesNull(columnAt: index) else { return value }
+        return nil
+    }
+
+    func refusesNull(columnAt index: Int) -> Bool {
+        columns.indices.contains(index) && columnNullable[columns[index]] == false
+    }
+
     func value(at row: Int, column: Int) -> PluginCellValue {
         guard row >= 0, row < rows.count else { return .null }
         return rows[row][column]

TablePro/Models/UI/MultiRowEditState.swift

@@ -44,6 +44,9 @@ struct FieldEditState: Identifiable {
     /// Save never writes.
     var isServerOwned: Bool = false
 
+    /// False for a NOT NULL column. Nil from the schema reads as nullable, as the grid's menus do.
+    var isNullable = true
+
     /// The value already differs from the loaded schema because the edit is recorded elsewhere.
     var hasCommittedEdit: Bool = false
 
@@ -130,6 +133,7 @@ final class MultiRowEditState: ObservableObject {
     @Published private(set) var absentCells: [Set<Int>] = []
     @Published private(set) var columns: [String] = []
     @Published private(set) var columnTypes: [ColumnType] = []
+    private var nonNullableColumns: Set<String> = []
 
     var hasEdits: Bool {
         fields.contains { $0.hasEdit }
@@ -146,6 +150,7 @@ final class MultiRowEditState: ObservableObject {
         primaryKeyColumns: Set<String> = [],
         foreignKeyColumns: Set<String> = [],
         serverOwnedColumns: Set<String> = [],
+        nonNullableColumns: Set<String> = [],
         displayFormats: [ValueDisplayFormat?] = [],
         binaryColumns: Set<Int> = []
     ) {
@@ -159,6 +164,7 @@ final class MultiRowEditState: ObservableObject {
         self.absentCells = absentCells
         self.columns = columns
         self.columnTypes = columnTypes
+        self.nonNullableColumns = nonNullableColumns
 
         var newFields: [FieldEditState] = []
 
@@ -206,11 +212,14 @@ final class MultiRowEditState: ObservableObject {
             }
 
             // Mark externally modified columns (e.g., edited in data grid). A modified field that
-            // is now missing is one an edit removed.
+            // is now missing is one an edit removed, and one that now reads NULL is a pending NULL:
+            // written as an empty pending value it read as `''`, and the boolean picker showed false.
             if externallyModifiedColumns.contains(colIndex), pendingValue == nil, !isPendingNull, !isPendingDefault,
                !isPendingRemoval {
                 if isAbsent {
                     isPendingRemoval = true
+                } else if originalValue == nil, !hasMultipleValues {
+                    isPendingNull = true
                 } else {
                     pendingValue = originalValue ?? ""
                 }
@@ -228,6 +237,7 @@ final class MultiRowEditState: ObservableObject {
                 isPrimaryKey: primaryKeyColumns.contains(columnName),
                 isForeignKey: foreignKeyColumns.contains(columnName),
                 isServerOwned: serverOwnedColumns.contains(columnName),
+                isNullable: !nonNullableColumns.contains(columnName),
                 originalValue: originalValue,
                 hasMultipleValues: hasMultipleValues,
                 pendingValue: pendingValue,
@@ -337,13 +347,22 @@ final class MultiRowEditState: ObservableObject {
         guard index < fields.count else { return }
         let hadPendingEdit = fields[index].hasEdit
         let original = fields[index].originalValue
-        let pending = Self.resolvePendingValue(value, original: original, isJson: fields[index].isJson)
+        var pending = Self.resolvePendingValue(value, original: original, isJson: fields[index].isJson)
+        var staged = pending.map { PluginCellValue.entered($0, columnType: fields[index].columnTypeEnum) }
+        /// A NOT NULL column whose type has no empty value holds nothing an emptied field could
+        /// mean, so the field goes back to its stored value, as the grid's cell does.
+        if staged == .null, !fields[index].isNullable {
+            pending = nil
+            staged = nil
+        }
         fields[index].pendingValue = pending
         fields[index].isPendingNull = false
         fields[index].isPendingDefault = false
         fields[index].isPendingRemoval = false
-        if pending != nil {
-            onFieldChanged?(index, PluginCellValue.fromOptional(pending), .typing)
+        /// The field keeps its empty text while the change it stages is NULL: swapping the editor
+        /// for the pending NULL pill would take the field away in the middle of typing a new value.
+        if let staged {
+            onFieldChanged?(index, staged, .typing)
         } else if hadPendingEdit {
             /// `originalValue` is nil for three different situations, and only one of them is a
             /// value: a stored NULL, a selection whose rows do not agree, and a field the rows do
@@ -391,7 +410,9 @@ final class MultiRowEditState: ObservableObject {
             updateField(at: columnIndex, value: value)
             return
         }
-        onDetachedFieldChanged?(columnIndex, PluginCellValue.fromOptional(value), rowIDs)
+        let staged = PluginCellValue.entered(value, columnType: columnTypes[safe: columnIndex])
+        if staged.isNull, columns[safe: columnIndex].map(nonNullableColumns.contains) == true { return }
+        onDetachedFieldChanged?(columnIndex, staged, rowIDs)
     }
 
     private static func resolvePendingValue(_ value: String?, original: String?, isJson: Bool) -> String? {
@@ -497,6 +518,7 @@ final class MultiRowEditState: ObservableObject {
         absentCells = []
         columns = []
         columnTypes = []
+        nonNullableColumns = []
     }
 
     /// Get all edited fields with their new values