feat(plugin-bigtable): add Google Cloud Bigtable driver

.github/plugin-registry.json

@@ -46,6 +46,20 @@
       "category": "database-driver",
       "homepage": "https://docs.tablepro.app/databases/bigquery"
     },
+    "bigtable": {
+      "target": "BigtableDriverPlugin",
+      "bundleName": "BigtableDriverPlugin",
+      "bundleId": "com.TablePro.BigtableDriverPlugin",
+      "bundled": false,
+      "displayName": "Bigtable Driver",
+      "summary": "Google Cloud Bigtable driver over the REST API",
+      "databaseTypeIds": [
+        "Bigtable"
+      ],
+      "icon": "bigtable-icon",
+      "category": "database-driver",
+      "homepage": "https://docs.tablepro.app/databases/bigtable"
+    },
     "cassandra": {
       "target": "CassandraDriver",
       "bundleName": "CassandraDriver",

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Setting in Settings > General to hide the version and Check for Updates on the Welcome window.
 - Install Parquet… in the Export dialog's format menu when the Parquet plugin is not installed.
 - Cloud Firestore and Firebase Realtime Database as a registry plugin over their REST APIs. (#2504)
+- Google Cloud Bigtable as a registry plugin over the REST API.
 - Warning banner and Test Connection notice for a server warning at login, such as an Oracle password about to expire. (#3401)
 
 ### Changed

Packages/TableProCore/Package.swift

@@ -35,6 +35,7 @@ let package = Package(
         .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]),
         .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]),
         .library(name: "TableProFirebaseCore", targets: ["TableProFirebaseCore"]),
+        .library(name: "TableProBigtableCore", targets: ["TableProBigtableCore"]),
         .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]),
         .library(name: "TableProDocumentPath", targets: ["TableProDocumentPath"]),
         .library(name: "TableProJavaScriptText", targets: ["TableProJavaScriptText"]),
@@ -168,9 +169,14 @@ let package = Package(
             dependencies: [],
             path: "Sources/TableProWeaviateCore"
         ),
+        .target(
+            name: "TableProBigtableCore",
+            dependencies: ["TableProGoogleCloud"],
+            path: "Sources/TableProBigtableCore"
+        ),
         .target(
             name: "TableProFirebaseCore",
-            dependencies: [],
+            dependencies: ["TableProGoogleCloud"],
             path: "Sources/TableProFirebaseCore"
         ),
         .target(
@@ -232,6 +238,11 @@ let package = Package(
             path: "Sources/TableProRedisCore",
             swiftSettings: pluginCoreSettings
         ),
+        .testTarget(
+            name: "TableProBigtableCoreTests",
+            dependencies: ["TableProBigtableCore", "TableProGoogleCloud"],
+            path: "Tests/TableProBigtableCoreTests"
+        ),
         .testTarget(
             name: "TableProTabularIOTests",
             dependencies: ["TableProTabularIO", "TableProPluginKit"],

Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift

@@ -0,0 +1,79 @@
+import Foundation
+import TableProGoogleCloud
+
+public enum BigtableFieldID {
+    public static let authMethod = "btAuthMethod"
+    public static let serviceAccountJson = "btServiceAccountJson"
+    public static let projectId = "btProjectId"
+    public static let instanceId = "btInstanceId"
+    public static let appProfileId = "btAppProfileId"
+}
+
+public enum BigtableAuthMethod: String, Sendable, CaseIterable {
+    case serviceAccount
+    case applicationDefault = "adc"
+}
+
+public struct BigtableConnectionSettings: Sendable, Equatable {
+    public let projectId: String
+    public let instanceId: String?
+    public let appProfileId: String?
+
+    public init(projectId: String, instanceId: String?, appProfileId: String? = nil) {
+        self.projectId = projectId
+        self.instanceId = instanceId
+        self.appProfileId = appProfileId
+    }
+
+    /// `database` wins over the Instance ID field: a pooled connection the app opens after a switch
+    /// names the instance it switched to there and never calls `switchDatabase`.
+    public static func parse(
+        fields: [String: String],
+        database: String? = nil,
+        credentialProject: String? = nil
+    ) throws -> BigtableConnectionSettings {
+        guard let projectId = trimmed(fields[BigtableFieldID.projectId]) ?? credentialProject else {
+            throw BigtableError.configuration(String(localized: "Enter a Project ID, or use a service account key that names one."))
+        }
+        guard isValidProjectId(projectId) else {
+            throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Google Cloud project ID."), projectId))
+        }
+        let instanceId = trimmed(database) ?? trimmed(fields[BigtableFieldID.instanceId])
+        if let instanceId, !isValidResourceId(instanceId) {
+            throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Bigtable instance ID."), instanceId))
+        }
+        let appProfileId = trimmed(fields[BigtableFieldID.appProfileId])
+        if let appProfileId, !isValidResourceId(appProfileId) {
+            throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not an app profile ID."), appProfileId))
+        }
+        return BigtableConnectionSettings(projectId: projectId, instanceId: instanceId, appProfileId: appProfileId)
+    }
+
+    public static func credentialSource(fields: [String: String], password: String?) throws -> GoogleCredentialSource {
+        let method = trimmed(fields[BigtableFieldID.authMethod]).flatMap(BigtableAuthMethod.init(rawValue:)) ?? .serviceAccount
+        switch method {
+        case .applicationDefault:
+            return .applicationDefault
+        case .serviceAccount:
+            guard let key = trimmed(fields[BigtableFieldID.serviceAccountJson]) ?? trimmed(password) else {
+                throw BigtableError.configuration(String(localized: "Choose a service account key file, or paste its JSON."))
+            }
+            return .serviceAccountKey(key)
+        }
+    }
+
+    static func isValidProjectId(_ text: String) -> Bool {
+        text.range(of: #"^[a-z][a-z0-9.:-]{2,62}$"#, options: .regularExpression) != nil
+    }
+
+    /// Instance, table and app profile IDs go into the request path, so anything outside the
+    /// characters Bigtable allows is refused before it can change the path.
+    static func isValidResourceId(_ text: String) -> Bool {
+        text.range(of: #"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,49}$"#, options: .regularExpression) != nil
+    }
+
+    static func trimmed(_ value: String?) -> String? {
+        guard let value = value?.trimmingCharacters(in: .whitespacesAndNewlines), !value.isEmpty else { return nil }
+        return value
+    }
+}

Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift

@@ -0,0 +1,155 @@
+import Foundation
+
+public struct BigtableColumn: Hashable, Sendable, Comparable {
+    public let family: String
+    public let qualifier: Data
+
+    public init(family: String, qualifier: Data) {
+        self.family = family
+        self.qualifier = qualifier
+    }
+
+    /// `family:qualifier`, the way `cbt` and the console name a column. A family name cannot hold a
+    /// colon, so the first one always splits the two.
+    public var name: String {
+        family + ":" + BigtableBytes.display(qualifier)
+    }
+
+    public init(name: String) throws {
+        guard let colon = name.firstIndex(of: ":"), colon != name.startIndex else {
+            throw BigtableError.configuration(String(
+                format: String(localized: "\"%@\" is not a column. Name it as family:qualifier."), name
+            ))
+        }
+        self.family = String(name[..<colon])
+        self.qualifier = try BigtableBytes.parse(String(name[name.index(after: colon)...]))
+    }
+
+    public static func < (lhs: BigtableColumn, rhs: BigtableColumn) -> Bool {
+        lhs.family == rhs.family ? lhs.qualifier.lexicographicallyPrecedes(rhs.qualifier) : lhs.family < rhs.family
+    }
+}
+
+public struct BigtableCell: Equatable, Sendable {
+    public let value: Data
+    public let timestampMicros: Int64
+
+    public init(value: Data, timestampMicros: Int64) {
+        self.value = value
+        self.timestampMicros = timestampMicros
+    }
+}
+
+/// A row with the newest version of each cell.
+public struct BigtableRow: Equatable, Sendable {
+    public let key: Data
+    public let cells: [BigtableColumn: BigtableCell]
+
+    public init(key: Data, cells: [BigtableColumn: BigtableCell]) {
+        self.key = key
+        self.cells = cells
+    }
+}
+
+/// Rebuilds rows from the `chunks` of a `ReadRows` response. Family and qualifier carry over from
+/// the chunk before when a chunk leaves them out, a value larger than one chunk arrives in pieces
+/// marked by `valueSize`, and `resetRow` throws away the row in progress.
+public struct BigtableChunkAssembler {
+    private var rows: [BigtableRow] = []
+    private var key: Data?
+    private var cells: [BigtableColumn: BigtableCell] = [:]
+    private var family: String?
+    private var qualifier: Data?
+    private var timestamp: Int64 = 0
+    private var value = Data()
+    private var inCell = false
+
+    public init() {}
+
+    public static func rows(fromResponse json: Any, operation: String) throws -> [BigtableRow] {
+        let messages = json as? [[String: Any]] ?? (json as? [String: Any]).map { [$0] } ?? []
+        var assembler = BigtableChunkAssembler()
+        for message in messages {
+            if let error = message["error"] as? [String: Any] {
+                throw BigtableError.from(streamError: error, operation: operation)
+            }
+            for chunk in message["chunks"] as? [[String: Any]] ?? [] {
+                try assembler.add(chunk)
+            }
+        }
+        return try assembler.finish()
+    }
+
+    public mutating func add(_ chunk: [String: Any]) throws {
+        if chunk["resetRow"] as? Bool == true {
+            resetRow()
+            return
+        }
+        if let encodedKey = chunk["rowKey"] as? String {
+            let newKey = try BigtableBytes.decodeBase64(encodedKey)
+            if let key, key != newKey, !cells.isEmpty || inCell {
+                throw BigtableError.malformedResponse(String(localized: "Bigtable started a new row before committing the last one."))
+            }
+            key = newKey
+        }
+        if let name = Self.string(chunk["familyName"]) {
+            family = name
+            qualifier = nil
+        }
+        if let encoded = Self.string(chunk["qualifier"]) {
+            qualifier = try BigtableBytes.decodeBase64(encoded)
+        }
+        if !inCell {
+            timestamp = Self.int64(chunk["timestampMicros"]) ?? 0
+            value = Data()
+        }
+        if let encoded = chunk["value"] as? String {
+            value.append(try BigtableBytes.decodeBase64(encoded))
+        }
+        inCell = (Self.int64(chunk["valueSize"]) ?? 0) > 0
+        if !inCell, let family, let qualifier {
+            let column = BigtableColumn(family: family, qualifier: qualifier)
+            if cells[column].map({ $0.timestampMicros < timestamp }) ?? true {
+                cells[column] = BigtableCell(value: value, timestampMicros: timestamp)
+            }
+        }
+        if chunk["commitRow"] as? Bool == true {
+            guard let key, !inCell else {
+                throw BigtableError.malformedResponse(String(localized: "Bigtable committed a row it never started."))
+            }
+            rows.append(BigtableRow(key: key, cells: cells))
+            resetRow()
+        }
+    }
+
+    public func finish() throws -> [BigtableRow] {
+        guard !inCell, cells.isEmpty else {
+            throw BigtableError.malformedResponse(String(localized: "Bigtable ended the response in the middle of a row."))
+        }
+        return rows
+    }
+
+    private mutating func resetRow() {
+        key = nil
+        cells = [:]
+        family = nil
+        qualifier = nil
+        value = Data()
+        inCell = false
+    }
+
+    /// `familyName` and `qualifier` are wrapper messages, which JSON writes as the bare value but a
+    /// proxy may still write as `{"value": ...}`.
+    private static func string(_ any: Any?) -> String? {
+        if let text = any as? String { return text }
+        return (any as? [String: Any])?["value"] as? String
+    }
+
+    private static func int64(_ any: Any?) -> Int64? {
+        switch any {
+        case let text as String: return Int64(text)
+        case let number as NSNumber: return number.int64Value
+        default: return nil
+        }
+    }
+}

Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift

@@ -0,0 +1,78 @@
+import Foundation
+
+/// Rows laid out for the grid: `_key` first, then one column per `family:qualifier` any row on the
+/// page holds, in family then qualifier order.
+public struct BigtableRowSet: Sendable, Equatable {
+    public static let keyColumn = "_key"
+    public static let extraColumn = "__extra__"
+
+    public enum Cell: Sendable, Equatable {
+        case absent
+        case text(String)
+        case bytes(Data)
+    }
+
+    public let columns: [String]
+    public let rows: [[Cell]]
+    public let locators: [String]
+    public let absentCells: [Int: Set<Int>]
+
+    public init(rows input: [BigtableRow]) {
+        let ordered = Set(input.flatMap { $0.cells.keys }).sorted()
+        var rows: [[Cell]] = []
+        var absent: [Int: Set<Int>] = [:]
+        rows.reserveCapacity(input.count)
+        for (rowIndex, row) in input.enumerated() {
+            var cells: [Cell] = [.text(BigtableBytes.display(row.key))]
+            for (offset, column) in ordered.enumerated() {
+                guard let cell = row.cells[column] else {
+                    cells.append(.absent)
+                    absent[rowIndex, default: []].insert(offset + 1)
+                    continue
+                }
+                cells.append(Self.cell(cell.value))
+            }
+            rows.append(cells)
+        }
+        self.columns = [Self.keyColumn] + ordered.map(\.name)
+        self.rows = rows
+        self.locators = input.map { $0.key.base64EncodedString() }
+        self.absentCells = absent
+    }
+
+    /// A value that is not text, or text a write would read back as base64, goes to the grid as
+    /// bytes, so writing it back reproduces the same bytes.
+    static func cell(_ value: Data) -> Cell {
+        BigtableBytes.printableText(value).map(Cell.text) ?? .bytes(value)
+    }
+
+    /// The page under `header`, the columns an export fixed from its first page, with `__extra__`
+    /// last. Cells of any column the header lacks go into `__extra__` as one JSON object.
+    public func rows(alignedTo header: [String]) -> [[Cell]] {
+        rows.map { row in
+            var byName: [String: Cell] = [:]
+            for (column, cell) in zip(columns, row) where cell != .absent {
+                byName[column] = cell
+            }
+            var aligned = header.filter { $0 != Self.extraColumn }.map { byName.removeValue(forKey: $0) ?? .absent }
+            aligned.append(Self.extraCell(byName))
+            return aligned
+        }
+    }
+
+    private static func extraCell(_ cells: [String: Cell]) -> Cell {
+        let extra = cells.compactMapValues { cell -> String? in
+            switch cell {
+            case .absent: return nil
+            case .text(let text): return text
+            case .bytes(let data): return BigtableBytes.base64Prefix + data.base64EncodedString()
+            }
+        }
+        guard !extra.isEmpty,
+              let data = try? JSONSerialization.data(withJSONObject: extra, options: [.sortedKeys, .withoutEscapingSlashes]),
+              let text = String(data: data, encoding: .utf8) else {
+            return .absent
+        }
+        return .text(text)
+    }
+}