diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index 717beaebcb..78d112ef4e 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -46,6 +46,20 @@ "category": "database-driver", "homepage": "https://docs.tablepro.app/databases/bigquery" }, + "bigtable": { + "target": "BigtableDriverPlugin", + "bundleName": "BigtableDriverPlugin", + "bundleId": "com.TablePro.BigtableDriverPlugin", + "bundled": false, + "displayName": "Bigtable Driver", + "summary": "Google Cloud Bigtable driver over the REST API", + "databaseTypeIds": [ + "Bigtable" + ], + "icon": "bigtable-icon", + "category": "database-driver", + "homepage": "https://docs.tablepro.app/databases/bigtable" + }, "cassandra": { "target": "CassandraDriver", "bundleName": "CassandraDriver", diff --git a/CHANGELOG.md b/CHANGELOG.md index bde337d5a6..6b8f210eb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Setting in Settings > General to hide the version and Check for Updates on the Welcome window. - Install Parquet… in the Export dialog's format menu when the Parquet plugin is not installed. - Cloud Firestore and Firebase Realtime Database as a registry plugin over their REST APIs. (#2504) +- Google Cloud Bigtable as a registry plugin over the REST API. - Warning banner and Test Connection notice for a server warning at login, such as an Oracle password about to expire. (#3401) ### Changed diff --git a/Packages/TableProCore/Package.swift b/Packages/TableProCore/Package.swift index ed70b9b600..65ee8c159a 100644 --- a/Packages/TableProCore/Package.swift +++ b/Packages/TableProCore/Package.swift @@ -35,6 +35,7 @@ let package = Package( .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]), .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), .library(name: "TableProFirebaseCore", targets: ["TableProFirebaseCore"]), + .library(name: "TableProBigtableCore", targets: ["TableProBigtableCore"]), .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), .library(name: "TableProDocumentPath", targets: ["TableProDocumentPath"]), .library(name: "TableProJavaScriptText", targets: ["TableProJavaScriptText"]), @@ -168,9 +169,14 @@ let package = Package( dependencies: [], path: "Sources/TableProWeaviateCore" ), + .target( + name: "TableProBigtableCore", + dependencies: ["TableProGoogleCloud"], + path: "Sources/TableProBigtableCore" + ), .target( name: "TableProFirebaseCore", - dependencies: [], + dependencies: ["TableProGoogleCloud"], path: "Sources/TableProFirebaseCore" ), .target( @@ -232,6 +238,11 @@ let package = Package( path: "Sources/TableProRedisCore", swiftSettings: pluginCoreSettings ), + .testTarget( + name: "TableProBigtableCoreTests", + dependencies: ["TableProBigtableCore", "TableProGoogleCloud"], + path: "Tests/TableProBigtableCoreTests" + ), .testTarget( name: "TableProTabularIOTests", dependencies: ["TableProTabularIO", "TableProPluginKit"], diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift new file mode 100644 index 0000000000..b4c9b94d0c --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift @@ -0,0 +1,79 @@ +import Foundation +import TableProGoogleCloud + +public enum BigtableFieldID { + public static let authMethod = "btAuthMethod" + public static let serviceAccountJson = "btServiceAccountJson" + public static let projectId = "btProjectId" + public static let instanceId = "btInstanceId" + public static let appProfileId = "btAppProfileId" +} + +public enum BigtableAuthMethod: String, Sendable, CaseIterable { + case serviceAccount + case applicationDefault = "adc" +} + +public struct BigtableConnectionSettings: Sendable, Equatable { + public let projectId: String + public let instanceId: String? + public let appProfileId: String? + + public init(projectId: String, instanceId: String?, appProfileId: String? = nil) { + self.projectId = projectId + self.instanceId = instanceId + self.appProfileId = appProfileId + } + + /// `database` wins over the Instance ID field: a pooled connection the app opens after a switch + /// names the instance it switched to there and never calls `switchDatabase`. + public static func parse( + fields: [String: String], + database: String? = nil, + credentialProject: String? = nil + ) throws -> BigtableConnectionSettings { + guard let projectId = trimmed(fields[BigtableFieldID.projectId]) ?? credentialProject else { + throw BigtableError.configuration(String(localized: "Enter a Project ID, or use a service account key that names one.")) + } + guard isValidProjectId(projectId) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Google Cloud project ID."), projectId)) + } + let instanceId = trimmed(database) ?? trimmed(fields[BigtableFieldID.instanceId]) + if let instanceId, !isValidResourceId(instanceId) { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Bigtable instance ID."), instanceId)) + } + let appProfileId = trimmed(fields[BigtableFieldID.appProfileId]) + if let appProfileId, !isValidResourceId(appProfileId) { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not an app profile ID."), appProfileId)) + } + return BigtableConnectionSettings(projectId: projectId, instanceId: instanceId, appProfileId: appProfileId) + } + + public static func credentialSource(fields: [String: String], password: String?) throws -> GoogleCredentialSource { + let method = trimmed(fields[BigtableFieldID.authMethod]).flatMap(BigtableAuthMethod.init(rawValue:)) ?? .serviceAccount + switch method { + case .applicationDefault: + return .applicationDefault + case .serviceAccount: + guard let key = trimmed(fields[BigtableFieldID.serviceAccountJson]) ?? trimmed(password) else { + throw BigtableError.configuration(String(localized: "Choose a service account key file, or paste its JSON.")) + } + return .serviceAccountKey(key) + } + } + + static func isValidProjectId(_ text: String) -> Bool { + text.range(of: #"^[a-z][a-z0-9.:-]{2,62}$"#, options: .regularExpression) != nil + } + + /// Instance, table and app profile IDs go into the request path, so anything outside the + /// characters Bigtable allows is refused before it can change the path. + static func isValidResourceId(_ text: String) -> Bool { + text.range(of: #"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,49}$"#, options: .regularExpression) != nil + } + + static func trimmed(_ value: String?) -> String? { + guard let value = value?.trimmingCharacters(in: .whitespacesAndNewlines), !value.isEmpty else { return nil } + return value + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift new file mode 100644 index 0000000000..d96c5be4f0 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift @@ -0,0 +1,155 @@ +import Foundation + +public struct BigtableColumn: Hashable, Sendable, Comparable { + public let family: String + public let qualifier: Data + + public init(family: String, qualifier: Data) { + self.family = family + self.qualifier = qualifier + } + + /// `family:qualifier`, the way `cbt` and the console name a column. A family name cannot hold a + /// colon, so the first one always splits the two. + public var name: String { + family + ":" + BigtableBytes.display(qualifier) + } + + public init(name: String) throws { + guard let colon = name.firstIndex(of: ":"), colon != name.startIndex else { + throw BigtableError.configuration(String( + format: String(localized: "\"%@\" is not a column. Name it as family:qualifier."), name + )) + } + self.family = String(name[.. Bool { + lhs.family == rhs.family ? lhs.qualifier.lexicographicallyPrecedes(rhs.qualifier) : lhs.family < rhs.family + } +} + +public struct BigtableCell: Equatable, Sendable { + public let value: Data + public let timestampMicros: Int64 + + public init(value: Data, timestampMicros: Int64) { + self.value = value + self.timestampMicros = timestampMicros + } +} + +/// A row with the newest version of each cell. +public struct BigtableRow: Equatable, Sendable { + public let key: Data + public let cells: [BigtableColumn: BigtableCell] + + public init(key: Data, cells: [BigtableColumn: BigtableCell]) { + self.key = key + self.cells = cells + } +} + +/// Rebuilds rows from the `chunks` of a `ReadRows` response. Family and qualifier carry over from +/// the chunk before when a chunk leaves them out, a value larger than one chunk arrives in pieces +/// marked by `valueSize`, and `resetRow` throws away the row in progress. +public struct BigtableChunkAssembler { + private var rows: [BigtableRow] = [] + private var key: Data? + private var cells: [BigtableColumn: BigtableCell] = [:] + private var family: String? + private var qualifier: Data? + private var timestamp: Int64 = 0 + private var value = Data() + private var inCell = false + + public init() {} + + public static func rows(fromResponse json: Any, operation: String) throws -> [BigtableRow] { + let messages = json as? [[String: Any]] ?? (json as? [String: Any]).map { [$0] } ?? [] + var assembler = BigtableChunkAssembler() + for message in messages { + if let error = message["error"] as? [String: Any] { + throw BigtableError.from(streamError: error, operation: operation) + } + for chunk in message["chunks"] as? [[String: Any]] ?? [] { + try assembler.add(chunk) + } + } + return try assembler.finish() + } + + public mutating func add(_ chunk: [String: Any]) throws { + if chunk["resetRow"] as? Bool == true { + resetRow() + return + } + if let encodedKey = chunk["rowKey"] as? String { + let newKey = try BigtableBytes.decodeBase64(encodedKey) + if let key, key != newKey, !cells.isEmpty || inCell { + throw BigtableError.malformedResponse(String(localized: "Bigtable started a new row before committing the last one.")) + } + key = newKey + } + if let name = Self.string(chunk["familyName"]) { + family = name + qualifier = nil + } + if let encoded = Self.string(chunk["qualifier"]) { + qualifier = try BigtableBytes.decodeBase64(encoded) + } + if !inCell { + timestamp = Self.int64(chunk["timestampMicros"]) ?? 0 + value = Data() + } + if let encoded = chunk["value"] as? String { + value.append(try BigtableBytes.decodeBase64(encoded)) + } + inCell = (Self.int64(chunk["valueSize"]) ?? 0) > 0 + if !inCell, let family, let qualifier { + let column = BigtableColumn(family: family, qualifier: qualifier) + if cells[column].map({ $0.timestampMicros < timestamp }) ?? true { + cells[column] = BigtableCell(value: value, timestampMicros: timestamp) + } + } + if chunk["commitRow"] as? Bool == true { + guard let key, !inCell else { + throw BigtableError.malformedResponse(String(localized: "Bigtable committed a row it never started.")) + } + rows.append(BigtableRow(key: key, cells: cells)) + resetRow() + } + } + + public func finish() throws -> [BigtableRow] { + guard !inCell, cells.isEmpty else { + throw BigtableError.malformedResponse(String(localized: "Bigtable ended the response in the middle of a row.")) + } + return rows + } + + private mutating func resetRow() { + key = nil + cells = [:] + family = nil + qualifier = nil + value = Data() + inCell = false + } + + /// `familyName` and `qualifier` are wrapper messages, which JSON writes as the bare value but a + /// proxy may still write as `{"value": ...}`. + private static func string(_ any: Any?) -> String? { + if let text = any as? String { return text } + return (any as? [String: Any])?["value"] as? String + } + + private static func int64(_ any: Any?) -> Int64? { + switch any { + case let text as String: return Int64(text) + case let number as NSNumber: return number.int64Value + default: return nil + } + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift new file mode 100644 index 0000000000..d2e4090875 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift @@ -0,0 +1,78 @@ +import Foundation + +/// Rows laid out for the grid: `_key` first, then one column per `family:qualifier` any row on the +/// page holds, in family then qualifier order. +public struct BigtableRowSet: Sendable, Equatable { + public static let keyColumn = "_key" + public static let extraColumn = "__extra__" + + public enum Cell: Sendable, Equatable { + case absent + case text(String) + case bytes(Data) + } + + public let columns: [String] + public let rows: [[Cell]] + public let locators: [String] + public let absentCells: [Int: Set] + + public init(rows input: [BigtableRow]) { + let ordered = Set(input.flatMap { $0.cells.keys }).sorted() + var rows: [[Cell]] = [] + var absent: [Int: Set] = [:] + rows.reserveCapacity(input.count) + for (rowIndex, row) in input.enumerated() { + var cells: [Cell] = [.text(BigtableBytes.display(row.key))] + for (offset, column) in ordered.enumerated() { + guard let cell = row.cells[column] else { + cells.append(.absent) + absent[rowIndex, default: []].insert(offset + 1) + continue + } + cells.append(Self.cell(cell.value)) + } + rows.append(cells) + } + self.columns = [Self.keyColumn] + ordered.map(\.name) + self.rows = rows + self.locators = input.map { $0.key.base64EncodedString() } + self.absentCells = absent + } + + /// A value that is not text, or text a write would read back as base64, goes to the grid as + /// bytes, so writing it back reproduces the same bytes. + static func cell(_ value: Data) -> Cell { + BigtableBytes.printableText(value).map(Cell.text) ?? .bytes(value) + } + + /// The page under `header`, the columns an export fixed from its first page, with `__extra__` + /// last. Cells of any column the header lacks go into `__extra__` as one JSON object. + public func rows(alignedTo header: [String]) -> [[Cell]] { + rows.map { row in + var byName: [String: Cell] = [:] + for (column, cell) in zip(columns, row) where cell != .absent { + byName[column] = cell + } + var aligned = header.filter { $0 != Self.extraColumn }.map { byName.removeValue(forKey: $0) ?? .absent } + aligned.append(Self.extraCell(byName)) + return aligned + } + } + + private static func extraCell(_ cells: [String: Cell]) -> Cell { + let extra = cells.compactMapValues { cell -> String? in + switch cell { + case .absent: return nil + case .text(let text): return text + case .bytes(let data): return BigtableBytes.base64Prefix + data.base64EncodedString() + } + } + guard !extra.isEmpty, + let data = try? JSONSerialization.data(withJSONObject: extra, options: [.sortedKeys, .withoutEscapingSlashes]), + let text = String(data: data, encoding: .utf8) else { + return .absent + } + return .text(text) + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift new file mode 100644 index 0000000000..6aef52ba87 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift @@ -0,0 +1,62 @@ +import Foundation + +/// The request lines a grid save sends, one atomic row mutation per changed row. `.absent` is a +/// cell the row has none of: left out of a new row, deleted from an existing one. +public enum BigtableRowWrites { + public static func insert(table: String, columns: [String], values: [BigtableRowSet.Cell]) throws -> BigtableRequest { + var key: Data? + var cells: [String: Any] = [:] + for (column, value) in zip(columns, values) { + if column == BigtableRowSet.keyColumn { + key = try self.key(value) + } else if let text = text(value) { + cells[column] = text + } + } + guard let key else { + throw BigtableError.configuration(String(localized: "Enter a _key for the new row. Bigtable has no generated keys.")) + } + guard !cells.isEmpty else { + throw BigtableError.configuration(String(localized: "A new row needs at least one cell. Bigtable does not store empty rows.")) + } + return BigtableRequest(verb: .insert, table: table, key: key, body: try body(cells)) + } + + public static func update(table: String, key: Data, changes: [(column: String, value: BigtableRowSet.Cell)]) throws -> BigtableRequest? { + guard !changes.contains(where: { $0.column == BigtableRowSet.keyColumn }) else { + throw BigtableError.unsupported(String(localized: "A row key cannot change. Duplicate the row, give the copy a new _key, then delete this one.")) + } + guard !changes.isEmpty else { return nil } + var cells: [String: Any] = [:] + for change in changes { + cells[change.column] = text(change.value) ?? NSNull() + } + return BigtableRequest(verb: .put, table: table, key: key, body: try body(cells)) + } + + public static func delete(table: String, key: Data) -> BigtableRequest { + BigtableRequest(verb: .delete, table: table, key: key) + } + + public static func key(_ cell: BigtableRowSet.Cell) throws -> Data? { + switch cell { + case .absent: return nil + case .bytes(let data): return data.isEmpty ? nil : data + case .text(let text): return text.isEmpty ? nil : try BigtableBytes.parse(text) + } + } + + /// Bytes go back as `b64:` text, which a write reads back to the same bytes. + private static func text(_ cell: BigtableRowSet.Cell) -> String? { + switch cell { + case .absent: return nil + case .text(let text): return text + case .bytes(let data): return BigtableBytes.base64Prefix + data.base64EncodedString() + } + } + + private static func body(_ cells: [String: Any]) throws -> String { + let data = try JSONSerialization.data(withJSONObject: cells, options: [.sortedKeys, .withoutEscapingSlashes]) + return String(bytes: data, encoding: .utf8) ?? "{}" + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableBrowseSpec.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableBrowseSpec.swift new file mode 100644 index 0000000000..366ca0ba8a --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableBrowseSpec.swift @@ -0,0 +1,64 @@ +import Foundation + +public struct BigtableFilterSpec: Codable, Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + public let isCaseSensitive: Bool + + public init(column: String, op: String, value: String, secondValue: String? = nil, isCaseSensitive: Bool = true) { + self.column = column + self.op = op.uppercased() + self.value = value + self.secondValue = secondValue + self.isCaseSensitive = isCaseSensitive + } +} + +/// A page of one table as the grid asked for it. Bigtable reads rows in key order and has no +/// offset, so a later page starts after the last key of the page before it. +public struct BigtableBrowseSpec: Codable, Sendable, Equatable { + public let table: String + public let filters: [BigtableFilterSpec] + public let logicMode: String + public let descending: Bool + /// Kept so a sort Bigtable cannot do is refused rather than silently ignored. + public let sortColumn: String? + public let limit: Int + public let offset: Int + + public init( + table: String, + filters: [BigtableFilterSpec] = [], + logicMode: String = "AND", + descending: Bool = false, + sortColumn: String? = nil, + limit: Int, + offset: Int + ) { + self.table = table + self.filters = filters + self.logicMode = logicMode + self.descending = descending + self.sortColumn = sortColumn + self.limit = limit + self.offset = offset + } + + public var usesOr: Bool { + logicMode.uppercased() == "OR" + } + + /// The spec without its page, so pages of the same view share one cursor cache. + public var viewKey: String { + let filterText = filters.map { + [$0.column, $0.op, $0.value, $0.secondValue ?? "", $0.isCaseSensitive ? "cs" : "ci"].joined(separator: "\u{1}") + }.joined(separator: "\u{2}") + return [table, logicMode.uppercased(), descending ? "desc" : "asc", filterText].joined(separator: "\u{3}") + } + + public static func viewKeyPrefix(table: String) -> String { + table + "\u{3}" + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableKeyBounds.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableKeyBounds.swift new file mode 100644 index 0000000000..54c43424fe --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableKeyBounds.swift @@ -0,0 +1,75 @@ +import Foundation + +/// The row keys a read may return: an interval, or a list of exact keys inside it. +struct BigtableKeyBounds { + struct Bound { + let key: Data + let isClosed: Bool + } + + var start: Bound? + var end: Bound? + var exact: [Data]? + + var isUnbounded: Bool { + start == nil && end == nil && exact == nil + } + + var isEmpty: Bool { + if let exact { + return !exact.contains(where: admits) + } + guard let start, let end else { return false } + if end.key.lexicographicallyPrecedes(start.key) { return true } + return start.key == end.key && !(start.isClosed && end.isClosed) + } + + mutating func raiseStart(_ key: Data, closed: Bool) { + guard let current = start else { + start = Bound(key: key, isClosed: closed) + return + } + if current.key.lexicographicallyPrecedes(key) { + start = Bound(key: key, isClosed: closed) + } else if current.key == key { + start = Bound(key: key, isClosed: closed && current.isClosed) + } + } + + mutating func lowerEnd(_ key: Data, closed: Bool) { + guard let current = end else { + end = Bound(key: key, isClosed: closed) + return + } + if key.lexicographicallyPrecedes(current.key) { + end = Bound(key: key, isClosed: closed) + } else if current.key == key { + end = Bound(key: key, isClosed: closed && current.isClosed) + } + } + + mutating func restrict(to keys: [Data]) { + exact = exact.map { existing in existing.filter(keys.contains) } ?? keys + } + + func admits(_ key: Data) -> Bool { + if let start { + if key.lexicographicallyPrecedes(start.key) || (key == start.key && !start.isClosed) { return false } + } + if let end { + if end.key.lexicographicallyPrecedes(key) || (key == end.key && !end.isClosed) { return false } + } + return true + } + + var admittedKeys: [Data] { + exact?.filter(admits) ?? [] + } + + var rangeJSON: [String: Any] { + var range: [String: Any] = [:] + if let start { range[start.isClosed ? "startKeyClosed" : "startKeyOpen"] = start.key.base64EncodedString() } + if let end { range[end.isClosed ? "endKeyClosed" : "endKeyOpen"] = end.key.base64EncodedString() } + return range + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift new file mode 100644 index 0000000000..3a080be2f3 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift @@ -0,0 +1,136 @@ +import Foundation + +/// `ReadRows` bodies. A nil body means the filters admit no row, which is not something a request +/// can say: an empty row set reads the whole table. +public enum BigtableReadRequest { + public static let rawFilterColumn = BigtableRowFilter.rawColumn + + /// `after` continues a page past that key. `keysOnly` reads one stripped cell per row, which is + /// how a page with no cursor yet skips the rows before it and how a count reads every key. + public static func body(spec: BigtableBrowseSpec, limit: Int, after: Data?, keysOnly: Bool = false) throws -> [String: Any]? { + if (spec.sortColumn.map { $0 != BigtableRowSet.keyColumn } ?? false) || spec.descending { + throw BigtableError.unsupported(String(localized: "The Bigtable REST API returns rows in ascending key order only.")) + } + var selection: [BigtableKeyBounds] + var rowFilters: [[String: Any]] = [] + let keyRanges = spec.filters.filter(isKeyRange) + if spec.usesOr, !keyRanges.isEmpty, keyRanges.count == spec.filters.count { + selection = try keyRanges.map { filter in + var bounds = BigtableKeyBounds() + try apply(filter, to: &bounds) + return bounds + } + } else { + var bounds = BigtableKeyBounds() + for filter in spec.filters { + if !spec.usesOr, isKeyRange(filter) { + try apply(filter, to: &bounds) + } else { + rowFilters.append(try BigtableRowFilter.make(filter, insideOr: spec.usesOr)) + } + } + selection = [bounds] + } + if let after { + for index in selection.indices { + selection[index].raiseStart(after, closed: false) + } + } + guard let rows = rowSet(selection) else { return nil } + + var body: [String: Any] = [:] + if !rows.isEmpty { body["rows"] = rows } + var filters: [[String: Any]] = [] + switch rowFilters.count { + case 0: break + case 1: filters.append(rowFilters[0]) + default: filters.append([spec.usesOr ? "interleave" : "chain": ["filters": rowFilters]]) + } + filters += keysOnly ? keysOnlyFilters : [["cellsPerColumnLimitFilter": 1]] + body["filter"] = filters.count == 1 ? filters[0] : ["chain": ["filters": filters]] + if limit > 0 { body["rowsLimit"] = String(limit) } + return body + } + + /// The first rows of a table with their values stripped, which is enough to learn its columns. + public static func columnSample(limit: Int) -> [String: Any] { + [ + "rowsLimit": String(limit), + "filter": ["chain": ["filters": [["cellsPerColumnLimitFilter": 1], ["stripValueTransformer": true]]]] + ] + } + + public static func keyLookup(_ key: Data) -> [String: Any] { + [ + "rows": ["rowKeys": [key.base64EncodedString()]], + "rowsLimit": "1", + "filter": ["chain": ["filters": keysOnlyFilters]] + ] + } + + public static func get(_ key: Data) -> [String: Any] { + ["rows": ["rowKeys": [key.base64EncodedString()]], "filter": ["cellsPerColumnLimitFilter": 1]] + } + + private static var keysOnlyFilters: [[String: Any]] { + [["cellsPerRowLimitFilter": 1], ["stripValueTransformer": true]] + } + + /// The union of `selection` as a `RowSet`: empty for the whole table, nil for no row at all. + private static func rowSet(_ selection: [BigtableKeyBounds]) -> [String: Any]? { + let live = selection.filter { !$0.isEmpty } + guard !live.isEmpty else { return nil } + if live.contains(where: \.isUnbounded) { return [:] } + var keys: [String] = [] + var ranges: [[String: Any]] = [] + for bounds in live { + if bounds.exact != nil { + for key in bounds.admittedKeys.map({ $0.base64EncodedString() }) where !keys.contains(key) { + keys.append(key) + } + } else { + ranges.append(bounds.rangeJSON) + } + } + var rows: [String: Any] = [:] + if !keys.isEmpty { rows["rowKeys"] = keys } + if !ranges.isEmpty { rows["rowRanges"] = ranges } + return rows + } + + /// A key test a row range can express. A case-insensitive equality or prefix cannot, so it runs + /// as a key regex instead. + static func isKeyRange(_ filter: BigtableFilterSpec) -> Bool { + guard filter.column == BigtableRowSet.keyColumn else { return false } + switch filter.op { + case ">", ">=", "<", "<=", "BETWEEN": return true + case "=", "IN", "STARTS WITH": return filter.isCaseSensitive + default: return false + } + } + + static func apply(_ filter: BigtableFilterSpec, to bounds: inout BigtableKeyBounds) throws { + switch filter.op { + case "=": + bounds.restrict(to: [try BigtableBytes.parse(filter.value)]) + case "IN": + bounds.restrict(to: try BigtableRowFilter.listItems(filter.value).map(BigtableBytes.parse)) + case "STARTS WITH": + let prefix = try BigtableBytes.parse(filter.value) + bounds.raiseStart(prefix, closed: true) + if let end = BigtableBytes.prefixEnd(prefix) { bounds.lowerEnd(end, closed: false) } + case ">": bounds.raiseStart(try BigtableBytes.parse(filter.value), closed: false) + case ">=": bounds.raiseStart(try BigtableBytes.parse(filter.value), closed: true) + case "<": bounds.lowerEnd(try BigtableBytes.parse(filter.value), closed: false) + case "<=": bounds.lowerEnd(try BigtableBytes.parse(filter.value), closed: true) + case "BETWEEN": + guard let upper = filter.secondValue else { + throw BigtableError.configuration(String(localized: "BETWEEN needs two values.")) + } + bounds.raiseStart(try BigtableBytes.parse(filter.value), closed: true) + bounds.lowerEnd(try BigtableBytes.parse(upper), closed: true) + default: + throw BigtableError.unsupported(BigtableRowFilter.unsupportedMessage(filter.op)) + } + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableRowFilter.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableRowFilter.swift new file mode 100644 index 0000000000..ada8351fd5 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableRowFilter.swift @@ -0,0 +1,128 @@ +import Foundation + +/// Turns one filter bar row into a `RowFilter` that passes a whole row or none of it. +enum BigtableRowFilter { + static let rawColumn = "__RAW__" + + static func make(_ filter: BigtableFilterSpec, insideOr: Bool) throws -> [String: Any] { + if filter.column == rawColumn { + return try raw(filter.value) + } + if filter.column == BigtableRowSet.keyColumn { + return try keyFilter(filter, insideOr: insideOr) + } + return try columnFilter(filter) + } + + private static func keyFilter(_ filter: BigtableFilterSpec, insideOr: Bool) throws -> [String: Any] { + guard let match = try textPattern(filter) else { + throw BigtableError.unsupported(insideOr + ? String(localized: "A key range cannot be ORed with a column filter. Use =, STARTS WITH or REGEX on _key, or AND.") + : unsupportedMessage(filter.op)) + } + let test: [String: Any] = ["rowKeyRegexFilter": match.pattern.base64EncodedString()] + return match.isNegated ? drop(test) : keep(test) + } + + private static func columnFilter(_ filter: BigtableFilterSpec) throws -> [String: Any] { + let column = try BigtableColumn(name: filter.column) + let base: [[String: Any]] = [ + ["familyNameRegexFilter": "^" + BigtableBytes.regexText(Data(column.family.utf8)) + "$"], + ["columnQualifierRegexFilter": (Data("(?s)^".utf8) + BigtableBytes.regexLiteral(column.qualifier) + Data("$".utf8)).base64EncodedString()], + ["cellsPerColumnLimitFilter": 1] + ] + func cellIn(_ test: [String: Any]?) -> [String: Any] { + ["chain": ["filters": base + (test.map { [$0] } ?? [])]] + } + let present = keep(cellIn(nil)) + + if let match = try textPattern(filter) { + let test = cellIn(["valueRegexFilter": match.pattern.base64EncodedString()]) + return match.isNegated ? ["chain": ["filters": [present, drop(test)]]] : keep(test) + } + switch filter.op { + case "IS NULL": + return drop(cellIn(nil)) + case "IS NOT NULL": + return present + case "IS EMPTY": + return keep(cellIn(["valueRegexFilter": Data("^$".utf8).base64EncodedString()])) + case "IS NOT EMPTY": + return ["chain": ["filters": [present, drop(cellIn(["valueRegexFilter": Data("^$".utf8).base64EncodedString()]))]]] + case ">", ">=", "<", "<=", "BETWEEN": + return keep(cellIn(["valueRangeFilter": try valueRange(filter)])) + default: + throw BigtableError.unsupported(unsupportedMessage(filter.op)) + } + } + + /// The regex a text comparison matches, for a key or a value alike. Nil for an operator that is + /// not a text comparison. + private static func textPattern(_ filter: BigtableFilterSpec) throws -> (pattern: Data, isNegated: Bool)? { + let anyText = Data(".*".utf8) + let flags = Data((filter.isCaseSensitive ? "(?s)" : "(?is)").utf8) + func literal() throws -> Data { + BigtableBytes.regexLiteral(try BigtableBytes.parse(filter.value)) + } + func alternatives() throws -> Data { + let items = try listItems(filter.value).map { BigtableBytes.regexText(try BigtableBytes.parse($0)) } + return flags + Data(("^(" + items.joined(separator: "|") + ")$").utf8) + } + switch filter.op { + case "=": return (flags + Data("^".utf8) + (try literal()) + Data("$".utf8), false) + case "!=", "<>": return (flags + Data("^".utf8) + (try literal()) + Data("$".utf8), true) + case "STARTS WITH": return (flags + Data("^".utf8) + (try literal()) + anyText, false) + case "ENDS WITH": return (flags + anyText + (try literal()) + Data("$".utf8), false) + case "CONTAINS": return (flags + anyText + (try literal()) + anyText, false) + case "NOT CONTAINS": return (flags + anyText + (try literal()) + anyText, true) + case "IN": return (try alternatives(), false) + case "NOT IN": return (try alternatives(), true) + case "REGEX": return ((filter.isCaseSensitive ? Data() : Data("(?i)".utf8)) + (try BigtableBytes.parse(filter.value)), false) + default: return nil + } + } + + private static func valueRange(_ filter: BigtableFilterSpec) throws -> [String: Any] { + let value = try BigtableBytes.parse(filter.value).base64EncodedString() + switch filter.op { + case ">": return ["startValueOpen": value] + case ">=": return ["startValueClosed": value] + case "<": return ["endValueOpen": value] + case "<=": return ["endValueClosed": value] + default: + guard let upper = filter.secondValue else { + throw BigtableError.configuration(String(localized: "BETWEEN needs two values.")) + } + return ["startValueClosed": value, "endValueClosed": try BigtableBytes.parse(upper).base64EncodedString()] + } + } + + private static func keep(_ predicate: [String: Any]) -> [String: Any] { + ["condition": ["predicateFilter": predicate, "trueFilter": ["passAllFilter": true]]] + } + + private static func drop(_ predicate: [String: Any]) -> [String: Any] { + ["condition": ["predicateFilter": predicate, "falseFilter": ["passAllFilter": true]]] + } + + /// The raw filter row takes a `RowFilter` as written in the Bigtable REST reference, with bytes + /// in base64, which reaches every filter the filter bar cannot draw. + private static func raw(_ text: String) throws -> [String: Any] { + guard let data = text.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], object.count == 1 else { + throw BigtableError.configuration(String(localized: "A raw Bigtable filter is a JSON RowFilter, such as {\"rowKeyRegexFilter\": \"\"}.")) + } + return object + } + + static func listItems(_ text: String) -> [String] { + if let data = text.data(using: .utf8), let array = try? JSONSerialization.jsonObject(with: data) as? [String] { + return array + } + return text.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + } + + static func unsupportedMessage(_ op: String) -> String { + String(format: String(localized: "Bigtable cannot filter with %@."), op) + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift new file mode 100644 index 0000000000..63240c11ed --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift @@ -0,0 +1,80 @@ +import Foundation + +/// Row keys, qualifiers and values are bytes. Text that is valid UTF-8 with no control characters +/// shows as itself; anything else, and text that itself starts with `b64:`, shows as `b64:` and +/// base64, which reads back to the same bytes. +public enum BigtableBytes { + public static let base64Prefix = "b64:" + + public static func display(_ data: Data) -> String { + if let text = printableText(data) { + return text + } + return base64Prefix + data.base64EncodedString() + } + + /// The bytes as text when `display` would show them as themselves. + public static func printableText(_ data: Data) -> String? { + guard let text = String(data: data, encoding: .utf8), isPrintable(text), !text.hasPrefix(base64Prefix) else { + return nil + } + return text + } + + public static func parse(_ text: String) throws -> Data { + guard text.hasPrefix(base64Prefix) else { return Data(text.utf8) } + guard let data = Data(base64Encoded: String(text.dropFirst(base64Prefix.count))) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not valid base64."), text)) + } + return data + } + + public static func isPrintable(_ text: String) -> Bool { + !text.unicodeScalars.contains { scalar in + scalar.value < 0x20 && scalar != "\t" && scalar != "\n" || scalar.value == 0x7F + } + } + + static func decodeBase64(_ text: String) throws -> Data { + guard let data = Data(base64Encoded: text) else { + throw BigtableError.malformedResponse(String(localized: "Bigtable sent bytes that are not base64.")) + } + return data + } + + /// The smallest key greater than every key starting with `prefix`. Nil when every byte is 0xFF, + /// which leaves the range open at the end. + public static func prefixEnd(_ prefix: Data) -> Data? { + var bytes = [UInt8](prefix) + while let last = bytes.last { + if last < 0xFF { + bytes[bytes.count - 1] = last + 1 + return Data(bytes) + } + bytes.removeLast() + } + return nil + } + + public static func regexText(_ data: Data) -> String { + String(bytes: regexLiteral(data), encoding: .ascii) ?? "" + } + + /// Escapes RE2 operators byte by byte and writes every other byte outside printable ASCII as + /// `\xNN`, so the result is always ASCII. + public static func regexLiteral(_ data: Data) -> Data { + let special = Set("\\^$.|?*+()[]{}".utf8) + var escaped = Data() + for byte in data { + if special.contains(byte) { + escaped.append(UInt8(ascii: "\\")) + escaped.append(byte) + } else if byte < 0x20 || byte >= 0x7F { + escaped.append(contentsOf: Array(String(format: "\\x%02X", byte).utf8)) + } else { + escaped.append(byte) + } + } + return escaped + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift new file mode 100644 index 0000000000..28513c9b52 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift @@ -0,0 +1,209 @@ +import Foundation +import TableProGoogleCloud + +public enum BigtableMutation: Sendable, Equatable { + case setCell(BigtableColumn, Data) + case deleteColumn(BigtableColumn) + case deleteRow + + /// A timestamp of -1 asks the server to stamp the cell with its own clock, which also keeps the + /// write valid on a table whose granularity is milliseconds. + var json: [String: Any] { + switch self { + case .setCell(let column, let value): + return ["setCell": [ + "familyName": column.family, + "columnQualifier": column.qualifier.base64EncodedString(), + "timestampMicros": "-1", + "value": value.base64EncodedString() + ]] + case .deleteColumn(let column): + return ["deleteFromColumn": [ + "familyName": column.family, + "columnQualifier": column.qualifier.base64EncodedString() + ]] + case .deleteRow: + return ["deleteFromRow": [String: Any]()] + } + } +} + +/// The Bigtable data and admin REST APIs (v2) for one project and one instance at a time. +public final class BigtableClient: @unchecked Sendable { + public static let dataEndpoint = "https://bigtable.googleapis.com/v2/" + public static let adminEndpoint = "https://bigtableadmin.googleapis.com/v2/" + public static let scopes = [GoogleOAuthClient.cloudPlatformScope] + + public let projectId: String + public let appProfileId: String? + private let requester: GoogleRESTRequester + private let lock = NSLock() + private var instance: String + + public init( + projectId: String, + instanceId: String, + appProfileId: String? = nil, + transport: any GoogleRESTTransport, + credential: GoogleRESTCredential, + timeout: @escaping @Sendable () -> TimeInterval + ) { + self.projectId = projectId + self.instance = instanceId + self.appProfileId = appProfileId + self.requester = GoogleRESTRequester(transport: transport, credential: credential, timeout: timeout) + } + + public var instanceId: String { + lock.withLock { instance } + } + + public func switchInstance(to instanceId: String) throws { + guard BigtableConnectionSettings.isValidResourceId(instanceId) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Bigtable instance ID."), instanceId)) + } + lock.withLock { instance = instanceId } + } + + // MARK: - Admin + + public func listInstances() async throws -> [String] { + let json = try await send("GET", Self.adminEndpoint, "projects/\(projectId)/instances", operation: String(localized: "list instances")) + let instances = json["instances"] as? [[String: Any]] ?? [] + return instances.compactMap { Self.lastSegment($0["name"]) }.sorted() + } + + public func listTables() async throws -> [String] { + var names: [String] = [] + var pageToken: String? + repeat { + var query = [URLQueryItem(name: "view", value: "NAME_ONLY"), URLQueryItem(name: "pageSize", value: "1000")] + if let pageToken { query.append(URLQueryItem(name: "pageToken", value: pageToken)) } + let json = try await send("GET", Self.adminEndpoint, instancePath + "/tables", query: query, operation: listTablesOperation) + names += (json["tables"] as? [[String: Any]] ?? []).compactMap { Self.lastSegment($0["name"]) } + pageToken = (json["nextPageToken"] as? String).flatMap { $0.isEmpty ? nil : $0 } + } while pageToken != nil + return names.sorted() + } + + /// One page of one table name, which proves the credential, the project and the instance. + public func ping() async throws { + let query = [URLQueryItem(name: "view", value: "NAME_ONLY"), URLQueryItem(name: "pageSize", value: "1")] + _ = try await send("GET", Self.adminEndpoint, instancePath + "/tables", query: query, operation: listTablesOperation) + } + + public func columnFamilies(table: String) async throws -> [String] { + let json = try await send( + "GET", Self.adminEndpoint, try tablePath(table), query: [URLQueryItem(name: "view", value: "SCHEMA_VIEW")], + operation: String(format: String(localized: "read the schema of %@"), table) + ) + return (json["columnFamilies"] as? [String: Any] ?? [:]).keys.sorted() + } + + // MARK: - Data + + /// A nil body is a read the filters ruled out, answered with no rows and no request. + public func readRows(table: String, body: [String: Any]?) async throws -> [BigtableRow] { + guard var body else { return [] } + if let appProfileId { body["appProfileId"] = appProfileId } + let operation = String(format: String(localized: "read %@"), table) + let response = try await requester.send( + method: "POST", url: try url(Self.dataEndpoint, try tablePath(table) + ":readRows"), body: try Self.encode(body) + ) + try Self.requireSuccess(response, operation: operation) + return try BigtableChunkAssembler.rows(fromResponse: try Self.decode(response), operation: operation) + } + + public func mutateRow(table: String, key: Data, mutations: [BigtableMutation]) async throws { + guard !mutations.isEmpty else { return } + guard !key.isEmpty else { + throw BigtableError.configuration(String(localized: "A row needs a key. Bigtable has no empty row key.")) + } + var body: [String: Any] = ["rowKey": key.base64EncodedString(), "mutations": mutations.map(\.json)] + if let appProfileId { body["appProfileId"] = appProfileId } + _ = try await send("POST", Self.dataEndpoint, try tablePath(table) + ":mutateRow", body: body, + operation: String(format: String(localized: "write a row of %@"), table)) + } + + /// Writes a new row, refusing a key that already has one. A conditional `checkAndMutateRow` + /// would close the gap between the check and the write, but Google refuses it under + /// multi-cluster routing, the default app profile of a replicated instance. + public func insertRow(table: String, key: Data, mutations: [BigtableMutation]) async throws { + guard try await readRows(table: table, body: BigtableReadRequest.keyLookup(key)).isEmpty else { + throw BigtableError.conflict(String( + format: String(localized: "Row %@ already exists. Edit that row, or give the new one another key."), + BigtableBytes.display(key) + )) + } + try await mutateRow(table: table, key: key, mutations: mutations) + } + + // MARK: - Requests + + private var listTablesOperation: String { + String(format: String(localized: "list the tables of %@"), instanceId) + } + + private func send( + _ method: String, + _ endpoint: String, + _ path: String, + query: [URLQueryItem] = [], + body: [String: Any]? = nil, + operation: String + ) async throws -> [String: Any] { + let response = try await requester.send(method: method, url: try url(endpoint, path, query: query), body: try body.map(Self.encode)) + try Self.requireSuccess(response, operation: operation) + return try Self.decode(response) as? [String: Any] ?? [:] + } + + private static func encode(_ body: [String: Any]) throws -> Data { + try JSONSerialization.data(withJSONObject: body, options: [.sortedKeys]) + } + + private static func decode(_ response: GoogleRESTResponse) throws -> Any { + guard !response.body.isEmpty else { return [String: Any]() } + do { + return try JSONSerialization.jsonObject(with: response.body, options: [.fragmentsAllowed]) + } catch { + throw BigtableError.malformedResponse(String(localized: "Bigtable sent a response that is not JSON.")) + } + } + + private static func requireSuccess(_ response: GoogleRESTResponse, operation: String) throws { + guard !response.isSuccess else { return } + throw BigtableError.from(status: response.statusCode, body: response.body, operation: operation) + } + + private static func lastSegment(_ name: Any?) -> String? { + (name as? String)?.split(separator: "/").last.map(String.init) + } + + var instancePath: String { + "projects/\(projectId)/instances/\(instanceId)" + } + + func tablePath(_ table: String) throws -> String { + guard BigtableConnectionSettings.isValidResourceId(table) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Bigtable table ID."), table)) + } + return instancePath + "/tables/" + table + } + + func url(_ endpoint: String, _ path: String, query: [URLQueryItem] = []) throws -> URL { + guard var components = URLComponents(string: endpoint + path) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) + } + if !query.isEmpty { components.queryItems = query } + guard let url = components.url else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) + } + return url + } +} + +public extension URLSessionGoogleRESTTransport { + static func bigtable(resourceTimeout: TimeInterval) -> URLSessionGoogleRESTTransport { + URLSessionGoogleRESTTransport(resourceTimeout: resourceTimeout) { BigtableError.transport($0) } + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift new file mode 100644 index 0000000000..6f6fe91732 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift @@ -0,0 +1,71 @@ +import Foundation + +public enum BigtableError: Error, LocalizedError, Equatable, Sendable { + case configuration(String) + case notConnected + case transport(String) + case authentication(String) + /// Kept apart from an empty result so a blocked read never looks like a table with no rows. + case permissionDenied(operation: String, detail: String) + case notFound(String) + case conflict(String) + case unsupported(String) + case api(status: Int, message: String) + case malformedResponse(String) + + public var errorDescription: String? { + switch self { + case .configuration(let detail), .transport(let detail), .malformedResponse(let detail), + .authentication(let detail), .notFound(let detail), .conflict(let detail), .unsupported(let detail): + return detail + case .notConnected: + return String(localized: "Not connected to Bigtable.") + case .permissionDenied(let operation, let detail): + let lead = String(format: String(localized: "Permission denied: %@."), operation) + return detail.isEmpty ? lead : lead + "\n" + detail + case .api(_, let message): + return message + } + } + + /// `operation` names what was attempted ("read events"), because Google's denial text names the + /// permission and not the table. + public static func from(status: Int, body: Data, operation: String) -> BigtableError { + from(status: status, message: apiMessage(from: body), operation: operation) + } + + /// A streamed read that fails after its first row has already answered 200, so the failure + /// arrives as the last element of the array instead. + static func from(streamError error: [String: Any], operation: String) -> BigtableError { + let status = (error["code"] as? NSNumber)?.intValue ?? 500 + let message = (error["message"] as? String).flatMap { $0.isEmpty ? nil : $0 } + return from(status: status, message: message, operation: operation) + } + + static func from(status: Int, message: String?, operation: String) -> BigtableError { + switch status { + case 401: + return .authentication(message ?? String(localized: "Google refused the credential.")) + case 403: + return .permissionDenied(operation: operation, detail: message ?? "") + case 404: + return .notFound(message ?? String(format: String(localized: "Not found: %@."), operation)) + default: + return .api(status: status, message: message ?? String(format: String(localized: "Bigtable returned HTTP %d."), status)) + } + } + + /// A streaming method answers an error as a one-element array, a unary one as an object. + public static func apiMessage(from body: Data) -> String? { + guard let json = try? JSONSerialization.jsonObject(with: body, options: [.fragmentsAllowed]) else { + let text = String(data: body, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) + return (text?.isEmpty ?? true) ? nil : text + } + let object = (json as? [[String: Any]])?.first ?? json as? [String: Any] + let error = object?["error"] as? [String: Any] + if let message = error?["message"] as? String, !message.isEmpty { + return message + } + return nil + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift new file mode 100644 index 0000000000..08b8966201 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift @@ -0,0 +1,282 @@ +import Foundation + +/// What the driver runs: a grid page as a tagged spec, or a request line such as +/// `GET events/user#42`, `SCAN events/user#?limit=20`, `READ events {"rowsLimit": "10"}`, +/// `INSERT events/user#42 {"profile:name": "Ann"}`, `PUT events/user#42 {"profile:name": null}`, +/// `DELETE events/user#42` or `DELETE events/user#42 profile:name`. +public enum BigtableStatement: Sendable, Equatable { + case browse(BigtableBrowseSpec) + case request(BigtableRequest) + + public static let browseTag = "BIGTABLE_BROWSE:" + + public static func encode(_ spec: BigtableBrowseSpec) -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return browseTag + ((try? encoder.encode(spec)) ?? Data()).base64EncodedString() + } + + public static func parse(_ text: String) throws -> BigtableStatement { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.hasPrefix(browseTag) { + guard let data = Data(base64Encoded: String(trimmed.dropFirst(browseTag.count))), + let spec = try? JSONDecoder().decode(BigtableBrowseSpec.self, from: data) else { + throw BigtableError.malformedResponse(String(localized: "The browse request is damaged. Reopen the table.")) + } + return .browse(spec) + } + return .request(try BigtableRequest.parse(trimmed)) + } +} + +public struct BigtableRequest: Sendable, Equatable { + public enum Verb: String, Sendable, CaseIterable { + case get = "GET" + case scan = "SCAN" + case read = "READ" + case insert = "INSERT" + case put = "PUT" + case delete = "DELETE" + } + + public let verb: Verb + public let table: String + public let key: Data? + public let column: String? + public let limit: Int? + public let body: String? + + public init(verb: Verb, table: String, key: Data? = nil, column: String? = nil, limit: Int? = nil, body: String? = nil) { + self.verb = verb + self.table = table + self.key = key + self.column = column + self.limit = limit + self.body = body + } + + /// The table comes before the first slash and the key after it. A key holding a space, a quote, + /// a slash or a `?` is written as a JSON string: `GET events/"a/b"`. + public static func parse(_ text: String) throws -> BigtableRequest { + var scanner = RequestScanner(text.trimmingCharacters(in: .whitespacesAndNewlines)) + guard let verbWord = scanner.word(), let verb = Verb(rawValue: verbWord.uppercased()) else { + throw BigtableError.configuration(usage) + } + let request: BigtableRequest + switch verb { + case .read: + guard let table = scanner.word(stoppingAt: "{") else { throw missingTable } + let body = scanner.rest() + guard body.hasPrefix("{") else { + throw BigtableError.configuration(String(localized: "READ needs a ReadRows JSON body, such as READ events {\"rowsLimit\": \"10\"}.")) + } + request = BigtableRequest(verb: .read, table: table, body: body) + case .scan: + let target = try scanner.target(tableOnly: true) + request = BigtableRequest(verb: .scan, table: target.table, key: target.key, limit: try scanner.limit()) + case .get, .delete, .insert, .put: + let target = try scanner.target(tableOnly: false) + guard let key = target.key else { + throw BigtableError.configuration(String(format: String(localized: "%@ needs a row key."), verb.rawValue)) + } + switch verb { + case .insert, .put: + let body = scanner.rest() + guard body.hasPrefix("{") else { + throw BigtableError.configuration(verb == .insert + ? String(localized: "INSERT needs a JSON object of cells, such as INSERT events/user#42 {\"profile:name\": \"Ann\"}.") + : String(localized: "PUT needs a JSON object of cells, such as PUT events/user#42 {\"profile:name\": \"Ann\"}.")) + } + request = BigtableRequest(verb: verb, table: target.table, key: key, body: body) + case .delete: + request = BigtableRequest(verb: .delete, table: target.table, key: key, column: try scanner.column()) + default: + request = BigtableRequest(verb: verb, table: target.table, key: key) + } + } + guard scanner.isAtEnd else { throw BigtableError.configuration(usage) } + return request + } + + /// The request line as the history shows it and as `parse` reads it back. + public var text: String { + var parts = [verb.rawValue] + if verb == .read { + parts.append(table) + parts.append(body ?? "{}") + return parts.joined(separator: " ") + } + var target = table + "/" + (key.map { Self.token(BigtableBytes.display($0), reserved: "/?") } ?? "") + if let limit { target += "?limit=\(limit)" } + parts.append(target) + if let column { parts.append(Self.token(column, reserved: "")) } + if let body { parts.append(body) } + return parts.joined(separator: " ") + } + + /// The cells a write sets or deletes, all of which land in one atomic row mutation. A body maps + /// `family:qualifier` to text, `b64:` text for bytes, or null to delete the cell. + public func mutations() throws -> [BigtableMutation] { + switch verb { + case .get, .scan, .read: + return [] + case .delete: + return [try column.map { .deleteColumn(try BigtableColumn(name: $0)) } ?? .deleteRow] + case .insert, .put: + guard let data = body?.data(using: .utf8), let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw BigtableError.configuration(String(localized: "Write the cells as a JSON object, such as {\"profile:name\": \"Ann\"}.")) + } + guard !object.isEmpty else { + throw BigtableError.configuration(String(localized: "A write needs at least one cell. Bigtable does not store empty rows.")) + } + return try object.keys.sorted().map { name in + try Self.mutation(column: try BigtableColumn(name: name), name: name, value: object[name], allowsDelete: verb == .put) + } + } + } + + private static func mutation(column: BigtableColumn, name: String, value: Any?, allowsDelete: Bool) throws -> BigtableMutation { + switch value { + case is NSNull: + guard allowsDelete else { + throw BigtableError.configuration(String(format: String(localized: "INSERT only sets cells. Leave %@ out, or delete it with PUT."), name)) + } + return .deleteColumn(column) + case let text as String: + return .setCell(column, try BigtableBytes.parse(text)) + case let number as NSNumber where CFGetTypeID(number) == CFBooleanGetTypeID(): + return .setCell(column, Data((number.boolValue ? "true" : "false").utf8)) + case let number as NSNumber: + return .setCell(column, Data(number.stringValue.utf8)) + default: + throw BigtableError.configuration(String( + format: String(localized: "The value of %@ must be text or null. Bigtable stores bytes, so a number is written as its digits."), name + )) + } + } + + /// Text that would not read back as one bare word goes in JSON quotes. + static func token(_ text: String, reserved: String) -> String { + let plain = !text.isEmpty && !text.hasPrefix("\"") && text.unicodeScalars.allSatisfy { scalar in + !CharacterSet.whitespacesAndNewlines.contains(scalar) && !reserved.unicodeScalars.contains(scalar) + } + if plain { return text } + let data = (try? JSONSerialization.data(withJSONObject: [text], options: [.withoutEscapingSlashes])) ?? Data() + let array = String(bytes: data, encoding: .utf8) ?? "[\"\"]" + return String(array.dropFirst().dropLast()) + } + + static let usage = String( + localized: "Enter a request such as GET events/user#42, SCAN events/user#?limit=20, PUT events/user#42 {\"profile:name\": \"Ann\"} or DELETE events/user#42." + ) + + static let missingTable = BigtableError.configuration(String(localized: "Name the table before the key, as in GET events/user#42.")) +} + +/// Reads a request line one token at a time. +private struct RequestScanner { + private var text: Substring + private var query: [String: String] = [:] + + init(_ text: String) { + self.text = Substring(text) + } + + var isAtEnd: Bool { + text.allSatisfy(\.isWhitespace) + } + + mutating func word(stoppingAt stop: Character? = nil) -> String? { + skipSpace() + let word = text.prefix { !$0.isWhitespace && $0 != stop } + guard !word.isEmpty else { return nil } + text = text.dropFirst(word.count) + return String(word) + } + + mutating func rest() -> String { + let remaining = String(text).trimmingCharacters(in: .whitespacesAndNewlines) + text = "" + return remaining + } + + /// `table/key`, with an optional `?name=value` query after it. With `tableOnly`, a target with + /// no slash is a table alone, as in `SCAN events`. + mutating func target(tableOnly: Bool) throws -> (table: String, key: Data?) { + skipSpace() + let head = text.prefix { !$0.isWhitespace && $0 != "\"" && $0 != "?" } + let table: String + if let slash = head.firstIndex(of: "/") { + table = String(text[.. Int? { + guard query.keys.allSatisfy({ $0 == "limit" }) else { + throw BigtableError.configuration(String(localized: "SCAN takes only ?limit=, as in SCAN events/user#?limit=20.")) + } + guard let text = query["limit"] else { return nil } + guard let limit = Int(text), limit > 0 else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a row limit. Use a whole number above 0."), text)) + } + return limit + } + + /// A column after the key: a JSON string, or the rest of the line, so a qualifier may hold a space. + mutating func column() throws -> String? { + skipSpace() + if text.first == "\"" { return try quoted() } + let column = rest() + return column.isEmpty ? nil : column + } + + private mutating func quoted() throws -> String { + var index = text.index(after: text.startIndex) + var escaped = false + while index < text.endIndex { + let character = text[index] + if character == "\"" && !escaped { break } + escaped = character == "\\" && !escaped + index = text.index(after: index) + } + guard index < text.endIndex else { + throw BigtableError.configuration(String(localized: "A quoted value is missing its closing quote.")) + } + let literal = String(text[text.startIndex...index]) + text = text[text.index(after: index)...] + guard let data = ("[" + literal + "]").data(using: .utf8), + let value = (try? JSONSerialization.jsonObject(with: data) as? [String])?.first else { + throw BigtableError.configuration(String(localized: "A quoted value is not a valid JSON string.")) + } + return value + } + + private mutating func skipSpace() { + text = text.drop { $0.isWhitespace } + } +} diff --git a/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift b/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift index 4a9d7b52be..d6962fdd63 100644 --- a/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift +++ b/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift @@ -45,6 +45,7 @@ public enum SQLDDLFallbackPolicy { "Beancount", "Firestore", "Firebase Realtime Database", + "Bigtable", ] /// True when `DROP`/`TRUNCATE` built by the app is something this engine could run. diff --git a/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift b/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift index 16a5b6a764..bb002a60ef 100644 --- a/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift +++ b/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift @@ -44,12 +44,13 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let weaviate = DatabaseType(rawValue: "Weaviate") public static let firestore = DatabaseType(rawValue: "Firestore") public static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") + public static let bigtable = DatabaseType(rawValue: "Bigtable") public static let allKnownTypes: [DatabaseType] = [ .mysql, .mariadb, .tidb, .databend, .oceanbase, .postgresql, .sqlite, .redis, .mongodb, .clickhouse, .mssql, .oracle, .dameng, .duckdb, .cassandra, .redshift, .etcd, .cloudflareD1, .dynamodb, .bigquery, .spanner, .snowflake, .libsql, .beancount, - .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .firestore, .firebaseRealtimeDatabase + .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .firestore, .firebaseRealtimeDatabase, .bigtable ] /// Icon name for this database type — asset catalog name (e.g. "mysql-icon") or SF Symbol fallback @@ -87,6 +88,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .cloudflareR2SQL: return "cloudflare-r2-sql-icon" case .weaviate: return "weaviate-icon" case .firestore, .firebaseRealtimeDatabase: return "firebase-icon" + case .bigtable: return "bigtable-icon" default: return "externaldrive" } } diff --git a/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift b/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift index efb4884497..c750649150 100644 --- a/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift +++ b/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift @@ -1,9 +1,10 @@ import Foundation +import TableProGoogleCloud public final class FirestoreClient: @unchecked Sendable { public let projectId: String private let baseURL: URL - private let requester: FirebaseRequester + private let requester: GoogleRESTRequester private let lock = NSLock() private var database: String @@ -11,14 +12,14 @@ public final class FirestoreClient: @unchecked Sendable { projectId: String, database: String, baseURL: URL, - transport: any FirebaseTransport, - credential: FirebaseCredential, + transport: any GoogleRESTTransport, + credential: GoogleRESTCredential, timeout: @escaping @Sendable () -> TimeInterval ) { self.projectId = projectId self.database = database self.baseURL = baseURL - self.requester = FirebaseRequester(transport: transport, credential: credential, timeout: timeout) + self.requester = GoogleRESTRequester(transport: transport, credential: credential, timeout: timeout) } public var currentDatabase: String { @@ -176,7 +177,7 @@ public final class FirestoreClient: @unchecked Sendable { guard var components = URLComponents(string: baseURL.absoluteString + path) else { throw FirebaseError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) } - components.percentEncodedQuery = FirebaseRequester.queryString(query) + components.percentEncodedQuery = FirebaseQueryString.encode(query) guard let url = components.url else { throw FirebaseError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) } diff --git a/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift b/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift index 18c553c61d..6f28e9214b 100644 --- a/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift +++ b/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift @@ -1,18 +1,19 @@ import Foundation +import TableProGoogleCloud public final class RealtimeDatabaseClient: @unchecked Sendable { - private let requester: FirebaseRequester + private let requester: GoogleRESTRequester private let lock = NSLock() private var instanceURL: URL public init( instanceURL: URL, - transport: any FirebaseTransport, - credential: FirebaseCredential, + transport: any GoogleRESTTransport, + credential: GoogleRESTCredential, timeout: @escaping @Sendable () -> TimeInterval ) { self.instanceURL = instanceURL - self.requester = FirebaseRequester(transport: transport, credential: credential, timeout: timeout) + self.requester = GoogleRESTRequester(transport: transport, credential: credential, timeout: timeout) } public var currentURL: URL { @@ -165,7 +166,7 @@ public final class RealtimeDatabaseClient: @unchecked Sendable { let joined = ([basePath].filter { !$0.isEmpty } + segments).joined(separator: "/") components.percentEncodedPath = "/" + joined + ".json" let namespace = components.queryItems?.filter { $0.name == "ns" } ?? [] - components.percentEncodedQuery = FirebaseRequester.queryString(namespace + query) + components.percentEncodedQuery = FirebaseQueryString.encode(namespace + query) guard let url = components.url else { throw FirebaseError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) } @@ -184,10 +185,10 @@ public final class RealtimeDatabaseClient: @unchecked Sendable { } public struct RealtimeDatabaseInstanceLister: Sendable { - private let requester: FirebaseRequester + private let requester: GoogleRESTRequester - public init(transport: any FirebaseTransport, credential: FirebaseCredential, timeout: @escaping @Sendable () -> TimeInterval) { - self.requester = FirebaseRequester(transport: transport, credential: credential, timeout: timeout) + public init(transport: any GoogleRESTTransport, credential: GoogleRESTCredential, timeout: @escaping @Sendable () -> TimeInterval) { + self.requester = GoogleRESTRequester(transport: transport, credential: credential, timeout: timeout) } public func instances(projectId: String) async throws -> [URL] { diff --git a/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift b/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift index f1c2825923..c60b7ab6b9 100644 --- a/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift +++ b/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift @@ -1,215 +1,33 @@ import Foundation +import TableProGoogleCloud -public struct FirebaseHTTPRequest: Sendable, Equatable { - public let method: String - public let url: URL - public let headers: [String: String] - public let body: Data? - public let timeoutInterval: TimeInterval - - public init(method: String, url: URL, headers: [String: String] = [:], body: Data? = nil, timeoutInterval: TimeInterval) { - self.method = method - self.url = url - self.headers = headers - self.body = body - self.timeoutInterval = timeoutInterval - } -} - -public struct FirebaseHTTPResponse: Sendable, Equatable { - public let statusCode: Int - public let headers: [String: String] - public let body: Data - - public init(statusCode: Int, headers: [String: String] = [:], body: Data) { - self.statusCode = statusCode - self.headers = headers - self.body = body - } - - public var isSuccess: Bool { - (200..<300).contains(statusCode) - } - - public func header(_ name: String) -> String? { - headers.first { $0.key.caseInsensitiveCompare(name) == .orderedSame }?.value - } - - public func json() throws -> FirebaseJSON { +public extension GoogleRESTResponse { + func json() throws -> FirebaseJSON { guard !body.isEmpty else { return .null } return try FirebaseJSON.parse(body) } +} +extension GoogleRESTResponse { func requireSuccess(_ operation: String) throws { guard !isSuccess else { return } throw FirebaseError.from(status: statusCode, body: body, operation: operation) } } -public protocol FirebaseTransport: Sendable { - func send(_ request: FirebaseHTTPRequest) async throws -> FirebaseHTTPResponse - func cancelAll() -} - -/// The emulators take the fixed token `owner`, which bypasses the security rules there. -public struct FirebaseCredential: Sendable { - public let token: @Sendable () async throws -> String? - public let invalidate: @Sendable () async -> Void - public let quotaProject: String? - - public init( - token: @escaping @Sendable () async throws -> String?, - invalidate: @escaping @Sendable () async -> Void = {}, - quotaProject: String? = nil - ) { - self.token = token - self.invalidate = invalidate - self.quotaProject = quotaProject - } - - public static let emulator = FirebaseCredential(token: { "owner" }) -} - -public final class URLSessionFirebaseTransport: FirebaseTransport, @unchecked Sendable { - private let session: URLSession - private let lock = NSLock() - private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] - - public init(resourceTimeout: TimeInterval) { - let configuration = URLSessionConfiguration.ephemeral - configuration.timeoutIntervalForResource = resourceTimeout - configuration.httpCookieStorage = nil - configuration.urlCache = nil - session = URLSession(configuration: configuration, delegate: FirebaseRedirectRefusal(), delegateQueue: nil) - } - - deinit { - session.invalidateAndCancel() - } - - public func cancelAll() { - let tasks = lock.withLock { Array(inFlight.values) } - tasks.forEach { $0.cancel() } - } - - public func send(_ request: FirebaseHTTPRequest) async throws -> FirebaseHTTPResponse { - var urlRequest = URLRequest(url: request.url) - urlRequest.httpMethod = request.method - urlRequest.httpBody = request.body - urlRequest.timeoutInterval = request.timeoutInterval - for (name, value) in request.headers { - urlRequest.setValue(value, forHTTPHeaderField: name) - } - let tracker = FirebaseTaskTracker(transport: self) - defer { tracker.finish() } - do { - let (data, response) = try await session.data(for: urlRequest, delegate: tracker) - guard let http = response as? HTTPURLResponse else { - throw FirebaseError.transport(String(localized: "Firebase answered with something other than HTTP.")) - } - var headers: [String: String] = [:] - for (key, value) in http.allHeaderFields { - if let key = key as? String, let value = value as? String { headers[key] = value } - } - return FirebaseHTTPResponse(statusCode: http.statusCode, headers: headers, body: data) - } catch let error as URLError where error.code == .cancelled { - throw CancellationError() - } catch let error as URLError { - throw FirebaseError.transport(error.localizedDescription) - } - } - - fileprivate func register(_ task: URLSessionTask) { - lock.withLock { inFlight[ObjectIdentifier(task)] = task } - } - - fileprivate func unregister(_ task: URLSessionTask) { - lock.withLock { inFlight[ObjectIdentifier(task)] = nil } - } -} - -/// A redirect would carry the bearer token to whatever host it names. -private final class FirebaseRedirectRefusal: NSObject, URLSessionTaskDelegate, Sendable { - func urlSession( - _ session: URLSession, - task: URLSessionTask, - willPerformHTTPRedirection response: HTTPURLResponse, - newRequest request: URLRequest - ) async -> URLRequest? { - nil - } +public extension GoogleRESTCredential { + /// The emulators take the fixed token `owner`, which bypasses the security rules there. + static let firebaseEmulator = GoogleRESTCredential(token: { "owner" }) } -private final class FirebaseTaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { - private weak var transport: URLSessionFirebaseTransport? - private let lock = NSLock() - private var task: URLSessionTask? - - init(transport: URLSessionFirebaseTransport) { - self.transport = transport - } - - func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { - lock.withLock { self.task = task } - transport?.register(task) - } - - func finish() { - guard let task = lock.withLock({ task }) else { return } - transport?.unregister(task) +public extension URLSessionGoogleRESTTransport { + static func firebase(resourceTimeout: TimeInterval) -> URLSessionGoogleRESTTransport { + URLSessionGoogleRESTTransport(resourceTimeout: resourceTimeout) { FirebaseError.transport($0) } } } -/// A 401 drops the cached token and retries once, since a token can expire between being cached -/// and being used. -struct FirebaseRequester: Sendable { - let transport: any FirebaseTransport - let credential: FirebaseCredential - let timeout: @Sendable () -> TimeInterval - - func send( - method: String, - url: URL, - body: Data? = nil, - headers: [String: String] = [:], - timeoutOverride: TimeInterval? = nil - ) async throws -> FirebaseHTTPResponse { - var response = try await attempt(method, url, body, headers, timeoutOverride) - if response.statusCode == 401 { - await credential.invalidate() - response = try await attempt(method, url, body, headers, timeoutOverride) - } - return response - } - - private func attempt( - _ method: String, - _ url: URL, - _ body: Data?, - _ extraHeaders: [String: String], - _ timeoutOverride: TimeInterval? - ) async throws -> FirebaseHTTPResponse { - var headers = extraHeaders - if let token = try await credential.token() { - headers["Authorization"] = "Bearer " + token - } - if let project = credential.quotaProject, headers["x-goog-user-project"] == nil { - headers["x-goog-user-project"] = project - } - if body != nil { - headers["Content-Type"] = "application/json" - } - try Task.checkCancellation() - return try await transport.send(FirebaseHTTPRequest( - method: method, - url: url, - headers: headers, - body: body, - timeoutInterval: timeoutOverride ?? timeout() - )) - } - - static func queryString(_ items: [URLQueryItem]) -> String? { +enum FirebaseQueryString { + static func encode(_ items: [URLQueryItem]) -> String? { guard !items.isEmpty else { return nil } var allowed = CharacterSet.urlQueryAllowed allowed.remove(charactersIn: "&=+?#") diff --git a/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTRequester.swift b/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTRequester.swift new file mode 100644 index 0000000000..62b177d372 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTRequester.swift @@ -0,0 +1,106 @@ +import Foundation + +public struct GoogleRESTCredential: Sendable { + public let token: @Sendable () async throws -> String? + public let invalidate: @Sendable () async -> Void + public let quotaProject: String? + + public init( + token: @escaping @Sendable () async throws -> String?, + invalidate: @escaping @Sendable () async -> Void = {}, + quotaProject: String? = nil + ) { + self.token = token + self.invalidate = invalidate + self.quotaProject = quotaProject + } + + public init(provider: any GoogleAccessTokenProviding, quotaProject: String?) { + self.init( + token: { try await provider.accessToken() }, + invalidate: { await provider.invalidateCachedToken() }, + quotaProject: quotaProject + ) + } + + /// `projectHint` is the project the key or the login names. Application Default Credentials + /// that name none fall back to `GOOGLE_CLOUD_PROJECT`, as the gcloud tools do. + public static func resolving( + _ source: GoogleCredentialSource, + scopes: [String], + environment: [String: String] = ProcessInfo.processInfo.environment, + readFile: (String) -> Data? = { FileManager.default.contents(atPath: $0) }, + http: any GoogleHTTPClient = URLSessionGoogleHTTPClient() + ) throws -> (credential: GoogleRESTCredential, projectHint: String?) { + let credentials = try GoogleTokenProviders.credentials( + from: source, + scopes: scopes, + readFile: readFile, + environment: environment, + http: http + ) + let environmentProject = source == .applicationDefault + ? GoogleCredentialDocument.trimmed(environment["GOOGLE_CLOUD_PROJECT"] ?? "") + : "" + return ( + GoogleRESTCredential(provider: credentials.tokenProvider, quotaProject: credentials.quotaProject), + credentials.projectHint ?? (environmentProject.isEmpty ? nil : environmentProject) + ) + } +} + +/// A 401 drops the cached token and retries once, since a token can expire between being cached +/// and being used. +public struct GoogleRESTRequester: Sendable { + public let transport: any GoogleRESTTransport + public let credential: GoogleRESTCredential + public let timeout: @Sendable () -> TimeInterval + + public init(transport: any GoogleRESTTransport, credential: GoogleRESTCredential, timeout: @escaping @Sendable () -> TimeInterval) { + self.transport = transport + self.credential = credential + self.timeout = timeout + } + + public func send( + method: String, + url: URL, + body: Data? = nil, + headers: [String: String] = [:], + timeoutOverride: TimeInterval? = nil + ) async throws -> GoogleRESTResponse { + var response = try await attempt(method, url, body, headers, timeoutOverride) + if response.statusCode == 401 { + await credential.invalidate() + response = try await attempt(method, url, body, headers, timeoutOverride) + } + return response + } + + private func attempt( + _ method: String, + _ url: URL, + _ body: Data?, + _ extraHeaders: [String: String], + _ timeoutOverride: TimeInterval? + ) async throws -> GoogleRESTResponse { + var headers = extraHeaders + if let token = try await credential.token() { + headers["Authorization"] = "Bearer " + token + } + if let project = credential.quotaProject, headers["x-goog-user-project"] == nil { + headers["x-goog-user-project"] = project + } + if body != nil { + headers["Content-Type"] = "application/json" + } + try Task.checkCancellation() + return try await transport.send(GoogleRESTRequest( + method: method, + url: url, + headers: headers, + body: body, + timeoutInterval: timeoutOverride ?? timeout() + )) + } +} diff --git a/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTTransport.swift b/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTTransport.swift new file mode 100644 index 0000000000..b398d77a0d --- /dev/null +++ b/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTTransport.swift @@ -0,0 +1,122 @@ +import Foundation + +public struct GoogleRESTRequest: Sendable, Equatable { + public let method: String + public let url: URL + public let headers: [String: String] + public let body: Data? + public let timeoutInterval: TimeInterval + + public init(method: String, url: URL, headers: [String: String] = [:], body: Data? = nil, timeoutInterval: TimeInterval) { + self.method = method + self.url = url + self.headers = headers + self.body = body + self.timeoutInterval = timeoutInterval + } +} + +public struct GoogleRESTResponse: Sendable, Equatable { + public let statusCode: Int + public let headers: [String: String] + public let body: Data + + public init(statusCode: Int, headers: [String: String] = [:], body: Data) { + self.statusCode = statusCode + self.headers = headers + self.body = body + } + + public var isSuccess: Bool { + (200..<300).contains(statusCode) + } + + public func header(_ name: String) -> String? { + headers.first { $0.key.caseInsensitiveCompare(name) == .orderedSame }?.value + } +} + +public protocol GoogleRESTTransport: Sendable { + func send(_ request: GoogleRESTRequest) async throws -> GoogleRESTResponse + func cancelAll() +} + +public final class URLSessionGoogleRESTTransport: GoogleRESTTransport, @unchecked Sendable { + private let session: URLSession + private let transportError: @Sendable (String) -> any Error + private let lock = NSLock() + private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] + + public init(resourceTimeout: TimeInterval, transportError: @escaping @Sendable (String) -> any Error) { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForResource = resourceTimeout + configuration.httpCookieStorage = nil + configuration.urlCache = nil + session = URLSession(configuration: configuration, delegate: GoogleRedirectRefusingDelegate(), delegateQueue: nil) + self.transportError = transportError + } + + deinit { + session.invalidateAndCancel() + } + + public func cancelAll() { + let tasks = lock.withLock { Array(inFlight.values) } + tasks.forEach { $0.cancel() } + } + + public func send(_ request: GoogleRESTRequest) async throws -> GoogleRESTResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = request.timeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + let tracker = GoogleRESTTaskTracker(transport: self) + defer { tracker.finish() } + do { + let (data, response) = try await session.data(for: urlRequest, delegate: tracker) + guard let http = response as? HTTPURLResponse else { + throw transportError(String(localized: "The server answered with something other than HTTP.")) + } + var headers: [String: String] = [:] + for (key, value) in http.allHeaderFields { + if let key = key as? String, let value = value as? String { headers[key] = value } + } + return GoogleRESTResponse(statusCode: http.statusCode, headers: headers, body: data) + } catch let error as URLError where error.code == .cancelled { + throw CancellationError() + } catch let error as URLError { + throw transportError(error.localizedDescription) + } + } + + fileprivate func register(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = task } + } + + fileprivate func unregister(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = nil } + } +} + +private final class GoogleRESTTaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { + private weak var transport: URLSessionGoogleRESTTransport? + private let lock = NSLock() + private var task: URLSessionTask? + + init(transport: URLSessionGoogleRESTTransport) { + self.transport = transport + } + + func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { + lock.withLock { self.task = task } + transport?.register(task) + } + + func finish() { + guard let task = lock.withLock({ task }) else { return } + transport?.unregister(task) + } +} diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index d60c9bd3f9..26c31c4877 100644 --- a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -285,6 +285,9 @@ public struct SQLLexicalProfile: Sendable, Hashable { "Typesense": commandLineFamily, "Weaviate": commandLineFamily, "Kafka": commandLineFamily, + "Firestore": commandLineFamily, + "Firebase Realtime Database": commandLineFamily, + "Bigtable": commandLineFamily, ] }() } diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableClientTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableClientTests.swift new file mode 100644 index 0000000000..e1f22a245e --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableClientTests.swift @@ -0,0 +1,113 @@ +import Foundation +@testable import TableProBigtableCore +import TableProGoogleCloud +import Testing + +struct BigtableConnectionSettingsTests { + @Test("Settings take the project from the credential and refuse IDs that would change the path") + func settings() throws { + let parsed = try BigtableConnectionSettings.parse(fields: ["btInstanceId": "ea-bt-01"], credentialProject: "my-proj") + #expect(parsed == BigtableConnectionSettings(projectId: "my-proj", instanceId: "ea-bt-01")) + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: [:]) } + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "p-123", "btInstanceId": "../x"]) } + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "Bad Project"]) } + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "p-123"], database: "a/b") } + } + + @Test("The database a connection opens on names its instance ahead of the Instance ID field") + func databaseNamesTheInstance() throws { + let fields = ["btProjectId": "my-proj", "btInstanceId": "first"] + #expect(try BigtableConnectionSettings.parse(fields: fields, database: "second").instanceId == "second") + #expect(try BigtableConnectionSettings.parse(fields: fields, database: " ").instanceId == "first") + #expect(try BigtableConnectionSettings.parse(fields: ["btProjectId": "my-proj"]).instanceId == nil) + } + + @Test("The credential comes from the key field, the password, or Application Default Credentials") + func credentialSource() throws { + #expect(try BigtableConnectionSettings.credentialSource(fields: ["btServiceAccountJson": "/k.json"], password: nil) == .serviceAccountKey("/k.json")) + #expect(try BigtableConnectionSettings.credentialSource(fields: [:], password: " {} ") == .serviceAccountKey("{}")) + #expect(try BigtableConnectionSettings.credentialSource(fields: ["btAuthMethod": "adc"], password: nil) == .applicationDefault) + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.credentialSource(fields: [:], password: "") } + } + + /// `PluginMetadataRegistry+BigtableDefaults.swift` in the app spells these IDs out, since the app + /// links no engine core. Renaming one here strands the connections saved under the old one. + @Test("Field IDs keep the spelling saved connections and the app's defaults use") + func fieldIDs() { + #expect([ + BigtableFieldID.authMethod, BigtableFieldID.serviceAccountJson, BigtableFieldID.projectId, + BigtableFieldID.instanceId, BigtableFieldID.appProfileId + ] == ["btAuthMethod", "btServiceAccountJson", "btProjectId", "btInstanceId", "btAppProfileId"]) + #expect(BigtableAuthMethod.applicationDefault.rawValue == "adc") + } +} + +struct BigtableClientTests { + private static let committedRow = """ + [{"chunks": [{"rowKey": "\(b64("k"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "", "commitRow": true}]}] + """ + + private func client(_ transport: RecordingTransport) -> BigtableClient { + BigtableClient( + projectId: "proj", instanceId: "inst", transport: transport, + credential: GoogleRESTCredential(token: { "token" }), timeout: { 30 } + ) + } + + private static func reply(_ body: String, status: Int = 200) -> GoogleRESTResponse { + GoogleRESTResponse(statusCode: status, body: Data(body.utf8)) + } + + @Test("An insert onto a key that already has a row is refused without writing") + func insertRefusesExistingKey() async throws { + let transport = RecordingTransport { _ in Self.reply(Self.committedRow) } + let column = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + + await #expect(throws: BigtableError.self) { + try await client(transport).insertRow(table: "t", key: Data("k".utf8), mutations: [.setCell(column, Data("1".utf8))]) + } + #expect(transport.requests.map { $0.path } == ["/v2/projects/proj/instances/inst/tables/t:readRows"]) + } + + @Test("An insert onto a free key looks it up, then writes the row") + func insertWritesFreeKey() async throws { + let transport = RecordingTransport { request in + Self.reply(request.path.hasSuffix(":readRows") ? "[]" : "{}") + } + let column = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + + try await client(transport).insertRow(table: "t", key: Data("k".utf8), mutations: [.setCell(column, Data("1".utf8))]) + + let requests = transport.requests + #expect(requests.map { $0.path } == [ + "/v2/projects/proj/instances/inst/tables/t:readRows", + "/v2/projects/proj/instances/inst/tables/t:mutateRow" + ]) + #expect(jsonText(requests[0].jsonBody["rows"] ?? [:]) == #"{"rowKeys":["\#(b64("k"))"]}"#) + #expect(requests[1].jsonBody["rowKey"] as? String == b64("k")) + } + + @Test("A read the filters ruled out sends no request") + func emptyReadSendsNothing() async throws { + let transport = RecordingTransport { _ in Self.reply("[]") } + #expect(try await client(transport).readRows(table: "t", body: nil).isEmpty) + #expect(transport.requests.isEmpty) + } + + @Test("A health check asks for one table name, not every page of them") + func pingReadsOnePage() async throws { + let transport = RecordingTransport { _ in Self.reply(#"{"tables": [{"name": "x/tables/a"}], "nextPageToken": "more"}"#) } + try await client(transport).ping() + let request = try #require(transport.requests.first) + #expect(transport.requests.count == 1) + #expect(URLComponents(url: request.url, resolvingAgainstBaseURL: false)?.queryItems?.first { $0.name == "pageSize" }?.value == "1") + } + + @Test("A denied read is a permission error naming the table, not an empty page") + func deniedRead() async { + let transport = RecordingTransport { _ in Self.reply(#"[{"error": {"code": 403, "message": "bigtable.tables.readRows denied"}}]"#, status: 403) } + await #expect(throws: BigtableError.permissionDenied(operation: "read t", detail: "bigtable.tables.readRows denied")) { + try await client(transport).readRows(table: "t", body: [:]) + } + } +} diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableReadRequestTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableReadRequestTests.swift new file mode 100644 index 0000000000..4fc77e55fc --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableReadRequestTests.swift @@ -0,0 +1,148 @@ +import Foundation +@testable import TableProBigtableCore +import Testing + +struct BigtableReadRequestTests { + private func body( + _ filters: [BigtableFilterSpec], + or: Bool = false, + descending: Bool = false, + after: String? = nil + ) throws -> [String: Any]? { + let spec = BigtableBrowseSpec(table: "t", filters: filters, logicMode: or ? "OR" : "AND", descending: descending, limit: 10, offset: 0) + return try BigtableReadRequest.body(spec: spec, limit: 10, after: after.map { Data($0.utf8) }) + } + + private func rows(_ filters: [BigtableFilterSpec], or: Bool = false, after: String? = nil) throws -> String { + jsonText(try #require(try body(filters, or: or, after: after))["rows"] ?? [String: Any]()) + } + + private func filter(_ filters: [BigtableFilterSpec], or: Bool = false) throws -> String { + jsonText(try #require(try body(filters, or: or))["filter"] ?? [String: Any]()) + } + + private func key(_ op: String, _ value: String, _ second: String? = nil, caseSensitive: Bool = true) -> BigtableFilterSpec { + BigtableFilterSpec(column: "_key", op: op, value: value, secondValue: second, isCaseSensitive: caseSensitive) + } + + @Test("A plain page reads the newest cells, with the limit as a string") + func plain() throws { + #expect(jsonText(try #require(try body([]))) == #"{"filter":{"cellsPerColumnLimitFilter":1},"rowsLimit":"10"}"#) + } + + @Test("Key filters become row ranges, intersected under AND") + func keyRanges() throws { + #expect(try rows([key("STARTS WITH", "user#")]) + == #"{"rowRanges":[{"endKeyOpen":"\#(b64("user$"))","startKeyClosed":"\#(b64("user#"))"}]}"#) + #expect(try rows([key(">=", "b"), key("<", "m"), key(">", "c")]) + == #"{"rowRanges":[{"endKeyOpen":"\#(b64("m"))","startKeyOpen":"\#(b64("c"))"}]}"#) + #expect(try rows([key("IN", "a, b, z"), key("<", "c")]) == #"{"rowKeys":["\#(b64("a"))","\#(b64("b"))"]}"#) + } + + @Test("A stricter bound keeps its own open or closed end") + func stricterBoundKeepsItsEnd() throws { + #expect(try rows([key("STARTS WITH", "user#"), key("<=", "user#5")]) + == #"{"rowRanges":[{"endKeyClosed":"\#(b64("user#5"))","startKeyClosed":"\#(b64("user#"))"}]}"#) + #expect(try rows([key(">", "a"), key(">=", "b")]) == #"{"rowRanges":[{"startKeyClosed":"\#(b64("b"))"}]}"#) + #expect(try rows([key(">=", "b"), key(">", "b")]) == #"{"rowRanges":[{"startKeyOpen":"\#(b64("b"))"}]}"#) + } + + @Test("Filters that admit no key read nothing instead of sending an empty or inverted range") + func emptySelections() throws { + #expect(try body([key("<=", "k")], after: "k") == nil) + #expect(try body([key(">", "m"), key("<", "c")]) == nil) + #expect(try body([key("=", "a"), key("=", "b")]) == nil) + #expect(try body([key("=", "a")], after: "a") == nil) + } + + @Test("A cursor continues after the last key, and a descending read is refused") + func cursors() throws { + #expect(try rows([], after: "k9") == #"{"rowRanges":[{"startKeyOpen":"\#(b64("k9"))"}]}"#) + #expect(throws: BigtableError.self) { try body([], descending: true) } + } + + @Test("Key filters ORed together become several ranges, and a range the cursor passed is dropped") + func orRanges() throws { + #expect(try rows([key("STARTS WITH", "a"), key("=", "q")], or: true, after: "a5") + == #"{"rowKeys":["\#(b64("q"))"],"rowRanges":[{"endKeyOpen":"\#(b64("b"))","startKeyOpen":"\#(b64("a5"))"}]}"#) + #expect(try rows([key("STARTS WITH", "a"), key("=", "q")], or: true, after: "b5") == #"{"rowKeys":["\#(b64("q"))"]}"#) + #expect(try body([key("STARTS WITH", "a"), key("=", "q")], or: true, after: "r") == nil) + } + + @Test("Base64 items in an IN list each read back as bytes") + func inListOfBytes() throws { + #expect(try rows([key("IN", "b64:AAE=, b64:AAI=")]) == #"{"rowKeys":["AAE=","AAI="]}"#) + let column = try filter([BigtableFilterSpec(column: "cf:a", op: "IN", value: "b64:AAE=, b64:AAI=")]) + #expect(column.contains(b64(#"(?s)^(\x00\x01|\x00\x02)$"#))) + } + + @Test("A case-insensitive key match runs as a key regex, since a row range is exact") + func caseInsensitiveKey() throws { + let object = try #require(try body([key("=", "Alice", caseSensitive: false)])) + #expect(object["rows"] == nil) + #expect(jsonText(object["filter"] ?? [:]).contains(#""rowKeyRegexFilter":"\#(b64("(?is)^Alice$"))""#)) + } + + @Test("Case-insensitive text filters fold case, case-sensitive ones do not") + func caseFolding() throws { + let insensitive = try filter([BigtableFilterSpec(column: "cf:name", op: "CONTAINS", value: "alice", isCaseSensitive: false)]) + #expect(insensitive.contains(b64("(?is).*alice.*"))) + let sensitive = try filter([BigtableFilterSpec(column: "cf:name", op: "CONTAINS", value: "alice")]) + #expect(sensitive.contains(b64("(?s).*alice.*"))) + let regex = try filter([BigtableFilterSpec(column: "cf:name", op: "REGEX", value: "^a", isCaseSensitive: false)]) + #expect(regex.contains(b64("(?i)^a"))) + } + + @Test("Column filters keep or drop whole rows through conditions") + func columnConditions() throws { + let equal = try filter([BigtableFilterSpec(column: "cf:name", op: "=", value: "A.n")]) + #expect(equal.contains(#""familyNameRegexFilter":"^cf$""#)) + #expect(equal.contains(b64(#"(?s)^A\.n$"#))) + #expect(equal.contains(#""trueFilter":{"passAllFilter":true}"#)) + let missing = try filter([BigtableFilterSpec(column: "cf:name", op: "IS NULL", value: "")]) + #expect(missing.contains(#""falseFilter":{"passAllFilter":true}"#) && !missing.contains("valueRegexFilter")) + let range = try filter([BigtableFilterSpec(column: "cf:n", op: "BETWEEN", value: "1", secondValue: "9")]) + #expect(range.contains(#""startValueClosed":"\#(b64("1"))""#) && range.contains(#""endValueClosed":"\#(b64("9"))""#)) + } + + @Test("A negative test on a column also needs the column, as in SQL a missing value matches neither side") + func negativeFiltersNeedTheColumn() throws { + for op in ["!=", "NOT CONTAINS", "NOT IN", "IS NOT EMPTY"] { + let text = try filter([BigtableFilterSpec(column: "cf:name", op: op, value: "x")]) + let present = try #require(text.range(of: #""trueFilter":{"passAllFilter":true}"#), "\(op)") + let excluded = try #require(text.range(of: #""falseFilter":{"passAllFilter":true}"#), "\(op)") + #expect(present.lowerBound < excluded.lowerBound, "\(op)") + } + } + + @Test("Several column filters chain under AND and interleave under OR") + func combinations() throws { + let filters = [BigtableFilterSpec(column: "cf:a", op: "=", value: "1"), BigtableFilterSpec(column: "cf:b", op: "CONTAINS", value: "x")] + #expect(try filter(filters).hasPrefix(#"{"chain":{"filters":[{"chain""#)) + #expect(try filter(filters, or: true).hasPrefix(#"{"chain":{"filters":[{"interleave""#)) + } + + @Test("What Bigtable cannot express is refused with the reason") + func refusals() { + #expect(throws: BigtableError.self) { + try body([key(">", "a"), BigtableFilterSpec(column: "cf:a", op: "=", value: "1")], or: true) + } + #expect(throws: BigtableError.self) { try body([BigtableFilterSpec(column: "nocolon", op: "=", value: "1")]) } + #expect(throws: BigtableError.self) { try body([BigtableFilterSpec(column: "__RAW__", op: "", value: "not json")]) } + let sorted = BigtableBrowseSpec(table: "t", sortColumn: "cf:a", limit: 1, offset: 0) + #expect(throws: BigtableError.self) { try BigtableReadRequest.body(spec: sorted, limit: 1, after: nil) } + } + + @Test("A keys-only read strips values and keeps one cell per row") + func keysOnly() throws { + let spec = BigtableBrowseSpec(table: "t", limit: 0, offset: 0) + let object = try #require(try BigtableReadRequest.body(spec: spec, limit: 500, after: nil, keysOnly: true)) + #expect(jsonText(object["filter"] ?? [:]) == #"{"chain":{"filters":[{"cellsPerRowLimitFilter":1},{"stripValueTransformer":true}]}}"#) + } + + @Test("A column sample strips values, since only the column names are wanted") + func columnSample() { + #expect(jsonText(BigtableReadRequest.columnSample(limit: 50)) + == #"{"filter":{"chain":{"filters":[{"cellsPerColumnLimitFilter":1},{"stripValueTransformer":true}]}},"rowsLimit":"50"}"#) + } +} diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift new file mode 100644 index 0000000000..36757a8d44 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift @@ -0,0 +1,122 @@ +import Foundation +@testable import TableProBigtableCore +import Testing + +struct BigtableChunkAssemblerTests { + @Test("Family and qualifier carry over, split values join, and only committed rows count") + func assembly() throws { + let response = json(""" + [{"chunks": [ + {"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "10", "value": "\(b64("he"))", "valueSize": 5}, + {"value": "\(b64("llo"))"}, + {"qualifier": "\(b64("b"))", "timestampMicros": "10", "value": "\(b64("x"))", "commitRow": true} + ]}, + {"chunks": [ + {"rowKey": "\(b64("r2"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "\(b64("old"))"}, + {"resetRow": true}, + {"rowKey": "\(b64("r2"))", "familyName": {"value": "cf"}, "qualifier": "\(b64("a"))", "timestampMicros": "2", "value": "\(b64("new"))", "commitRow": true} + ]}] + """) + let rows = try BigtableChunkAssembler.rows(fromResponse: response, operation: "read t") + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + #expect(rows.map(\.key) == [Data("r1".utf8), Data("r2".utf8)]) + #expect(rows[0].cells[a]?.value == Data("hello".utf8)) + #expect(rows[0].cells[BigtableColumn(family: "cf", qualifier: Data("b".utf8))]?.value == Data("x".utf8)) + #expect(rows[1].cells[a] == BigtableCell(value: Data("new".utf8), timestampMicros: 2)) + } + + @Test("A response that stops inside a row is refused") + func truncated() { + let response = json(""" + [{"chunks": [{"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "\(b64("v"))"}]}] + """) + #expect(throws: BigtableError.self) { try BigtableChunkAssembler.rows(fromResponse: response, operation: "read t") } + } + + @Test("An error that ends a stream after its first rows fails the read instead of shortening it") + func streamError() { + let response = json(""" + [{"chunks": [{"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "", "commitRow": true}]}, + {"error": {"code": 503, "message": "The service is unavailable.", "status": "UNAVAILABLE"}}] + """) + #expect(throws: BigtableError.api(status: 503, message: "The service is unavailable.")) { + try BigtableChunkAssembler.rows(fromResponse: response, operation: "read t") + } + } +} + +struct BigtableRowSetTests { + @Test("Rows lay out with _key first, columns in order, and missing cells absent") + func layout() { + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + let z = BigtableColumn(family: "meta", qualifier: Data([0xFF, 0x00])) + let set = BigtableRowSet(rows: [ + BigtableRow(key: Data("k1".utf8), cells: [a: BigtableCell(value: Data("v".utf8), timestampMicros: 1)]), + BigtableRow(key: Data([0x00, 0x01]), cells: [z: BigtableCell(value: Data([0xC3]), timestampMicros: 1)]) + ]) + #expect(set.columns == ["_key", "cf:a", "meta:b64:/wA="]) + #expect(set.rows[0] == [.text("k1"), .text("v"), .absent]) + #expect(set.rows[1] == [.text("b64:AAE="), .absent, .bytes(Data([0xC3]))]) + #expect(set.absentCells == [0: [2], 1: [1]]) + #expect(set.locators == [b64("k1"), Data([0x00, 0x01]).base64EncodedString()]) + } + + @Test("Text that starts with b64: reaches the grid as bytes, so writing it back keeps it as text") + func taggedTextRoundTrips() throws { + let stored = Data("b64:AAE=".utf8) + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + let set = BigtableRowSet(rows: [BigtableRow(key: Data("k".utf8), cells: [a: BigtableCell(value: stored, timestampMicros: 1)])]) + #expect(set.rows[0][1] == .bytes(stored)) + + let write = try #require(try BigtableRowWrites.update(table: "t", key: Data("k".utf8), changes: [("cf:a", set.rows[0][1])])) + #expect(try write.mutations() == [.setCell(a, stored)]) + } + + @Test("An export page lines up under the first page's columns and keeps the rest in __extra__") + func exportAlignment() { + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + let b = BigtableColumn(family: "cf", qualifier: Data("b".utf8)) + let page = BigtableRowSet(rows: [ + BigtableRow(key: Data("k1".utf8), cells: [a: BigtableCell(value: Data("1".utf8), timestampMicros: 1)]), + BigtableRow(key: Data("k2".utf8), cells: [b: BigtableCell(value: Data("x/y".utf8), timestampMicros: 1)]) + ]) + let header = ["_key", "cf:a", BigtableRowSet.extraColumn] + #expect(page.rows(alignedTo: header) == [ + [.text("k1"), .text("1"), .absent], + [.text("k2"), .absent, .text(#"{"cf:b":"x/y"}"#)] + ]) + } +} + +struct BigtableBytesTests { + @Test("Printable text shows as itself, anything else as b64 that reads back") + func displayRoundTrip() throws { + for data in [Data("user#42".utf8), Data([0, 1, 255]), Data("b64:looks-tagged".utf8), Data("line\u{07}bell".utf8)] { + #expect(try BigtableBytes.parse(BigtableBytes.display(data)) == data) + } + #expect(BigtableBytes.display(Data("user#42".utf8)) == "user#42") + #expect(throws: BigtableError.self) { try BigtableBytes.parse("b64:***") } + } + + @Test("Prefix end increments the last byte that can grow") + func prefixEnd() { + #expect(BigtableBytes.prefixEnd(Data("ab".utf8)) == Data("ac".utf8)) + #expect(BigtableBytes.prefixEnd(Data([0x61, 0xFF])) == Data([0x62])) + #expect(BigtableBytes.prefixEnd(Data([0xFF, 0xFF])) == nil) + } + + @Test("Regex literals escape operators and non-printable bytes") + func regexLiteral() { + #expect(BigtableBytes.regexText(Data("a.b*(c)".utf8)) == #"a\.b\*\(c\)"#) + #expect(BigtableBytes.regexText(Data([0x00, 0x41])) == #"\x00A"#) + } + + @Test("A column name splits at its first colon") + func columnNames() throws { + let column = try BigtableColumn(name: "cf:a:b") + #expect(column.family == "cf" && column.qualifier == Data("a:b".utf8)) + #expect(try BigtableColumn(name: "cf:b64:/wA=").qualifier == Data([0xFF, 0x00])) + #expect(throws: BigtableError.self) { try BigtableColumn(name: "nocolon") } + #expect(throws: BigtableError.self) { try BigtableColumn(name: ":q") } + } +} diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift new file mode 100644 index 0000000000..f8bc8ba3f0 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift @@ -0,0 +1,135 @@ +import Foundation +@testable import TableProBigtableCore +import Testing + +struct BigtableStatementTests { + private let nameColumn = BigtableColumn(family: "cf", qualifier: Data("name".utf8)) + + @Test("Request lines read table, key, column, query and body") + func requests() throws { + let get = try BigtableRequest.parse("get events/user#42") + #expect(get.verb == .get && get.table == "events" && get.key == Data("user#42".utf8)) + let quoted = try BigtableRequest.parse(#"DELETE events/"a b/c" profile:name"#) + #expect(quoted.key == Data("a b/c".utf8) && quoted.column == "profile:name") + let scan = try BigtableRequest.parse("SCAN events/user#?limit=20") + #expect(scan.key == Data("user#".utf8) && scan.limit == 20) + let put = try BigtableRequest.parse(#"PUT events/b64:AAE= {"cf:a": "1"}"#) + #expect(put.key == Data([0, 1]) && put.body == #"{"cf:a": "1"}"#) + let insert = try BigtableRequest.parse(#"INSERT events/k {"cf:a": "1"}"#) + #expect(insert.verb == .insert && insert.key == Data("k".utf8)) + let read = try BigtableRequest.parse(#"READ events {"rowsLimit": "5"}"#) + #expect(read.table == "events" && read.body == #"{"rowsLimit": "5"}"#) + } + + @Test("A scan names a whole table with or without a slash") + func scanWholeTable() throws { + for text in ["SCAN events", "SCAN events/"] { + let scan = try BigtableRequest.parse(text) + #expect(scan.table == "events" && scan.key == nil && scan.limit == nil) + } + #expect(try BigtableRequest.parse("SCAN events?limit=5").limit == 5) + } + + @Test("A scan limit is a whole number above zero, and no other query is taken") + func scanLimits() { + for text in ["SCAN events/?limit=0", "SCAN events/?limit=-5", "SCAN events/?limit=x", "SCAN events/?rows=5"] { + #expect(throws: BigtableError.self, "\(text)") { try BigtableRequest.parse(text) } + } + } + + @Test("A column after the key may hold a space, and nothing after a request is ignored") + func columnsAndTrailingText() throws { + let spaced = try BigtableRequest.parse("DELETE events/k cf:first name") + #expect(spaced.column == "cf:first name") + #expect(try spaced.mutations() == [.deleteColumn(BigtableColumn(family: "cf", qualifier: Data("first name".utf8)))]) + for text in ["GET events/k extra", #"DELETE events/k "cf:a" extra"#, "SCAN events/a?limit=5 more"] { + #expect(throws: BigtableError.self, "\(text)") { try BigtableRequest.parse(text) } + } + } + + @Test("Malformed requests explain the syntax") + func malformed() { + for text in [ + "SELECT * FROM t", "GET events/", "PUT events/k", #"GET events/"open"#, "READ events", + #"READ {"rowsLimit": "5"}"#, "GET user#42" + ] { + #expect(throws: BigtableError.self, "\(text)") { try BigtableRequest.parse(text) } + } + } + + @Test("A request reads back from its own text, quoting keys and columns that need it") + func roundTrip() throws { + for key in [Data("user#42".utf8), Data("a b".utf8), Data([0xFF]), Data("x/y".utf8), Data("\"q".utf8)] { + for column in ["cf:a", "cf:first name"] { + let request = BigtableRequest(verb: .delete, table: "events", key: key, column: column) + #expect(try BigtableRequest.parse(request.text) == request) + } + } + let scan = BigtableRequest(verb: .scan, table: "events", key: Data("user#".utf8), limit: 20) + #expect(scan.text == "SCAN events/user#?limit=20") + #expect(try BigtableRequest.parse(scan.text) == scan) + let spec = BigtableBrowseSpec(table: "t", filters: [BigtableFilterSpec(column: "_key", op: "=", value: "k")], limit: 5, offset: 10) + #expect(try BigtableStatement.parse(BigtableStatement.encode(spec)) == .browse(spec)) + } + + @Test("A write body sets cells, deletes nulled ones under PUT, and writes numbers and booleans as text") + func mutations() throws { + let put = BigtableRequest(verb: .put, table: "t", key: Data("k".utf8), body: #"{"cf:name": null, "cf:n": 42, "cf:on": true, "cf:raw": "b64:AAE="}"#) + #expect(try put.mutations() == [ + .setCell(BigtableColumn(family: "cf", qualifier: Data("n".utf8)), Data("42".utf8)), + .deleteColumn(nameColumn), + .setCell(BigtableColumn(family: "cf", qualifier: Data("on".utf8)), Data("true".utf8)), + .setCell(BigtableColumn(family: "cf", qualifier: Data("raw".utf8)), Data([0, 1])) + ]) + #expect(try BigtableRequest(verb: .delete, table: "t", key: Data("k".utf8)).mutations() == [.deleteRow]) + } + + @Test("INSERT only sets cells, and no write sends an empty mutation") + func writeRefusals() { + let key = Data("k".utf8) + #expect(throws: BigtableError.self) { try BigtableRequest(verb: .insert, table: "t", key: key, body: #"{"cf:a": null}"#).mutations() } + #expect(throws: BigtableError.self) { try BigtableRequest(verb: .put, table: "t", key: key, body: "{}").mutations() } + #expect(throws: BigtableError.self) { try BigtableRequest(verb: .put, table: "t", key: key, body: #"{"cf:a": [1]}"#).mutations() } + } +} + +struct BigtableRowWritesTests { + private let key = Data("k".utf8) + + @Test("A new row is an INSERT of its non-empty cells under its _key") + func insert() throws { + let request = try BigtableRowWrites.insert( + table: "events", columns: ["_key", "cf:a", "cf:b", "cf:c"], values: [.text("k"), .text("1"), .absent, .bytes(Data([0xFF]))] + ) + #expect(request.text == #"INSERT events/k {"cf:a":"1","cf:c":"b64:/w=="}"#) + } + + @Test("A new row without a _key, or with nothing but one, is refused") + func insertRefusals() { + #expect(throws: BigtableError.self) { + try BigtableRowWrites.insert(table: "events", columns: ["_key", "cf:a"], values: [.absent, .text("1")]) + } + #expect(throws: BigtableError.self) { + try BigtableRowWrites.insert(table: "events", columns: ["_key", "cf:a"], values: [.text("k"), .absent]) + } + } + + @Test("An edit sends only the changed cells, and a removed cell as null") + func update() throws { + let request = try #require(try BigtableRowWrites.update(table: "events", key: key, changes: [("cf:a", .text("2")), ("cf:b", .absent)])) + #expect(request.text == #"PUT events/k {"cf:a":"2","cf:b":null}"#) + #expect(try BigtableRowWrites.update(table: "events", key: key, changes: []) == nil) + } + + @Test("An edit to an existing row's key is refused, since Bigtable cannot rename a row") + func keyChangeRefused() { + #expect(throws: BigtableError.self) { + try BigtableRowWrites.update(table: "events", key: key, changes: [("_key", .text("k2"))]) + } + } + + @Test("A delete names the row's key") + func delete() { + #expect(BigtableRowWrites.delete(table: "events", key: Data("a b".utf8)).text == #"DELETE events/"a b""#) + } +} diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableTestSupport.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableTestSupport.swift new file mode 100644 index 0000000000..f4c7223de5 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableTestSupport.swift @@ -0,0 +1,45 @@ +import Foundation +import TableProGoogleCloud + +func b64(_ text: String) -> String { + Data(text.utf8).base64EncodedString() +} + +func json(_ text: String) -> Any { + (try? JSONSerialization.jsonObject(with: Data(text.utf8))) ?? [String: Any]() +} + +func jsonText(_ object: Any) -> String { + (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys, .withoutEscapingSlashes])).flatMap { String(bytes: $0, encoding: .utf8) } ?? "" +} + +final class RecordingTransport: GoogleRESTTransport, @unchecked Sendable { + private let lock = NSLock() + private var sent: [GoogleRESTRequest] = [] + private let reply: @Sendable (GoogleRESTRequest) -> GoogleRESTResponse + + init(reply: @escaping @Sendable (GoogleRESTRequest) -> GoogleRESTResponse) { + self.reply = reply + } + + var requests: [GoogleRESTRequest] { + lock.withLock { sent } + } + + func send(_ request: GoogleRESTRequest) async throws -> GoogleRESTResponse { + lock.withLock { sent.append(request) } + return reply(request) + } + + func cancelAll() {} +} + +extension GoogleRESTRequest { + var path: String { + url.path + } + + var jsonBody: [String: Any] { + body.flatMap { try? JSONSerialization.jsonObject(with: $0) as? [String: Any] } ?? [:] + } +} diff --git a/Packages/TableProCore/Tests/TableProGoogleCloudTests/GoogleRESTRequesterTests.swift b/Packages/TableProCore/Tests/TableProGoogleCloudTests/GoogleRESTRequesterTests.swift new file mode 100644 index 0000000000..eff1db09ee --- /dev/null +++ b/Packages/TableProCore/Tests/TableProGoogleCloudTests/GoogleRESTRequesterTests.swift @@ -0,0 +1,100 @@ +import Foundation +@testable import TableProGoogleCloud +import Testing + +private final class ScriptedTransport: GoogleRESTTransport, @unchecked Sendable { + private let lock = NSLock() + private var statuses: [Int] + private var sent: [GoogleRESTRequest] = [] + + init(statuses: [Int]) { + self.statuses = statuses + } + + var requests: [GoogleRESTRequest] { + lock.withLock { sent } + } + + func send(_ request: GoogleRESTRequest) async throws -> GoogleRESTResponse { + let status = lock.withLock { + sent.append(request) + return statuses.isEmpty ? 200 : statuses.removeFirst() + } + return GoogleRESTResponse(statusCode: status, body: Data()) + } + + func cancelAll() {} +} + +struct GoogleRESTRequesterTests { + private let url = URL(string: "https://bigtable.googleapis.com/v2/x") ?? URL(fileURLWithPath: "/") + + @Test("A 401 drops the cached token and retries once with a fresh one") + func unauthorizedRetriesOnce() async throws { + let transport = ScriptedTransport(statuses: [401, 401]) + let tokens = LockedBox(["stale", "fresh"]) + let credential = GoogleRESTCredential( + token: { tokens.value.first }, + invalidate: { tokens.mutate { $0.removeFirst() } } + ) + let response = try await GoogleRESTRequester(transport: transport, credential: credential, timeout: { 5 }) + .send(method: "GET", url: url) + + #expect(response.statusCode == 401) + #expect(transport.requests.map { $0.headers["Authorization"] } == ["Bearer stale", "Bearer fresh"]) + } + + @Test("A body is sent as JSON, a login's quota project is billed, and the caller's own quota header wins") + func headers() async throws { + let transport = ScriptedTransport(statuses: []) + let requester = GoogleRESTRequester( + transport: transport, credential: GoogleRESTCredential(token: { "t" }, quotaProject: "quota"), timeout: { 7 } + ) + _ = try await requester.send(method: "POST", url: url, body: Data("{}".utf8)) + _ = try await requester.send(method: "GET", url: url, headers: ["x-goog-user-project": "browsed"], timeoutOverride: 2) + + let first = try #require(transport.requests.first) + #expect(first.headers["Content-Type"] == "application/json") + #expect(first.headers["x-goog-user-project"] == "quota") + #expect(first.timeoutInterval == 7) + let second = try #require(transport.requests.last) + #expect(second.headers["Content-Type"] == nil) + #expect(second.headers["x-goog-user-project"] == "browsed") + #expect(second.timeoutInterval == 2) + } + + @Test("A credential with no token sends no Authorization header") + func tokenlessCredential() async throws { + let transport = ScriptedTransport(statuses: []) + _ = try await GoogleRESTRequester(transport: transport, credential: GoogleRESTCredential(token: { nil }), timeout: { 5 }) + .send(method: "GET", url: url) + #expect(transport.requests.first?.headers["Authorization"] == nil) + } + + @Test("Application Default Credentials that name no project fall back to GOOGLE_CLOUD_PROJECT") + func projectHintFromEnvironment() throws { + let login = #"{"type":"authorized_user","client_id":"cid","client_secret":"cs","refresh_token":"1//rt"}"# + let resolved = try GoogleRESTCredential.resolving( + .applicationDefault, + scopes: [GoogleOAuthClient.cloudPlatformScope], + environment: ["GOOGLE_CLOUD_PROJECT": " env-proj "], + readFile: { $0 == GoogleApplicationDefaultCredentials.defaultPath ? Data(login.utf8) : nil }, + http: StubGoogleHTTPClient(json: "{}") + ) + #expect(resolved.projectHint == "env-proj") + #expect(resolved.credential.quotaProject == nil) + } + + @Test("A service account key names its own project, whatever the environment says") + func projectHintFromKey() throws { + let rsa = try #require(TestRSAKey.shared) + let resolved = try GoogleRESTCredential.resolving( + .serviceAccountKey(rsa.serviceAccountJSON(projectId: "key-project")), + scopes: [GoogleOAuthClient.cloudPlatformScope], + environment: ["GOOGLE_CLOUD_PROJECT": "env-proj"], + readFile: { _ in nil }, + http: StubGoogleHTTPClient(json: "{}") + ) + #expect(resolved.projectHint == "key-project") + } +} diff --git a/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift b/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift index 5a410e066b..ca8b6a75bd 100644 --- a/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift +++ b/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift @@ -61,7 +61,7 @@ struct DatabaseTypeTests { @Test("allKnownTypes contains all expected types") func allKnownTypesComplete() { - #expect(DatabaseType.allKnownTypes.count == 32) + #expect(DatabaseType.allKnownTypes.count == 33) #expect(DatabaseType.allKnownTypes.contains(.mysql)) #expect(DatabaseType.allKnownTypes.contains(.tidb)) #expect(DatabaseType.allKnownTypes.contains(.databend)) @@ -80,6 +80,7 @@ struct DatabaseTypeTests { #expect(DatabaseType.allKnownTypes.contains(.weaviate)) #expect(DatabaseType.allKnownTypes.contains(.firestore)) #expect(DatabaseType.allKnownTypes.contains(.firebaseRealtimeDatabase)) + #expect(DatabaseType.allKnownTypes.contains(.bigtable)) } /// The list has no duplicates, which a count alone would not catch: adding a type twice @@ -113,6 +114,13 @@ struct DatabaseTypeTests { #expect(DatabaseType.firebaseRealtimeDatabase.pluginTypeId == "Firebase Realtime Database") } + @Test("Bigtable resolves its icon and plugin type id") + func bigtableIdentity() { + #expect(DatabaseType.bigtable.rawValue == "Bigtable") + #expect(DatabaseType.bigtable.iconName == "bigtable-icon") + #expect(DatabaseType.bigtable.pluginTypeId == "Bigtable") + } + @Test("ScyllaDB resolves its own icon") func scyllaDBIdentity() { #expect(DatabaseType.scylladb.rawValue == "ScyllaDB") diff --git a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift index 966b805fe9..8d194f955e 100644 --- a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift +++ b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift @@ -11,6 +11,7 @@ struct SQLLexicalReadingsTests { "SQLite", "libSQL", "Turso", "Cloudflare D1", "DuckDB", "Oracle", "Dameng", "SQL Server", "ClickHouse", "Snowflake", "BigQuery", "Spanner", "Trino", "Teradata", "SAP HANA", "Cassandra", "ScyllaDB", "DynamoDB", "SurrealDB", "Redis", "MongoDB", "etcd", "Elasticsearch", "Typesense", "Weaviate", "Kafka", + "Firestore", "Firebase Realtime Database", "Bigtable", ] let missing = shipped.filter { SQLLexicalProfile.curated(forDatabaseTypeId: $0) == nil } #expect(missing.isEmpty) diff --git a/Plugins/BigtableDriverPlugin/BigtablePlugin.swift b/Plugins/BigtableDriverPlugin/BigtablePlugin.swift new file mode 100644 index 0000000000..024d94cf5d --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePlugin.swift @@ -0,0 +1,110 @@ +import Foundation +import TableProBigtableCore +import TableProPluginKit + +final class BigtablePlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Bigtable Driver" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Google Cloud Bigtable over the REST API" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "Bigtable" + static let databaseDisplayName = "Google Cloud Bigtable" + static let iconName = "bigtable-icon" + static let defaultPort = 0 + static let isDownloadable = true + + static let connectionMode: ConnectionMode = .apiOnly + static let navigationModel: NavigationModel = .standard + static let pathFieldRole: PathFieldRole = .database + static let requiresAuthentication = true + static let brandColorHex = "#4285F4" + static let queryLanguageName = "REST" + static let editorLanguage: EditorLanguage = .javascript + static let supportsForeignKeys = false + static let supportsSchemaEditing = false + static let supportsDatabaseSwitching = true + static let supportsImport = false + static let supportsExport = true + static let supportsSSH = false + static let supportsSSL = false + static let supportsReadOnlyMode = true + static let supportsForeignKeyDisable = false + static let supportsAddColumn = false + static let supportsModifyColumn = false + static let supportsDropColumn = false + static let supportsAddIndex = false + static let supportsDropIndex = false + static let supportsModifyPrimaryKey = false + static let supportsFieldRemoval = true + static let databaseGroupingStrategy: GroupingStrategy = .byDatabase + static let defaultGroupName = "default" + static let tableEntityName = "Tables" + static let containerEntityName = "Instance" + static let defaultSchemaName = "" + static let defaultPrimaryKeyColumn: String? = BigtableRowSet.keyColumn + static let structureColumnFields: [StructureColumnField] = [.name, .type, .nullable] + static let sqlDialect: SQLDialectDescriptor? = nil + static let caseSensitivityStyle: SQLDialectDescriptor.CaseSensitivityStyle = .driverManaged + + static let columnTypesByCategory: [String: [String]] = ["Binary": ["bytes"]] + static let additionalConnectionFields: [ConnectionField] = bigtableConnectionFields() + static var statementCompletions: [CompletionEntry] { bigtableCompletions } + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + BigtablePluginDriver(config: config) + } +} + +func bigtableConnectionFields() -> [ConnectionField] { + [ + ConnectionField( + id: BigtableFieldID.authMethod, + label: String(localized: "Auth Method"), + defaultValue: BigtableAuthMethod.serviceAccount.rawValue, + fieldType: .dropdown(options: [ + .init(value: BigtableAuthMethod.serviceAccount.rawValue, label: "Service Account Key"), + .init(value: BigtableAuthMethod.applicationDefault.rawValue, label: "Application Default Credentials") + ]), + section: .authentication + ), + ConnectionField( + id: BigtableFieldID.serviceAccountJson, + label: String(localized: "Service Account Key"), + placeholder: "File path or paste JSON", + required: true, + fieldType: .secure, + section: .authentication, + hidesPassword: true, + visibleWhen: FieldVisibilityRule(fieldId: BigtableFieldID.authMethod, values: [BigtableAuthMethod.serviceAccount.rawValue]) + ).withHidesUsername(true), + ConnectionField( + id: BigtableFieldID.projectId, + label: String(localized: "Project ID"), + placeholder: "Taken from the key when empty", + section: .authentication + ), + ConnectionField( + id: BigtableFieldID.instanceId, + label: String(localized: "Instance ID"), + placeholder: "The first instance when empty", + section: .authentication + ), + ConnectionField( + id: BigtableFieldID.appProfileId, + label: String(localized: "App Profile ID"), + placeholder: "default", + section: .advanced + ) + ] +} + +let bigtableCompletions: [CompletionEntry] = [ + CompletionEntry(label: "GET row", insertText: "GET events/user#42"), + CompletionEntry(label: "SCAN prefix", insertText: "SCAN events/user#?limit=20"), + CompletionEntry(label: "READ", insertText: "READ events {\"rowsLimit\": \"20\", \"filter\": {\"cellsPerColumnLimitFilter\": 1}}"), + CompletionEntry(label: "INSERT row", insertText: "INSERT events/user#42 {\"profile:name\": \"Ann\"}"), + CompletionEntry(label: "PUT cells", insertText: "PUT events/user#42 {\"profile:name\": \"Ann\"}"), + CompletionEntry(label: "DELETE row", insertText: "DELETE events/user#42"), + CompletionEntry(label: "DELETE column", insertText: "DELETE events/user#42 profile:name") +] diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift new file mode 100644 index 0000000000..6f92b8c0a4 --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift @@ -0,0 +1,92 @@ +import Foundation +import TableProBigtableCore +import TableProPluginKit + +extension BigtablePluginDriver { + private static let defaultMarker = "__DEFAULT__" + + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try changes.compactMap { change in + do { + return try request(for: change, table: table, columns: columns, insertedRowData: insertedRowData) + .map { PluginRowWrite(statement: $0.text, rowIndices: [change.rowIndex]) } + } catch { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: error.localizedDescription) + } + } + } + + /// Restoring a deleted row writes it back under its own key, refused if that key is taken again. + func generateIdentityPreservingInsert( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + rows: [[PluginCellValue]], + absentCells: [Int: Set] + ) -> [(statement: String, parameters: [PluginCellValue])]? { + let statements = rows.enumerated().compactMap { index, row in + try? BigtableRowWrites.insert(table: table, columns: columns, values: cells(row, absent: absentCells[index] ?? [])).text + } + return statements.count == rows.count ? statements.map { ($0, []) } : nil + } + + private func request( + for change: PluginRowChange, + table: String, + columns: [String], + insertedRowData: [Int: [PluginCellValue]] + ) throws -> BigtableRequest? { + let absent = change.absentColumns ?? [] + switch change.type { + case .insert: + return try BigtableRowWrites.insert( + table: table, columns: columns, values: cells(insertedRowData[change.rowIndex] ?? [], absent: absent) + ) + case .update: + let changes = try change.cellChanges.map { cell in + guard cell.newValue.asText != Self.defaultMarker else { + throw BigtableError.unsupported(String(localized: "Bigtable has no default values. Delete the cell, or enter a value.")) + } + return (column: cell.columnName, value: absent.contains(cell.columnIndex) ? BigtableRowSet.Cell.absent : Self.cell(cell.newValue)) + } + return try BigtableRowWrites.update(table: table, key: try originalKey(change, columns: columns), changes: changes) + case .delete: + return BigtableRowWrites.delete(table: table, key: try originalKey(change, columns: columns)) + } + } + + private func originalKey(_ change: PluginRowChange, columns: [String]) throws -> Data { + guard let index = columns.firstIndex(of: BigtableRowSet.keyColumn), + let row = change.originalRow, index < row.count, + let key = try BigtableRowWrites.key(Self.cell(row[index])) else { + throw BigtableError.configuration(String(localized: "The row has no _key, so the row it came from is unknown. Reload the table.")) + } + return key + } + + /// A new row holds the host's Default marker where it has no value yet. Bigtable generates + /// nothing, so that cell is left out, and a `_key` left that way is refused. + private func cells(_ row: [PluginCellValue], absent: Set) -> [BigtableRowSet.Cell] { + row.enumerated().map { index, value in + absent.contains(index) || value.asText == Self.defaultMarker ? .absent : Self.cell(value) + } + } + + private static func cell(_ value: PluginCellValue) -> BigtableRowSet.Cell { + switch value { + case .null: return .absent + case .text(let text): return .text(text) + case .bytes(let data): return .bytes(data) + } + } +} diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift new file mode 100644 index 0000000000..1b3c890c81 --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift @@ -0,0 +1,196 @@ +import Foundation +import TableProBigtableCore +import TableProPluginKit + +extension BigtablePluginDriver { + static let scanPageSize = 100 + static let exportPageSize = 500 + static let skipBatchSize = 5_000 + + func execute(query: String) async throws -> PluginQueryResult { + try await run(query, rowCap: PluginRowLimits.emergencyMax) + } + + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { + try await run(query, rowCap: rowCap ?? PluginRowLimits.emergencyMax) + } + + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + try await run(query, rowCap: rowCap) + } + + /// Every read asks the server for one row past `rowCap`, which is how a truncated result is + /// told apart from one that ended there, without buffering the rest of the table. + private func run(_ query: String, rowCap: Int) async throws -> PluginQueryResult { + let started = Date() + let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.lowercased() == "select 1" { + try await ping() + return PluginQueryResult(columns: ["ok"], columnTypeNames: ["integer"], rows: [[.text("1")]], + rowsAffected: 0, timing: PluginQueryTiming(total: Date().timeIntervalSince(started))) + } + let cap = min(max(rowCap, 1), PluginRowLimits.emergencyMax) + let rows: [BigtableRow] + switch try BigtableStatement.parse(trimmed) { + case .browse(let spec): + rows = try await browse(spec, rowCap: cap) + case .request(let request): + if let written = try await write(request, started: started) { + return written + } + rows = try await read(request, rowCap: cap) + } + return result(BigtableRowSet(rows: Array(rows.prefix(cap))), isTruncated: rows.count > cap, started: started) + } + + // MARK: - Browse + + func browse(_ spec: BigtableBrowseSpec, rowCap: Int) async throws -> [BigtableRow] { + let client = try requireClient() + let after = try await startKey(for: spec, client: client) + if spec.offset > 0, after == nil { + return [] + } + let limit = spec.limit > 0 ? min(spec.limit, rowCap + 1) : rowCap + 1 + let rows = try await client.readRows(table: spec.table, body: try BigtableReadRequest.body(spec: spec, limit: limit, after: after)) + if let last = rows.last?.key, spec.limit > 0, rows.count == spec.limit { + rememberCursor(view: spec.viewKey, offset: spec.offset + rows.count, key: last) + } + return rows + } + + /// Page 1 needs no start key; a later page starts from the nearest cursor an earlier page left + /// and skips the rest by reading keys alone. + private func startKey(for spec: BigtableBrowseSpec, client: BigtableClient) async throws -> Data? { + guard spec.offset > 0 else { return nil } + var position = cursor(view: spec.viewKey, offset: spec.offset) + while (position?.offset ?? 0) < spec.offset { + try Task.checkCancellation() + let start = position?.offset ?? 0 + let step = min(Self.skipBatchSize, spec.offset - start) + let body = try BigtableReadRequest.body(spec: spec, limit: step, after: position?.key, keysOnly: true) + let keys = try await client.readRows(table: spec.table, body: body) + guard let last = keys.last?.key else { return nil } + position = (start + keys.count, last) + rememberCursor(view: spec.viewKey, offset: start + keys.count, key: last) + if keys.count < step { break } + } + guard let position, position.offset == spec.offset else { return nil } + return position.key + } + + func result(_ rows: BigtableRowSet, isTruncated: Bool = false, started: Date) -> PluginQueryResult { + var result = PluginQueryResult( + columns: rows.columns, + columnTypeNames: rows.columns.map { _ in "bytes" }, + rows: rows.rows.map { $0.map(Self.cell) }, + rowsAffected: 0, + timing: PluginQueryTiming(total: Date().timeIntervalSince(started)), + isTruncated: isTruncated + ) + result.rowLocators = rows.locators + result.absentCells = rows.absentCells + return result + } + + static func cell(_ cell: BigtableRowSet.Cell) -> PluginCellValue { + switch cell { + case .absent: return .null + case .text(let text): return .text(text) + case .bytes(let data): return .bytes(data) + } + } + + // MARK: - Request lines + + private func read(_ request: BigtableRequest, rowCap: Int) async throws -> [BigtableRow] { + let client = try requireClient() + switch request.verb { + case .get: + return try await client.readRows(table: request.table, body: BigtableReadRequest.get(request.key ?? Data())) + case .scan: + let filters = request.key.map { + [BigtableFilterSpec(column: BigtableRowSet.keyColumn, op: "STARTS WITH", value: BigtableBytes.display($0))] + } ?? [] + let spec = BigtableBrowseSpec(table: request.table, filters: filters, limit: request.limit ?? Self.scanPageSize, offset: 0) + return try await browse(spec, rowCap: rowCap) + case .read: + guard let data = request.body?.data(using: .utf8), + var body = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw BigtableError.configuration(String(localized: "READ needs a ReadRows JSON body, such as READ events {\"rowsLimit\": \"10\"}.")) + } + let asked = (body["rowsLimit"] as? String).flatMap(Int.init) ?? (body["rowsLimit"] as? NSNumber)?.intValue ?? 0 + body["rowsLimit"] = String(asked > 0 ? min(asked, rowCap + 1) : rowCap + 1) + return try await client.readRows(table: request.table, body: body) + case .insert, .put, .delete: + return [] + } + } + + private func write(_ request: BigtableRequest, started: Date) async throws -> PluginQueryResult? { + let client = try requireClient() + let key = request.key ?? Data() + switch request.verb { + case .get, .scan, .read: + return nil + case .insert: + try await client.insertRow(table: request.table, key: key, mutations: try request.mutations()) + case .put, .delete: + try await client.mutateRow(table: request.table, key: key, mutations: try request.mutations()) + } + forgetCursors(table: request.table) + return PluginQueryResult( + columns: ["row"], columnTypeNames: ["bytes"], + rows: [[.text(request.table + "/" + BigtableBytes.display(key))]], + rowsAffected: 1, timing: PluginQueryTiming(total: Date().timeIntervalSince(started)) + ) + } + + // MARK: - Export + + func streamRows(query: String) -> AsyncThrowingStream { + AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + let task = Task { + do { + if case .browse(let spec)? = try? BigtableStatement.parse(query), spec.limit == 0 { + try await self.export(spec, continuation: continuation) + } else { + let result = try await self.execute(query: query) + continuation.yield(.header(PluginStreamHeader( + columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: nil + ))) + if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { @Sendable _ in task.cancel() } + } + } + + /// The columns come from the first page; a column only later rows hold goes into `__extra__`. + private func export(_ spec: BigtableBrowseSpec, continuation: AsyncThrowingStream.Continuation) async throws { + let client = try requireClient() + var header: [String]? + var after: Data? + repeat { + try Task.checkCancellation() + let body = try BigtableReadRequest.body(spec: spec, limit: Self.exportPageSize, after: after) + let rows = try await client.readRows(table: spec.table, body: body) + let page = BigtableRowSet(rows: rows) + let columns = header ?? page.columns + [BigtableRowSet.extraColumn] + if header == nil { + header = columns + continuation.yield(.header(PluginStreamHeader( + columns: columns, columnTypeNames: columns.map { _ in "bytes" }, estimatedRowCount: nil + ))) + } + if !rows.isEmpty { + continuation.yield(.rows(page.rows(alignedTo: columns).map { $0.map(Self.cell) })) + } + after = rows.count == Self.exportPageSize ? rows.last?.key : nil + } while after != nil + } +} diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift new file mode 100644 index 0000000000..6a3fb17afc --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift @@ -0,0 +1,112 @@ +import Foundation +import TableProBigtableCore +import TableProPluginKit + +extension BigtablePluginDriver { + static let sampleSize = 50 + static let countBatchSize = 10_000 + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + try await requireClient().listTables().map { PluginTableInfo(name: $0, type: "TABLE") } + } + + /// Bigtable has no fixed columns, so these are the `family:qualifier` pairs a sample of rows + /// holds. A family with no sampled cell has no column until a row writes one. + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + try await sampledColumns(table: table).map { name in + let isKey = name == BigtableRowSet.keyColumn + return PluginColumnInfo(name: name, dataType: "bytes", isNullable: !isKey, isPrimaryKey: isKey) + } + } + + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { + let key = BigtableRowSet.keyColumn + return [PluginIndexInfo(name: key, columns: [key], isUnique: true, isPrimary: true, type: "PRIMARY KEY")] + } + + func fetchTableDDL(table: String, schema: String?) async throws -> String { + let families = try await requireClient().columnFamilies(table: table) + let columns = try await sampledColumns(table: table) + let description: [String: Any] = [ + "table": table, + "columnFamilies": families, + "sampledColumns": Array(columns.dropFirst()) + ] + let data = (try? JSONSerialization.data(withJSONObject: description, options: [.prettyPrinted, .sortedKeys])) ?? Data() + return String(bytes: data, encoding: .utf8) ?? "{}" + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table, engine: "Bigtable") + } + + private func sampledColumns(table: String) async throws -> [String] { + let rows = try await requireClient().readRows(table: table, body: BigtableReadRequest.columnSample(limit: Self.sampleSize)) + return BigtableRowSet(rows: rows).columns + } + + // MARK: - Browse statements + + func buildBrowseQuery( + table: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int + ) -> String? { + buildFilteredQuery( + table: table, schema: nil, queryFilters: [], logicMode: "AND", + sortColumns: sortColumns, columns: columns, limit: limit, offset: offset, columnKinds: [:] + ) + } + + func buildFilteredQuery( + table: String, + schema: String?, + queryFilters filters: [PluginQueryFilter], + logicMode: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int, + columnKinds: [String: PluginColumnKind] + ) -> String? { + let sort = sortColumns.first.flatMap { sort -> (column: String, ascending: Bool)? in + guard sort.columnIndex >= 0, sort.columnIndex < columns.count else { return nil } + return (columns[sort.columnIndex], sort.ascending) + } + return BigtableStatement.encode(BigtableBrowseSpec( + table: table, + filters: filters.map(Self.filterSpec), + logicMode: logicMode, + descending: sort.map { !$0.ascending } ?? false, + sortColumn: sort?.column, + limit: limit, + offset: offset + )) + } + + /// Bigtable keeps no row count, so this reads every key the filters admit, without values. The + /// app runs it only when asked, since it is a full scan. + func fetchExactRowCount(table: String, schema: String?, queryFilters filters: [PluginQueryFilter], logicMode: String) async throws -> Int? { + let client = try requireClient() + let spec = BigtableBrowseSpec(table: table, filters: filters.map(Self.filterSpec), logicMode: logicMode, limit: 0, offset: 0) + var total = 0 + var after: Data? + repeat { + try Task.checkCancellation() + let body = try BigtableReadRequest.body(spec: spec, limit: Self.countBatchSize, after: after, keysOnly: true) + let keys = try await client.readRows(table: table, body: body) + total += keys.count + after = keys.count == Self.countBatchSize ? keys.last?.key : nil + } while after != nil + return total + } + + private static func filterSpec(_ filter: PluginQueryFilter) -> BigtableFilterSpec { + BigtableFilterSpec( + column: filter.column, op: filter.op, value: filter.value, + secondValue: filter.secondValue, isCaseSensitive: filter.isCaseSensitive + ) + } +} diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift new file mode 100644 index 0000000000..6a6b3de9c0 --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift @@ -0,0 +1,180 @@ +import Foundation +import os +import TableProBigtableCore +import TableProGoogleCloud +import TableProLogRedaction +import TableProPluginKit + +internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Sendable { + static let logger = Logger(subsystem: "com.TablePro", category: "BigtablePluginDriver") + static let cursorViewLimit = 64 + + let config: DriverConnectionConfig + let queryTimeout = HttpQueryTimeoutBox() + private let lock = NSLock() + private var client: BigtableClient? + private var transport: URLSessionGoogleRESTTransport? + private var cursors: [String: [Int: Data]] = [:] + private var cursorOrder: [String] = [] + + init(config: DriverConnectionConfig) { + self.config = config + } + + var serverVersion: String? { "Google Cloud Bigtable" } + var capabilities: PluginCapabilities { [.cancelQuery] } + var supportsTransactions: Bool { false } + var parameterStyle: ParameterStyle { .questionMark } + + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + + // MARK: - Lifecycle + + /// The probe runs under the connect deadline and the client after it under the query timeout. + func connect() async throws { + let resolved = try GoogleRESTCredential.resolving( + try BigtableConnectionSettings.credentialSource(fields: config.additionalFields, password: config.password), + scopes: BigtableClient.scopes + ) + let settings = try BigtableConnectionSettings.parse( + fields: config.additionalFields, database: config.database, credentialProject: resolved.projectHint + ) + let deadline = PluginConnectDeadline(milliseconds: PluginConnectTimeout.milliseconds( + in: config.additionalFields, + default: Int(HttpQueryTimeout().requestTimeoutInterval * 1_000) + )) + let transport = URLSessionGoogleRESTTransport.bigtable(resourceTimeout: HttpQueryTimeout.sessionResourceTimeout) + let timeoutBox = queryTimeout + + let probe = BigtableClient( + projectId: settings.projectId, instanceId: settings.instanceId ?? "-", appProfileId: settings.appProfileId, + transport: transport, credential: resolved.credential, timeout: { deadline.remainingSeconds() } + ) + if settings.instanceId == nil { + guard let first = try await probe.listInstances().first else { + throw BigtableError.notFound(String(format: String(localized: "Project %@ has no Bigtable instance."), settings.projectId)) + } + try probe.switchInstance(to: first) + } + try await probe.ping() + + let client = BigtableClient( + projectId: settings.projectId, instanceId: probe.instanceId, appProfileId: settings.appProfileId, + transport: transport, credential: resolved.credential, timeout: { timeoutBox.requestTimeoutInterval } + ) + lock.withLock { + self.client = client + self.transport = transport + cursors.removeAll() + cursorOrder.removeAll() + } + } + + func disconnect() { + lock.withLock { + transport?.cancelAll() + client = nil + transport = nil + } + } + + func ping() async throws { + try await requireClient().ping() + } + + func cancelQuery() throws { + lock.withLock { transport?.cancelAll() } + } + + func applyQueryTimeout(_ seconds: Int) async throws { + queryTimeout.set(serverTimeoutSeconds: seconds) + } + + func requireClient() throws -> BigtableClient { + guard let client = lock.withLock({ client }) else { throw BigtableError.notConnected } + return client + } + + // MARK: - Instances + + var currentDatabase: String? { + lock.withLock { client?.instanceId } + } + + func fetchDatabases() async throws -> [String] { + let client = try requireClient() + do { + let instances = try await client.listInstances() + return instances.isEmpty ? [client.instanceId] : instances + } catch let error as BigtableError { + Self.logger.info( + "Could not list Bigtable instances: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + return [client.instanceId] + } + } + + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } + + func switchDatabase(to database: String) async throws { + try requireClient().switchInstance(to: database) + lock.withLock { + cursors.removeAll() + cursorOrder.removeAll() + } + } + + // MARK: - Page cursors + + func cursor(view: String, offset: Int) -> (offset: Int, key: Data)? { + lock.withLock { + guard let known = cursors[view] else { return nil } + let best = known.keys.filter { $0 <= offset }.max() + return best.flatMap { start in known[start].map { (start, $0) } } + } + } + + func rememberCursor(view: String, offset: Int, key: Data) { + lock.withLock { + if cursors[view] == nil { + cursorOrder.append(view) + if cursorOrder.count > Self.cursorViewLimit { + cursors[cursorOrder.removeFirst()] = nil + } + } + cursors[view, default: [:]][offset] = key + } + } + + func forgetCursors(table: String) { + let prefix = BigtableBrowseSpec.viewKeyPrefix(table: table) + lock.withLock { + for view in cursors.keys where view.hasPrefix(prefix) { + cursors[view] = nil + } + cursorOrder.removeAll { $0.hasPrefix(prefix) } + } + } + + // MARK: - Table operations + + func defaultExportQuery(table: String) -> String? { + BigtableStatement.encode(BigtableBrowseSpec(table: table, limit: 0, offset: 0)) + } + + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + throw BigtableError.unsupported(String(localized: "Bigtable has no views.")) + } + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + [] + } + + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { + nil + } +} diff --git a/Plugins/BigtableDriverPlugin/Info.plist b/Plugins/BigtableDriverPlugin/Info.plist new file mode 100644 index 0000000000..2da24fdd81 --- /dev/null +++ b/Plugins/BigtableDriverPlugin/Info.plist @@ -0,0 +1,14 @@ + + + + + TableProMinAppVersion + 0.81.0 + TableProPluginKitVersion + 38 + TableProProvidesDatabaseTypeIds + + Bigtable + + + diff --git a/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift b/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift index 79b79c28f8..d7966f88a5 100644 --- a/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift +++ b/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift @@ -11,45 +11,21 @@ internal enum FirebaseCredentialFactory { "https://www.googleapis.com/auth/userinfo.email" ] - struct Resolved { - let credential: FirebaseCredential - let projectHint: String? - } - - static func resolve(config: DriverConnectionConfig) throws -> Resolved { + static func resolve(config: DriverConnectionConfig) throws -> (credential: GoogleRESTCredential, projectHint: String?) { let fields = config.additionalFields let method = trimmed(fields[FirebaseFieldID.authMethod]).flatMap(FirebaseAuthMethod.init(rawValue:)) ?? .serviceAccount - let source: GoogleCredentialSource switch method { case .emulator: - return Resolved(credential: .emulator, projectHint: nil) + return (.firebaseEmulator, nil) case .serviceAccount: guard let value = trimmed(fields[FirebaseFieldID.serviceAccountJson]) ?? trimmed(config.password) else { throw FirebaseError.configuration(String(localized: "Choose a service account key file, or paste its JSON.")) } - source = .serviceAccountKey(value) + return try GoogleRESTCredential.resolving(.serviceAccountKey(value), scopes: scopes) case .applicationDefault: - source = .applicationDefault + return try GoogleRESTCredential.resolving(.applicationDefault, scopes: scopes) } - let environment = ProcessInfo.processInfo.environment - let credentials = try GoogleTokenProviders.credentials( - from: source, - scopes: scopes, - readFile: { FileManager.default.contents(atPath: $0) }, - environment: environment, - http: URLSessionGoogleHTTPClient() - ) - let provider = credentials.tokenProvider - let environmentProject = source == .applicationDefault ? trimmed(environment["GOOGLE_CLOUD_PROJECT"]) : nil - return Resolved( - credential: FirebaseCredential( - token: { try await provider.accessToken() }, - invalidate: { await provider.invalidateCachedToken() }, - quotaProject: credentials.quotaProject - ), - projectHint: credentials.projectHint ?? environmentProject - ) } private static func trimmed(_ value: String?) -> String? { diff --git a/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift b/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift index ace0cbad58..abad5c8f93 100644 --- a/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift +++ b/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift @@ -1,6 +1,7 @@ import Foundation import os import TableProFirebaseCore +import TableProGoogleCloud import TableProLogRedaction import TableProPluginKit @@ -51,7 +52,7 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send default: Int(HttpQueryTimeout().requestTimeoutInterval * 1_000) ) let deadline = PluginConnectDeadline(milliseconds: connectMilliseconds) - let transport = URLSessionFirebaseTransport(resourceTimeout: HttpQueryTimeout.sessionResourceTimeout) + let transport = URLSessionGoogleRESTTransport.firebase(resourceTimeout: HttpQueryTimeout.sessionResourceTimeout) let timeoutBox = queryTimeout let timeout: @Sendable () -> TimeInterval = { timeoutBox.requestTimeoutInterval } let connectTimeout: @Sendable () -> TimeInterval = { deadline.remainingSeconds() } @@ -74,8 +75,8 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send /// The probe runs under the connect deadline and the client after it under the query timeout. private func connectFirestore( settings: FirebaseConnectionSettings, - transport: URLSessionFirebaseTransport, - credential: FirebaseCredential, + transport: URLSessionGoogleRESTTransport, + credential: GoogleRESTCredential, connectTimeout: @escaping @Sendable () -> TimeInterval, timeout: @escaping @Sendable () -> TimeInterval ) async throws -> FirebaseSession { @@ -93,8 +94,8 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send private func connectRealtimeDatabase( settings: FirebaseConnectionSettings, - transport: URLSessionFirebaseTransport, - credential: FirebaseCredential, + transport: URLSessionGoogleRESTTransport, + credential: GoogleRESTCredential, connectTimeout: @escaping @Sendable () -> TimeInterval, timeout: @escaping @Sendable () -> TimeInterval ) async throws -> FirebaseSession { @@ -298,8 +299,8 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send struct FirebaseSession: Sendable { let settings: FirebaseConnectionSettings - let transport: URLSessionFirebaseTransport - let credential: FirebaseCredential + let transport: URLSessionGoogleRESTTransport + let credential: GoogleRESTCredential let timeout: @Sendable () -> TimeInterval let firestore: FirestoreClient? let database: RealtimeDatabaseClient? diff --git a/TablePro/Assets.xcassets/bigtable-icon.imageset/Contents.json b/TablePro/Assets.xcassets/bigtable-icon.imageset/Contents.json new file mode 100644 index 0000000000..85f3b0a3bf --- /dev/null +++ b/TablePro/Assets.xcassets/bigtable-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "bigtable.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/TablePro/Assets.xcassets/bigtable-icon.imageset/bigtable.svg b/TablePro/Assets.xcassets/bigtable-icon.imageset/bigtable.svg new file mode 100644 index 0000000000..1367fc5d70 --- /dev/null +++ b/TablePro/Assets.xcassets/bigtable-icon.imageset/bigtable.svg @@ -0,0 +1 @@ +Google Bigtable \ No newline at end of file diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift new file mode 100644 index 0000000000..3e10fedf55 --- /dev/null +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift @@ -0,0 +1,126 @@ +import Foundation +import TableProPluginKit + +extension PluginMetadataRegistry { + /// Rows come only in key order and Bigtable keeps no count, so a column sort is the grid's own + /// and a count reads every key, only when asked. `_key` stays writable because a new row has + /// no other way to get one; the driver refuses a change to an existing row's key. + func bigtablePluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { + [ + ("Bigtable", PluginMetadataSnapshot( + displayName: "Google Cloud Bigtable", iconName: "bigtable-icon", defaultPort: 0, + requiresAuthentication: true, supportsForeignKeys: false, supportsSchemaEditing: false, + isDownloadable: true, primaryUrlScheme: "", parameterStyle: .questionMark, + navigationModel: .standard, explainVariants: [], pathFieldRole: .database, + supportsHealthMonitor: true, urlSchemes: [], postConnectActions: [], + brandColorHex: "#4285F4", + queryLanguageName: "REST", editorLanguage: .javascript, + connectionMode: .apiOnly, supportsDatabaseSwitching: true, + capabilities: PluginMetadataSnapshot.CapabilityFlags( + supportsSchemaSwitching: false, + supportsImport: false, + supportsExport: true, + supportsSSH: false, + supportsSSL: false, + supportsCascadeDrop: false, + supportsForeignKeyDisable: false, + supportsReadOnlyMode: true, + supportsQueryProgress: false, + requiresReconnectForDatabaseSwitch: false, + supportsDropDatabase: false, + supportsFieldRemoval: true, + supportsAddColumn: false, + supportsModifyColumn: false, + supportsDropColumn: false, + supportsAddIndex: false, + supportsDropIndex: false, + supportsModifyPrimaryKey: false, + supportsOpportunisticTLS: false, + exactRowCountIsFullScan: true, + supportsColumnSort: false, + columnsAreSampled: true, + supportsPrincipalConnectionLimit: false + ), + schema: PluginMetadataSnapshot.SchemaInfo( + defaultSchemaName: "", + defaultGroupName: "default", + tableEntityName: "Tables", + containerEntityName: "Instance", + defaultPrimaryKeyColumn: "_key", + immutableColumns: [], + systemDatabaseNames: [], + systemSchemaNames: [], + fileExtensions: [], + databaseGroupingStrategy: .byDatabase, + structureColumnFields: [.name, .type, .nullable] + ), + editor: PluginMetadataSnapshot.EditorConfig( + sqlDialect: nil, + statementCompletions: bigtableCompletions, + columnTypesByCategory: ["Binary": ["bytes"]] + ), + connection: PluginMetadataSnapshot.ConnectionConfig( + additionalConnectionFields: bigtableConnectionFields(), + category: .wideColumn, + tagline: String(localized: "Google Cloud wide-column store for large data"), + hidesBuiltInPassword: true, + hidesBuiltInDatabase: true + ) + )) + ] + } +} + +private let bigtableCompletions: [CompletionEntry] = [ + CompletionEntry(label: "GET row", insertText: "GET events/user#42"), + CompletionEntry(label: "SCAN prefix", insertText: "SCAN events/user#?limit=20"), + CompletionEntry(label: "READ", insertText: "READ events {\"rowsLimit\": \"20\", \"filter\": {\"cellsPerColumnLimitFilter\": 1}}"), + CompletionEntry(label: "INSERT row", insertText: "INSERT events/user#42 {\"profile:name\": \"Ann\"}"), + CompletionEntry(label: "PUT cells", insertText: "PUT events/user#42 {\"profile:name\": \"Ann\"}"), + CompletionEntry(label: "DELETE row", insertText: "DELETE events/user#42"), + CompletionEntry(label: "DELETE column", insertText: "DELETE events/user#42 profile:name") +] + +func bigtableConnectionFields() -> [ConnectionField] { + let authMethod = "btAuthMethod" + return [ + ConnectionField( + id: authMethod, + label: String(localized: "Auth Method"), + defaultValue: "serviceAccount", + fieldType: .dropdown(options: [ + .init(value: "serviceAccount", label: "Service Account Key"), + .init(value: "adc", label: "Application Default Credentials") + ]), + section: .authentication + ), + ConnectionField( + id: "btServiceAccountJson", + label: String(localized: "Service Account Key"), + placeholder: "File path or paste JSON", + required: true, + fieldType: .secure, + section: .authentication, + hidesPassword: true, + visibleWhen: FieldVisibilityRule(fieldId: authMethod, values: ["serviceAccount"]) + ).withHidesUsername(true), + ConnectionField( + id: "btProjectId", + label: String(localized: "Project ID"), + placeholder: "Taken from the key when empty", + section: .authentication + ), + ConnectionField( + id: "btInstanceId", + label: String(localized: "Instance ID"), + placeholder: "The first instance when empty", + section: .authentication + ), + ConnectionField( + id: "btAppProfileId", + label: String(localized: "App Profile ID"), + placeholder: "default", + section: .advanced + ) + ] +} diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 05e19de268..b73471a73a 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -1149,7 +1149,7 @@ extension PluginMetadataRegistry { + duckdbPluginDefaults(dialect: duckdbDialect, columnTypes: duckdbColumnTypes) + cloudPluginDefaults() + elasticsearchPluginDefaults() + surrealDBPluginDefaults() + kafkaPluginDefaults() + typesensePluginDefaults() + r2SQLPluginDefaults() - + weaviatePluginDefaults() + hanaPluginDefaults() + firebasePluginDefaults() + + weaviatePluginDefaults() + hanaPluginDefaults() + firebasePluginDefaults() + bigtablePluginDefaults() } // swiftlint:enable function_body_length } diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/TablePro/Core/Plugins/PluginMetadataRegistry.swift index aa4c56ecd0..09a2952a0c 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -830,7 +830,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { return .document case "Redis": return .keyValue - case "Cassandra", "ScyllaDB": + case "Cassandra", "ScyllaDB", "Bigtable": return .wideColumn case "etcd", "Etcd": return .coordination @@ -870,6 +870,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { case "Typesense": return String(localized: "Typo-tolerant open-source search engine") case "Weaviate": return String(localized: "Open-source vector database") case "Firestore": return String(localized: "Serverless document database from Google") + case "Bigtable": return String(localized: "Google Cloud wide-column store for large data") case "Firebase Realtime Database": return String(localized: "JSON tree database that syncs in real time") default: return "" } diff --git a/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift b/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift index 081fca3892..81d0b4b02e 100644 --- a/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift +++ b/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift @@ -159,7 +159,7 @@ enum CatalogChangeClassifier { } private static let documentStoreTypes: Set = [ - .mongodb, .elasticsearch, .typesense, .weaviate, .etcd, .firestore, .firebaseRealtimeDatabase + .mongodb, .elasticsearch, .typesense, .weaviate, .etcd, .firestore, .firebaseRealtimeDatabase, .bigtable ] private static let transactionEndKeywords: Set = ["COMMIT", "END", "ROLLBACK", "ABORT"] diff --git a/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/TablePro/Core/Utilities/SQL/QueryClassifier.swift index 3d3fb0fccb..f44e6ab55d 100644 --- a/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -861,6 +861,8 @@ private extension QueryClassifier { return trimmed.isEmpty ? .safe : weaviateClassification(trimmed) case .firestore, .firebaseRealtimeDatabase: return trimmed.isEmpty ? .safe : firebaseClassification(trimmed, databaseType: databaseType) + case .bigtable: + return trimmed.isEmpty ? .safe : bigtableClassification(trimmed) default: return nil } @@ -1091,4 +1093,35 @@ private extension QueryClassifier { return false } } + + /// A PUT that sets a cell to null deletes that cell, so it ranks with DELETE. INSERT refuses a + /// key that already has a row, so it never overwrites one. + static func bigtableClassification(_ trimmed: String) -> QueryClassification { + if trimmed.hasPrefix("BIGTABLE_BROWSE:") { + return .safe + } + let write = QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + let destructive = QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) + switch trimmed.prefix(while: { !$0.isWhitespace }).uppercased() { + case "GET", "SCAN", "READ": + return .safe + case "DELETE": + return destructive + case "PUT": + return bigtableBodyDeletesCell(trimmed) ? destructive : write + default: + return write + } + } + + /// The body is the JSON object that runs to the end of the text. A quoted row key may itself + /// hold a brace, so each brace is tried until one starts an object that parses. + private static func bigtableBodyDeletesCell(_ trimmed: String) -> Bool { + for brace in trimmed.indices where trimmed[brace] == "{" { + guard let data = String(trimmed[brace...]).data(using: .utf8), + let cells = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { continue } + return cells.values.contains { $0 is NSNull } + } + return false + } } diff --git a/TablePro/Models/Connection/DatabaseType.swift b/TablePro/Models/Connection/DatabaseType.swift index 0cb79fe8fb..ce72173b7d 100644 --- a/TablePro/Models/Connection/DatabaseType.swift +++ b/TablePro/Models/Connection/DatabaseType.swift @@ -51,6 +51,7 @@ extension DatabaseType { static let teradata = DatabaseType(rawValue: "Teradata") static let trino = DatabaseType(rawValue: "Trino") static let weaviate = DatabaseType(rawValue: "Weaviate") + static let bigtable = DatabaseType(rawValue: "Bigtable") static let firestore = DatabaseType(rawValue: "Firestore") static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") static let kafka = DatabaseType(rawValue: "Kafka") diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 62261146b7..79f5e0814a 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -784,10 +784,22 @@ } } }, + "\"%@\" is not a Bigtable instance ID." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "\"%@\" is not a Bigtable table ID." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "\"%@\" is not a collection path." : { "extractionState" : "manual", "generatesSymbol" : false }, + "\"%@\" is not a column. Name it as family:qualifier." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "\"%@\" is not a DynamoDB action this editor runs" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -861,6 +873,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "\"%@\" is not a row limit. Use a whole number above 0." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "\"%@\" is not a URL. Enter an endpoint such as http://localhost:8000." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -924,6 +940,10 @@ }, "\"%@\" is not a valid theme identifier." : { + }, + "\"%@\" is not an app profile ID." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "\"%@\" is not an AWS region" : { "extractionState" : "manual", @@ -953,6 +973,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "\"%@\" is not valid base64." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "“%@” is still in use" : { "localizations" : { "fr" : { @@ -4236,6 +4260,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "%@ needs a row key." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ needs a table. Pass 'table'." : { "localizations" : { "fr" : { @@ -15397,6 +15425,10 @@ } } }, + "A key range cannot be ORed with a column filter. Use =, STARTS WITH or REGEX on _key, or AND." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A Kubernetes resource is required, such as service/postgres" : { "localizations" : { "fr" : { @@ -15569,6 +15601,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "A new row needs at least one cell. Bigtable does not store empty rows." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A new version downloads in the background and installs the next time you quit TablePro." : { "localizations" : { "fr" : { @@ -15841,6 +15877,14 @@ } } }, + "A quoted value is missing its closing quote." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "A quoted value is not a valid JSON string." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A quoted value was never closed." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -15853,6 +15897,10 @@ } } }, + "A raw Bigtable filter is a JSON RowFilter, such as {\"rowKeyRegexFilter\": \"\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A raw Firestore filter is a JSON Filter object, such as %@." : { "extractionState" : "manual", "generatesSymbol" : false @@ -15905,6 +15953,14 @@ } } }, + "A row key cannot change. Duplicate the row, give the copy a new _key, then delete this one." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "A row needs a key. Bigtable has no empty row key." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A row the filter hides holds %@ in column “%@”, which %@ cannot store. Clear the filter to find it, or choose Save As and pick UTF-8." : { "localizations" : { "fr" : { @@ -16772,6 +16828,10 @@ } } }, + "A write needs at least one cell. Bigtable does not store empty rows." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "about %@ row" : { "localizations" : { "fr" : { @@ -24928,6 +24988,10 @@ } } }, + "App Profile ID" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "app_reader" : { "localizations" : { "fr" : { @@ -30423,6 +30487,42 @@ } } }, + "Bigtable cannot filter with %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable committed a row it never started." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable ended the response in the middle of a row." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable has no default values. Delete the cell, or enter a value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable has no views." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable returned HTTP %d." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable sent a response that is not JSON." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable sent bytes that are not base64." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable started a new row before committing the last one." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Billing" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -69840,6 +69940,10 @@ } } }, + "Enter a _key for the new row. Bigtable has no generated keys." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Enter a collection name." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -69992,6 +70096,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Enter a request such as GET events/user#42, SCAN events/user#?limit=20, PUT events/user#42 {\"profile:name\": \"Ann\"} or DELETE events/user#42." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Enter a value for the primary key column %@." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -81184,10 +81292,6 @@ } } }, - "Firebase answered with something other than HTTP." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "Firebase has no views." : { "extractionState" : "manual", "generatesSymbol" : false @@ -84658,6 +84762,10 @@ } } }, + "Google refused the credential." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Google rejected the OAuth client ID or secret." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -92147,6 +92255,14 @@ } } }, + "INSERT needs a JSON object of cells, such as INSERT events/user#42 {\"profile:name\": \"Ann\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "INSERT only sets cells. Leave %@ out, or delete it with PUT." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Insert Row" : { "localizations" : { "fr" : { @@ -100607,6 +100723,10 @@ } } }, + "list instances" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "List Maintenance Operations" : { "localizations" : { "fr" : { @@ -101411,6 +101531,10 @@ } } }, + "list the tables of %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "List the tables the user opened most recently on a connection, newest first." : { "localizations" : { "fr" : { @@ -109918,6 +110042,10 @@ } } }, + "Name the table before the key, as in GET events/user#42." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Name this selection" : { "localizations" : { "fr" : { @@ -118870,6 +118998,10 @@ } } }, + "Not connected to Bigtable." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Not connected to database" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -135046,6 +135178,10 @@ } } }, + "Project %@ has no Bigtable instance." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Project ID" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -136050,6 +136186,10 @@ } } }, + "PUT needs a JSON object of cells, such as PUT events/user#42 {\"profile:name\": \"Ann\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Put the credential in the Token field, not in the address." : { "localizations" : { "fr" : { @@ -139195,6 +139335,10 @@ } } }, + "READ needs a ReadRows JSON body, such as READ events {\"rowsLimit\": \"10\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Read Only" : { "localizations" : { "fr" : { @@ -139488,6 +139632,10 @@ } } }, + "read the schema of %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Read-only" : { "extractionState" : "stale", "localizations" : { @@ -148753,6 +148901,10 @@ } } }, + "Row %@ already exists. Edit that row, or give the new one another key." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Row %@ is not an object." : { "localizations" : { "fr" : { @@ -153451,6 +153603,10 @@ } } }, + "SCAN takes only ?limit=, as in SCAN events/user#?limit=20." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Scan with filters" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -179692,6 +179848,10 @@ } } }, + "The Bigtable REST API returns rows in ascending key order only." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The binary subtype of %@ is not known, so saving could store a different kind of binary. Change this field with a query." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -179972,6 +180132,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The browse request is damaged. Reopen the table." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The bundled sample database is missing from the app." : { "localizations" : { "fr" : { @@ -185377,6 +185541,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The row has no _key, so the row it came from is unknown. Reload the table." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The row has no $key, so the node it came from is unknown. Reload the table." : { "extractionState" : "manual", "generatesSymbol" : false @@ -186054,6 +186222,10 @@ } } }, + "The server answered with something other than HTTP." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The server chose this row's key." : { "localizations" : { "fr" : { @@ -188848,6 +189020,10 @@ } } }, + "The value of %@ must be text or null. Bigtable stores bytes, so a number is written as its digits." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The value of a %@ key cannot be edited in the grid. Change it with a command in the query editor." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -209834,6 +210010,10 @@ } } }, + "write a row of %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Write and Run SQL" : { "localizations" : { "fr" : { @@ -209989,6 +210169,10 @@ } } }, + "Write the cells as a JSON object, such as {\"profile:name\": \"Ann\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Writes %lld files, one for each database." : { "localizations" : { "fr" : { diff --git a/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/Contents.json b/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/Contents.json new file mode 100644 index 0000000000..85f3b0a3bf --- /dev/null +++ b/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "bigtable.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/bigtable.svg b/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/bigtable.svg new file mode 100644 index 0000000000..1367fc5d70 --- /dev/null +++ b/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/bigtable.svg @@ -0,0 +1 @@ +Google Bigtable \ No newline at end of file diff --git a/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/Contents.json b/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/Contents.json new file mode 100644 index 0000000000..85f3b0a3bf --- /dev/null +++ b/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "bigtable.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/bigtable.svg b/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/bigtable.svg new file mode 100644 index 0000000000..1367fc5d70 --- /dev/null +++ b/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/bigtable.svg @@ -0,0 +1 @@ +Google Bigtable \ No newline at end of file diff --git a/TableProTests/Core/ChangeTracking/DataChangeManagerTests.swift b/TableProTests/Core/ChangeTracking/DataChangeManagerTests.swift index 5d0232bd08..94410b44dc 100644 --- a/TableProTests/Core/ChangeTracking/DataChangeManagerTests.swift +++ b/TableProTests/Core/ChangeTracking/DataChangeManagerTests.swift @@ -684,4 +684,18 @@ struct DataChangeManagerImmutableColumnTests { #expect(!manager.isColumnWritable("total")) } + + @Test("Bigtable's _key is writable, since a new row has no other way to get a key") + func bigtableRowKeyWritable() { + let manager = DataChangeManager() + manager.configureForTable( + tableName: "events", + columns: ["_key", "cf:name"], + primaryKeyColumns: ["_key"], + databaseType: .bigtable, + generatedColumns: [] + ) + + #expect(manager.isColumnWritable("_key")) + } } diff --git a/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift b/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift index e549d93fce..c5b464367b 100644 --- a/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift +++ b/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift @@ -11,7 +11,7 @@ import Testing /// this registry, `snippets/driver-counts.mdx` in TableProApp/docs, and the marketing site. Nothing at runtime /// reconciles them, and by August 2026 they read 28, 27 and 25 at once. /// -/// The answer is 39, and the reason it once read 28 is worth keeping. Turso is served by +/// The answer is 40, and the reason it once read 28 is worth keeping. Turso is served by /// the libSQL plugin and was the only alias in `reverseTypeIndex` with no curated entry of its /// own, so it was the only type the picker could not offer before its plugin was installed. /// ScyllaDB is the shape every other alias already had: an alias of Cassandra with a curated @@ -29,7 +29,7 @@ struct PluginMetadataRegistryTypeCountTests { "CockroachDB", "Dameng", "Databend", "DuckDB", "DynamoDB", "Elasticsearch", "etcd", "Kafka", "libSQL", "MariaDB", "MongoDB", "MySQL", "Oracle", "PGlite", "PostgreSQL", "Redis", "Redshift", "SAP HANA", "ScyllaDB", "Snowflake", "Spanner", "SQL Server", "SQLite", "SurrealDB", "Teradata", "TiDB", - "Trino", "Turso", "Typesense", "Weaviate", "OceanBase", "Firestore", "Firebase Realtime Database" + "Trino", "Turso", "Typesense", "Weaviate", "OceanBase", "Firestore", "Firebase Realtime Database", "Bigtable" ] private static func builtInTypeIds() -> Set { diff --git a/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift b/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift index ef1632788c..6d44d13d70 100644 --- a/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift +++ b/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift @@ -152,6 +152,13 @@ struct CatalogChangeClassifierTests { #expect(kinds("db.dropDatabase()", .mongodb) == .everything) } + @Test("A Bigtable write refreshes the columns, since it can add one") + func bigtableWritesRefresh() { + #expect(kinds("SCAN events/user#", .bigtable).isEmpty) + #expect(kinds("BIGTABLE_BROWSE:e30=", .bigtable).isEmpty) + #expect(kinds("PUT events/k {\"cf:new\": \"1\"}", .bigtable) == .everything) + } + @Test("A Firebase write refreshes the sidebar, since it can create a collection or a node") func firebaseWritesRefresh() { for type in [DatabaseType.firestore, .firebaseRealtimeDatabase] { diff --git a/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 3cb1445390..3a3b85a50d 100644 --- a/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -690,6 +690,20 @@ struct QueryClassifierNonSqlTests { #expect(!QueryClassifier.isWriteQuery(search, databaseType: .weaviate)) } + @Test("Bigtable request lines are tiered by their verb, and a PUT that nulls a cell is destructive") + func bigtableTiers() { + #expect(!QueryClassifier.isWriteQuery("BIGTABLE_BROWSE:e30=", databaseType: .bigtable)) + #expect(!QueryClassifier.isWriteQuery("GET events/user#42", databaseType: .bigtable)) + #expect(!QueryClassifier.isWriteQuery("scan events/user#?limit=5", databaseType: .bigtable)) + #expect(!QueryClassifier.isWriteQuery("READ events {\"rowsLimit\": \"5\"}", databaseType: .bigtable)) + #expect(QueryClassifier.classifyTier("INSERT events/k {\"cf:a\": \"1\"}", databaseType: .bigtable) == .write) + #expect(QueryClassifier.classifyTier("PUT events/k {\"cf:a\": \"1\"}", databaseType: .bigtable) == .write) + #expect(QueryClassifier.classifyTier("PUT events/k {\"cf:a\": null}", databaseType: .bigtable) == .destructive) + #expect(QueryClassifier.classifyTier("PUT events/\"a{b\" {\"cf:a\": null}", databaseType: .bigtable) == .destructive) + #expect(QueryClassifier.classifyTier("DELETE events/k", databaseType: .bigtable) == .destructive) + #expect(QueryClassifier.isWriteQuery("SELECT * FROM events", databaseType: .bigtable)) + } + @Test("Firebase request lines are tiered by their verb, and a grid page is a read") func firebaseTiers() { for type in [DatabaseType.firestore, .firebaseRealtimeDatabase] { diff --git a/TableProTests/Models/Database/SQLDDLFallbackPolicyTests.swift b/TableProTests/Models/Database/SQLDDLFallbackPolicyTests.swift index cfb685d6ec..1228d0f36d 100644 --- a/TableProTests/Models/Database/SQLDDLFallbackPolicyTests.swift +++ b/TableProTests/Models/Database/SQLDDLFallbackPolicyTests.swift @@ -24,7 +24,7 @@ struct SQLDDLFallbackPolicyTests { func nonSQLEnginesRefuse() { for type in [ DatabaseType.elasticsearch, .kafka, .weaviate, .etcd, - .redis, .mongodb, .typesense, .surrealdb, .dynamodb, .beancount, + .redis, .mongodb, .typesense, .surrealdb, .dynamodb, .beancount, .bigtable, ] { #expect(!SQLDDLFallbackPolicy.allowsGeneratedDDL(for: type), "\(type.rawValue) should refuse") } diff --git a/project.yml b/project.yml index 9a03c82530..b773fd6e30 100644 --- a/project.yml +++ b/project.yml @@ -1168,6 +1168,19 @@ targets: - package: TableProCore product: TableProDocumentPath + BigtableDriverPlugin: + templates: [DriverPlugin] + templateAttributes: + folder: BigtableDriverPlugin + principalClass: BigtablePlugin + dependencies: + - package: TableProCore + product: TableProGoogleCloud + - package: TableProCore + product: TableProBigtableCore + - package: TableProCore + product: TableProLogRedaction + FirebaseDriverPlugin: templates: [DriverPlugin] templateAttributes: @@ -1197,6 +1210,7 @@ aggregateTargets: targets: - BeancountDriver - BigQueryDriverPlugin + - BigtableDriverPlugin - CSVExport - CSVImport - CassandraDriver