From a63efd6cbf8d6f4d19d2e00cc0716a273b5821ed Mon Sep 17 00:00:00 2001 From: "alessio.vigilante" Date: Wed, 14 Oct 2026 14:19:25 +0200 Subject: [PATCH 1/3] feat(plugin-bigtable): add Google Cloud Bigtable driver Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/plugin-registry.json | 14 + CHANGELOG.md | 1 + Packages/TableProCore/Package.swift | 11 + .../Config/BigtableConnectionSettings.swift | 65 ++++ .../Data/BigtableRow.swift | 153 ++++++++ .../Data/BigtableRowSet.swift | 50 +++ .../Query/BigtableReadRequest.swift | 347 ++++++++++++++++++ .../Wire/BigtableBytes.swift | 75 ++++ .../Wire/BigtableClient.swift | 153 ++++++++ .../Wire/BigtableError.swift | 63 ++++ .../Wire/BigtableStatement.swift | 238 ++++++++++++ .../Wire/BigtableTransport.swift | 184 ++++++++++ .../TableProCoreTypes/DatabaseType.swift | 4 +- .../BigtableCoreTests.swift | 234 ++++++++++++ .../DatabaseTypeTests.swift | 10 +- .../BigtableCredentialFactory.swift | 51 +++ .../BigtableDriverPlugin/BigtablePlugin.swift | 109 ++++++ .../BigtablePluginDriver+Editing.swift | 116 ++++++ .../BigtablePluginDriver+Execution.swift | 220 +++++++++++ .../BigtablePluginDriver+Metadata.swift | 105 ++++++ .../BigtablePluginDriver.swift | 192 ++++++++++ Plugins/BigtableDriverPlugin/Info.plist | 14 + .../bigtable-icon.imageset/Contents.json | 16 + .../bigtable-icon.imageset/bigtable.svg | 1 + ...ginMetadataRegistry+BigtableDefaults.swift | 124 +++++++ ...ginMetadataRegistry+RegistryDefaults.swift | 2 +- .../Core/Plugins/PluginMetadataRegistry.swift | 3 +- .../SQL/CatalogChangeClassifier.swift | 2 +- .../Core/Utilities/SQL/QueryClassifier.swift | 33 ++ TablePro/Models/Connection/DatabaseType.swift | 1 + TablePro/Resources/Localizable.xcstrings | 156 ++++++++ .../bigtable-icon.imageset/Contents.json | 16 + .../bigtable-icon.imageset/bigtable.svg | 1 + .../bigtable-icon.imageset/Contents.json | 16 + .../bigtable-icon.imageset/bigtable.svg | 1 + ...PluginMetadataRegistryTypeCountTests.swift | 4 +- .../SQL/CatalogChangeClassifierTests.swift | 7 + .../SQL/QueryClassifierHardeningTests.swift | 13 + project.yml | 14 + 39 files changed, 2812 insertions(+), 7 deletions(-) create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableTransport.swift create mode 100644 Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableCoreTests.swift create mode 100644 Plugins/BigtableDriverPlugin/BigtableCredentialFactory.swift create mode 100644 Plugins/BigtableDriverPlugin/BigtablePlugin.swift create mode 100644 Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift create mode 100644 Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift create mode 100644 Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift create mode 100644 Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift create mode 100644 Plugins/BigtableDriverPlugin/Info.plist create mode 100644 TablePro/Assets.xcassets/bigtable-icon.imageset/Contents.json create mode 100644 TablePro/Assets.xcassets/bigtable-icon.imageset/bigtable.svg create mode 100644 TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift create mode 100644 TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/Contents.json create mode 100644 TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/bigtable.svg create mode 100644 TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/Contents.json create mode 100644 TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/bigtable.svg diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index 717beaebcb..78d112ef4e 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -46,6 +46,20 @@ "category": "database-driver", "homepage": "https://docs.tablepro.app/databases/bigquery" }, + "bigtable": { + "target": "BigtableDriverPlugin", + "bundleName": "BigtableDriverPlugin", + "bundleId": "com.TablePro.BigtableDriverPlugin", + "bundled": false, + "displayName": "Bigtable Driver", + "summary": "Google Cloud Bigtable driver over the REST API", + "databaseTypeIds": [ + "Bigtable" + ], + "icon": "bigtable-icon", + "category": "database-driver", + "homepage": "https://docs.tablepro.app/databases/bigtable" + }, "cassandra": { "target": "CassandraDriver", "bundleName": "CassandraDriver", diff --git a/CHANGELOG.md b/CHANGELOG.md index bde337d5a6..6b8f210eb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Setting in Settings > General to hide the version and Check for Updates on the Welcome window. - Install Parquet… in the Export dialog's format menu when the Parquet plugin is not installed. - Cloud Firestore and Firebase Realtime Database as a registry plugin over their REST APIs. (#2504) +- Google Cloud Bigtable as a registry plugin over the REST API. - Warning banner and Test Connection notice for a server warning at login, such as an Oracle password about to expire. (#3401) ### Changed diff --git a/Packages/TableProCore/Package.swift b/Packages/TableProCore/Package.swift index 1431520c03..370379cc14 100644 --- a/Packages/TableProCore/Package.swift +++ b/Packages/TableProCore/Package.swift @@ -28,6 +28,7 @@ let package = Package( .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]), .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), .library(name: "TableProFirebaseCore", targets: ["TableProFirebaseCore"]), + .library(name: "TableProBigtableCore", targets: ["TableProBigtableCore"]), .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), .library(name: "TableProDocumentPath", targets: ["TableProDocumentPath"]), .library(name: "TableProJavaScriptText", targets: ["TableProJavaScriptText"]), @@ -152,6 +153,11 @@ let package = Package( dependencies: [], path: "Sources/TableProWeaviateCore" ), + .target( + name: "TableProBigtableCore", + dependencies: [], + path: "Sources/TableProBigtableCore" + ), .target( name: "TableProFirebaseCore", dependencies: [], @@ -197,6 +203,11 @@ let package = Package( dependencies: ["CSQLite"], path: "Sources/TableProSQLiteCore" ), + .testTarget( + name: "TableProBigtableCoreTests", + dependencies: ["TableProBigtableCore"], + path: "Tests/TableProBigtableCoreTests" + ), .testTarget( name: "TableProTabularIOTests", dependencies: ["TableProTabularIO", "TableProPluginKit"], diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift new file mode 100644 index 0000000000..001f75d5bc --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift @@ -0,0 +1,65 @@ +import Foundation + +public enum BigtableFieldID { + public static let authMethod = "btAuthMethod" + public static let serviceAccountJson = "btServiceAccountJson" + public static let projectId = "btProjectId" + public static let instanceId = "btInstanceId" + public static let appProfileId = "btAppProfileId" +} + +public enum BigtableAuthMethod: String, Sendable, CaseIterable { + case serviceAccount + case applicationDefault = "adc" +} + +public struct BigtableConnectionSettings: Sendable, Equatable { + public let authMethod: BigtableAuthMethod + public let projectId: String + public let instanceId: String? + public let appProfileId: String? + + public init(authMethod: BigtableAuthMethod, projectId: String, instanceId: String?, appProfileId: String? = nil) { + self.authMethod = authMethod + self.projectId = projectId + self.instanceId = instanceId + self.appProfileId = appProfileId + } + + /// `credentialProject` is the project the key or Application Default Credentials name, used + /// when the Project ID field is empty. With no Instance ID the driver opens the project's only + /// instance, or the first one, and lists the rest in the database switcher. + public static func parse(fields: [String: String], credentialProject: String? = nil) throws -> BigtableConnectionSettings { + let authMethod = trimmed(fields[BigtableFieldID.authMethod]).flatMap(BigtableAuthMethod.init(rawValue:)) ?? .serviceAccount + guard let projectId = trimmed(fields[BigtableFieldID.projectId]) ?? credentialProject else { + throw BigtableError.configuration(String(localized: "Enter a Project ID, or use a service account key that names one.")) + } + guard isValidProjectId(projectId) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Google Cloud project ID."), projectId)) + } + let instanceId = trimmed(fields[BigtableFieldID.instanceId]) + if let instanceId, !isValidResourceId(instanceId) { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Bigtable instance ID."), instanceId)) + } + let appProfileId = trimmed(fields[BigtableFieldID.appProfileId]) + if let appProfileId, !isValidResourceId(appProfileId) { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not an app profile ID."), appProfileId)) + } + return BigtableConnectionSettings(authMethod: authMethod, projectId: projectId, instanceId: instanceId, appProfileId: appProfileId) + } + + static func isValidProjectId(_ text: String) -> Bool { + text.range(of: #"^[a-z][a-z0-9.:-]{2,62}$"#, options: .regularExpression) != nil + } + + /// Instance, table and app profile IDs all go into the request path, so anything outside the + /// characters Bigtable allows is refused before it can change the path. + static func isValidResourceId(_ text: String) -> Bool { + text.range(of: #"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,49}$"#, options: .regularExpression) != nil + } + + static func trimmed(_ value: String?) -> String? { + guard let value = value?.trimmingCharacters(in: .whitespacesAndNewlines), !value.isEmpty else { return nil } + return value + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift new file mode 100644 index 0000000000..113f431376 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift @@ -0,0 +1,153 @@ +import Foundation + +public struct BigtableColumn: Hashable, Sendable, Comparable { + public let family: String + public let qualifier: Data + + public init(family: String, qualifier: Data) { + self.family = family + self.qualifier = qualifier + } + + /// `family:qualifier`, the way `cbt` and the console name a column. A family name cannot hold a + /// colon, so the first one always splits the two. + public var name: String { + family + ":" + BigtableBytes.display(qualifier) + } + + public init(name: String) throws { + guard let colon = name.firstIndex(of: ":"), colon != name.startIndex else { + throw BigtableError.configuration(String( + format: String(localized: "\"%@\" is not a column. Name it as family:qualifier."), name + )) + } + self.family = String(name[.. Bool { + lhs.family == rhs.family ? lhs.qualifier.lexicographicallyPrecedes(rhs.qualifier) : lhs.family < rhs.family + } +} + +public struct BigtableCell: Equatable, Sendable { + public let value: Data + public let timestampMicros: Int64 + + public init(value: Data, timestampMicros: Int64) { + self.value = value + self.timestampMicros = timestampMicros + } +} + +/// A row with the newest version of each cell. Older versions are filtered out on the server. +public struct BigtableRow: Equatable, Sendable { + public let key: Data + public let cells: [BigtableColumn: BigtableCell] + + public init(key: Data, cells: [BigtableColumn: BigtableCell]) { + self.key = key + self.cells = cells + } +} + +/// Rebuilds rows from the `chunks` of a `ReadRows` response. Family and qualifier carry over from +/// the chunk before when a chunk leaves them out, a value larger than one chunk arrives in pieces +/// marked by `valueSize`, and `resetRow` throws away the row in progress. +public struct BigtableChunkAssembler { + private var rows: [BigtableRow] = [] + private var key: Data? + private var cells: [BigtableColumn: BigtableCell] = [:] + private var family: String? + private var qualifier: Data? + private var timestamp: Int64 = 0 + private var value = Data() + private var inCell = false + + public init() {} + + public static func rows(fromResponse json: Any) throws -> [BigtableRow] { + let messages = json as? [[String: Any]] ?? (json as? [String: Any]).map { [$0] } ?? [] + var assembler = BigtableChunkAssembler() + for message in messages { + for chunk in message["chunks"] as? [[String: Any]] ?? [] { + try assembler.add(chunk) + } + } + return try assembler.finish() + } + + public mutating func add(_ chunk: [String: Any]) throws { + if chunk["resetRow"] as? Bool == true { + resetRow() + return + } + if let encodedKey = chunk["rowKey"] as? String { + let newKey = try BigtableBytes.decodeBase64(encodedKey) + if let key, key != newKey, !cells.isEmpty || inCell { + throw BigtableError.malformedResponse(String(localized: "Bigtable started a new row before committing the last one.")) + } + key = newKey + } + if let name = Self.string(chunk["familyName"]) { + family = name + qualifier = nil + } + if let encoded = Self.string(chunk["qualifier"]) { + qualifier = try BigtableBytes.decodeBase64(encoded) + } + if !inCell { + timestamp = Self.int64(chunk["timestampMicros"]) ?? 0 + value = Data() + } + if let encoded = chunk["value"] as? String { + value.append(try BigtableBytes.decodeBase64(encoded)) + } + let hasCell = family != nil && qualifier != nil + inCell = (Self.int64(chunk["valueSize"]) ?? 0) > 0 + if hasCell, !inCell, let family, let qualifier { + let column = BigtableColumn(family: family, qualifier: qualifier) + if cells[column].map({ $0.timestampMicros < timestamp }) ?? true { + cells[column] = BigtableCell(value: value, timestampMicros: timestamp) + } + } + if chunk["commitRow"] as? Bool == true { + guard let key, !inCell else { + throw BigtableError.malformedResponse(String(localized: "Bigtable committed a row it never started.")) + } + rows.append(BigtableRow(key: key, cells: cells)) + resetRow() + } + } + + public func finish() throws -> [BigtableRow] { + guard !inCell, cells.isEmpty else { + throw BigtableError.malformedResponse(String(localized: "Bigtable ended the response in the middle of a row.")) + } + return rows + } + + private mutating func resetRow() { + key = nil + cells = [:] + family = nil + qualifier = nil + value = Data() + inCell = false + } + + /// `familyName` and `qualifier` are wrapper messages, which JSON writes as the bare value but a + /// proxy may still write as `{"value": ...}`. + private static func string(_ any: Any?) -> String? { + if let text = any as? String { return text } + return (any as? [String: Any])?["value"] as? String + } + + private static func int64(_ any: Any?) -> Int64? { + switch any { + case let text as String: return Int64(text) + case let number as NSNumber: return number.int64Value + default: return nil + } + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift new file mode 100644 index 0000000000..57eb156d19 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift @@ -0,0 +1,50 @@ +import Foundation + +/// Rows laid out for the grid: `_key` first, then one column per `family:qualifier` any row on the +/// page holds, in family then qualifier order. A cell a row does not have is reported absent. +public struct BigtableRowSet: Sendable, Equatable { + public static let keyColumn = "_key" + + public enum Cell: Sendable, Equatable { + case absent + case text(String) + case bytes(Data) + } + + public let columns: [String] + public let rows: [[Cell]] + public let locators: [String] + public let absentCells: [Int: Set] + + public init(rows input: [BigtableRow]) { + let ordered = Set(input.flatMap { $0.cells.keys }).sorted() + var rows: [[Cell]] = [] + var absent: [Int: Set] = [:] + rows.reserveCapacity(input.count) + for (rowIndex, row) in input.enumerated() { + var cells: [Cell] = [.text(BigtableBytes.display(row.key))] + for (offset, column) in ordered.enumerated() { + guard let cell = row.cells[column] else { + cells.append(.absent) + absent[rowIndex, default: []].insert(offset + 1) + continue + } + cells.append(Self.cell(cell.value)) + } + rows.append(cells) + } + self.columns = [Self.keyColumn] + ordered.map(\.name) + self.rows = rows + self.locators = input.map { $0.key.base64EncodedString() } + self.absentCells = absent + } + + /// A value that is not text goes to the grid as bytes, so it shows as hex and an edit of + /// another cell in the row cannot rewrite it as a string. + static func cell(_ value: Data) -> Cell { + if let text = String(data: value, encoding: .utf8), BigtableBytes.isPrintable(text) { + return .text(text) + } + return .bytes(value) + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift new file mode 100644 index 0000000000..9937a447e0 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift @@ -0,0 +1,347 @@ +import Foundation + +public struct BigtableFilterSpec: Codable, Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + + public init(column: String, op: String, value: String, secondValue: String? = nil) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + } +} + +/// A page of one table as the grid asked for it. `after` is the last key of the page before, which +/// is how a page starts: Bigtable reads rows in key order and has no offset. +public struct BigtableBrowseSpec: Codable, Sendable, Equatable { + public let table: String + public let filters: [BigtableFilterSpec] + public let logicMode: String + public let descending: Bool + /// The column the grid sorted by, kept so a sort Bigtable cannot do is refused rather than + /// silently ignored. Rows only come in key order. + public let sortColumn: String? + public let limit: Int + public let offset: Int + + public init( + table: String, + filters: [BigtableFilterSpec] = [], + logicMode: String = "AND", + descending: Bool = false, + sortColumn: String? = nil, + limit: Int, + offset: Int + ) { + self.table = table + self.filters = filters + self.logicMode = logicMode + self.descending = descending + self.sortColumn = sortColumn + self.limit = limit + self.offset = offset + } + + public var usesOr: Bool { + logicMode.uppercased() == "OR" + } + + /// The spec with the page left out, so pages of the same view share one cursor cache. + public var viewKey: String { + let filterText = filters.map { "\($0.column)\u{1}\($0.op)\u{1}\($0.value)\u{1}\($0.secondValue ?? "")" }.joined(separator: "\u{2}") + return [table, logicMode.uppercased(), descending ? "desc" : "asc", filterText].joined(separator: "\u{3}") + } +} + +/// The bounds a row key may take: an interval, or a list of exact keys. +struct BigtableKeyBounds: Equatable { + var start: (key: Data, closed: Bool)? + var end: (key: Data, closed: Bool)? + var exact: [Data]? + + static func == (lhs: BigtableKeyBounds, rhs: BigtableKeyBounds) -> Bool { + lhs.start?.key == rhs.start?.key && lhs.start?.closed == rhs.start?.closed + && lhs.end?.key == rhs.end?.key && lhs.end?.closed == rhs.end?.closed && lhs.exact == rhs.exact + } + + mutating func raiseStart(_ key: Data, closed: Bool) { + guard let current = start else { start = (key, closed); return } + if current.key.lexicographicallyPrecedes(key) || (current.key == key && !closed) { + start = (key, closed && current.closed) + } + } + + mutating func lowerEnd(_ key: Data, closed: Bool) { + guard let current = end else { end = (key, closed); return } + if key.lexicographicallyPrecedes(current.key) || (current.key == key && !closed) { + end = (key, closed && current.closed) + } + } + + func admits(_ key: Data) -> Bool { + if let start { + if key.lexicographicallyPrecedes(start.key) || (key == start.key && !start.closed) { return false } + } + if let end { + if end.key.lexicographicallyPrecedes(key) || (key == end.key && !end.closed) { return false } + } + return true + } + + func json() -> [String: Any] { + if let exact { + return ["rowKeys": exact.filter(admits).map { $0.base64EncodedString() }] + } + var range: [String: Any] = [:] + if let start { range[start.closed ? "startKeyClosed" : "startKeyOpen"] = start.key.base64EncodedString() } + if let end { range[end.closed ? "endKeyClosed" : "endKeyOpen"] = end.key.base64EncodedString() } + return range.isEmpty ? [:] : ["rowRanges": [range]] + } +} + +public enum BigtableReadRequest { + public static let rawFilterColumn = "__RAW__" + + /// `after` continues a page past that key. `keysOnly` reads keys + /// without values, which is how a page with no cursor yet skips the rows before it cheaply. + public static func body(spec: BigtableBrowseSpec, limit: Int, after: Data?, keysOnly: Bool = false) throws -> [String: Any] { + if (spec.sortColumn.map { $0 != BigtableRowSet.keyColumn } ?? false) || spec.descending { + throw BigtableError.unsupported(String(localized: "The Bigtable REST API returns rows in ascending key order only.")) + } + var bounds = BigtableKeyBounds() + var rowFilters: [[String: Any]] = [] + var unions: [[String: Any]]? + + let keyFilters = spec.filters.filter { $0.column == BigtableRowSet.keyColumn && isRangeOperator($0.op) } + if spec.usesOr, keyFilters.count == spec.filters.count, !keyFilters.isEmpty { + unions = try keyFilters.map { filter in + var single = BigtableKeyBounds() + try applyKeyRange(filter, to: &single) + return single.json() + } + } else { + for filter in spec.filters { + if filter.column == BigtableRowSet.keyColumn, isRangeOperator(filter.op), !spec.usesOr { + try applyKeyRange(filter, to: &bounds) + } else { + rowFilters.append(try rowFilter(filter, insideOr: spec.usesOr)) + } + } + } + if let after { + bounds.raiseStart(after, closed: false) + } + + var body: [String: Any] = [:] + if let unions { + body["rows"] = mergeUnions(unions, within: bounds) + } else { + let rows = bounds.json() + if !rows.isEmpty { body["rows"] = rows } + if let exact = rows["rowKeys"] as? [String], exact.isEmpty { + body["rows"] = ["rowKeys": [String]()] + } + } + var filters: [[String: Any]] = [] + switch rowFilters.count { + case 0: break + case 1: filters.append(rowFilters[0]) + default: filters.append([spec.usesOr ? "interleave" : "chain": ["filters": rowFilters]]) + } + if keysOnly { + filters += [["cellsPerRowLimitFilter": 1], ["stripValueTransformer": true]] + } else { + filters.append(["cellsPerColumnLimitFilter": 1]) + } + body["filter"] = filters.count == 1 ? filters[0] : ["chain": ["filters": filters]] + if limit > 0 { body["rowsLimit"] = String(limit) } + return body + } + + /// A list of exact keys or ranges asked for with no matching row is an empty `rowKeys`, which + /// the API refuses, so the read is skipped by the caller when it would ask for nothing. + public static func asksForNothing(_ body: [String: Any]) -> Bool { + guard let rows = body["rows"] as? [String: Any] else { return false } + let keys = rows["rowKeys"] as? [Any] ?? [] + let ranges = rows["rowRanges"] as? [Any] ?? [] + return keys.isEmpty && ranges.isEmpty + } + + private static func mergeUnions(_ unions: [[String: Any]], within bounds: BigtableKeyBounds) -> [String: Any] { + var keys: [String] = [] + var ranges: [[String: Any]] = [] + for union in unions { + keys += union["rowKeys"] as? [String] ?? [] + ranges += union["rowRanges"] as? [[String: Any]] ?? [] + } + var rows: [String: Any] = [:] + let admitted = keys.compactMap { Data(base64Encoded: $0) }.filter(bounds.admits).map { $0.base64EncodedString() } + if !admitted.isEmpty { rows["rowKeys"] = admitted } + if !ranges.isEmpty { + rows["rowRanges"] = ranges.map { range in clamp(range, to: bounds) } + } + return rows + } + + /// A cursor cuts every range of an OR at the key the last page ended on. + private static func clamp(_ range: [String: Any], to bounds: BigtableKeyBounds) -> [String: Any] { + var single = BigtableKeyBounds() + if let key = (range["startKeyClosed"] as? String).flatMap({ Data(base64Encoded: $0) }) { single.start = (key, true) } + if let key = (range["startKeyOpen"] as? String).flatMap({ Data(base64Encoded: $0) }) { single.start = (key, false) } + if let key = (range["endKeyClosed"] as? String).flatMap({ Data(base64Encoded: $0) }) { single.end = (key, true) } + if let key = (range["endKeyOpen"] as? String).flatMap({ Data(base64Encoded: $0) }) { single.end = (key, false) } + if let start = bounds.start { single.raiseStart(start.key, closed: start.closed) } + if let end = bounds.end { single.lowerEnd(end.key, closed: end.closed) } + return (single.json()["rowRanges"] as? [[String: Any]])?.first ?? [:] + } + + static func isRangeOperator(_ op: String) -> Bool { + ["=", "IN", "STARTS WITH", ">", ">=", "<", "<=", "BETWEEN"].contains(op.uppercased()) + } + + static func applyKeyRange(_ filter: BigtableFilterSpec, to bounds: inout BigtableKeyBounds) throws { + let key = try BigtableBytes.parse(filter.value) + switch filter.op.uppercased() { + case "=": + bounds.exact = (bounds.exact ?? [key]).filter { $0 == key } + case "IN": + let keys = try listItems(filter.value).map(BigtableBytes.parse) + bounds.exact = bounds.exact.map { existing in existing.filter(keys.contains) } ?? keys + case "STARTS WITH": + bounds.raiseStart(key, closed: true) + if let end = BigtableBytes.prefixEnd(key) { bounds.lowerEnd(end, closed: false) } + case ">": bounds.raiseStart(key, closed: false) + case ">=": bounds.raiseStart(key, closed: true) + case "<": bounds.lowerEnd(key, closed: false) + case "<=": bounds.lowerEnd(key, closed: true) + case "BETWEEN": + guard let upper = filter.secondValue else { + throw BigtableError.configuration(String(localized: "BETWEEN needs two values.")) + } + bounds.raiseStart(key, closed: true) + bounds.lowerEnd(try BigtableBytes.parse(upper), closed: true) + default: + throw unsupported(filter.op) + } + } + + // MARK: - Row filters + + static func rowFilter(_ filter: BigtableFilterSpec, insideOr: Bool) throws -> [String: Any] { + if filter.column == rawFilterColumn { + return try rawFilter(filter.value) + } + if filter.column == BigtableRowSet.keyColumn { + return try keyRegexFilter(filter, insideOr: insideOr) + } + let column = try BigtableColumn(name: filter.column) + let base: [[String: Any]] = [ + ["familyNameRegexFilter": "^" + BigtableBytes.regexText(Data(column.family.utf8)) + "$"], + ["columnQualifierRegexFilter": anchored(column.qualifier).base64EncodedString()], + ["cellsPerColumnLimitFilter": 1] + ] + let value = try BigtableBytes.parse(filter.value) + func predicate(_ extra: [String: Any]?) -> [String: Any] { + ["chain": ["filters": base + (extra.map { [$0] } ?? [])]] + } + func regex(_ pattern: Data) -> [String: Any] { + ["valueRegexFilter": pattern.base64EncodedString()] + } + let literal = BigtableBytes.regexLiteral(value) + let anyText = Data("(?s).*".utf8) + switch filter.op.uppercased() { + case "=": return keep(predicate(regex(Data("(?s)^".utf8) + literal + Data("$".utf8)))) + case "!=", "<>": return drop(predicate(regex(Data("(?s)^".utf8) + literal + Data("$".utf8)))) + case "CONTAINS": return keep(predicate(regex(anyText + literal + anyText))) + case "NOT CONTAINS": return drop(predicate(regex(anyText + literal + anyText))) + case "STARTS WITH": return keep(predicate(regex(Data("(?s)^".utf8) + literal + anyText))) + case "ENDS WITH": return keep(predicate(regex(anyText + literal + Data("$".utf8)))) + case "REGEX": return keep(predicate(regex(value))) + case "IS EMPTY": return keep(predicate(regex(Data("^$".utf8)))) + case "IS NOT EMPTY": return drop(predicate(regex(Data("^$".utf8)))) + case "IS NULL": return drop(predicate(nil)) + case "IS NOT NULL": return keep(predicate(nil)) + case ">": return keep(predicate(["valueRangeFilter": ["startValueOpen": value.base64EncodedString()]])) + case ">=": return keep(predicate(["valueRangeFilter": ["startValueClosed": value.base64EncodedString()]])) + case "<": return keep(predicate(["valueRangeFilter": ["endValueOpen": value.base64EncodedString()]])) + case "<=": return keep(predicate(["valueRangeFilter": ["endValueClosed": value.base64EncodedString()]])) + case "BETWEEN": + guard let upper = filter.secondValue else { + throw BigtableError.configuration(String(localized: "BETWEEN needs two values.")) + } + return keep(predicate(["valueRangeFilter": [ + "startValueClosed": value.base64EncodedString(), + "endValueClosed": try BigtableBytes.parse(upper).base64EncodedString() + ]])) + case "IN": + let alternatives = try listItems(filter.value).map { BigtableBytes.regexText(try BigtableBytes.parse($0)) } + return keep(predicate(regex(Data(("(?s)^(" + alternatives.joined(separator: "|") + ")$").utf8)))) + default: + throw unsupported(filter.op) + } + } + + /// A key test that no row range can express, or one inside an OR with column tests, runs as a + /// key regex instead. + private static func keyRegexFilter(_ filter: BigtableFilterSpec, insideOr: Bool) throws -> [String: Any] { + let literal = BigtableBytes.regexLiteral(try BigtableBytes.parse(filter.value)) + let anyText = Data("(?s).*".utf8) + let pattern: Data + var negate = false + switch filter.op.uppercased() { + case "=": pattern = Data("(?s)^".utf8) + literal + Data("$".utf8) + case "!=", "<>": pattern = Data("(?s)^".utf8) + literal + Data("$".utf8); negate = true + case "STARTS WITH": pattern = Data("(?s)^".utf8) + literal + anyText + case "ENDS WITH": pattern = anyText + literal + Data("$".utf8) + case "CONTAINS": pattern = anyText + literal + anyText + case "NOT CONTAINS": pattern = anyText + literal + anyText; negate = true + case "REGEX": pattern = try BigtableBytes.parse(filter.value) + default: + throw BigtableError.unsupported(insideOr + ? String(localized: "A key range cannot be ORed with a column filter. Use =, STARTS WITH or REGEX on _key, or AND.") + : unsupportedMessage(filter.op)) + } + let test: [String: Any] = ["rowKeyRegexFilter": pattern.base64EncodedString()] + return negate ? drop(test) : keep(test) + } + + private static func keep(_ predicate: [String: Any]) -> [String: Any] { + ["condition": ["predicateFilter": predicate, "trueFilter": ["passAllFilter": true]]] + } + + private static func drop(_ predicate: [String: Any]) -> [String: Any] { + ["condition": ["predicateFilter": predicate, "falseFilter": ["passAllFilter": true]]] + } + + private static func anchored(_ qualifier: Data) -> Data { + Data("(?s)^".utf8) + BigtableBytes.regexLiteral(qualifier) + Data("$".utf8) + } + + /// The raw filter row takes a `RowFilter` as written in the Bigtable REST reference, with bytes + /// in base64, which reaches every filter the filter bar cannot draw. + private static func rawFilter(_ text: String) throws -> [String: Any] { + guard let data = text.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], object.count == 1 else { + throw BigtableError.configuration(String(localized: "A raw Bigtable filter is a JSON RowFilter, such as {\"rowKeyRegexFilter\": \"\"}.")) + } + return object + } + + static func listItems(_ text: String) -> [String] { + if let data = text.data(using: .utf8), let array = try? JSONSerialization.jsonObject(with: data) as? [String] { + return array + } + return text.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + } + + private static func unsupported(_ op: String) -> BigtableError { + .unsupported(unsupportedMessage(op)) + } + + private static func unsupportedMessage(_ op: String) -> String { + String(format: String(localized: "Bigtable cannot filter with %@."), op) + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift new file mode 100644 index 0000000000..6cf2745015 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift @@ -0,0 +1,75 @@ +import Foundation + +/// Row keys, qualifiers and values are bytes. Text that is valid UTF-8 with no control characters +/// shows as itself; anything else shows as `b64:` followed by base64, which reads back to the same +/// bytes, so a binary key can still be named in a request line and edited from the grid. +public enum BigtableBytes { + public static let base64Prefix = "b64:" + + public static func display(_ data: Data) -> String { + if let text = String(data: data, encoding: .utf8), isPrintable(text), !text.hasPrefix(base64Prefix) { + return text + } + return base64Prefix + data.base64EncodedString() + } + + public static func parse(_ text: String) throws -> Data { + guard text.hasPrefix(base64Prefix) else { return Data(text.utf8) } + guard let data = Data(base64Encoded: String(text.dropFirst(base64Prefix.count))) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not valid base64."), text)) + } + return data + } + + public static func isPrintable(_ text: String) -> Bool { + !text.unicodeScalars.contains { scalar in + scalar.value < 0x20 && scalar != "\t" && scalar != "\n" || scalar.value == 0x7F + } + } + + static func decodeBase64(_ text: String?) throws -> Data { + guard let text else { return Data() } + guard let data = Data(base64Encoded: text) else { + throw BigtableError.malformedResponse(String(localized: "Bigtable sent bytes that are not base64.")) + } + return data + } + + /// The smallest key greater than every key starting with `prefix`: the prefix with its last + /// byte below 0xFF incremented and what follows dropped. Nil when every byte is 0xFF, which + /// leaves the range open at the end. + public static func prefixEnd(_ prefix: Data) -> Data? { + var bytes = [UInt8](prefix) + while let last = bytes.last { + if last < 0xFF { + bytes[bytes.count - 1] = last + 1 + return Data(bytes) + } + bytes.removeLast() + } + return nil + } + + /// `regexLiteral` writes only printable ASCII, so its text form always exists. + public static func regexText(_ data: Data) -> String { + String(bytes: regexLiteral(data), encoding: .ascii) ?? "" + } + + /// RE2 reads these as operators, so a literal is escaped byte by byte before it goes into a + /// regex filter. + public static func regexLiteral(_ data: Data) -> Data { + let special = Set("\\^$.|?*+()[]{}".utf8) + var escaped = Data() + for byte in data { + if special.contains(byte) { + escaped.append(UInt8(ascii: "\\")) + escaped.append(byte) + } else if byte < 0x20 || byte >= 0x7F { + escaped.append(contentsOf: Array(String(format: "\\x%02X", byte).utf8)) + } else { + escaped.append(byte) + } + } + return escaped + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift new file mode 100644 index 0000000000..1fbc448af0 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift @@ -0,0 +1,153 @@ +import Foundation + +public enum BigtableMutation: Sendable, Equatable { + case setCell(BigtableColumn, Data) + case deleteColumn(BigtableColumn) + case deleteRow + + /// A timestamp of -1 asks the server to stamp the cell with its own clock, which also keeps the + /// write valid on a table whose granularity is milliseconds. + var json: [String: Any] { + switch self { + case .setCell(let column, let value): + return ["setCell": [ + "familyName": column.family, + "columnQualifier": column.qualifier.base64EncodedString(), + "timestampMicros": "-1", + "value": value.base64EncodedString() + ]] + case .deleteColumn(let column): + return ["deleteFromColumn": [ + "familyName": column.family, + "columnQualifier": column.qualifier.base64EncodedString() + ]] + case .deleteRow: + return ["deleteFromRow": [String: Any]()] + } + } +} + +/// The Bigtable data and admin REST APIs (v2) for one project and one instance at a time. +public final class BigtableClient: @unchecked Sendable { + public static let dataEndpoint = "https://bigtable.googleapis.com/v2/" + public static let adminEndpoint = "https://bigtableadmin.googleapis.com/v2/" + + public let projectId: String + public let appProfileId: String? + private let requester: BigtableRequester + private let lock = NSLock() + private var instance: String + + public init( + projectId: String, + instanceId: String, + appProfileId: String? = nil, + transport: any BigtableTransport, + credential: BigtableCredential, + timeout: @escaping @Sendable () -> TimeInterval + ) { + self.projectId = projectId + self.instance = instanceId + self.appProfileId = appProfileId + self.requester = BigtableRequester(transport: transport, credential: credential, timeout: timeout) + } + + public var instanceId: String { + lock.withLock { instance } + } + + public func switchInstance(to instanceId: String) throws { + guard BigtableConnectionSettings.isValidResourceId(instanceId) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Bigtable instance ID."), instanceId)) + } + lock.withLock { instance = instanceId } + } + + // MARK: - Admin + + public func listInstances() async throws -> [String] { + let response = try await requester.send(method: "GET", url: try url(Self.adminEndpoint, "projects/\(projectId)/instances")) + try response.requireSuccess(String(localized: "list instances")) + let instances = (try response.json() as? [String: Any])?["instances"] as? [[String: Any]] ?? [] + return instances.compactMap { ($0["name"] as? String)?.split(separator: "/").last.map(String.init) }.sorted() + } + + public func listTables() async throws -> [String] { + var names: [String] = [] + var pageToken: String? + repeat { + var query = [URLQueryItem(name: "view", value: "NAME_ONLY"), URLQueryItem(name: "pageSize", value: "1000")] + if let pageToken { query.append(URLQueryItem(name: "pageToken", value: pageToken)) } + let response = try await requester.send(method: "GET", url: try url(Self.adminEndpoint, instancePath + "/tables", query: query)) + try response.requireSuccess(String(format: String(localized: "list the tables of %@"), instanceId)) + let json = try response.json() as? [String: Any] + let tables = json?["tables"] as? [[String: Any]] ?? [] + names += tables.compactMap { ($0["name"] as? String)?.split(separator: "/").last.map(String.init) } + pageToken = (json?["nextPageToken"] as? String)?.nonEmpty + } while pageToken != nil + return names.sorted() + } + + public func columnFamilies(table: String) async throws -> [String] { + let response = try await requester.send( + method: "GET", + url: try url(Self.adminEndpoint, try tablePath(table), query: [URLQueryItem(name: "view", value: "SCHEMA_VIEW")]) + ) + try response.requireSuccess(String(format: String(localized: "read the schema of %@"), table)) + let families = (try response.json() as? [String: Any])?["columnFamilies"] as? [String: Any] ?? [:] + return families.keys.sorted() + } + + // MARK: - Data + + public func readRows(table: String, body: [String: Any]) async throws -> [BigtableRow] { + if BigtableReadRequest.asksForNothing(body) { return [] } + var request = body + if let appProfileId { request["appProfileId"] = appProfileId } + let response = try await requester.send(method: "POST", url: try url(Self.dataEndpoint, try tablePath(table) + ":readRows"), json: request) + try response.requireSuccess(String(format: String(localized: "read %@"), table)) + return try BigtableChunkAssembler.rows(fromResponse: try response.json()) + } + + public func mutateRow(table: String, key: Data, mutations: [BigtableMutation]) async throws { + guard !mutations.isEmpty else { return } + guard !key.isEmpty else { + throw BigtableError.configuration(String(localized: "A row needs a key. Bigtable has no empty row key.")) + } + var body: [String: Any] = [ + "rowKey": key.base64EncodedString(), + "mutations": mutations.map(\.json) + ] + if let appProfileId { body["appProfileId"] = appProfileId } + let response = try await requester.send(method: "POST", url: try url(Self.dataEndpoint, try tablePath(table) + ":mutateRow"), json: body) + try response.requireSuccess(String(format: String(localized: "write a row of %@"), table)) + } + + public func ping() async throws { + _ = try await listTables() + } + + // MARK: - Paths + + var instancePath: String { + "projects/\(projectId)/instances/\(instanceId)" + } + + func tablePath(_ table: String) throws -> String { + guard BigtableConnectionSettings.isValidResourceId(table) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Bigtable table ID."), table)) + } + return instancePath + "/tables/" + table + } + + func url(_ endpoint: String, _ path: String, query: [URLQueryItem] = []) throws -> URL { + guard var components = URLComponents(string: endpoint + path) else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) + } + if !query.isEmpty { components.queryItems = query } + guard let url = components.url else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) + } + return url + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift new file mode 100644 index 0000000000..0c84262172 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift @@ -0,0 +1,63 @@ +import Foundation + +public enum BigtableError: Error, LocalizedError, Equatable, Sendable { + case configuration(String) + case notConnected + case transport(String) + case authentication(String) + /// IAM refused the operation. Kept apart from an empty result so a blocked read never looks + /// like a table with no rows. + case permissionDenied(operation: String, detail: String) + case notFound(String) + case unsupported(String) + case api(status: Int, message: String) + case malformedResponse(String) + + public var errorDescription: String? { + switch self { + case .configuration(let detail), .transport(let detail), .malformedResponse(let detail), + .authentication(let detail), .notFound(let detail), .unsupported(let detail): + return detail + case .notConnected: + return String(localized: "Not connected to Bigtable.") + case .permissionDenied(let operation, let detail): + let lead = String(format: String(localized: "Permission denied: %@."), operation) + return detail.isEmpty ? lead : lead + "\n" + detail + case .api(_, let message): + return message + } + } + + /// `operation` names what was attempted ("read events", "write a row of events"), because + /// Google's denial text names the permission and not the table. + public static func from(status: Int, body: Data, operation: String) -> BigtableError { + let message = apiMessage(from: body) + switch status { + case 401: + return .authentication(message ?? String(localized: "Google refused the credential.")) + case 403: + return .permissionDenied(operation: operation, detail: message ?? "") + case 404: + return .notFound(message ?? String(format: String(localized: "Not found: %@."), operation)) + default: + return .api( + status: status, + message: message ?? String(format: String(localized: "Bigtable returned HTTP %d."), status) + ) + } + } + + /// A streaming method answers an error as a one-element array, a unary one as an object. + public static func apiMessage(from body: Data) -> String? { + guard let json = try? JSONSerialization.jsonObject(with: body, options: [.fragmentsAllowed]) else { + let text = String(data: body, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) + return (text?.isEmpty ?? true) ? nil : text + } + let object = (json as? [[String: Any]])?.first ?? json as? [String: Any] + let error = object?["error"] as? [String: Any] + if let message = error?["message"] as? String, !message.isEmpty { + return message + } + return nil + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift new file mode 100644 index 0000000000..133c55a0a1 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift @@ -0,0 +1,238 @@ +import Foundation + +/// The text the driver runs. A page the grid asks for travels as a tagged spec; everything else is a +/// request line the history shows and the editor takes: +/// +/// GET user#42 SCAN user#?limit=20 READ {"rowsLimit": "10"} +/// PUT user#42 {"profile:name": "Ann"} DELETE user#42 DELETE user#42 profile:name +/// +/// A key with spaces is written as a JSON string, and binary bytes as `b64:`. +public enum BigtableStatement: Sendable, Equatable { + case browse(BigtableBrowseSpec) + case request(BigtableRequest) + + public static let browseTag = "BIGTABLE_BROWSE:" + + public static func encode(_ spec: BigtableBrowseSpec) -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return browseTag + ((try? encoder.encode(spec)) ?? Data()).base64EncodedString() + } + + public static func parse(_ text: String) throws -> BigtableStatement { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.hasPrefix(browseTag) { + guard let data = Data(base64Encoded: String(trimmed.dropFirst(browseTag.count))), + let spec = try? JSONDecoder().decode(BigtableBrowseSpec.self, from: data) else { + throw BigtableError.malformedResponse(String(localized: "The browse request is damaged. Reopen the table.")) + } + return .browse(spec) + } + return .request(try BigtableRequest.parse(trimmed)) + } +} + +public struct BigtableRequest: Sendable, Equatable { + public enum Verb: String, Sendable, CaseIterable { + case get = "GET" + case scan = "SCAN" + case read = "READ" + case put = "PUT" + case delete = "DELETE" + } + + public let verb: Verb + /// The table the request names with `table/`, or nil for the table open in the editor's tab. + public let table: String? + public let key: Data? + public let column: String? + public let limit: Int? + public let body: String? + + public init(verb: Verb, table: String? = nil, key: Data? = nil, column: String? = nil, limit: Int? = nil, body: String? = nil) { + self.verb = verb + self.table = table + self.key = key + self.column = column + self.limit = limit + self.body = body + } + + /// `GET events/user#42` names the table before the first slash; a key holding a slash is + /// written as a JSON string, `GET events/"a/b"`. + public static func parse(_ text: String) throws -> BigtableRequest { + var scanner = RequestScanner(text.trimmingCharacters(in: .whitespacesAndNewlines)) + guard let verbWord = scanner.word(), let verb = Verb(rawValue: verbWord.uppercased()) else { + throw BigtableError.configuration(usage) + } + switch verb { + case .read: + let rest = scanner.rest() + var table: String? + var body = rest + if let brace = rest.firstIndex(of: "{"), brace != rest.startIndex { + table = rest[.. String { + var parts = [verb.rawValue] + if verb == .read { + parts.append(contentsOf: [table, body].compactMap { $0 }) + return parts.joined(separator: " ") + } + let keyText = key.map(Self.keyToken) ?? "" + let tableName = table ?? defaultTable + parts.append(tableName.map { $0 + "/" + keyText } ?? keyText) + if let column { parts.append(column) } + if let body { parts.append(body) } + return parts.joined(separator: " ") + } + + /// A key that would not read back as one bare word goes in JSON quotes. + static func keyToken(_ key: Data) -> String { + let display = BigtableBytes.display(key) + let plain = !display.isEmpty && display.unicodeScalars.allSatisfy { scalar in + !CharacterSet.whitespacesAndNewlines.contains(scalar) && scalar != "\"" && scalar != "/" && scalar != "?" + } + if plain { return display } + let data = (try? JSONSerialization.data(withJSONObject: [display], options: [])) ?? Data() + let array = String(bytes: data, encoding: .utf8) ?? "" + return String(array.dropFirst().dropLast()) + } + + static let usage = String( + localized: "Enter a request such as GET events/user#42, SCAN events/user#?limit=20, PUT events/user#42 {\"profile:name\": \"Ann\"} or DELETE events/user#42." + ) +} + +/// Reads a request line one token at a time: words, a `table/key` target where the key may be a +/// JSON string, a `?name=value` query on the target, and whatever is left as the body. +private struct RequestScanner { + private var text: Substring + var query: [String: String] = [:] + + init(_ text: String) { + self.text = Substring(text) + } + + mutating func word() -> String? { + skipSpace() + let word = text.prefix { !$0.isWhitespace } + guard !word.isEmpty else { return nil } + text = text.dropFirst(word.count) + return String(word) + } + + mutating func rest() -> String { + skipSpace() + let remaining = String(text).trimmingCharacters(in: .whitespacesAndNewlines) + text = "" + return remaining + } + + mutating func target() throws -> (table: String?, key: Data?) { + skipSpace() + var table: String? + if let slash = text.firstIndex(of: "/"), !text[.. String { + var index = text.index(after: text.startIndex) + var escaped = false + while index < text.endIndex { + let character = text[index] + if character == "\"" && !escaped { break } + escaped = character == "\\" && !escaped + index = text.index(after: index) + } + guard index < text.endIndex else { + throw BigtableError.configuration(String(localized: "A quoted row key is missing its closing quote.")) + } + let literal = String(text[text.startIndex...index]) + text = text[text.index(after: index)...] + guard let data = ("[" + literal + "]").data(using: .utf8), + let value = (try? JSONSerialization.jsonObject(with: data) as? [String])?.first else { + throw BigtableError.configuration(String(localized: "A quoted row key is not a valid JSON string.")) + } + return value + } + + private mutating func skipSpace() { + text = text.drop { $0.isWhitespace } + } +} + +public enum BigtableStatementClassifier { + public enum Tier: Sendable, Equatable { + case read + case write + case destructive + } + + /// Text that is neither a browse nor a request line counts as a write, so the read-only gate + /// refuses what it cannot read. + public static func classify(_ text: String) -> Tier { + guard let statement = try? BigtableStatement.parse(text) else { return .write } + switch statement { + case .browse: + return .read + case .request(let request): + switch request.verb { + case .get, .scan, .read: return .read + case .put: return .write + case .delete: return .destructive + } + } + } +} + +extension String { + var nonEmpty: String? { isEmpty ? nil : self } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableTransport.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableTransport.swift new file mode 100644 index 0000000000..28160f481f --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableTransport.swift @@ -0,0 +1,184 @@ +import Foundation + +public struct BigtableHTTPRequest: Sendable, Equatable { + public let method: String + public let url: URL + public let headers: [String: String] + public let body: Data? + public let timeoutInterval: TimeInterval + + public init(method: String, url: URL, headers: [String: String] = [:], body: Data? = nil, timeoutInterval: TimeInterval) { + self.method = method + self.url = url + self.headers = headers + self.body = body + self.timeoutInterval = timeoutInterval + } +} + +public struct BigtableHTTPResponse: Sendable, Equatable { + public let statusCode: Int + public let body: Data + + public init(statusCode: Int, body: Data) { + self.statusCode = statusCode + self.body = body + } + + public var isSuccess: Bool { + (200..<300).contains(statusCode) + } + + func json() throws -> Any { + guard !body.isEmpty else { return [String: Any]() } + do { + return try JSONSerialization.jsonObject(with: body, options: [.fragmentsAllowed]) + } catch { + throw BigtableError.malformedResponse(String(localized: "Bigtable sent a response that is not JSON.")) + } + } + + func requireSuccess(_ operation: String) throws { + guard !isSuccess else { return } + throw BigtableError.from(status: statusCode, body: body, operation: operation) + } +} + +public protocol BigtableTransport: Sendable { + func send(_ request: BigtableHTTPRequest) async throws -> BigtableHTTPResponse + func cancelAll() +} + +public struct BigtableCredential: Sendable { + public let token: @Sendable () async throws -> String + public let invalidate: @Sendable () async -> Void + public let quotaProject: String? + + public init( + token: @escaping @Sendable () async throws -> String, + invalidate: @escaping @Sendable () async -> Void = {}, + quotaProject: String? = nil + ) { + self.token = token + self.invalidate = invalidate + self.quotaProject = quotaProject + } +} + +public final class URLSessionBigtableTransport: BigtableTransport, @unchecked Sendable { + private let session: URLSession + private let lock = NSLock() + private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] + + public init(resourceTimeout: TimeInterval) { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForResource = resourceTimeout + configuration.httpCookieStorage = nil + configuration.urlCache = nil + session = URLSession(configuration: configuration, delegate: BigtableRedirectRefusal(), delegateQueue: nil) + } + + deinit { + session.invalidateAndCancel() + } + + public func cancelAll() { + let tasks = lock.withLock { Array(inFlight.values) } + tasks.forEach { $0.cancel() } + } + + public func send(_ request: BigtableHTTPRequest) async throws -> BigtableHTTPResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = request.timeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + let tracker = BigtableTaskTracker(transport: self) + defer { tracker.finish() } + do { + let (data, response) = try await session.data(for: urlRequest, delegate: tracker) + guard let http = response as? HTTPURLResponse else { + throw BigtableError.transport(String(localized: "Bigtable answered with something other than HTTP.")) + } + return BigtableHTTPResponse(statusCode: http.statusCode, body: data) + } catch let error as URLError where error.code == .cancelled { + throw CancellationError() + } catch let error as URLError { + throw BigtableError.transport(error.localizedDescription) + } + } + + fileprivate func register(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = task } + } + + fileprivate func unregister(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = nil } + } +} + +/// A redirect would carry the bearer token to whatever host it names. +private final class BigtableRedirectRefusal: NSObject, URLSessionTaskDelegate, Sendable { + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest + ) async -> URLRequest? { + nil + } +} + +private final class BigtableTaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { + private weak var transport: URLSessionBigtableTransport? + private let lock = NSLock() + private var task: URLSessionTask? + + init(transport: URLSessionBigtableTransport) { + self.transport = transport + } + + func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { + lock.withLock { self.task = task } + transport?.register(task) + } + + func finish() { + guard let task = lock.withLock({ task }) else { return } + transport?.unregister(task) + } +} + +/// A 401 drops the cached token and retries once, since a token can expire between being cached +/// and being used. +struct BigtableRequester: Sendable { + let transport: any BigtableTransport + let credential: BigtableCredential + let timeout: @Sendable () -> TimeInterval + + func send(method: String, url: URL, json: Any? = nil) async throws -> BigtableHTTPResponse { + let body = try json.map { try JSONSerialization.data(withJSONObject: $0, options: [.sortedKeys]) } + var response = try await attempt(method, url, body) + if response.statusCode == 401 { + await credential.invalidate() + response = try await attempt(method, url, body) + } + return response + } + + private func attempt(_ method: String, _ url: URL, _ body: Data?) async throws -> BigtableHTTPResponse { + var headers = ["Authorization": "Bearer " + (try await credential.token())] + if let project = credential.quotaProject { + headers["x-goog-user-project"] = project + } + if body != nil { + headers["Content-Type"] = "application/json" + } + try Task.checkCancellation() + return try await transport.send(BigtableHTTPRequest( + method: method, url: url, headers: headers, body: body, timeoutInterval: timeout() + )) + } +} diff --git a/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift b/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift index 16a5b6a764..bb002a60ef 100644 --- a/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift +++ b/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift @@ -44,12 +44,13 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let weaviate = DatabaseType(rawValue: "Weaviate") public static let firestore = DatabaseType(rawValue: "Firestore") public static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") + public static let bigtable = DatabaseType(rawValue: "Bigtable") public static let allKnownTypes: [DatabaseType] = [ .mysql, .mariadb, .tidb, .databend, .oceanbase, .postgresql, .sqlite, .redis, .mongodb, .clickhouse, .mssql, .oracle, .dameng, .duckdb, .cassandra, .redshift, .etcd, .cloudflareD1, .dynamodb, .bigquery, .spanner, .snowflake, .libsql, .beancount, - .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .firestore, .firebaseRealtimeDatabase + .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .firestore, .firebaseRealtimeDatabase, .bigtable ] /// Icon name for this database type — asset catalog name (e.g. "mysql-icon") or SF Symbol fallback @@ -87,6 +88,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .cloudflareR2SQL: return "cloudflare-r2-sql-icon" case .weaviate: return "weaviate-icon" case .firestore, .firebaseRealtimeDatabase: return "firebase-icon" + case .bigtable: return "bigtable-icon" default: return "externaldrive" } } diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableCoreTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableCoreTests.swift new file mode 100644 index 0000000000..0fe0ef0549 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableCoreTests.swift @@ -0,0 +1,234 @@ +import Foundation +@testable import TableProBigtableCore +import Testing + +private func b64(_ text: String) -> String { Data(text.utf8).base64EncodedString() } +private func json(_ text: String) -> Any { (try? JSONSerialization.jsonObject(with: Data(text.utf8))) ?? [:] } + +@Suite("Bigtable chunk assembly") +struct BigtableChunkAssemblerTests { + @Test("Family and qualifier carry over, split values join, and only committed rows count") + func assembly() throws { + let response = json(""" + [{"chunks": [ + {"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "10", "value": "\(b64("he"))", "valueSize": 5}, + {"value": "\(b64("llo"))"}, + {"qualifier": "\(b64("b"))", "timestampMicros": "10", "value": "\(b64("x"))", "commitRow": true} + ]}, + {"chunks": [ + {"rowKey": "\(b64("r2"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "\(b64("old"))"}, + {"resetRow": true}, + {"rowKey": "\(b64("r2"))", "familyName": {"value": "cf"}, "qualifier": "\(b64("a"))", "timestampMicros": "2", "value": "\(b64("new"))", "commitRow": true} + ]}] + """) + let rows = try BigtableChunkAssembler.rows(fromResponse: response) + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + #expect(rows.map(\.key) == [Data("r1".utf8), Data("r2".utf8)]) + #expect(rows[0].cells[a]?.value == Data("hello".utf8)) + #expect(rows[0].cells[BigtableColumn(family: "cf", qualifier: Data("b".utf8))]?.value == Data("x".utf8)) + #expect(rows[1].cells[a] == BigtableCell(value: Data("new".utf8), timestampMicros: 2)) + } + + @Test("A response that stops inside a row is refused") + func truncated() { + let response = json(""" + [{"chunks": [{"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "\(b64("v"))"}]}] + """) + #expect(throws: BigtableError.self) { try BigtableChunkAssembler.rows(fromResponse: response) } + } + + @Test("Rows lay out with _key first, columns in order, and missing cells absent") + func rowSet() { + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + let z = BigtableColumn(family: "meta", qualifier: Data([0xFF, 0x00])) + let set = BigtableRowSet(rows: [ + BigtableRow(key: Data("k1".utf8), cells: [a: BigtableCell(value: Data("v".utf8), timestampMicros: 1)]), + BigtableRow(key: Data([0x00, 0x01]), cells: [z: BigtableCell(value: Data([0xC3]), timestampMicros: 1)]) + ]) + #expect(set.columns == ["_key", "cf:a", "meta:b64:/wA="]) + #expect(set.rows[0] == [.text("k1"), .text("v"), .absent]) + #expect(set.rows[1] == [.text("b64:AAE="), .absent, .bytes(Data([0xC3]))]) + #expect(set.absentCells == [0: [2], 1: [1]]) + #expect(set.locators == [b64("k1"), Data([0x00, 0x01]).base64EncodedString()]) + } +} + +@Suite("Bigtable bytes and columns") +struct BigtableBytesTests { + @Test("Printable text shows as itself, anything else as b64 that reads back") + func displayRoundTrip() throws { + for data in [Data("user#42".utf8), Data([0, 1, 255]), Data("b64:looks-tagged".utf8), Data("line\u{07}bell".utf8)] { + #expect(try BigtableBytes.parse(BigtableBytes.display(data)) == data) + } + #expect(BigtableBytes.display(Data("user#42".utf8)) == "user#42") + #expect(throws: BigtableError.self) { try BigtableBytes.parse("b64:***") } + } + + @Test("Prefix end increments the last byte that can grow") + func prefixEnd() { + #expect(BigtableBytes.prefixEnd(Data("ab".utf8)) == Data("ac".utf8)) + #expect(BigtableBytes.prefixEnd(Data([0x61, 0xFF])) == Data([0x62])) + #expect(BigtableBytes.prefixEnd(Data([0xFF, 0xFF])) == nil) + } + + @Test("Regex literals escape operators and non-printable bytes") + func regexLiteral() { + #expect(BigtableBytes.regexText(Data("a.b*(c)".utf8)) == #"a\.b\*\(c\)"#) + #expect(BigtableBytes.regexText(Data([0x00, 0x41])) == #"\x00A"#) + } + + @Test("A column name splits at its first colon") + func columnNames() throws { + let column = try BigtableColumn(name: "cf:a:b") + #expect(column.family == "cf" && column.qualifier == Data("a:b".utf8)) + #expect(try BigtableColumn(name: "cf:b64:/wA=").qualifier == Data([0xFF, 0x00])) + #expect(throws: BigtableError.self) { try BigtableColumn(name: "nocolon") } + #expect(throws: BigtableError.self) { try BigtableColumn(name: ":q") } + } +} + +@Suite("Bigtable read requests") +struct BigtableReadRequestTests { + private func body(_ filters: [BigtableFilterSpec], or: Bool = false, descending: Bool = false, after: String? = nil) throws -> [String: Any] { + let spec = BigtableBrowseSpec(table: "t", filters: filters, logicMode: or ? "OR" : "AND", descending: descending, limit: 10, offset: 0) + return try BigtableReadRequest.body(spec: spec, limit: 10, after: after.map { Data($0.utf8) }) + } + + private func text(_ object: Any) -> String { + (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys])).flatMap { String(bytes: $0, encoding: .utf8) } ?? "" + } + + @Test("A plain page reads the newest cells, with the limit as a string") + func plain() throws { + #expect(text(try body([])) == #"{"filter":{"cellsPerColumnLimitFilter":1},"rowsLimit":"10"}"#) + } + + @Test("Key filters become row ranges, intersected under AND") + func keyRanges() throws { + let prefix = try body([BigtableFilterSpec(column: "_key", op: "STARTS WITH", value: "user#")]) + #expect(text(prefix["rows"] ?? [:]) == #"{"rowRanges":[{"endKeyOpen":"\#(b64("user$"))","startKeyClosed":"\#(b64("user#"))"}]}"#) + let both = try body([ + BigtableFilterSpec(column: "_key", op: ">=", value: "b"), + BigtableFilterSpec(column: "_key", op: "<", value: "m"), + BigtableFilterSpec(column: "_key", op: ">", value: "c") + ]) + #expect(text(both["rows"] ?? [:]) == #"{"rowRanges":[{"endKeyOpen":"\#(b64("m"))","startKeyOpen":"\#(b64("c"))"}]}"#) + let exact = try body([BigtableFilterSpec(column: "_key", op: "IN", value: "a, b, z"), BigtableFilterSpec(column: "_key", op: "<", value: "c")]) + #expect(text(exact["rows"] ?? [:]) == #"{"rowKeys":["\#(b64("a"))","\#(b64("b"))"]}"#) + } + + @Test("A cursor continues after the last key, and a descending read is refused") + func cursors() throws { + #expect(text(try body([], after: "k9")["rows"] ?? [:]) == #"{"rowRanges":[{"startKeyOpen":"\#(b64("k9"))"}]}"#) + #expect(throws: BigtableError.self) { try body([], descending: true) } + } + + @Test("Key filters ORed together become several ranges, cut by the cursor") + func orRanges() throws { + let rows = try body([ + BigtableFilterSpec(column: "_key", op: "STARTS WITH", value: "a"), + BigtableFilterSpec(column: "_key", op: "=", value: "q") + ], or: true, after: "a5")["rows"] as? [String: Any] + #expect(text(rows ?? [:]) == #"{"rowKeys":["\#(b64("q"))"],"rowRanges":[{"endKeyOpen":"\#(b64("b"))","startKeyOpen":"\#(b64("a5"))"}]}"#) + } + + @Test("Column filters keep or drop whole rows through conditions") + func columnConditions() throws { + let equal = try body([BigtableFilterSpec(column: "cf:name", op: "=", value: "A.n")]) + let filter = text(equal["filter"] ?? [:]) + #expect(filter.contains(#""familyNameRegexFilter":"^cf$""#)) + #expect(filter.contains(b64(#"(?s)^A\.n$"#))) + #expect(filter.contains(#""trueFilter":{"passAllFilter":true}"#)) + let missing = text(try body([BigtableFilterSpec(column: "cf:name", op: "IS NULL", value: "")])["filter"] ?? [:]) + #expect(missing.contains(#""falseFilter":{"passAllFilter":true}"#) && !missing.contains("valueRegexFilter")) + let range = text(try body([BigtableFilterSpec(column: "cf:n", op: "BETWEEN", value: "1", secondValue: "9")])["filter"] ?? [:]) + #expect(range.contains(#""startValueClosed":"\#(b64("1"))""#) && range.contains(#""endValueClosed":"\#(b64("9"))""#)) + } + + @Test("Several column filters chain under AND and interleave under OR") + func combinations() throws { + let filters = [BigtableFilterSpec(column: "cf:a", op: "=", value: "1"), BigtableFilterSpec(column: "cf:b", op: "CONTAINS", value: "x")] + #expect(text(try body(filters)["filter"] ?? [:]).hasPrefix(#"{"chain":{"filters":[{"chain""#)) + #expect(text(try body(filters, or: true)["filter"] ?? [:]).hasPrefix(#"{"chain":{"filters":[{"interleave""#)) + } + + @Test("What Bigtable cannot express is refused with the reason") + func refusals() { + #expect(throws: BigtableError.self) { + try body([BigtableFilterSpec(column: "_key", op: ">", value: "a"), BigtableFilterSpec(column: "cf:a", op: "=", value: "1")], or: true) + } + #expect(throws: BigtableError.self) { try body([BigtableFilterSpec(column: "nocolon", op: "=", value: "1")]) } + #expect(throws: BigtableError.self) { try body([BigtableFilterSpec(column: "__RAW__", op: "", value: "not json")]) } + let sorted = BigtableBrowseSpec(table: "t", sortColumn: "cf:a", limit: 1, offset: 0) + #expect(throws: BigtableError.self) { try BigtableReadRequest.body(spec: sorted, limit: 1, after: nil) } + } + + @Test("An empty key list asks for nothing and is skipped") + func asksForNothing() throws { + let impossible = try body([BigtableFilterSpec(column: "_key", op: "=", value: "a"), BigtableFilterSpec(column: "_key", op: "=", value: "b")]) + #expect(BigtableReadRequest.asksForNothing(impossible)) + #expect(!BigtableReadRequest.asksForNothing(try body([]))) + } + + @Test("A keys-only read strips values and keeps one cell per row") + func keysOnly() throws { + let spec = BigtableBrowseSpec(table: "t", limit: 0, offset: 0) + let object = try BigtableReadRequest.body(spec: spec, limit: 500, after: nil, keysOnly: true) + #expect(text(object["filter"] ?? [:]) == #"{"chain":{"filters":[{"cellsPerRowLimitFilter":1},{"stripValueTransformer":true}]}}"#) + } +} + +@Suite("Bigtable statements and settings") +struct BigtableStatementTests { + @Test("Request lines read table, key, column, query and body") + func requests() throws { + let get = try BigtableRequest.parse("get events/user#42") + #expect(get.verb == .get && get.table == "events" && get.key == Data("user#42".utf8)) + let quoted = try BigtableRequest.parse(#"DELETE events/"a b/c" profile:name"#) + #expect(quoted.key == Data("a b/c".utf8) && quoted.column == "profile:name") + let scan = try BigtableRequest.parse("SCAN events/user#?limit=20") + #expect(scan.key == Data("user#".utf8) && scan.limit == 20) + let all = try BigtableRequest.parse("SCAN events/") + #expect(all.table == "events" && all.key == nil) + let put = try BigtableRequest.parse(#"PUT events/b64:AAE= {"cf:a": "1"}"#) + #expect(put.key == Data([0, 1]) && put.body == #"{"cf:a": "1"}"#) + let read = try BigtableRequest.parse(#"READ events {"rowsLimit": "5"}"#) + #expect(read.table == "events" && read.body == #"{"rowsLimit": "5"}"#) + } + + @Test("Malformed requests explain the syntax") + func malformed() { + for text in ["SELECT * FROM t", "GET events/", "PUT events/k", #"GET events/"open"#, "READ events"] { + #expect(throws: BigtableError.self) { try BigtableRequest.parse(text) } + } + } + + @Test("A request reads back from its own text, quoting keys that need it") + func roundTrip() throws { + for key in [Data("user#42".utf8), Data("a b".utf8), Data([0xFF]), Data("x/y".utf8)] { + let request = BigtableRequest(verb: .delete, table: "events", key: key, column: "cf:a") + #expect(try BigtableRequest.parse(request.text(defaultTable: nil)) == request) + } + let spec = BigtableBrowseSpec(table: "t", filters: [BigtableFilterSpec(column: "_key", op: "=", value: "k")], limit: 5, offset: 10) + #expect(try BigtableStatement.parse(BigtableStatement.encode(spec)) == .browse(spec)) + } + + @Test("The read-only gate sees reads, writes and deletes") + func classify() { + #expect(BigtableStatementClassifier.classify(BigtableStatement.encode(BigtableBrowseSpec(table: "t", limit: 1, offset: 0))) == .read) + #expect(BigtableStatementClassifier.classify("SCAN events/a") == .read) + #expect(BigtableStatementClassifier.classify(#"READ events {}"#) == .read) + #expect(BigtableStatementClassifier.classify(#"PUT events/k {"cf:a": "1"}"#) == .write) + #expect(BigtableStatementClassifier.classify("DELETE events/k") == .destructive) + #expect(BigtableStatementClassifier.classify("DROP TABLE events") == .write) + } + + @Test("Settings take the project from the credential and refuse IDs that would change the path") + func settings() throws { + let parsed = try BigtableConnectionSettings.parse(fields: ["btInstanceId": "ea-bt-01"], credentialProject: "my-proj") + #expect(parsed.projectId == "my-proj" && parsed.instanceId == "ea-bt-01" && parsed.authMethod == .serviceAccount) + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: [:]) } + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "p-123", "btInstanceId": "../x"]) } + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "Bad Project"]) } + } +} diff --git a/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift b/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift index 5a410e066b..ca8b6a75bd 100644 --- a/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift +++ b/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift @@ -61,7 +61,7 @@ struct DatabaseTypeTests { @Test("allKnownTypes contains all expected types") func allKnownTypesComplete() { - #expect(DatabaseType.allKnownTypes.count == 32) + #expect(DatabaseType.allKnownTypes.count == 33) #expect(DatabaseType.allKnownTypes.contains(.mysql)) #expect(DatabaseType.allKnownTypes.contains(.tidb)) #expect(DatabaseType.allKnownTypes.contains(.databend)) @@ -80,6 +80,7 @@ struct DatabaseTypeTests { #expect(DatabaseType.allKnownTypes.contains(.weaviate)) #expect(DatabaseType.allKnownTypes.contains(.firestore)) #expect(DatabaseType.allKnownTypes.contains(.firebaseRealtimeDatabase)) + #expect(DatabaseType.allKnownTypes.contains(.bigtable)) } /// The list has no duplicates, which a count alone would not catch: adding a type twice @@ -113,6 +114,13 @@ struct DatabaseTypeTests { #expect(DatabaseType.firebaseRealtimeDatabase.pluginTypeId == "Firebase Realtime Database") } + @Test("Bigtable resolves its icon and plugin type id") + func bigtableIdentity() { + #expect(DatabaseType.bigtable.rawValue == "Bigtable") + #expect(DatabaseType.bigtable.iconName == "bigtable-icon") + #expect(DatabaseType.bigtable.pluginTypeId == "Bigtable") + } + @Test("ScyllaDB resolves its own icon") func scyllaDBIdentity() { #expect(DatabaseType.scylladb.rawValue == "ScyllaDB") diff --git a/Plugins/BigtableDriverPlugin/BigtableCredentialFactory.swift b/Plugins/BigtableDriverPlugin/BigtableCredentialFactory.swift new file mode 100644 index 0000000000..d7078eee59 --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtableCredentialFactory.swift @@ -0,0 +1,51 @@ +import Foundation +import TableProBigtableCore +import TableProGoogleCloud +import TableProPluginKit + +internal enum BigtableCredentialFactory { + static let scopes = [GoogleOAuthClient.cloudPlatformScope] + + struct Resolved { + let credential: BigtableCredential + let projectHint: String? + } + + static func resolve(config: DriverConnectionConfig) throws -> Resolved { + let fields = config.additionalFields + let method = trimmed(fields[BigtableFieldID.authMethod]).flatMap(BigtableAuthMethod.init(rawValue:)) ?? .serviceAccount + let source: GoogleCredentialSource + switch method { + case .serviceAccount: + guard let value = trimmed(fields[BigtableFieldID.serviceAccountJson]) ?? trimmed(config.password) else { + throw BigtableError.configuration(String(localized: "Choose a service account key file, or paste its JSON.")) + } + source = .serviceAccountKey(value) + case .applicationDefault: + source = .applicationDefault + } + let environment = ProcessInfo.processInfo.environment + let credentials = try GoogleTokenProviders.credentials( + from: source, + scopes: scopes, + readFile: { FileManager.default.contents(atPath: $0) }, + environment: environment, + http: URLSessionGoogleHTTPClient() + ) + let provider = credentials.tokenProvider + let environmentProject = source == .applicationDefault ? trimmed(environment["GOOGLE_CLOUD_PROJECT"]) : nil + return Resolved( + credential: BigtableCredential( + token: { try await provider.accessToken() }, + invalidate: { await provider.invalidateCachedToken() }, + quotaProject: credentials.quotaProject + ), + projectHint: credentials.projectHint ?? environmentProject + ) + } + + private static func trimmed(_ value: String?) -> String? { + guard let value = value?.trimmingCharacters(in: .whitespacesAndNewlines), !value.isEmpty else { return nil } + return value + } +} diff --git a/Plugins/BigtableDriverPlugin/BigtablePlugin.swift b/Plugins/BigtableDriverPlugin/BigtablePlugin.swift new file mode 100644 index 0000000000..ffe8a49a3b --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePlugin.swift @@ -0,0 +1,109 @@ +import Foundation +import TableProBigtableCore +import TableProPluginKit + +final class BigtablePlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Bigtable Driver" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Google Cloud Bigtable over the REST API" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "Bigtable" + static let databaseDisplayName = "Google Cloud Bigtable" + static let iconName = "bigtable-icon" + static let defaultPort = 0 + static let isDownloadable = true + + static let connectionMode: ConnectionMode = .apiOnly + static let navigationModel: NavigationModel = .standard + static let pathFieldRole: PathFieldRole = .database + static let requiresAuthentication = true + static let brandColorHex = "#4285F4" + static let queryLanguageName = "REST" + static let editorLanguage: EditorLanguage = .javascript + static let supportsForeignKeys = false + static let supportsSchemaEditing = false + static let supportsDatabaseSwitching = true + static let supportsImport = false + static let supportsExport = true + static let supportsSSH = false + static let supportsSSL = false + static let supportsReadOnlyMode = true + static let supportsForeignKeyDisable = false + static let supportsAddColumn = false + static let supportsModifyColumn = false + static let supportsDropColumn = false + static let supportsAddIndex = false + static let supportsDropIndex = false + static let supportsModifyPrimaryKey = false + static let supportsFieldRemoval = true + static let databaseGroupingStrategy: GroupingStrategy = .byDatabase + static let defaultGroupName = "default" + static let tableEntityName = "Tables" + static let containerEntityName = "Instance" + static let defaultSchemaName = "" + static let immutableColumns: [String] = ["_key"] + static let defaultPrimaryKeyColumn: String? = "_key" + static let structureColumnFields: [StructureColumnField] = [.name, .type, .nullable] + static let sqlDialect: SQLDialectDescriptor? = nil + + static let columnTypesByCategory: [String: [String]] = ["Binary": ["bytes"]] + static let additionalConnectionFields: [ConnectionField] = bigtableConnectionFields() + static var statementCompletions: [CompletionEntry] { bigtableCompletions } + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + BigtablePluginDriver(config: config) + } +} + +func bigtableConnectionFields() -> [ConnectionField] { + [ + ConnectionField( + id: BigtableFieldID.authMethod, + label: String(localized: "Auth Method"), + defaultValue: BigtableAuthMethod.serviceAccount.rawValue, + fieldType: .dropdown(options: [ + .init(value: BigtableAuthMethod.serviceAccount.rawValue, label: "Service Account Key"), + .init(value: BigtableAuthMethod.applicationDefault.rawValue, label: "Application Default Credentials") + ]), + section: .authentication + ), + ConnectionField( + id: BigtableFieldID.serviceAccountJson, + label: String(localized: "Service Account Key"), + placeholder: "File path or paste JSON", + required: true, + fieldType: .secure, + section: .authentication, + hidesPassword: true, + visibleWhen: FieldVisibilityRule(fieldId: BigtableFieldID.authMethod, values: [BigtableAuthMethod.serviceAccount.rawValue]) + ).withHidesUsername(true), + ConnectionField( + id: BigtableFieldID.projectId, + label: String(localized: "Project ID"), + placeholder: "Taken from the key when empty", + section: .authentication + ), + ConnectionField( + id: BigtableFieldID.instanceId, + label: String(localized: "Instance ID"), + placeholder: "The first instance when empty", + section: .authentication + ), + ConnectionField( + id: BigtableFieldID.appProfileId, + label: String(localized: "App Profile ID"), + placeholder: "default", + section: .advanced + ) + ] +} + +let bigtableCompletions: [CompletionEntry] = [ + CompletionEntry(label: "GET row", insertText: "GET events/user#42"), + CompletionEntry(label: "SCAN prefix", insertText: "SCAN events/user#?limit=20"), + CompletionEntry(label: "READ", insertText: "READ events {\"rowsLimit\": \"20\", \"filter\": {\"cellsPerColumnLimitFilter\": 1}}"), + CompletionEntry(label: "PUT cells", insertText: "PUT events/user#42 {\"profile:name\": \"Ann\"}"), + CompletionEntry(label: "DELETE row", insertText: "DELETE events/user#42"), + CompletionEntry(label: "DELETE column", insertText: "DELETE events/user#42 profile:name") +] diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift new file mode 100644 index 0000000000..6dc838d62d --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift @@ -0,0 +1,116 @@ +import Foundation +import TableProBigtableCore +import TableProPluginKit + +extension BigtablePluginDriver { + /// Each change becomes one request line, the text the editor takes, so the history shows what + /// was sent. All cells of a row change in one atomic `mutateRow`. + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try changes.compactMap { change in + do { + return try request(for: change, table: table, columns: columns, insertedRowData: insertedRowData) + .map { PluginRowWrite(statement: $0.text(defaultTable: table), rowIndices: [change.rowIndex]) } + } catch { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: error.localizedDescription) + } + } + } + + func generateIdentityPreservingInsert( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + rows: [[PluginCellValue]] + ) -> [(statement: String, parameters: [PluginCellValue])]? { + let statements = rows.enumerated().compactMap { index, row -> String? in + let change = PluginRowChange(rowIndex: index, type: .insert, cellChanges: [], originalRow: nil) + return (try? request(for: change, table: table, columns: columns, insertedRowData: [index: row]))??.text(defaultTable: table) + } + return statements.count == rows.count ? statements.map { ($0, []) } : nil + } + + private func request( + for change: PluginRowChange, + table: String, + columns: [String], + insertedRowData: [Int: [PluginCellValue]] + ) throws -> BigtableRequest? { + let keyColumn = BigtableRowSet.keyColumn + switch change.type { + case .insert: + let row = insertedRowData[change.rowIndex] ?? [] + let absent = change.absentColumns ?? [] + var cells: [String: Any] = [:] + var key: Data? + for (index, column) in columns.enumerated() where index < row.count && !absent.contains(index) { + if column == keyColumn { + key = try keyBytes(row[index]) + continue + } + guard let value = Self.text(row[index]) else { continue } + cells[column] = value + } + guard let key, !key.isEmpty else { + throw BigtableError.configuration(String(localized: "A new row needs a _key. Bigtable has no generated keys.")) + } + guard !cells.isEmpty else { + throw BigtableError.configuration(String(localized: "A new row needs at least one cell. Bigtable does not store empty rows.")) + } + return BigtableRequest(verb: .put, key: key, body: try Self.body(cells)) + case .update: + let key = try originalKey(change, columns: columns) + let absent = change.absentColumns ?? [] + var cells: [String: Any] = [:] + for cell in change.cellChanges where cell.columnName != keyColumn { + cells[cell.columnName] = absent.contains(cell.columnIndex) ? NSNull() : (Self.text(cell.newValue) ?? NSNull()) + } + for index in absent where index < columns.count && columns[index] != keyColumn && cells[columns[index]] == nil { + cells[columns[index]] = NSNull() + } + guard !cells.isEmpty else { return nil } + return BigtableRequest(verb: .put, key: key, body: try Self.body(cells)) + case .delete: + return BigtableRequest(verb: .delete, key: try originalKey(change, columns: columns)) + } + } + + private func originalKey(_ change: PluginRowChange, columns: [String]) throws -> Data { + guard let index = columns.firstIndex(of: BigtableRowSet.keyColumn), + let row = change.originalRow, index < row.count, let key = try keyBytes(row[index]), !key.isEmpty else { + throw BigtableError.configuration(String(localized: "The row has no _key, so the row it came from is unknown. Reload the table.")) + } + return key + } + + private func keyBytes(_ cell: PluginCellValue) throws -> Data? { + switch cell { + case .null: return nil + case .bytes(let data): return data + case .text(let text): return text.isEmpty ? nil : try BigtableBytes.parse(text) + } + } + + /// Bytes go back as `b64:` text so a binary value survives the JSON body unchanged. + private static func text(_ cell: PluginCellValue) -> String? { + switch cell { + case .null: return nil + case .text(let text): return text + case .bytes(let data): return BigtableBytes.base64Prefix + data.base64EncodedString() + } + } + + private static func body(_ cells: [String: Any]) throws -> String { + let data = try JSONSerialization.data(withJSONObject: cells, options: [.sortedKeys, .withoutEscapingSlashes]) + return String(bytes: data, encoding: .utf8) ?? "{}" + } +} diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift new file mode 100644 index 0000000000..4f73dddd7a --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift @@ -0,0 +1,220 @@ +import Foundation +import TableProBigtableCore +import TableProPluginKit + +extension BigtablePluginDriver { + static let scanPageSize = 100 + static let exportPageSize = 500 + /// Rows skipped per keys-only read when a page has no cursor yet. + static let skipBatchSize = 5_000 + /// Export fixes its columns from the first page; a column only later rows have goes here as + /// JSON rather than being dropped. + static let extraColumn = "__extra__" + + func execute(query: String) async throws -> PluginQueryResult { + let started = Date() + let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.lowercased() == "select 1" { + try await ping() + return PluginQueryResult(columns: ["ok"], columnTypeNames: ["integer"], rows: [[.text("1")]], + rowsAffected: 0, executionTime: elapsed(since: started)) + } + switch try BigtableStatement.parse(trimmed) { + case .browse(let spec): + return try await browse(spec, started: started) + case .request(let request): + return try await execute(request, started: started) + } + } + + // MARK: - Browse + + func browse(_ spec: BigtableBrowseSpec, started: Date = Date()) async throws -> PluginQueryResult { + let client = try requireClient() + let after = try await startKey(for: spec, client: client) + if spec.offset > 0, after == nil { + return result(BigtableRowSet(rows: []), started: started) + } + let body = try BigtableReadRequest.body(spec: spec, limit: spec.limit, after: after?.key) + let rows = try await client.readRows(table: spec.table, body: body) + if let last = rows.last?.key, spec.limit > 0, rows.count == spec.limit { + rememberCursor(view: spec.viewKey, offset: spec.offset + rows.count, key: last) + } + return result(BigtableRowSet(rows: rows), started: started) + } + + /// The key a page starts after. Page 1 needs none; a later page starts from the nearest cursor an + /// earlier page left and skips the rest by reading keys alone, which costs no values. + private func startKey(for spec: BigtableBrowseSpec, client: BigtableClient) async throws -> (key: Data, offset: Int)? { + guard spec.offset > 0 else { return nil } + var position = cursor(view: spec.viewKey, offset: spec.offset) + if position?.offset == spec.offset { return position.map { ($0.key, $0.offset) } } + while (position?.offset ?? 0) < spec.offset { + try Task.checkCancellation() + let start = position?.offset ?? 0 + let step = min(Self.skipBatchSize, spec.offset - start) + let body = try BigtableReadRequest.body(spec: spec, limit: step, after: position?.key, keysOnly: true) + let keys = try await client.readRows(table: spec.table, body: body) + guard let last = keys.last?.key else { return nil } + position = (start + keys.count, last) + rememberCursor(view: spec.viewKey, offset: start + keys.count, key: last) + if keys.count < step { break } + } + guard let position, position.offset == spec.offset else { return nil } + return (position.key, position.offset) + } + + func result(_ rows: BigtableRowSet, started: Date) -> PluginQueryResult { + var result = PluginQueryResult( + columns: rows.columns, + columnTypeNames: rows.columns.map { _ in "bytes" }, + rows: rows.rows.map { $0.map(Self.cell) }, + rowsAffected: 0, + executionTime: elapsed(since: started) + ) + result.rowLocators = rows.locators + result.absentCells = rows.absentCells + return result + } + + static func cell(_ cell: BigtableRowSet.Cell) -> PluginCellValue { + switch cell { + case .absent: return .null + case .text(let text): return .text(text) + case .bytes(let data): return .bytes(data) + } + } + + // MARK: - Request lines + + private func execute(_ request: BigtableRequest, started: Date) async throws -> PluginQueryResult { + let client = try requireClient() + guard let table = request.table else { + throw BigtableError.configuration(String(localized: "Name the table before the key, as in GET events/user#42.")) + } + switch request.verb { + case .get: + let rows = try await client.readRows(table: table, body: [ + "rows": ["rowKeys": [request.key?.base64EncodedString() ?? ""]], + "filter": ["cellsPerColumnLimitFilter": 1] + ]) + return result(BigtableRowSet(rows: rows), started: started) + case .scan: + var filters: [BigtableFilterSpec] = [] + if let key = request.key { + filters.append(BigtableFilterSpec(column: Self.keyColumn, op: "STARTS WITH", value: BigtableBytes.display(key))) + } + let spec = BigtableBrowseSpec(table: table, filters: filters, limit: request.limit ?? Self.scanPageSize, offset: 0) + let rows = try await client.readRows(table: table, body: try BigtableReadRequest.body(spec: spec, limit: spec.limit, after: nil)) + return result(BigtableRowSet(rows: rows), started: started) + case .read: + guard let data = request.body?.data(using: .utf8), + let body = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw BigtableError.configuration(String(localized: "READ needs a ReadRows JSON body, such as READ events {\"rowsLimit\": \"10\"}.")) + } + return result(BigtableRowSet(rows: try await client.readRows(table: table, body: body)), started: started) + case .put: + let mutations = try Self.mutations(fromBody: request.body ?? "{}") + try await client.mutateRow(table: table, key: request.key ?? Data(), mutations: mutations) + forgetCursors(table: table) + return writeResult(table: table, key: request.key, started: started) + case .delete: + let mutation: BigtableMutation = try request.column.map { .deleteColumn(try BigtableColumn(name: $0)) } ?? .deleteRow + try await client.mutateRow(table: table, key: request.key ?? Data(), mutations: [mutation]) + forgetCursors(table: table) + return writeResult(table: table, key: request.key, started: started) + } + } + + /// A `PUT` body maps `family:qualifier` to the new value as text, `b64:` text for bytes, or + /// null to delete that cell. All of it lands in one atomic row mutation. + static func mutations(fromBody body: String) throws -> [BigtableMutation] { + guard let data = body.data(using: .utf8), let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw BigtableError.configuration(String(localized: "PUT needs a JSON object of cells, such as {\"profile:name\": \"Ann\"}.")) + } + return try object.keys.sorted().map { name in + let column = try BigtableColumn(name: name) + switch object[name] { + case is NSNull: + return .deleteColumn(column) + case let text as String: + return .setCell(column, try BigtableBytes.parse(text)) + case let number as NSNumber: + return .setCell(column, Data(number.stringValue.utf8)) + default: + throw BigtableError.configuration(String( + format: String(localized: "The value of %@ must be text or null. Bigtable stores bytes, so a number is written as its digits."), name + )) + } + } + } + + private func writeResult(table: String, key: Data?, started: Date) -> PluginQueryResult { + PluginQueryResult( + columns: ["row"], columnTypeNames: ["bytes"], + rows: [[.text(table + "/" + (key.map(BigtableBytes.display) ?? ""))]], + rowsAffected: 1, executionTime: elapsed(since: started) + ) + } + + // MARK: - Export + + func streamRows(query: String) -> AsyncThrowingStream { + AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + let task = Task { + do { + if case .browse(let spec)? = try? BigtableStatement.parse(query), spec.limit == 0 { + try await self.export(spec, continuation: continuation) + } else { + let result = try await self.execute(query: query) + continuation.yield(.header(PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames, + estimatedRowCount: nil))) + if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { @Sendable _ in task.cancel() } + } + } + + private func export(_ spec: BigtableBrowseSpec, continuation: AsyncThrowingStream.Continuation) async throws { + let client = try requireClient() + var header: [String]? + var after: Data? + repeat { + try Task.checkCancellation() + let body = try BigtableReadRequest.body(spec: spec, limit: Self.exportPageSize, after: after) + let rows = try await client.readRows(table: spec.table, body: body) + let page = BigtableRowSet(rows: rows) + if header == nil { + header = page.columns + [Self.extraColumn] + continuation.yield(.header(PluginStreamHeader( + columns: header ?? [], columnTypeNames: (header ?? []).map { _ in "bytes" }, estimatedRowCount: nil + ))) + } + let fixed = header ?? [] + continuation.yield(.rows(page.rows.map { Self.align($0.map(Self.cell), columns: page.columns, to: fixed) })) + after = rows.count == Self.exportPageSize ? rows.last?.key : nil + } while after != nil + } + + /// Places a page's cells under the export's columns and gathers the rest into `__extra__`. + static func align(_ row: [PluginCellValue], columns: [String], to header: [String]) -> [PluginCellValue] { + var byName: [String: PluginCellValue] = [:] + for (column, value) in zip(columns, row) { byName[column] = value } + var aligned = header.dropLast().map { byName.removeValue(forKey: $0) ?? .null } + let extra = byName.filter { !$0.value.isNull }.mapValues { value -> String in + value.asText ?? (value.asBytes.map { BigtableBytes.base64Prefix + $0.base64EncodedString() } ?? "") + } + if extra.isEmpty { + aligned.append(.null) + } else { + let data = (try? JSONSerialization.data(withJSONObject: extra, options: [.sortedKeys])) ?? Data() + aligned.append(String(bytes: data, encoding: .utf8).map(PluginCellValue.text) ?? .null) + } + return aligned + } +} diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift new file mode 100644 index 0000000000..d27667cf4f --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift @@ -0,0 +1,105 @@ +import Foundation +import TableProBigtableCore +import TableProPluginKit + +extension BigtablePluginDriver { + static let sampleSize = 50 + static let keyColumn = BigtableRowSet.keyColumn + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + try await requireClient().listTables().map { PluginTableInfo(name: $0, type: "TABLE") } + } + + /// Bigtable has no fixed columns, so these are the `family:qualifier` pairs a sample of rows + /// holds. A family with no sampled cell has no column until a row writes one. + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + let sample = try await browse(BigtableBrowseSpec(table: table, limit: Self.sampleSize, offset: 0)) + return sample.columns.map { name in + PluginColumnInfo(name: name, dataType: "bytes", isNullable: name != Self.keyColumn, isPrimaryKey: name == Self.keyColumn) + } + } + + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { + [PluginIndexInfo(name: Self.keyColumn, columns: [Self.keyColumn], isUnique: true, isPrimary: true, type: "PRIMARY KEY")] + } + + func fetchTableDDL(table: String, schema: String?) async throws -> String { + let client = try requireClient() + let families = try await client.columnFamilies(table: table) + let sample = try await browse(BigtableBrowseSpec(table: table, limit: Self.sampleSize, offset: 0)) + let description: [String: Any] = [ + "table": table, + "columnFamilies": families, + "sampledRows": sample.rows.count, + "sampledColumns": Array(sample.columns.dropFirst()) + ] + let data = (try? JSONSerialization.data(withJSONObject: description, options: [.prettyPrinted, .sortedKeys])) ?? Data() + return String(bytes: data, encoding: .utf8) ?? "{}" + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table, engine: "Bigtable") + } + + // MARK: - Browse statements + + func buildBrowseQuery( + table: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int + ) -> String? { + buildFilteredQuery( + table: table, schema: nil, queryFilters: [], logicMode: "AND", + sortColumns: sortColumns, columns: columns, limit: limit, offset: offset, columnKinds: [:] + ) + } + + func buildFilteredQuery( + table: String, + schema: String?, + queryFilters filters: [PluginQueryFilter], + logicMode: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int, + columnKinds: [String: PluginColumnKind] + ) -> String? { + let sort = sortColumns.first.flatMap { sort -> (column: String, ascending: Bool)? in + guard sort.columnIndex >= 0, sort.columnIndex < columns.count else { return nil } + return (columns[sort.columnIndex], sort.ascending) + } + let specs = filters.map { BigtableFilterSpec(column: $0.column, op: $0.op, value: $0.value, secondValue: $0.secondValue) } + return BigtableStatement.encode(BigtableBrowseSpec( + table: table, + filters: specs, + logicMode: logicMode, + descending: sort.map { !$0.ascending } ?? false, + sortColumn: sort?.column, + limit: limit, + offset: offset + )) + } + + /// Bigtable keeps no row count, so this reads every key the filters admit, without values. The + /// app runs it only when asked, since it is a full scan. + func fetchExactRowCount(table: String, schema: String?, queryFilters filters: [PluginQueryFilter], logicMode: String) async throws -> Int? { + let client = try requireClient() + let specs = filters.map { BigtableFilterSpec(column: $0.column, op: $0.op, value: $0.value, secondValue: $0.secondValue) } + let spec = BigtableBrowseSpec(table: table, filters: specs, logicMode: logicMode, limit: 0, offset: 0) + var total = 0 + var after: Data? + repeat { + try Task.checkCancellation() + let body = try BigtableReadRequest.body(spec: spec, limit: Self.countBatchSize, after: after, keysOnly: true) + let keys = try await client.readRows(table: table, body: body) + total += keys.count + after = keys.count == Self.countBatchSize ? keys.last?.key : nil + } while after != nil + return total + } + + static let countBatchSize = 10_000 +} diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift new file mode 100644 index 0000000000..2a73f449de --- /dev/null +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift @@ -0,0 +1,192 @@ +import Foundation +import os +import TableProBigtableCore +import TableProLogRedaction +import TableProPluginKit + +internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Sendable { + static let logger = Logger(subsystem: "com.TablePro", category: "BigtablePluginDriver") + + /// Page cursors of this many views are kept; a view is one table with one set of filters. + static let cursorViewLimit = 64 + + let config: DriverConnectionConfig + let queryTimeout = HttpQueryTimeoutBox() + private let lock = NSLock() + private var client: BigtableClient? + private var transport: URLSessionBigtableTransport? + private var cursors: [String: [Int: Data]] = [:] + private var cursorOrder: [String] = [] + + init(config: DriverConnectionConfig) { + self.config = config + } + + var serverVersion: String? { "Google Cloud Bigtable" } + var capabilities: PluginCapabilities { [.cancelQuery] } + var supportsTransactions: Bool { false } + var parameterStyle: ParameterStyle { .questionMark } + + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + + // MARK: - Lifecycle + + /// The first calls run under the connect deadline; once they answer, the client switches to the + /// query timeout the user set. + func connect() async throws { + let resolved = try BigtableCredentialFactory.resolve(config: config) + let settings = try BigtableConnectionSettings.parse(fields: config.additionalFields, credentialProject: resolved.projectHint) + let connectMilliseconds = PluginConnectTimeout.milliseconds( + in: config.additionalFields, + default: Int(HttpQueryTimeout().requestTimeoutInterval * 1_000) + ) + let deadline = PluginConnectDeadline(milliseconds: connectMilliseconds) + let transport = URLSessionBigtableTransport(resourceTimeout: HttpQueryTimeout.sessionResourceTimeout) + let timeoutBox = queryTimeout + + let probe = BigtableClient( + projectId: settings.projectId, instanceId: settings.instanceId ?? "-", appProfileId: settings.appProfileId, + transport: transport, credential: resolved.credential, timeout: { deadline.remainingSeconds() } + ) + var instanceId = settings.instanceId + if instanceId == nil { + guard let first = try await probe.listInstances().first else { + throw BigtableError.notFound(String(format: String(localized: "Project %@ has no Bigtable instance."), settings.projectId)) + } + instanceId = first + try probe.switchInstance(to: first) + } + try await probe.ping() + + let client = BigtableClient( + projectId: settings.projectId, instanceId: instanceId ?? probe.instanceId, appProfileId: settings.appProfileId, + transport: transport, credential: resolved.credential, timeout: { timeoutBox.requestTimeoutInterval } + ) + lock.withLock { + self.client = client + self.transport = transport + cursors.removeAll() + cursorOrder.removeAll() + } + } + + func disconnect() { + lock.withLock { + transport?.cancelAll() + client = nil + transport = nil + } + } + + func ping() async throws { + try await requireClient().ping() + } + + func cancelQuery() throws { + lock.withLock { transport?.cancelAll() } + } + + func applyQueryTimeout(_ seconds: Int) async throws { + queryTimeout.set(serverTimeoutSeconds: seconds) + } + + func requireClient() throws -> BigtableClient { + guard let client = lock.withLock({ client }) else { throw BigtableError.notConnected } + return client + } + + // MARK: - Instances + + var currentDatabase: String? { + lock.withLock { client?.instanceId } + } + + func fetchDatabases() async throws -> [String] { + let client = try requireClient() + do { + let instances = try await client.listInstances() + return instances.isEmpty ? [client.instanceId] : instances + } catch let error as BigtableError { + Self.logger.info( + "Could not list Bigtable instances: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + return [client.instanceId] + } + } + + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } + + func switchDatabase(to database: String) async throws { + try requireClient().switchInstance(to: database) + lock.withLock { + cursors.removeAll() + cursorOrder.removeAll() + } + } + + // MARK: - Page cursors + + func cursor(view: String, offset: Int) -> (offset: Int, key: Data)? { + lock.withLock { + guard let known = cursors[view] else { return nil } + let best = known.keys.filter { $0 <= offset }.max() + return best.flatMap { start in known[start].map { (start, $0) } } + } + } + + func rememberCursor(view: String, offset: Int, key: Data) { + lock.withLock { + if cursors[view] == nil { + cursorOrder.append(view) + if cursorOrder.count > Self.cursorViewLimit { + cursors[cursorOrder.removeFirst()] = nil + } + } + cursors[view, default: [:]][offset] = key + } + } + + func forgetCursors(table: String) { + lock.withLock { + let prefix = table + "\u{3}" + for view in cursors.keys where view.hasPrefix(prefix) { + cursors[view] = nil + } + cursorOrder.removeAll { $0.hasPrefix(prefix) } + } + } + + // MARK: - Table operations + + func defaultExportQuery(table: String) -> String? { + BigtableStatement.encode(BigtableBrowseSpec(table: table, limit: 0, offset: 0)) + } + + func dropObjectStatement(name: String, objectType: String, schema: String?, cascade: Bool) -> String? { + nil + } + + func truncateTableStatements(table: String, schema: String?, cascade: Bool) -> [String]? { + nil + } + + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + throw BigtableError.unsupported(String(localized: "Bigtable has no views.")) + } + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + [] + } + + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { + nil + } + + func elapsed(since start: Date) -> TimeInterval { + Date().timeIntervalSince(start) + } +} diff --git a/Plugins/BigtableDriverPlugin/Info.plist b/Plugins/BigtableDriverPlugin/Info.plist new file mode 100644 index 0000000000..2c5227ec99 --- /dev/null +++ b/Plugins/BigtableDriverPlugin/Info.plist @@ -0,0 +1,14 @@ + + + + + TableProMinAppVersion + 0.80.0 + TableProPluginKitVersion + 38 + TableProProvidesDatabaseTypeIds + + Bigtable + + + diff --git a/TablePro/Assets.xcassets/bigtable-icon.imageset/Contents.json b/TablePro/Assets.xcassets/bigtable-icon.imageset/Contents.json new file mode 100644 index 0000000000..85f3b0a3bf --- /dev/null +++ b/TablePro/Assets.xcassets/bigtable-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "bigtable.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/TablePro/Assets.xcassets/bigtable-icon.imageset/bigtable.svg b/TablePro/Assets.xcassets/bigtable-icon.imageset/bigtable.svg new file mode 100644 index 0000000000..1367fc5d70 --- /dev/null +++ b/TablePro/Assets.xcassets/bigtable-icon.imageset/bigtable.svg @@ -0,0 +1 @@ +Google Bigtable \ No newline at end of file diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift new file mode 100644 index 0000000000..6e399ed175 --- /dev/null +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift @@ -0,0 +1,124 @@ +import Foundation +import TableProPluginKit + +extension PluginMetadataRegistry { + /// Rows come only in key order and Bigtable keeps no count, so a column sort is the grid's own + /// and a count reads every key, only when asked. + func bigtablePluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { + [ + ("Bigtable", PluginMetadataSnapshot( + displayName: "Google Cloud Bigtable", iconName: "bigtable-icon", defaultPort: 0, + requiresAuthentication: true, supportsForeignKeys: false, supportsSchemaEditing: false, + isDownloadable: true, primaryUrlScheme: "", parameterStyle: .questionMark, + navigationModel: .standard, explainVariants: [], pathFieldRole: .database, + supportsHealthMonitor: true, urlSchemes: [], postConnectActions: [], + brandColorHex: "#4285F4", + queryLanguageName: "REST", editorLanguage: .javascript, + connectionMode: .apiOnly, supportsDatabaseSwitching: true, + capabilities: PluginMetadataSnapshot.CapabilityFlags( + supportsSchemaSwitching: false, + supportsImport: false, + supportsExport: true, + supportsSSH: false, + supportsSSL: false, + supportsCascadeDrop: false, + supportsForeignKeyDisable: false, + supportsReadOnlyMode: true, + supportsQueryProgress: false, + requiresReconnectForDatabaseSwitch: false, + supportsDropDatabase: false, + supportsFieldRemoval: true, + supportsAddColumn: false, + supportsModifyColumn: false, + supportsDropColumn: false, + supportsAddIndex: false, + supportsDropIndex: false, + supportsModifyPrimaryKey: false, + supportsOpportunisticTLS: false, + exactRowCountIsFullScan: true, + supportsColumnSort: false, + columnsAreSampled: true, + supportsPrincipalConnectionLimit: false + ), + schema: PluginMetadataSnapshot.SchemaInfo( + defaultSchemaName: "", + defaultGroupName: "default", + tableEntityName: "Tables", + containerEntityName: "Instance", + defaultPrimaryKeyColumn: "_key", + immutableColumns: ["_key"], + systemDatabaseNames: [], + systemSchemaNames: [], + fileExtensions: [], + databaseGroupingStrategy: .byDatabase, + structureColumnFields: [.name, .type, .nullable] + ), + editor: PluginMetadataSnapshot.EditorConfig( + sqlDialect: nil, + statementCompletions: bigtableCompletions, + columnTypesByCategory: ["Binary": ["bytes"]] + ), + connection: PluginMetadataSnapshot.ConnectionConfig( + additionalConnectionFields: bigtableConnectionFields(), + category: .wideColumn, + tagline: String(localized: "Google Cloud wide-column store for large data"), + hidesBuiltInPassword: true, + hidesBuiltInDatabase: true + ) + )) + ] + } +} + +private let bigtableCompletions: [CompletionEntry] = [ + CompletionEntry(label: "GET row", insertText: "GET events/user#42"), + CompletionEntry(label: "SCAN prefix", insertText: "SCAN events/user#?limit=20"), + CompletionEntry(label: "READ", insertText: "READ events {\"rowsLimit\": \"20\", \"filter\": {\"cellsPerColumnLimitFilter\": 1}}"), + CompletionEntry(label: "PUT cells", insertText: "PUT events/user#42 {\"profile:name\": \"Ann\"}"), + CompletionEntry(label: "DELETE row", insertText: "DELETE events/user#42"), + CompletionEntry(label: "DELETE column", insertText: "DELETE events/user#42 profile:name") +] + +func bigtableConnectionFields() -> [ConnectionField] { + let authMethod = "btAuthMethod" + return [ + ConnectionField( + id: authMethod, + label: String(localized: "Auth Method"), + defaultValue: "serviceAccount", + fieldType: .dropdown(options: [ + .init(value: "serviceAccount", label: "Service Account Key"), + .init(value: "adc", label: "Application Default Credentials") + ]), + section: .authentication + ), + ConnectionField( + id: "btServiceAccountJson", + label: String(localized: "Service Account Key"), + placeholder: "File path or paste JSON", + required: true, + fieldType: .secure, + section: .authentication, + hidesPassword: true, + visibleWhen: FieldVisibilityRule(fieldId: authMethod, values: ["serviceAccount"]) + ).withHidesUsername(true), + ConnectionField( + id: "btProjectId", + label: String(localized: "Project ID"), + placeholder: "Taken from the key when empty", + section: .authentication + ), + ConnectionField( + id: "btInstanceId", + label: String(localized: "Instance ID"), + placeholder: "The first instance when empty", + section: .authentication + ), + ConnectionField( + id: "btAppProfileId", + label: String(localized: "App Profile ID"), + placeholder: "default", + section: .advanced + ) + ] +} diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 05e19de268..b73471a73a 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -1149,7 +1149,7 @@ extension PluginMetadataRegistry { + duckdbPluginDefaults(dialect: duckdbDialect, columnTypes: duckdbColumnTypes) + cloudPluginDefaults() + elasticsearchPluginDefaults() + surrealDBPluginDefaults() + kafkaPluginDefaults() + typesensePluginDefaults() + r2SQLPluginDefaults() - + weaviatePluginDefaults() + hanaPluginDefaults() + firebasePluginDefaults() + + weaviatePluginDefaults() + hanaPluginDefaults() + firebasePluginDefaults() + bigtablePluginDefaults() } // swiftlint:enable function_body_length } diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/TablePro/Core/Plugins/PluginMetadataRegistry.swift index 27015e7fe5..7b86c79f13 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -829,7 +829,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { return .document case "Redis": return .keyValue - case "Cassandra", "ScyllaDB": + case "Cassandra", "ScyllaDB", "Bigtable": return .wideColumn case "etcd", "Etcd": return .coordination @@ -869,6 +869,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { case "Typesense": return String(localized: "Typo-tolerant open-source search engine") case "Weaviate": return String(localized: "Open-source vector database") case "Firestore": return String(localized: "Serverless document database from Google") + case "Bigtable": return String(localized: "Google Cloud wide-column store for large data") case "Firebase Realtime Database": return String(localized: "JSON tree database that syncs in real time") default: return "" } diff --git a/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift b/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift index 081fca3892..81d0b4b02e 100644 --- a/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift +++ b/TablePro/Core/Utilities/SQL/CatalogChangeClassifier.swift @@ -159,7 +159,7 @@ enum CatalogChangeClassifier { } private static let documentStoreTypes: Set = [ - .mongodb, .elasticsearch, .typesense, .weaviate, .etcd, .firestore, .firebaseRealtimeDatabase + .mongodb, .elasticsearch, .typesense, .weaviate, .etcd, .firestore, .firebaseRealtimeDatabase, .bigtable ] private static let transactionEndKeywords: Set = ["COMMIT", "END", "ROLLBACK", "ABORT"] diff --git a/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/TablePro/Core/Utilities/SQL/QueryClassifier.swift index a4bf3b2f74..e3fd46eb34 100644 --- a/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -859,6 +859,8 @@ private extension QueryClassifier { return trimmed.isEmpty ? .safe : weaviateClassification(trimmed) case .firestore, .firebaseRealtimeDatabase: return trimmed.isEmpty ? .safe : firebaseClassification(trimmed, databaseType: databaseType) + case .bigtable: + return trimmed.isEmpty ? .safe : bigtableClassification(trimmed) default: return nil } @@ -1089,4 +1091,35 @@ private extension QueryClassifier { return false } } + + /// A grid page is a tagged spec and anything else a request line led by its verb. A PUT that sets + /// a cell to null deletes that cell, so it ranks with DELETE. + static func bigtableClassification(_ trimmed: String) -> QueryClassification { + if trimmed.hasPrefix("BIGTABLE_BROWSE:") { + return .safe + } + let destructive = QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) + let verb = trimmed.prefix { !$0.isWhitespace }.uppercased() + switch verb { + case "GET", "SCAN", "READ": + return .safe + case "DELETE": + return destructive + case "PUT" where bigtableBodyDeletesCell(trimmed): + return destructive + default: + return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + } + } + + /// The body is the JSON object that runs to the end of the text. A quoted row key may itself + /// hold a brace, so each brace is tried until one starts an object that parses. + private static func bigtableBodyDeletesCell(_ trimmed: String) -> Bool { + for brace in trimmed.indices where trimmed[brace] == "{" { + guard let data = String(trimmed[brace...]).data(using: .utf8), + let cells = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { continue } + return cells.values.contains { $0 is NSNull } + } + return false + } } diff --git a/TablePro/Models/Connection/DatabaseType.swift b/TablePro/Models/Connection/DatabaseType.swift index 0cb79fe8fb..ce72173b7d 100644 --- a/TablePro/Models/Connection/DatabaseType.swift +++ b/TablePro/Models/Connection/DatabaseType.swift @@ -51,6 +51,7 @@ extension DatabaseType { static let teradata = DatabaseType(rawValue: "Teradata") static let trino = DatabaseType(rawValue: "Trino") static let weaviate = DatabaseType(rawValue: "Weaviate") + static let bigtable = DatabaseType(rawValue: "Bigtable") static let firestore = DatabaseType(rawValue: "Firestore") static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") static let kafka = DatabaseType(rawValue: "Kafka") diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index d17f021e34..ba95eb9782 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -784,10 +784,22 @@ } } }, + "\"%@\" is not a Bigtable instance ID." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "\"%@\" is not a Bigtable table ID." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "\"%@\" is not a collection path." : { "extractionState" : "manual", "generatesSymbol" : false }, + "\"%@\" is not a column. Name it as family:qualifier." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "\"%@\" is not a DynamoDB action this editor runs" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -924,6 +936,10 @@ }, "\"%@\" is not a valid theme identifier." : { + }, + "\"%@\" is not an app profile ID." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "\"%@\" is not an AWS region" : { "extractionState" : "manual", @@ -953,6 +969,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "\"%@\" is not valid base64." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "“%@” is still in use" : { "localizations" : { "fr" : { @@ -4236,6 +4256,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "%@ needs a row key." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%@ needs a table. Pass 'table'." : { "localizations" : { "fr" : { @@ -15393,6 +15417,10 @@ } } }, + "A key range cannot be ORed with a column filter. Use =, STARTS WITH or REGEX on _key, or AND." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A Kubernetes resource is required, such as service/postgres" : { "localizations" : { "fr" : { @@ -15565,6 +15593,14 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "A new row needs a _key. Bigtable has no generated keys." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "A new row needs at least one cell. Bigtable does not store empty rows." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A new version downloads in the background and installs the next time you quit TablePro." : { "localizations" : { "fr" : { @@ -15837,6 +15873,14 @@ } } }, + "A quoted row key is missing its closing quote." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "A quoted row key is not a valid JSON string." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A quoted value was never closed." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -15849,6 +15893,10 @@ } } }, + "A raw Bigtable filter is a JSON RowFilter, such as {\"rowKeyRegexFilter\": \"\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A raw Firestore filter is a JSON Filter object, such as %@." : { "extractionState" : "manual", "generatesSymbol" : false @@ -15901,6 +15949,10 @@ } } }, + "A row needs a key. Bigtable has no empty row key." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A row the filter hides holds %@ in column “%@”, which %@ cannot store. Clear the filter to find it, or choose Save As and pick UTF-8." : { "localizations" : { "fr" : { @@ -24922,6 +24974,10 @@ } } }, + "App Profile ID" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "app_reader" : { "localizations" : { "fr" : { @@ -30409,6 +30465,42 @@ } } }, + "Bigtable answered with something other than HTTP." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable cannot filter with %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable committed a row it never started." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable ended the response in the middle of a row." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable has no views." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable returned HTTP %d." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable sent a response that is not JSON." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable sent bytes that are not base64." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Bigtable started a new row before committing the last one." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Billing" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -69936,6 +70028,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Enter a request such as GET events/user#42, SCAN events/user#?limit=20, PUT events/user#42 {\"profile:name\": \"Ann\"} or DELETE events/user#42." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Enter a value for the primary key column %@." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -84594,6 +84690,10 @@ } } }, + "Google refused the credential." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Google rejected the OAuth client ID or secret." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -100535,6 +100635,10 @@ } } }, + "list instances" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "List Maintenance Operations" : { "localizations" : { "fr" : { @@ -101339,6 +101443,10 @@ } } }, + "list the tables of %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "List the tables the user opened most recently on a connection, newest first." : { "localizations" : { "fr" : { @@ -109840,6 +109948,10 @@ } } }, + "Name the table before the key, as in GET events/user#42." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Name this selection" : { "localizations" : { "fr" : { @@ -118792,6 +118904,10 @@ } } }, + "Not connected to Bigtable." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Not connected to database" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -134944,6 +135060,10 @@ } } }, + "Project %@ has no Bigtable instance." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Project ID" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -135946,6 +136066,14 @@ } } }, + "PUT needs a JSON object of cells, such as {\"profile:name\": \"Ann\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "PUT needs a JSON object of cells, such as PUT events/user#42 {\"profile:name\": \"Ann\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Put the credential in the Token field, not in the address." : { "localizations" : { "fr" : { @@ -139087,6 +139215,10 @@ } } }, + "READ needs a ReadRows JSON body, such as READ events {\"rowsLimit\": \"10\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Read Only" : { "localizations" : { "fr" : { @@ -139380,6 +139512,10 @@ } } }, + "read the schema of %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Read-only" : { "extractionState" : "stale", "localizations" : { @@ -179530,6 +179666,10 @@ } } }, + "The Bigtable REST API returns rows in ascending key order only." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The binary subtype of %@ is not known, so saving could store a different kind of binary. Change this field with a query." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -179810,6 +179950,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The browse request is damaged. Reopen the table." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The bundled sample database is missing from the app." : { "localizations" : { "fr" : { @@ -185211,6 +185355,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The row has no _key, so the row it came from is unknown. Reload the table." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The row has no $key, so the node it came from is unknown. Reload the table." : { "extractionState" : "manual", "generatesSymbol" : false @@ -188678,6 +188826,10 @@ } } }, + "The value of %@ must be text or null. Bigtable stores bytes, so a number is written as its digits." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The value of a %@ key cannot be edited in the grid. Change it with a command in the query editor." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -209638,6 +209790,10 @@ } } }, + "write a row of %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Write and Run SQL" : { "localizations" : { "fr" : { diff --git a/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/Contents.json b/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/Contents.json new file mode 100644 index 0000000000..85f3b0a3bf --- /dev/null +++ b/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "bigtable.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/bigtable.svg b/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/bigtable.svg new file mode 100644 index 0000000000..1367fc5d70 --- /dev/null +++ b/TableProMobile/TableProMobile/Assets.xcassets/bigtable-icon.imageset/bigtable.svg @@ -0,0 +1 @@ +Google Bigtable \ No newline at end of file diff --git a/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/Contents.json b/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/Contents.json new file mode 100644 index 0000000000..85f3b0a3bf --- /dev/null +++ b/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "bigtable.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/bigtable.svg b/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/bigtable.svg new file mode 100644 index 0000000000..1367fc5d70 --- /dev/null +++ b/TableProMobile/TableProWidget/Assets.xcassets/bigtable-icon.imageset/bigtable.svg @@ -0,0 +1 @@ +Google Bigtable \ No newline at end of file diff --git a/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift b/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift index e549d93fce..c5b464367b 100644 --- a/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift +++ b/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift @@ -11,7 +11,7 @@ import Testing /// this registry, `snippets/driver-counts.mdx` in TableProApp/docs, and the marketing site. Nothing at runtime /// reconciles them, and by August 2026 they read 28, 27 and 25 at once. /// -/// The answer is 39, and the reason it once read 28 is worth keeping. Turso is served by +/// The answer is 40, and the reason it once read 28 is worth keeping. Turso is served by /// the libSQL plugin and was the only alias in `reverseTypeIndex` with no curated entry of its /// own, so it was the only type the picker could not offer before its plugin was installed. /// ScyllaDB is the shape every other alias already had: an alias of Cassandra with a curated @@ -29,7 +29,7 @@ struct PluginMetadataRegistryTypeCountTests { "CockroachDB", "Dameng", "Databend", "DuckDB", "DynamoDB", "Elasticsearch", "etcd", "Kafka", "libSQL", "MariaDB", "MongoDB", "MySQL", "Oracle", "PGlite", "PostgreSQL", "Redis", "Redshift", "SAP HANA", "ScyllaDB", "Snowflake", "Spanner", "SQL Server", "SQLite", "SurrealDB", "Teradata", "TiDB", - "Trino", "Turso", "Typesense", "Weaviate", "OceanBase", "Firestore", "Firebase Realtime Database" + "Trino", "Turso", "Typesense", "Weaviate", "OceanBase", "Firestore", "Firebase Realtime Database", "Bigtable" ] private static func builtInTypeIds() -> Set { diff --git a/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift b/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift index ef1632788c..6d44d13d70 100644 --- a/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift +++ b/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift @@ -152,6 +152,13 @@ struct CatalogChangeClassifierTests { #expect(kinds("db.dropDatabase()", .mongodb) == .everything) } + @Test("A Bigtable write refreshes the columns, since it can add one") + func bigtableWritesRefresh() { + #expect(kinds("SCAN events/user#", .bigtable).isEmpty) + #expect(kinds("BIGTABLE_BROWSE:e30=", .bigtable).isEmpty) + #expect(kinds("PUT events/k {\"cf:new\": \"1\"}", .bigtable) == .everything) + } + @Test("A Firebase write refreshes the sidebar, since it can create a collection or a node") func firebaseWritesRefresh() { for type in [DatabaseType.firestore, .firebaseRealtimeDatabase] { diff --git a/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 3cb1445390..980c62bffe 100644 --- a/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -690,6 +690,19 @@ struct QueryClassifierNonSqlTests { #expect(!QueryClassifier.isWriteQuery(search, databaseType: .weaviate)) } + @Test("Bigtable request lines are tiered by their verb, and a PUT that nulls a cell is destructive") + func bigtableTiers() { + #expect(!QueryClassifier.isWriteQuery("BIGTABLE_BROWSE:e30=", databaseType: .bigtable)) + #expect(!QueryClassifier.isWriteQuery("GET events/user#42", databaseType: .bigtable)) + #expect(!QueryClassifier.isWriteQuery("scan events/user#?limit=5", databaseType: .bigtable)) + #expect(!QueryClassifier.isWriteQuery("READ events {\"rowsLimit\": \"5\"}", databaseType: .bigtable)) + #expect(QueryClassifier.classifyTier("PUT events/k {\"cf:a\": \"1\"}", databaseType: .bigtable) == .write) + #expect(QueryClassifier.classifyTier("PUT events/k {\"cf:a\": null}", databaseType: .bigtable) == .destructive) + #expect(QueryClassifier.classifyTier("PUT events/\"a{b\" {\"cf:a\": null}", databaseType: .bigtable) == .destructive) + #expect(QueryClassifier.classifyTier("DELETE events/k", databaseType: .bigtable) == .destructive) + #expect(QueryClassifier.isWriteQuery("SELECT * FROM events", databaseType: .bigtable)) + } + @Test("Firebase request lines are tiered by their verb, and a grid page is a read") func firebaseTiers() { for type in [DatabaseType.firestore, .firebaseRealtimeDatabase] { diff --git a/project.yml b/project.yml index 949c4709c7..9638584284 100644 --- a/project.yml +++ b/project.yml @@ -1288,6 +1288,19 @@ targets: - package: TableProCore product: TableProDocumentPath + BigtableDriverPlugin: + templates: [DriverPlugin] + templateAttributes: + folder: BigtableDriverPlugin + principalClass: BigtablePlugin + dependencies: + - package: TableProCore + product: TableProGoogleCloud + - package: TableProCore + product: TableProBigtableCore + - package: TableProCore + product: TableProLogRedaction + FirebaseDriverPlugin: templates: [DriverPlugin] templateAttributes: @@ -1317,6 +1330,7 @@ aggregateTargets: targets: - BeancountDriver - BigQueryDriverPlugin + - BigtableDriverPlugin - CSVExport - CSVImport - CassandraDriver From 1a3ebef3667e476c513b05a11434e0575c597fc4 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 15 Oct 2026 10:04:07 +0700 Subject: [PATCH 2/3] refactor(plugin-bigtable): fix review findings, share the Google REST transport --- Packages/TableProCore/Package.swift | 6 +- .../Config/BigtableConnectionSettings.swift | 36 +- .../Data/BigtableRow.swift | 10 +- .../Data/BigtableRowSet.swift | 40 +- .../Data/BigtableRowWrites.swift | 62 +++ .../Query/BigtableBrowseSpec.swift | 64 +++ .../Query/BigtableKeyBounds.swift | 75 ++++ .../Query/BigtableReadRequest.swift | 377 ++++-------------- .../Query/BigtableRowFilter.swift | 128 ++++++ .../Wire/BigtableBytes.swift | 27 +- .../Wire/BigtableClient.swift | 128 ++++-- .../Wire/BigtableError.swift | 28 +- .../Wire/BigtableStatement.swift | 216 ++++++---- .../Wire/BigtableTransport.swift | 184 --------- .../SQLDDLFallbackPolicy.swift | 1 + .../Firestore/FirestoreClient.swift | 11 +- .../RealtimeDatabaseClient.swift | 17 +- .../Wire/FirebaseTransport.swift | 208 +--------- .../REST/GoogleRESTRequester.swift | 106 +++++ .../REST/GoogleRESTTransport.swift | 122 ++++++ .../SQLLexicalProfile.swift | 3 + .../BigtableClientTests.swift | 113 ++++++ .../BigtableCoreTests.swift | 234 ----------- .../BigtableReadRequestTests.swift | 148 +++++++ .../BigtableRowTests.swift | 122 ++++++ .../BigtableStatementTests.swift | 135 +++++++ .../BigtableTestSupport.swift | 45 +++ .../GoogleRESTRequesterTests.swift | 100 +++++ .../SQLLexicalReadingsTests.swift | 1 + .../BigtableCredentialFactory.swift | 51 --- .../BigtableDriverPlugin/BigtablePlugin.swift | 4 +- .../BigtablePluginDriver+Editing.swift | 86 ++-- .../BigtablePluginDriver+Execution.swift | 168 ++++---- .../BigtablePluginDriver+Metadata.swift | 37 +- .../BigtablePluginDriver.swift | 44 +- Plugins/BigtableDriverPlugin/Info.plist | 2 +- .../FirebaseCredentialFactory.swift | 32 +- .../FirebasePluginDriver.swift | 15 +- ...ginMetadataRegistry+BigtableDefaults.swift | 6 +- .../Core/Utilities/SQL/QueryClassifier.swift | 14 +- TablePro/Resources/Localizable.xcstrings | 64 ++- .../DataChangeManagerTests.swift | 14 + .../SQL/QueryClassifierHardeningTests.swift | 1 + .../Database/SQLDDLFallbackPolicyTests.swift | 2 +- 44 files changed, 1889 insertions(+), 1398 deletions(-) create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableBrowseSpec.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableKeyBounds.swift create mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableRowFilter.swift delete mode 100644 Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableTransport.swift create mode 100644 Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTRequester.swift create mode 100644 Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTTransport.swift create mode 100644 Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableClientTests.swift delete mode 100644 Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableCoreTests.swift create mode 100644 Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableReadRequestTests.swift create mode 100644 Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift create mode 100644 Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift create mode 100644 Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableTestSupport.swift create mode 100644 Packages/TableProCore/Tests/TableProGoogleCloudTests/GoogleRESTRequesterTests.swift delete mode 100644 Plugins/BigtableDriverPlugin/BigtableCredentialFactory.swift diff --git a/Packages/TableProCore/Package.swift b/Packages/TableProCore/Package.swift index c08ddf82b9..65ee8c159a 100644 --- a/Packages/TableProCore/Package.swift +++ b/Packages/TableProCore/Package.swift @@ -171,12 +171,12 @@ let package = Package( ), .target( name: "TableProBigtableCore", - dependencies: [], + dependencies: ["TableProGoogleCloud"], path: "Sources/TableProBigtableCore" ), .target( name: "TableProFirebaseCore", - dependencies: [], + dependencies: ["TableProGoogleCloud"], path: "Sources/TableProFirebaseCore" ), .target( @@ -240,7 +240,7 @@ let package = Package( ), .testTarget( name: "TableProBigtableCoreTests", - dependencies: ["TableProBigtableCore"], + dependencies: ["TableProBigtableCore", "TableProGoogleCloud"], path: "Tests/TableProBigtableCoreTests" ), .testTarget( diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift index 001f75d5bc..b4c9b94d0c 100644 --- a/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Config/BigtableConnectionSettings.swift @@ -1,4 +1,5 @@ import Foundation +import TableProGoogleCloud public enum BigtableFieldID { public static let authMethod = "btAuthMethod" @@ -14,30 +15,30 @@ public enum BigtableAuthMethod: String, Sendable, CaseIterable { } public struct BigtableConnectionSettings: Sendable, Equatable { - public let authMethod: BigtableAuthMethod public let projectId: String public let instanceId: String? public let appProfileId: String? - public init(authMethod: BigtableAuthMethod, projectId: String, instanceId: String?, appProfileId: String? = nil) { - self.authMethod = authMethod + public init(projectId: String, instanceId: String?, appProfileId: String? = nil) { self.projectId = projectId self.instanceId = instanceId self.appProfileId = appProfileId } - /// `credentialProject` is the project the key or Application Default Credentials name, used - /// when the Project ID field is empty. With no Instance ID the driver opens the project's only - /// instance, or the first one, and lists the rest in the database switcher. - public static func parse(fields: [String: String], credentialProject: String? = nil) throws -> BigtableConnectionSettings { - let authMethod = trimmed(fields[BigtableFieldID.authMethod]).flatMap(BigtableAuthMethod.init(rawValue:)) ?? .serviceAccount + /// `database` wins over the Instance ID field: a pooled connection the app opens after a switch + /// names the instance it switched to there and never calls `switchDatabase`. + public static func parse( + fields: [String: String], + database: String? = nil, + credentialProject: String? = nil + ) throws -> BigtableConnectionSettings { guard let projectId = trimmed(fields[BigtableFieldID.projectId]) ?? credentialProject else { throw BigtableError.configuration(String(localized: "Enter a Project ID, or use a service account key that names one.")) } guard isValidProjectId(projectId) else { throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Google Cloud project ID."), projectId)) } - let instanceId = trimmed(fields[BigtableFieldID.instanceId]) + let instanceId = trimmed(database) ?? trimmed(fields[BigtableFieldID.instanceId]) if let instanceId, !isValidResourceId(instanceId) { throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a Bigtable instance ID."), instanceId)) } @@ -45,14 +46,27 @@ public struct BigtableConnectionSettings: Sendable, Equatable { if let appProfileId, !isValidResourceId(appProfileId) { throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not an app profile ID."), appProfileId)) } - return BigtableConnectionSettings(authMethod: authMethod, projectId: projectId, instanceId: instanceId, appProfileId: appProfileId) + return BigtableConnectionSettings(projectId: projectId, instanceId: instanceId, appProfileId: appProfileId) + } + + public static func credentialSource(fields: [String: String], password: String?) throws -> GoogleCredentialSource { + let method = trimmed(fields[BigtableFieldID.authMethod]).flatMap(BigtableAuthMethod.init(rawValue:)) ?? .serviceAccount + switch method { + case .applicationDefault: + return .applicationDefault + case .serviceAccount: + guard let key = trimmed(fields[BigtableFieldID.serviceAccountJson]) ?? trimmed(password) else { + throw BigtableError.configuration(String(localized: "Choose a service account key file, or paste its JSON.")) + } + return .serviceAccountKey(key) + } } static func isValidProjectId(_ text: String) -> Bool { text.range(of: #"^[a-z][a-z0-9.:-]{2,62}$"#, options: .regularExpression) != nil } - /// Instance, table and app profile IDs all go into the request path, so anything outside the + /// Instance, table and app profile IDs go into the request path, so anything outside the /// characters Bigtable allows is refused before it can change the path. static func isValidResourceId(_ text: String) -> Bool { text.range(of: #"^[A-Za-z0-9_][A-Za-z0-9_.-]{0,49}$"#, options: .regularExpression) != nil diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift index 113f431376..d96c5be4f0 100644 --- a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift @@ -40,7 +40,7 @@ public struct BigtableCell: Equatable, Sendable { } } -/// A row with the newest version of each cell. Older versions are filtered out on the server. +/// A row with the newest version of each cell. public struct BigtableRow: Equatable, Sendable { public let key: Data public let cells: [BigtableColumn: BigtableCell] @@ -66,10 +66,13 @@ public struct BigtableChunkAssembler { public init() {} - public static func rows(fromResponse json: Any) throws -> [BigtableRow] { + public static func rows(fromResponse json: Any, operation: String) throws -> [BigtableRow] { let messages = json as? [[String: Any]] ?? (json as? [String: Any]).map { [$0] } ?? [] var assembler = BigtableChunkAssembler() for message in messages { + if let error = message["error"] as? [String: Any] { + throw BigtableError.from(streamError: error, operation: operation) + } for chunk in message["chunks"] as? [[String: Any]] ?? [] { try assembler.add(chunk) } @@ -103,9 +106,8 @@ public struct BigtableChunkAssembler { if let encoded = chunk["value"] as? String { value.append(try BigtableBytes.decodeBase64(encoded)) } - let hasCell = family != nil && qualifier != nil inCell = (Self.int64(chunk["valueSize"]) ?? 0) > 0 - if hasCell, !inCell, let family, let qualifier { + if !inCell, let family, let qualifier { let column = BigtableColumn(family: family, qualifier: qualifier) if cells[column].map({ $0.timestampMicros < timestamp }) ?? true { cells[column] = BigtableCell(value: value, timestampMicros: timestamp) diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift index 57eb156d19..d2e4090875 100644 --- a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowSet.swift @@ -1,9 +1,10 @@ import Foundation /// Rows laid out for the grid: `_key` first, then one column per `family:qualifier` any row on the -/// page holds, in family then qualifier order. A cell a row does not have is reported absent. +/// page holds, in family then qualifier order. public struct BigtableRowSet: Sendable, Equatable { public static let keyColumn = "_key" + public static let extraColumn = "__extra__" public enum Cell: Sendable, Equatable { case absent @@ -39,12 +40,39 @@ public struct BigtableRowSet: Sendable, Equatable { self.absentCells = absent } - /// A value that is not text goes to the grid as bytes, so it shows as hex and an edit of - /// another cell in the row cannot rewrite it as a string. + /// A value that is not text, or text a write would read back as base64, goes to the grid as + /// bytes, so writing it back reproduces the same bytes. static func cell(_ value: Data) -> Cell { - if let text = String(data: value, encoding: .utf8), BigtableBytes.isPrintable(text) { - return .text(text) + BigtableBytes.printableText(value).map(Cell.text) ?? .bytes(value) + } + + /// The page under `header`, the columns an export fixed from its first page, with `__extra__` + /// last. Cells of any column the header lacks go into `__extra__` as one JSON object. + public func rows(alignedTo header: [String]) -> [[Cell]] { + rows.map { row in + var byName: [String: Cell] = [:] + for (column, cell) in zip(columns, row) where cell != .absent { + byName[column] = cell + } + var aligned = header.filter { $0 != Self.extraColumn }.map { byName.removeValue(forKey: $0) ?? .absent } + aligned.append(Self.extraCell(byName)) + return aligned + } + } + + private static func extraCell(_ cells: [String: Cell]) -> Cell { + let extra = cells.compactMapValues { cell -> String? in + switch cell { + case .absent: return nil + case .text(let text): return text + case .bytes(let data): return BigtableBytes.base64Prefix + data.base64EncodedString() + } + } + guard !extra.isEmpty, + let data = try? JSONSerialization.data(withJSONObject: extra, options: [.sortedKeys, .withoutEscapingSlashes]), + let text = String(data: data, encoding: .utf8) else { + return .absent } - return .bytes(value) + return .text(text) } } diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift new file mode 100644 index 0000000000..6aef52ba87 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift @@ -0,0 +1,62 @@ +import Foundation + +/// The request lines a grid save sends, one atomic row mutation per changed row. `.absent` is a +/// cell the row has none of: left out of a new row, deleted from an existing one. +public enum BigtableRowWrites { + public static func insert(table: String, columns: [String], values: [BigtableRowSet.Cell]) throws -> BigtableRequest { + var key: Data? + var cells: [String: Any] = [:] + for (column, value) in zip(columns, values) { + if column == BigtableRowSet.keyColumn { + key = try self.key(value) + } else if let text = text(value) { + cells[column] = text + } + } + guard let key else { + throw BigtableError.configuration(String(localized: "Enter a _key for the new row. Bigtable has no generated keys.")) + } + guard !cells.isEmpty else { + throw BigtableError.configuration(String(localized: "A new row needs at least one cell. Bigtable does not store empty rows.")) + } + return BigtableRequest(verb: .insert, table: table, key: key, body: try body(cells)) + } + + public static func update(table: String, key: Data, changes: [(column: String, value: BigtableRowSet.Cell)]) throws -> BigtableRequest? { + guard !changes.contains(where: { $0.column == BigtableRowSet.keyColumn }) else { + throw BigtableError.unsupported(String(localized: "A row key cannot change. Duplicate the row, give the copy a new _key, then delete this one.")) + } + guard !changes.isEmpty else { return nil } + var cells: [String: Any] = [:] + for change in changes { + cells[change.column] = text(change.value) ?? NSNull() + } + return BigtableRequest(verb: .put, table: table, key: key, body: try body(cells)) + } + + public static func delete(table: String, key: Data) -> BigtableRequest { + BigtableRequest(verb: .delete, table: table, key: key) + } + + public static func key(_ cell: BigtableRowSet.Cell) throws -> Data? { + switch cell { + case .absent: return nil + case .bytes(let data): return data.isEmpty ? nil : data + case .text(let text): return text.isEmpty ? nil : try BigtableBytes.parse(text) + } + } + + /// Bytes go back as `b64:` text, which a write reads back to the same bytes. + private static func text(_ cell: BigtableRowSet.Cell) -> String? { + switch cell { + case .absent: return nil + case .text(let text): return text + case .bytes(let data): return BigtableBytes.base64Prefix + data.base64EncodedString() + } + } + + private static func body(_ cells: [String: Any]) throws -> String { + let data = try JSONSerialization.data(withJSONObject: cells, options: [.sortedKeys, .withoutEscapingSlashes]) + return String(bytes: data, encoding: .utf8) ?? "{}" + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableBrowseSpec.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableBrowseSpec.swift new file mode 100644 index 0000000000..366ca0ba8a --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableBrowseSpec.swift @@ -0,0 +1,64 @@ +import Foundation + +public struct BigtableFilterSpec: Codable, Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + public let isCaseSensitive: Bool + + public init(column: String, op: String, value: String, secondValue: String? = nil, isCaseSensitive: Bool = true) { + self.column = column + self.op = op.uppercased() + self.value = value + self.secondValue = secondValue + self.isCaseSensitive = isCaseSensitive + } +} + +/// A page of one table as the grid asked for it. Bigtable reads rows in key order and has no +/// offset, so a later page starts after the last key of the page before it. +public struct BigtableBrowseSpec: Codable, Sendable, Equatable { + public let table: String + public let filters: [BigtableFilterSpec] + public let logicMode: String + public let descending: Bool + /// Kept so a sort Bigtable cannot do is refused rather than silently ignored. + public let sortColumn: String? + public let limit: Int + public let offset: Int + + public init( + table: String, + filters: [BigtableFilterSpec] = [], + logicMode: String = "AND", + descending: Bool = false, + sortColumn: String? = nil, + limit: Int, + offset: Int + ) { + self.table = table + self.filters = filters + self.logicMode = logicMode + self.descending = descending + self.sortColumn = sortColumn + self.limit = limit + self.offset = offset + } + + public var usesOr: Bool { + logicMode.uppercased() == "OR" + } + + /// The spec without its page, so pages of the same view share one cursor cache. + public var viewKey: String { + let filterText = filters.map { + [$0.column, $0.op, $0.value, $0.secondValue ?? "", $0.isCaseSensitive ? "cs" : "ci"].joined(separator: "\u{1}") + }.joined(separator: "\u{2}") + return [table, logicMode.uppercased(), descending ? "desc" : "asc", filterText].joined(separator: "\u{3}") + } + + public static func viewKeyPrefix(table: String) -> String { + table + "\u{3}" + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableKeyBounds.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableKeyBounds.swift new file mode 100644 index 0000000000..54c43424fe --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableKeyBounds.swift @@ -0,0 +1,75 @@ +import Foundation + +/// The row keys a read may return: an interval, or a list of exact keys inside it. +struct BigtableKeyBounds { + struct Bound { + let key: Data + let isClosed: Bool + } + + var start: Bound? + var end: Bound? + var exact: [Data]? + + var isUnbounded: Bool { + start == nil && end == nil && exact == nil + } + + var isEmpty: Bool { + if let exact { + return !exact.contains(where: admits) + } + guard let start, let end else { return false } + if end.key.lexicographicallyPrecedes(start.key) { return true } + return start.key == end.key && !(start.isClosed && end.isClosed) + } + + mutating func raiseStart(_ key: Data, closed: Bool) { + guard let current = start else { + start = Bound(key: key, isClosed: closed) + return + } + if current.key.lexicographicallyPrecedes(key) { + start = Bound(key: key, isClosed: closed) + } else if current.key == key { + start = Bound(key: key, isClosed: closed && current.isClosed) + } + } + + mutating func lowerEnd(_ key: Data, closed: Bool) { + guard let current = end else { + end = Bound(key: key, isClosed: closed) + return + } + if key.lexicographicallyPrecedes(current.key) { + end = Bound(key: key, isClosed: closed) + } else if current.key == key { + end = Bound(key: key, isClosed: closed && current.isClosed) + } + } + + mutating func restrict(to keys: [Data]) { + exact = exact.map { existing in existing.filter(keys.contains) } ?? keys + } + + func admits(_ key: Data) -> Bool { + if let start { + if key.lexicographicallyPrecedes(start.key) || (key == start.key && !start.isClosed) { return false } + } + if let end { + if end.key.lexicographicallyPrecedes(key) || (key == end.key && !end.isClosed) { return false } + } + return true + } + + var admittedKeys: [Data] { + exact?.filter(admits) ?? [] + } + + var rangeJSON: [String: Any] { + var range: [String: Any] = [:] + if let start { range[start.isClosed ? "startKeyClosed" : "startKeyOpen"] = start.key.base64EncodedString() } + if let end { range[end.isClosed ? "endKeyClosed" : "endKeyOpen"] = end.key.base64EncodedString() } + return range + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift index 9937a447e0..3a080be2f3 100644 --- a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableReadRequest.swift @@ -1,347 +1,136 @@ import Foundation -public struct BigtableFilterSpec: Codable, Sendable, Equatable { - public let column: String - public let op: String - public let value: String - public let secondValue: String? - - public init(column: String, op: String, value: String, secondValue: String? = nil) { - self.column = column - self.op = op - self.value = value - self.secondValue = secondValue - } -} - -/// A page of one table as the grid asked for it. `after` is the last key of the page before, which -/// is how a page starts: Bigtable reads rows in key order and has no offset. -public struct BigtableBrowseSpec: Codable, Sendable, Equatable { - public let table: String - public let filters: [BigtableFilterSpec] - public let logicMode: String - public let descending: Bool - /// The column the grid sorted by, kept so a sort Bigtable cannot do is refused rather than - /// silently ignored. Rows only come in key order. - public let sortColumn: String? - public let limit: Int - public let offset: Int - - public init( - table: String, - filters: [BigtableFilterSpec] = [], - logicMode: String = "AND", - descending: Bool = false, - sortColumn: String? = nil, - limit: Int, - offset: Int - ) { - self.table = table - self.filters = filters - self.logicMode = logicMode - self.descending = descending - self.sortColumn = sortColumn - self.limit = limit - self.offset = offset - } - - public var usesOr: Bool { - logicMode.uppercased() == "OR" - } - - /// The spec with the page left out, so pages of the same view share one cursor cache. - public var viewKey: String { - let filterText = filters.map { "\($0.column)\u{1}\($0.op)\u{1}\($0.value)\u{1}\($0.secondValue ?? "")" }.joined(separator: "\u{2}") - return [table, logicMode.uppercased(), descending ? "desc" : "asc", filterText].joined(separator: "\u{3}") - } -} - -/// The bounds a row key may take: an interval, or a list of exact keys. -struct BigtableKeyBounds: Equatable { - var start: (key: Data, closed: Bool)? - var end: (key: Data, closed: Bool)? - var exact: [Data]? - - static func == (lhs: BigtableKeyBounds, rhs: BigtableKeyBounds) -> Bool { - lhs.start?.key == rhs.start?.key && lhs.start?.closed == rhs.start?.closed - && lhs.end?.key == rhs.end?.key && lhs.end?.closed == rhs.end?.closed && lhs.exact == rhs.exact - } - - mutating func raiseStart(_ key: Data, closed: Bool) { - guard let current = start else { start = (key, closed); return } - if current.key.lexicographicallyPrecedes(key) || (current.key == key && !closed) { - start = (key, closed && current.closed) - } - } - - mutating func lowerEnd(_ key: Data, closed: Bool) { - guard let current = end else { end = (key, closed); return } - if key.lexicographicallyPrecedes(current.key) || (current.key == key && !closed) { - end = (key, closed && current.closed) - } - } - - func admits(_ key: Data) -> Bool { - if let start { - if key.lexicographicallyPrecedes(start.key) || (key == start.key && !start.closed) { return false } - } - if let end { - if end.key.lexicographicallyPrecedes(key) || (key == end.key && !end.closed) { return false } - } - return true - } - - func json() -> [String: Any] { - if let exact { - return ["rowKeys": exact.filter(admits).map { $0.base64EncodedString() }] - } - var range: [String: Any] = [:] - if let start { range[start.closed ? "startKeyClosed" : "startKeyOpen"] = start.key.base64EncodedString() } - if let end { range[end.closed ? "endKeyClosed" : "endKeyOpen"] = end.key.base64EncodedString() } - return range.isEmpty ? [:] : ["rowRanges": [range]] - } -} - +/// `ReadRows` bodies. A nil body means the filters admit no row, which is not something a request +/// can say: an empty row set reads the whole table. public enum BigtableReadRequest { - public static let rawFilterColumn = "__RAW__" + public static let rawFilterColumn = BigtableRowFilter.rawColumn - /// `after` continues a page past that key. `keysOnly` reads keys - /// without values, which is how a page with no cursor yet skips the rows before it cheaply. - public static func body(spec: BigtableBrowseSpec, limit: Int, after: Data?, keysOnly: Bool = false) throws -> [String: Any] { + /// `after` continues a page past that key. `keysOnly` reads one stripped cell per row, which is + /// how a page with no cursor yet skips the rows before it and how a count reads every key. + public static func body(spec: BigtableBrowseSpec, limit: Int, after: Data?, keysOnly: Bool = false) throws -> [String: Any]? { if (spec.sortColumn.map { $0 != BigtableRowSet.keyColumn } ?? false) || spec.descending { throw BigtableError.unsupported(String(localized: "The Bigtable REST API returns rows in ascending key order only.")) } - var bounds = BigtableKeyBounds() + var selection: [BigtableKeyBounds] var rowFilters: [[String: Any]] = [] - var unions: [[String: Any]]? - - let keyFilters = spec.filters.filter { $0.column == BigtableRowSet.keyColumn && isRangeOperator($0.op) } - if spec.usesOr, keyFilters.count == spec.filters.count, !keyFilters.isEmpty { - unions = try keyFilters.map { filter in - var single = BigtableKeyBounds() - try applyKeyRange(filter, to: &single) - return single.json() + let keyRanges = spec.filters.filter(isKeyRange) + if spec.usesOr, !keyRanges.isEmpty, keyRanges.count == spec.filters.count { + selection = try keyRanges.map { filter in + var bounds = BigtableKeyBounds() + try apply(filter, to: &bounds) + return bounds } } else { + var bounds = BigtableKeyBounds() for filter in spec.filters { - if filter.column == BigtableRowSet.keyColumn, isRangeOperator(filter.op), !spec.usesOr { - try applyKeyRange(filter, to: &bounds) + if !spec.usesOr, isKeyRange(filter) { + try apply(filter, to: &bounds) } else { - rowFilters.append(try rowFilter(filter, insideOr: spec.usesOr)) + rowFilters.append(try BigtableRowFilter.make(filter, insideOr: spec.usesOr)) } } + selection = [bounds] } if let after { - bounds.raiseStart(after, closed: false) + for index in selection.indices { + selection[index].raiseStart(after, closed: false) + } } + guard let rows = rowSet(selection) else { return nil } var body: [String: Any] = [:] - if let unions { - body["rows"] = mergeUnions(unions, within: bounds) - } else { - let rows = bounds.json() - if !rows.isEmpty { body["rows"] = rows } - if let exact = rows["rowKeys"] as? [String], exact.isEmpty { - body["rows"] = ["rowKeys": [String]()] - } - } + if !rows.isEmpty { body["rows"] = rows } var filters: [[String: Any]] = [] switch rowFilters.count { case 0: break case 1: filters.append(rowFilters[0]) default: filters.append([spec.usesOr ? "interleave" : "chain": ["filters": rowFilters]]) } - if keysOnly { - filters += [["cellsPerRowLimitFilter": 1], ["stripValueTransformer": true]] - } else { - filters.append(["cellsPerColumnLimitFilter": 1]) - } + filters += keysOnly ? keysOnlyFilters : [["cellsPerColumnLimitFilter": 1]] body["filter"] = filters.count == 1 ? filters[0] : ["chain": ["filters": filters]] if limit > 0 { body["rowsLimit"] = String(limit) } return body } - /// A list of exact keys or ranges asked for with no matching row is an empty `rowKeys`, which - /// the API refuses, so the read is skipped by the caller when it would ask for nothing. - public static func asksForNothing(_ body: [String: Any]) -> Bool { - guard let rows = body["rows"] as? [String: Any] else { return false } - let keys = rows["rowKeys"] as? [Any] ?? [] - let ranges = rows["rowRanges"] as? [Any] ?? [] - return keys.isEmpty && ranges.isEmpty + /// The first rows of a table with their values stripped, which is enough to learn its columns. + public static func columnSample(limit: Int) -> [String: Any] { + [ + "rowsLimit": String(limit), + "filter": ["chain": ["filters": [["cellsPerColumnLimitFilter": 1], ["stripValueTransformer": true]]]] + ] + } + + public static func keyLookup(_ key: Data) -> [String: Any] { + [ + "rows": ["rowKeys": [key.base64EncodedString()]], + "rowsLimit": "1", + "filter": ["chain": ["filters": keysOnlyFilters]] + ] } - private static func mergeUnions(_ unions: [[String: Any]], within bounds: BigtableKeyBounds) -> [String: Any] { + public static func get(_ key: Data) -> [String: Any] { + ["rows": ["rowKeys": [key.base64EncodedString()]], "filter": ["cellsPerColumnLimitFilter": 1]] + } + + private static var keysOnlyFilters: [[String: Any]] { + [["cellsPerRowLimitFilter": 1], ["stripValueTransformer": true]] + } + + /// The union of `selection` as a `RowSet`: empty for the whole table, nil for no row at all. + private static func rowSet(_ selection: [BigtableKeyBounds]) -> [String: Any]? { + let live = selection.filter { !$0.isEmpty } + guard !live.isEmpty else { return nil } + if live.contains(where: \.isUnbounded) { return [:] } var keys: [String] = [] var ranges: [[String: Any]] = [] - for union in unions { - keys += union["rowKeys"] as? [String] ?? [] - ranges += union["rowRanges"] as? [[String: Any]] ?? [] + for bounds in live { + if bounds.exact != nil { + for key in bounds.admittedKeys.map({ $0.base64EncodedString() }) where !keys.contains(key) { + keys.append(key) + } + } else { + ranges.append(bounds.rangeJSON) + } } var rows: [String: Any] = [:] - let admitted = keys.compactMap { Data(base64Encoded: $0) }.filter(bounds.admits).map { $0.base64EncodedString() } - if !admitted.isEmpty { rows["rowKeys"] = admitted } - if !ranges.isEmpty { - rows["rowRanges"] = ranges.map { range in clamp(range, to: bounds) } - } + if !keys.isEmpty { rows["rowKeys"] = keys } + if !ranges.isEmpty { rows["rowRanges"] = ranges } return rows } - /// A cursor cuts every range of an OR at the key the last page ended on. - private static func clamp(_ range: [String: Any], to bounds: BigtableKeyBounds) -> [String: Any] { - var single = BigtableKeyBounds() - if let key = (range["startKeyClosed"] as? String).flatMap({ Data(base64Encoded: $0) }) { single.start = (key, true) } - if let key = (range["startKeyOpen"] as? String).flatMap({ Data(base64Encoded: $0) }) { single.start = (key, false) } - if let key = (range["endKeyClosed"] as? String).flatMap({ Data(base64Encoded: $0) }) { single.end = (key, true) } - if let key = (range["endKeyOpen"] as? String).flatMap({ Data(base64Encoded: $0) }) { single.end = (key, false) } - if let start = bounds.start { single.raiseStart(start.key, closed: start.closed) } - if let end = bounds.end { single.lowerEnd(end.key, closed: end.closed) } - return (single.json()["rowRanges"] as? [[String: Any]])?.first ?? [:] - } - - static func isRangeOperator(_ op: String) -> Bool { - ["=", "IN", "STARTS WITH", ">", ">=", "<", "<=", "BETWEEN"].contains(op.uppercased()) + /// A key test a row range can express. A case-insensitive equality or prefix cannot, so it runs + /// as a key regex instead. + static func isKeyRange(_ filter: BigtableFilterSpec) -> Bool { + guard filter.column == BigtableRowSet.keyColumn else { return false } + switch filter.op { + case ">", ">=", "<", "<=", "BETWEEN": return true + case "=", "IN", "STARTS WITH": return filter.isCaseSensitive + default: return false + } } - static func applyKeyRange(_ filter: BigtableFilterSpec, to bounds: inout BigtableKeyBounds) throws { - let key = try BigtableBytes.parse(filter.value) - switch filter.op.uppercased() { + static func apply(_ filter: BigtableFilterSpec, to bounds: inout BigtableKeyBounds) throws { + switch filter.op { case "=": - bounds.exact = (bounds.exact ?? [key]).filter { $0 == key } + bounds.restrict(to: [try BigtableBytes.parse(filter.value)]) case "IN": - let keys = try listItems(filter.value).map(BigtableBytes.parse) - bounds.exact = bounds.exact.map { existing in existing.filter(keys.contains) } ?? keys + bounds.restrict(to: try BigtableRowFilter.listItems(filter.value).map(BigtableBytes.parse)) case "STARTS WITH": - bounds.raiseStart(key, closed: true) - if let end = BigtableBytes.prefixEnd(key) { bounds.lowerEnd(end, closed: false) } - case ">": bounds.raiseStart(key, closed: false) - case ">=": bounds.raiseStart(key, closed: true) - case "<": bounds.lowerEnd(key, closed: false) - case "<=": bounds.lowerEnd(key, closed: true) + let prefix = try BigtableBytes.parse(filter.value) + bounds.raiseStart(prefix, closed: true) + if let end = BigtableBytes.prefixEnd(prefix) { bounds.lowerEnd(end, closed: false) } + case ">": bounds.raiseStart(try BigtableBytes.parse(filter.value), closed: false) + case ">=": bounds.raiseStart(try BigtableBytes.parse(filter.value), closed: true) + case "<": bounds.lowerEnd(try BigtableBytes.parse(filter.value), closed: false) + case "<=": bounds.lowerEnd(try BigtableBytes.parse(filter.value), closed: true) case "BETWEEN": guard let upper = filter.secondValue else { throw BigtableError.configuration(String(localized: "BETWEEN needs two values.")) } - bounds.raiseStart(key, closed: true) + bounds.raiseStart(try BigtableBytes.parse(filter.value), closed: true) bounds.lowerEnd(try BigtableBytes.parse(upper), closed: true) default: - throw unsupported(filter.op) - } - } - - // MARK: - Row filters - - static func rowFilter(_ filter: BigtableFilterSpec, insideOr: Bool) throws -> [String: Any] { - if filter.column == rawFilterColumn { - return try rawFilter(filter.value) - } - if filter.column == BigtableRowSet.keyColumn { - return try keyRegexFilter(filter, insideOr: insideOr) - } - let column = try BigtableColumn(name: filter.column) - let base: [[String: Any]] = [ - ["familyNameRegexFilter": "^" + BigtableBytes.regexText(Data(column.family.utf8)) + "$"], - ["columnQualifierRegexFilter": anchored(column.qualifier).base64EncodedString()], - ["cellsPerColumnLimitFilter": 1] - ] - let value = try BigtableBytes.parse(filter.value) - func predicate(_ extra: [String: Any]?) -> [String: Any] { - ["chain": ["filters": base + (extra.map { [$0] } ?? [])]] - } - func regex(_ pattern: Data) -> [String: Any] { - ["valueRegexFilter": pattern.base64EncodedString()] - } - let literal = BigtableBytes.regexLiteral(value) - let anyText = Data("(?s).*".utf8) - switch filter.op.uppercased() { - case "=": return keep(predicate(regex(Data("(?s)^".utf8) + literal + Data("$".utf8)))) - case "!=", "<>": return drop(predicate(regex(Data("(?s)^".utf8) + literal + Data("$".utf8)))) - case "CONTAINS": return keep(predicate(regex(anyText + literal + anyText))) - case "NOT CONTAINS": return drop(predicate(regex(anyText + literal + anyText))) - case "STARTS WITH": return keep(predicate(regex(Data("(?s)^".utf8) + literal + anyText))) - case "ENDS WITH": return keep(predicate(regex(anyText + literal + Data("$".utf8)))) - case "REGEX": return keep(predicate(regex(value))) - case "IS EMPTY": return keep(predicate(regex(Data("^$".utf8)))) - case "IS NOT EMPTY": return drop(predicate(regex(Data("^$".utf8)))) - case "IS NULL": return drop(predicate(nil)) - case "IS NOT NULL": return keep(predicate(nil)) - case ">": return keep(predicate(["valueRangeFilter": ["startValueOpen": value.base64EncodedString()]])) - case ">=": return keep(predicate(["valueRangeFilter": ["startValueClosed": value.base64EncodedString()]])) - case "<": return keep(predicate(["valueRangeFilter": ["endValueOpen": value.base64EncodedString()]])) - case "<=": return keep(predicate(["valueRangeFilter": ["endValueClosed": value.base64EncodedString()]])) - case "BETWEEN": - guard let upper = filter.secondValue else { - throw BigtableError.configuration(String(localized: "BETWEEN needs two values.")) - } - return keep(predicate(["valueRangeFilter": [ - "startValueClosed": value.base64EncodedString(), - "endValueClosed": try BigtableBytes.parse(upper).base64EncodedString() - ]])) - case "IN": - let alternatives = try listItems(filter.value).map { BigtableBytes.regexText(try BigtableBytes.parse($0)) } - return keep(predicate(regex(Data(("(?s)^(" + alternatives.joined(separator: "|") + ")$").utf8)))) - default: - throw unsupported(filter.op) - } - } - - /// A key test that no row range can express, or one inside an OR with column tests, runs as a - /// key regex instead. - private static func keyRegexFilter(_ filter: BigtableFilterSpec, insideOr: Bool) throws -> [String: Any] { - let literal = BigtableBytes.regexLiteral(try BigtableBytes.parse(filter.value)) - let anyText = Data("(?s).*".utf8) - let pattern: Data - var negate = false - switch filter.op.uppercased() { - case "=": pattern = Data("(?s)^".utf8) + literal + Data("$".utf8) - case "!=", "<>": pattern = Data("(?s)^".utf8) + literal + Data("$".utf8); negate = true - case "STARTS WITH": pattern = Data("(?s)^".utf8) + literal + anyText - case "ENDS WITH": pattern = anyText + literal + Data("$".utf8) - case "CONTAINS": pattern = anyText + literal + anyText - case "NOT CONTAINS": pattern = anyText + literal + anyText; negate = true - case "REGEX": pattern = try BigtableBytes.parse(filter.value) - default: - throw BigtableError.unsupported(insideOr - ? String(localized: "A key range cannot be ORed with a column filter. Use =, STARTS WITH or REGEX on _key, or AND.") - : unsupportedMessage(filter.op)) + throw BigtableError.unsupported(BigtableRowFilter.unsupportedMessage(filter.op)) } - let test: [String: Any] = ["rowKeyRegexFilter": pattern.base64EncodedString()] - return negate ? drop(test) : keep(test) - } - - private static func keep(_ predicate: [String: Any]) -> [String: Any] { - ["condition": ["predicateFilter": predicate, "trueFilter": ["passAllFilter": true]]] - } - - private static func drop(_ predicate: [String: Any]) -> [String: Any] { - ["condition": ["predicateFilter": predicate, "falseFilter": ["passAllFilter": true]]] - } - - private static func anchored(_ qualifier: Data) -> Data { - Data("(?s)^".utf8) + BigtableBytes.regexLiteral(qualifier) + Data("$".utf8) - } - - /// The raw filter row takes a `RowFilter` as written in the Bigtable REST reference, with bytes - /// in base64, which reaches every filter the filter bar cannot draw. - private static func rawFilter(_ text: String) throws -> [String: Any] { - guard let data = text.data(using: .utf8), - let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], object.count == 1 else { - throw BigtableError.configuration(String(localized: "A raw Bigtable filter is a JSON RowFilter, such as {\"rowKeyRegexFilter\": \"\"}.")) - } - return object - } - - static func listItems(_ text: String) -> [String] { - if let data = text.data(using: .utf8), let array = try? JSONSerialization.jsonObject(with: data) as? [String] { - return array - } - return text.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } - } - - private static func unsupported(_ op: String) -> BigtableError { - .unsupported(unsupportedMessage(op)) - } - - private static func unsupportedMessage(_ op: String) -> String { - String(format: String(localized: "Bigtable cannot filter with %@."), op) } } diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableRowFilter.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableRowFilter.swift new file mode 100644 index 0000000000..ada8351fd5 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Query/BigtableRowFilter.swift @@ -0,0 +1,128 @@ +import Foundation + +/// Turns one filter bar row into a `RowFilter` that passes a whole row or none of it. +enum BigtableRowFilter { + static let rawColumn = "__RAW__" + + static func make(_ filter: BigtableFilterSpec, insideOr: Bool) throws -> [String: Any] { + if filter.column == rawColumn { + return try raw(filter.value) + } + if filter.column == BigtableRowSet.keyColumn { + return try keyFilter(filter, insideOr: insideOr) + } + return try columnFilter(filter) + } + + private static func keyFilter(_ filter: BigtableFilterSpec, insideOr: Bool) throws -> [String: Any] { + guard let match = try textPattern(filter) else { + throw BigtableError.unsupported(insideOr + ? String(localized: "A key range cannot be ORed with a column filter. Use =, STARTS WITH or REGEX on _key, or AND.") + : unsupportedMessage(filter.op)) + } + let test: [String: Any] = ["rowKeyRegexFilter": match.pattern.base64EncodedString()] + return match.isNegated ? drop(test) : keep(test) + } + + private static func columnFilter(_ filter: BigtableFilterSpec) throws -> [String: Any] { + let column = try BigtableColumn(name: filter.column) + let base: [[String: Any]] = [ + ["familyNameRegexFilter": "^" + BigtableBytes.regexText(Data(column.family.utf8)) + "$"], + ["columnQualifierRegexFilter": (Data("(?s)^".utf8) + BigtableBytes.regexLiteral(column.qualifier) + Data("$".utf8)).base64EncodedString()], + ["cellsPerColumnLimitFilter": 1] + ] + func cellIn(_ test: [String: Any]?) -> [String: Any] { + ["chain": ["filters": base + (test.map { [$0] } ?? [])]] + } + let present = keep(cellIn(nil)) + + if let match = try textPattern(filter) { + let test = cellIn(["valueRegexFilter": match.pattern.base64EncodedString()]) + return match.isNegated ? ["chain": ["filters": [present, drop(test)]]] : keep(test) + } + switch filter.op { + case "IS NULL": + return drop(cellIn(nil)) + case "IS NOT NULL": + return present + case "IS EMPTY": + return keep(cellIn(["valueRegexFilter": Data("^$".utf8).base64EncodedString()])) + case "IS NOT EMPTY": + return ["chain": ["filters": [present, drop(cellIn(["valueRegexFilter": Data("^$".utf8).base64EncodedString()]))]]] + case ">", ">=", "<", "<=", "BETWEEN": + return keep(cellIn(["valueRangeFilter": try valueRange(filter)])) + default: + throw BigtableError.unsupported(unsupportedMessage(filter.op)) + } + } + + /// The regex a text comparison matches, for a key or a value alike. Nil for an operator that is + /// not a text comparison. + private static func textPattern(_ filter: BigtableFilterSpec) throws -> (pattern: Data, isNegated: Bool)? { + let anyText = Data(".*".utf8) + let flags = Data((filter.isCaseSensitive ? "(?s)" : "(?is)").utf8) + func literal() throws -> Data { + BigtableBytes.regexLiteral(try BigtableBytes.parse(filter.value)) + } + func alternatives() throws -> Data { + let items = try listItems(filter.value).map { BigtableBytes.regexText(try BigtableBytes.parse($0)) } + return flags + Data(("^(" + items.joined(separator: "|") + ")$").utf8) + } + switch filter.op { + case "=": return (flags + Data("^".utf8) + (try literal()) + Data("$".utf8), false) + case "!=", "<>": return (flags + Data("^".utf8) + (try literal()) + Data("$".utf8), true) + case "STARTS WITH": return (flags + Data("^".utf8) + (try literal()) + anyText, false) + case "ENDS WITH": return (flags + anyText + (try literal()) + Data("$".utf8), false) + case "CONTAINS": return (flags + anyText + (try literal()) + anyText, false) + case "NOT CONTAINS": return (flags + anyText + (try literal()) + anyText, true) + case "IN": return (try alternatives(), false) + case "NOT IN": return (try alternatives(), true) + case "REGEX": return ((filter.isCaseSensitive ? Data() : Data("(?i)".utf8)) + (try BigtableBytes.parse(filter.value)), false) + default: return nil + } + } + + private static func valueRange(_ filter: BigtableFilterSpec) throws -> [String: Any] { + let value = try BigtableBytes.parse(filter.value).base64EncodedString() + switch filter.op { + case ">": return ["startValueOpen": value] + case ">=": return ["startValueClosed": value] + case "<": return ["endValueOpen": value] + case "<=": return ["endValueClosed": value] + default: + guard let upper = filter.secondValue else { + throw BigtableError.configuration(String(localized: "BETWEEN needs two values.")) + } + return ["startValueClosed": value, "endValueClosed": try BigtableBytes.parse(upper).base64EncodedString()] + } + } + + private static func keep(_ predicate: [String: Any]) -> [String: Any] { + ["condition": ["predicateFilter": predicate, "trueFilter": ["passAllFilter": true]]] + } + + private static func drop(_ predicate: [String: Any]) -> [String: Any] { + ["condition": ["predicateFilter": predicate, "falseFilter": ["passAllFilter": true]]] + } + + /// The raw filter row takes a `RowFilter` as written in the Bigtable REST reference, with bytes + /// in base64, which reaches every filter the filter bar cannot draw. + private static func raw(_ text: String) throws -> [String: Any] { + guard let data = text.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], object.count == 1 else { + throw BigtableError.configuration(String(localized: "A raw Bigtable filter is a JSON RowFilter, such as {\"rowKeyRegexFilter\": \"\"}.")) + } + return object + } + + static func listItems(_ text: String) -> [String] { + if let data = text.data(using: .utf8), let array = try? JSONSerialization.jsonObject(with: data) as? [String] { + return array + } + return text.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + } + + static func unsupportedMessage(_ op: String) -> String { + String(format: String(localized: "Bigtable cannot filter with %@."), op) + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift index 6cf2745015..63240c11ed 100644 --- a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift @@ -1,18 +1,26 @@ import Foundation /// Row keys, qualifiers and values are bytes. Text that is valid UTF-8 with no control characters -/// shows as itself; anything else shows as `b64:` followed by base64, which reads back to the same -/// bytes, so a binary key can still be named in a request line and edited from the grid. +/// shows as itself; anything else, and text that itself starts with `b64:`, shows as `b64:` and +/// base64, which reads back to the same bytes. public enum BigtableBytes { public static let base64Prefix = "b64:" public static func display(_ data: Data) -> String { - if let text = String(data: data, encoding: .utf8), isPrintable(text), !text.hasPrefix(base64Prefix) { + if let text = printableText(data) { return text } return base64Prefix + data.base64EncodedString() } + /// The bytes as text when `display` would show them as themselves. + public static func printableText(_ data: Data) -> String? { + guard let text = String(data: data, encoding: .utf8), isPrintable(text), !text.hasPrefix(base64Prefix) else { + return nil + } + return text + } + public static func parse(_ text: String) throws -> Data { guard text.hasPrefix(base64Prefix) else { return Data(text.utf8) } guard let data = Data(base64Encoded: String(text.dropFirst(base64Prefix.count))) else { @@ -27,17 +35,15 @@ public enum BigtableBytes { } } - static func decodeBase64(_ text: String?) throws -> Data { - guard let text else { return Data() } + static func decodeBase64(_ text: String) throws -> Data { guard let data = Data(base64Encoded: text) else { throw BigtableError.malformedResponse(String(localized: "Bigtable sent bytes that are not base64.")) } return data } - /// The smallest key greater than every key starting with `prefix`: the prefix with its last - /// byte below 0xFF incremented and what follows dropped. Nil when every byte is 0xFF, which - /// leaves the range open at the end. + /// The smallest key greater than every key starting with `prefix`. Nil when every byte is 0xFF, + /// which leaves the range open at the end. public static func prefixEnd(_ prefix: Data) -> Data? { var bytes = [UInt8](prefix) while let last = bytes.last { @@ -50,13 +56,12 @@ public enum BigtableBytes { return nil } - /// `regexLiteral` writes only printable ASCII, so its text form always exists. public static func regexText(_ data: Data) -> String { String(bytes: regexLiteral(data), encoding: .ascii) ?? "" } - /// RE2 reads these as operators, so a literal is escaped byte by byte before it goes into a - /// regex filter. + /// Escapes RE2 operators byte by byte and writes every other byte outside printable ASCII as + /// `\xNN`, so the result is always ASCII. public static func regexLiteral(_ data: Data) -> Data { let special = Set("\\^$.|?*+()[]{}".utf8) var escaped = Data() diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift index 1fbc448af0..28513c9b52 100644 --- a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableClient.swift @@ -1,4 +1,5 @@ import Foundation +import TableProGoogleCloud public enum BigtableMutation: Sendable, Equatable { case setCell(BigtableColumn, Data) @@ -31,10 +32,11 @@ public enum BigtableMutation: Sendable, Equatable { public final class BigtableClient: @unchecked Sendable { public static let dataEndpoint = "https://bigtable.googleapis.com/v2/" public static let adminEndpoint = "https://bigtableadmin.googleapis.com/v2/" + public static let scopes = [GoogleOAuthClient.cloudPlatformScope] public let projectId: String public let appProfileId: String? - private let requester: BigtableRequester + private let requester: GoogleRESTRequester private let lock = NSLock() private var instance: String @@ -42,14 +44,14 @@ public final class BigtableClient: @unchecked Sendable { projectId: String, instanceId: String, appProfileId: String? = nil, - transport: any BigtableTransport, - credential: BigtableCredential, + transport: any GoogleRESTTransport, + credential: GoogleRESTCredential, timeout: @escaping @Sendable () -> TimeInterval ) { self.projectId = projectId self.instance = instanceId self.appProfileId = appProfileId - self.requester = BigtableRequester(transport: transport, credential: credential, timeout: timeout) + self.requester = GoogleRESTRequester(transport: transport, credential: credential, timeout: timeout) } public var instanceId: String { @@ -66,10 +68,9 @@ public final class BigtableClient: @unchecked Sendable { // MARK: - Admin public func listInstances() async throws -> [String] { - let response = try await requester.send(method: "GET", url: try url(Self.adminEndpoint, "projects/\(projectId)/instances")) - try response.requireSuccess(String(localized: "list instances")) - let instances = (try response.json() as? [String: Any])?["instances"] as? [[String: Any]] ?? [] - return instances.compactMap { ($0["name"] as? String)?.split(separator: "/").last.map(String.init) }.sorted() + let json = try await send("GET", Self.adminEndpoint, "projects/\(projectId)/instances", operation: String(localized: "list instances")) + let instances = json["instances"] as? [[String: Any]] ?? [] + return instances.compactMap { Self.lastSegment($0["name"]) }.sorted() } public func listTables() async throws -> [String] { @@ -78,35 +79,39 @@ public final class BigtableClient: @unchecked Sendable { repeat { var query = [URLQueryItem(name: "view", value: "NAME_ONLY"), URLQueryItem(name: "pageSize", value: "1000")] if let pageToken { query.append(URLQueryItem(name: "pageToken", value: pageToken)) } - let response = try await requester.send(method: "GET", url: try url(Self.adminEndpoint, instancePath + "/tables", query: query)) - try response.requireSuccess(String(format: String(localized: "list the tables of %@"), instanceId)) - let json = try response.json() as? [String: Any] - let tables = json?["tables"] as? [[String: Any]] ?? [] - names += tables.compactMap { ($0["name"] as? String)?.split(separator: "/").last.map(String.init) } - pageToken = (json?["nextPageToken"] as? String)?.nonEmpty + let json = try await send("GET", Self.adminEndpoint, instancePath + "/tables", query: query, operation: listTablesOperation) + names += (json["tables"] as? [[String: Any]] ?? []).compactMap { Self.lastSegment($0["name"]) } + pageToken = (json["nextPageToken"] as? String).flatMap { $0.isEmpty ? nil : $0 } } while pageToken != nil return names.sorted() } + /// One page of one table name, which proves the credential, the project and the instance. + public func ping() async throws { + let query = [URLQueryItem(name: "view", value: "NAME_ONLY"), URLQueryItem(name: "pageSize", value: "1")] + _ = try await send("GET", Self.adminEndpoint, instancePath + "/tables", query: query, operation: listTablesOperation) + } + public func columnFamilies(table: String) async throws -> [String] { - let response = try await requester.send( - method: "GET", - url: try url(Self.adminEndpoint, try tablePath(table), query: [URLQueryItem(name: "view", value: "SCHEMA_VIEW")]) + let json = try await send( + "GET", Self.adminEndpoint, try tablePath(table), query: [URLQueryItem(name: "view", value: "SCHEMA_VIEW")], + operation: String(format: String(localized: "read the schema of %@"), table) ) - try response.requireSuccess(String(format: String(localized: "read the schema of %@"), table)) - let families = (try response.json() as? [String: Any])?["columnFamilies"] as? [String: Any] ?? [:] - return families.keys.sorted() + return (json["columnFamilies"] as? [String: Any] ?? [:]).keys.sorted() } // MARK: - Data - public func readRows(table: String, body: [String: Any]) async throws -> [BigtableRow] { - if BigtableReadRequest.asksForNothing(body) { return [] } - var request = body - if let appProfileId { request["appProfileId"] = appProfileId } - let response = try await requester.send(method: "POST", url: try url(Self.dataEndpoint, try tablePath(table) + ":readRows"), json: request) - try response.requireSuccess(String(format: String(localized: "read %@"), table)) - return try BigtableChunkAssembler.rows(fromResponse: try response.json()) + /// A nil body is a read the filters ruled out, answered with no rows and no request. + public func readRows(table: String, body: [String: Any]?) async throws -> [BigtableRow] { + guard var body else { return [] } + if let appProfileId { body["appProfileId"] = appProfileId } + let operation = String(format: String(localized: "read %@"), table) + let response = try await requester.send( + method: "POST", url: try url(Self.dataEndpoint, try tablePath(table) + ":readRows"), body: try Self.encode(body) + ) + try Self.requireSuccess(response, operation: operation) + return try BigtableChunkAssembler.rows(fromResponse: try Self.decode(response), operation: operation) } public func mutateRow(table: String, key: Data, mutations: [BigtableMutation]) async throws { @@ -114,20 +119,65 @@ public final class BigtableClient: @unchecked Sendable { guard !key.isEmpty else { throw BigtableError.configuration(String(localized: "A row needs a key. Bigtable has no empty row key.")) } - var body: [String: Any] = [ - "rowKey": key.base64EncodedString(), - "mutations": mutations.map(\.json) - ] + var body: [String: Any] = ["rowKey": key.base64EncodedString(), "mutations": mutations.map(\.json)] if let appProfileId { body["appProfileId"] = appProfileId } - let response = try await requester.send(method: "POST", url: try url(Self.dataEndpoint, try tablePath(table) + ":mutateRow"), json: body) - try response.requireSuccess(String(format: String(localized: "write a row of %@"), table)) + _ = try await send("POST", Self.dataEndpoint, try tablePath(table) + ":mutateRow", body: body, + operation: String(format: String(localized: "write a row of %@"), table)) } - public func ping() async throws { - _ = try await listTables() + /// Writes a new row, refusing a key that already has one. A conditional `checkAndMutateRow` + /// would close the gap between the check and the write, but Google refuses it under + /// multi-cluster routing, the default app profile of a replicated instance. + public func insertRow(table: String, key: Data, mutations: [BigtableMutation]) async throws { + guard try await readRows(table: table, body: BigtableReadRequest.keyLookup(key)).isEmpty else { + throw BigtableError.conflict(String( + format: String(localized: "Row %@ already exists. Edit that row, or give the new one another key."), + BigtableBytes.display(key) + )) + } + try await mutateRow(table: table, key: key, mutations: mutations) + } + + // MARK: - Requests + + private var listTablesOperation: String { + String(format: String(localized: "list the tables of %@"), instanceId) } - // MARK: - Paths + private func send( + _ method: String, + _ endpoint: String, + _ path: String, + query: [URLQueryItem] = [], + body: [String: Any]? = nil, + operation: String + ) async throws -> [String: Any] { + let response = try await requester.send(method: method, url: try url(endpoint, path, query: query), body: try body.map(Self.encode)) + try Self.requireSuccess(response, operation: operation) + return try Self.decode(response) as? [String: Any] ?? [:] + } + + private static func encode(_ body: [String: Any]) throws -> Data { + try JSONSerialization.data(withJSONObject: body, options: [.sortedKeys]) + } + + private static func decode(_ response: GoogleRESTResponse) throws -> Any { + guard !response.body.isEmpty else { return [String: Any]() } + do { + return try JSONSerialization.jsonObject(with: response.body, options: [.fragmentsAllowed]) + } catch { + throw BigtableError.malformedResponse(String(localized: "Bigtable sent a response that is not JSON.")) + } + } + + private static func requireSuccess(_ response: GoogleRESTResponse, operation: String) throws { + guard !response.isSuccess else { return } + throw BigtableError.from(status: response.statusCode, body: response.body, operation: operation) + } + + private static func lastSegment(_ name: Any?) -> String? { + (name as? String)?.split(separator: "/").last.map(String.init) + } var instancePath: String { "projects/\(projectId)/instances/\(instanceId)" @@ -151,3 +201,9 @@ public final class BigtableClient: @unchecked Sendable { return url } } + +public extension URLSessionGoogleRESTTransport { + static func bigtable(resourceTimeout: TimeInterval) -> URLSessionGoogleRESTTransport { + URLSessionGoogleRESTTransport(resourceTimeout: resourceTimeout) { BigtableError.transport($0) } + } +} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift index 0c84262172..6f6fe91732 100644 --- a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableError.swift @@ -5,10 +5,10 @@ public enum BigtableError: Error, LocalizedError, Equatable, Sendable { case notConnected case transport(String) case authentication(String) - /// IAM refused the operation. Kept apart from an empty result so a blocked read never looks - /// like a table with no rows. + /// Kept apart from an empty result so a blocked read never looks like a table with no rows. case permissionDenied(operation: String, detail: String) case notFound(String) + case conflict(String) case unsupported(String) case api(status: Int, message: String) case malformedResponse(String) @@ -16,7 +16,7 @@ public enum BigtableError: Error, LocalizedError, Equatable, Sendable { public var errorDescription: String? { switch self { case .configuration(let detail), .transport(let detail), .malformedResponse(let detail), - .authentication(let detail), .notFound(let detail), .unsupported(let detail): + .authentication(let detail), .notFound(let detail), .conflict(let detail), .unsupported(let detail): return detail case .notConnected: return String(localized: "Not connected to Bigtable.") @@ -28,10 +28,21 @@ public enum BigtableError: Error, LocalizedError, Equatable, Sendable { } } - /// `operation` names what was attempted ("read events", "write a row of events"), because - /// Google's denial text names the permission and not the table. + /// `operation` names what was attempted ("read events"), because Google's denial text names the + /// permission and not the table. public static func from(status: Int, body: Data, operation: String) -> BigtableError { - let message = apiMessage(from: body) + from(status: status, message: apiMessage(from: body), operation: operation) + } + + /// A streamed read that fails after its first row has already answered 200, so the failure + /// arrives as the last element of the array instead. + static func from(streamError error: [String: Any], operation: String) -> BigtableError { + let status = (error["code"] as? NSNumber)?.intValue ?? 500 + let message = (error["message"] as? String).flatMap { $0.isEmpty ? nil : $0 } + return from(status: status, message: message, operation: operation) + } + + static func from(status: Int, message: String?, operation: String) -> BigtableError { switch status { case 401: return .authentication(message ?? String(localized: "Google refused the credential.")) @@ -40,10 +51,7 @@ public enum BigtableError: Error, LocalizedError, Equatable, Sendable { case 404: return .notFound(message ?? String(format: String(localized: "Not found: %@."), operation)) default: - return .api( - status: status, - message: message ?? String(format: String(localized: "Bigtable returned HTTP %d."), status) - ) + return .api(status: status, message: message ?? String(format: String(localized: "Bigtable returned HTTP %d."), status)) } } diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift index 133c55a0a1..08b8966201 100644 --- a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift +++ b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift @@ -1,12 +1,9 @@ import Foundation -/// The text the driver runs. A page the grid asks for travels as a tagged spec; everything else is a -/// request line the history shows and the editor takes: -/// -/// GET user#42 SCAN user#?limit=20 READ {"rowsLimit": "10"} -/// PUT user#42 {"profile:name": "Ann"} DELETE user#42 DELETE user#42 profile:name -/// -/// A key with spaces is written as a JSON string, and binary bytes as `b64:`. +/// What the driver runs: a grid page as a tagged spec, or a request line such as +/// `GET events/user#42`, `SCAN events/user#?limit=20`, `READ events {"rowsLimit": "10"}`, +/// `INSERT events/user#42 {"profile:name": "Ann"}`, `PUT events/user#42 {"profile:name": null}`, +/// `DELETE events/user#42` or `DELETE events/user#42 profile:name`. public enum BigtableStatement: Sendable, Equatable { case browse(BigtableBrowseSpec) case request(BigtableRequest) @@ -37,19 +34,19 @@ public struct BigtableRequest: Sendable, Equatable { case get = "GET" case scan = "SCAN" case read = "READ" + case insert = "INSERT" case put = "PUT" case delete = "DELETE" } public let verb: Verb - /// The table the request names with `table/`, or nil for the table open in the editor's tab. - public let table: String? + public let table: String public let key: Data? public let column: String? public let limit: Int? public let body: String? - public init(verb: Verb, table: String? = nil, key: Data? = nil, column: String? = nil, limit: Int? = nil, body: String? = nil) { + public init(verb: Verb, table: String, key: Data? = nil, column: String? = nil, limit: Int? = nil, body: String? = nil) { self.verb = verb self.table = table self.key = key @@ -58,111 +55,168 @@ public struct BigtableRequest: Sendable, Equatable { self.body = body } - /// `GET events/user#42` names the table before the first slash; a key holding a slash is - /// written as a JSON string, `GET events/"a/b"`. + /// The table comes before the first slash and the key after it. A key holding a space, a quote, + /// a slash or a `?` is written as a JSON string: `GET events/"a/b"`. public static func parse(_ text: String) throws -> BigtableRequest { var scanner = RequestScanner(text.trimmingCharacters(in: .whitespacesAndNewlines)) guard let verbWord = scanner.word(), let verb = Verb(rawValue: verbWord.uppercased()) else { throw BigtableError.configuration(usage) } + let request: BigtableRequest switch verb { case .read: - let rest = scanner.rest() - var table: String? - var body = rest - if let brace = rest.firstIndex(of: "{"), brace != rest.startIndex { - table = rest[.. String { + public var text: String { var parts = [verb.rawValue] if verb == .read { - parts.append(contentsOf: [table, body].compactMap { $0 }) + parts.append(table) + parts.append(body ?? "{}") return parts.joined(separator: " ") } - let keyText = key.map(Self.keyToken) ?? "" - let tableName = table ?? defaultTable - parts.append(tableName.map { $0 + "/" + keyText } ?? keyText) - if let column { parts.append(column) } + var target = table + "/" + (key.map { Self.token(BigtableBytes.display($0), reserved: "/?") } ?? "") + if let limit { target += "?limit=\(limit)" } + parts.append(target) + if let column { parts.append(Self.token(column, reserved: "")) } if let body { parts.append(body) } return parts.joined(separator: " ") } - /// A key that would not read back as one bare word goes in JSON quotes. - static func keyToken(_ key: Data) -> String { - let display = BigtableBytes.display(key) - let plain = !display.isEmpty && display.unicodeScalars.allSatisfy { scalar in - !CharacterSet.whitespacesAndNewlines.contains(scalar) && scalar != "\"" && scalar != "/" && scalar != "?" + /// The cells a write sets or deletes, all of which land in one atomic row mutation. A body maps + /// `family:qualifier` to text, `b64:` text for bytes, or null to delete the cell. + public func mutations() throws -> [BigtableMutation] { + switch verb { + case .get, .scan, .read: + return [] + case .delete: + return [try column.map { .deleteColumn(try BigtableColumn(name: $0)) } ?? .deleteRow] + case .insert, .put: + guard let data = body?.data(using: .utf8), let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw BigtableError.configuration(String(localized: "Write the cells as a JSON object, such as {\"profile:name\": \"Ann\"}.")) + } + guard !object.isEmpty else { + throw BigtableError.configuration(String(localized: "A write needs at least one cell. Bigtable does not store empty rows.")) + } + return try object.keys.sorted().map { name in + try Self.mutation(column: try BigtableColumn(name: name), name: name, value: object[name], allowsDelete: verb == .put) + } + } + } + + private static func mutation(column: BigtableColumn, name: String, value: Any?, allowsDelete: Bool) throws -> BigtableMutation { + switch value { + case is NSNull: + guard allowsDelete else { + throw BigtableError.configuration(String(format: String(localized: "INSERT only sets cells. Leave %@ out, or delete it with PUT."), name)) + } + return .deleteColumn(column) + case let text as String: + return .setCell(column, try BigtableBytes.parse(text)) + case let number as NSNumber where CFGetTypeID(number) == CFBooleanGetTypeID(): + return .setCell(column, Data((number.boolValue ? "true" : "false").utf8)) + case let number as NSNumber: + return .setCell(column, Data(number.stringValue.utf8)) + default: + throw BigtableError.configuration(String( + format: String(localized: "The value of %@ must be text or null. Bigtable stores bytes, so a number is written as its digits."), name + )) + } + } + + /// Text that would not read back as one bare word goes in JSON quotes. + static func token(_ text: String, reserved: String) -> String { + let plain = !text.isEmpty && !text.hasPrefix("\"") && text.unicodeScalars.allSatisfy { scalar in + !CharacterSet.whitespacesAndNewlines.contains(scalar) && !reserved.unicodeScalars.contains(scalar) } - if plain { return display } - let data = (try? JSONSerialization.data(withJSONObject: [display], options: [])) ?? Data() - let array = String(bytes: data, encoding: .utf8) ?? "" + if plain { return text } + let data = (try? JSONSerialization.data(withJSONObject: [text], options: [.withoutEscapingSlashes])) ?? Data() + let array = String(bytes: data, encoding: .utf8) ?? "[\"\"]" return String(array.dropFirst().dropLast()) } static let usage = String( localized: "Enter a request such as GET events/user#42, SCAN events/user#?limit=20, PUT events/user#42 {\"profile:name\": \"Ann\"} or DELETE events/user#42." ) + + static let missingTable = BigtableError.configuration(String(localized: "Name the table before the key, as in GET events/user#42.")) } -/// Reads a request line one token at a time: words, a `table/key` target where the key may be a -/// JSON string, a `?name=value` query on the target, and whatever is left as the body. +/// Reads a request line one token at a time. private struct RequestScanner { private var text: Substring - var query: [String: String] = [:] + private var query: [String: String] = [:] init(_ text: String) { self.text = Substring(text) } - mutating func word() -> String? { + var isAtEnd: Bool { + text.allSatisfy(\.isWhitespace) + } + + mutating func word(stoppingAt stop: Character? = nil) -> String? { skipSpace() - let word = text.prefix { !$0.isWhitespace } + let word = text.prefix { !$0.isWhitespace && $0 != stop } guard !word.isEmpty else { return nil } text = text.dropFirst(word.count) return String(word) } mutating func rest() -> String { - skipSpace() let remaining = String(text).trimmingCharacters(in: .whitespacesAndNewlines) text = "" return remaining } - mutating func target() throws -> (table: String?, key: Data?) { + /// `table/key`, with an optional `?name=value` query after it. With `tableOnly`, a target with + /// no slash is a table alone, as in `SCAN events`. + mutating func target(tableOnly: Bool) throws -> (table: String, key: Data?) { skipSpace() - var table: String? - if let slash = text.firstIndex(of: "/"), !text[.. 1 ? parts[1] : "" } } } - guard let keyText else { return (table, nil) } - return (table, try BigtableBytes.parse(keyText)) + return (table, try keyText.map(BigtableBytes.parse)) + } + + func limit() throws -> Int? { + guard query.keys.allSatisfy({ $0 == "limit" }) else { + throw BigtableError.configuration(String(localized: "SCAN takes only ?limit=, as in SCAN events/user#?limit=20.")) + } + guard let text = query["limit"] else { return nil } + guard let limit = Int(text), limit > 0 else { + throw BigtableError.configuration(String(format: String(localized: "\"%@\" is not a row limit. Use a whole number above 0."), text)) + } + return limit + } + + /// A column after the key: a JSON string, or the rest of the line, so a qualifier may hold a space. + mutating func column() throws -> String? { + skipSpace() + if text.first == "\"" { return try quoted() } + let column = rest() + return column.isEmpty ? nil : column } private mutating func quoted() throws -> String { @@ -193,13 +265,13 @@ private struct RequestScanner { index = text.index(after: index) } guard index < text.endIndex else { - throw BigtableError.configuration(String(localized: "A quoted row key is missing its closing quote.")) + throw BigtableError.configuration(String(localized: "A quoted value is missing its closing quote.")) } let literal = String(text[text.startIndex...index]) text = text[text.index(after: index)...] guard let data = ("[" + literal + "]").data(using: .utf8), let value = (try? JSONSerialization.jsonObject(with: data) as? [String])?.first else { - throw BigtableError.configuration(String(localized: "A quoted row key is not a valid JSON string.")) + throw BigtableError.configuration(String(localized: "A quoted value is not a valid JSON string.")) } return value } @@ -208,31 +280,3 @@ private struct RequestScanner { text = text.drop { $0.isWhitespace } } } - -public enum BigtableStatementClassifier { - public enum Tier: Sendable, Equatable { - case read - case write - case destructive - } - - /// Text that is neither a browse nor a request line counts as a write, so the read-only gate - /// refuses what it cannot read. - public static func classify(_ text: String) -> Tier { - guard let statement = try? BigtableStatement.parse(text) else { return .write } - switch statement { - case .browse: - return .read - case .request(let request): - switch request.verb { - case .get, .scan, .read: return .read - case .put: return .write - case .delete: return .destructive - } - } - } -} - -extension String { - var nonEmpty: String? { isEmpty ? nil : self } -} diff --git a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableTransport.swift b/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableTransport.swift deleted file mode 100644 index 28160f481f..0000000000 --- a/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableTransport.swift +++ /dev/null @@ -1,184 +0,0 @@ -import Foundation - -public struct BigtableHTTPRequest: Sendable, Equatable { - public let method: String - public let url: URL - public let headers: [String: String] - public let body: Data? - public let timeoutInterval: TimeInterval - - public init(method: String, url: URL, headers: [String: String] = [:], body: Data? = nil, timeoutInterval: TimeInterval) { - self.method = method - self.url = url - self.headers = headers - self.body = body - self.timeoutInterval = timeoutInterval - } -} - -public struct BigtableHTTPResponse: Sendable, Equatable { - public let statusCode: Int - public let body: Data - - public init(statusCode: Int, body: Data) { - self.statusCode = statusCode - self.body = body - } - - public var isSuccess: Bool { - (200..<300).contains(statusCode) - } - - func json() throws -> Any { - guard !body.isEmpty else { return [String: Any]() } - do { - return try JSONSerialization.jsonObject(with: body, options: [.fragmentsAllowed]) - } catch { - throw BigtableError.malformedResponse(String(localized: "Bigtable sent a response that is not JSON.")) - } - } - - func requireSuccess(_ operation: String) throws { - guard !isSuccess else { return } - throw BigtableError.from(status: statusCode, body: body, operation: operation) - } -} - -public protocol BigtableTransport: Sendable { - func send(_ request: BigtableHTTPRequest) async throws -> BigtableHTTPResponse - func cancelAll() -} - -public struct BigtableCredential: Sendable { - public let token: @Sendable () async throws -> String - public let invalidate: @Sendable () async -> Void - public let quotaProject: String? - - public init( - token: @escaping @Sendable () async throws -> String, - invalidate: @escaping @Sendable () async -> Void = {}, - quotaProject: String? = nil - ) { - self.token = token - self.invalidate = invalidate - self.quotaProject = quotaProject - } -} - -public final class URLSessionBigtableTransport: BigtableTransport, @unchecked Sendable { - private let session: URLSession - private let lock = NSLock() - private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] - - public init(resourceTimeout: TimeInterval) { - let configuration = URLSessionConfiguration.ephemeral - configuration.timeoutIntervalForResource = resourceTimeout - configuration.httpCookieStorage = nil - configuration.urlCache = nil - session = URLSession(configuration: configuration, delegate: BigtableRedirectRefusal(), delegateQueue: nil) - } - - deinit { - session.invalidateAndCancel() - } - - public func cancelAll() { - let tasks = lock.withLock { Array(inFlight.values) } - tasks.forEach { $0.cancel() } - } - - public func send(_ request: BigtableHTTPRequest) async throws -> BigtableHTTPResponse { - var urlRequest = URLRequest(url: request.url) - urlRequest.httpMethod = request.method - urlRequest.httpBody = request.body - urlRequest.timeoutInterval = request.timeoutInterval - for (name, value) in request.headers { - urlRequest.setValue(value, forHTTPHeaderField: name) - } - let tracker = BigtableTaskTracker(transport: self) - defer { tracker.finish() } - do { - let (data, response) = try await session.data(for: urlRequest, delegate: tracker) - guard let http = response as? HTTPURLResponse else { - throw BigtableError.transport(String(localized: "Bigtable answered with something other than HTTP.")) - } - return BigtableHTTPResponse(statusCode: http.statusCode, body: data) - } catch let error as URLError where error.code == .cancelled { - throw CancellationError() - } catch let error as URLError { - throw BigtableError.transport(error.localizedDescription) - } - } - - fileprivate func register(_ task: URLSessionTask) { - lock.withLock { inFlight[ObjectIdentifier(task)] = task } - } - - fileprivate func unregister(_ task: URLSessionTask) { - lock.withLock { inFlight[ObjectIdentifier(task)] = nil } - } -} - -/// A redirect would carry the bearer token to whatever host it names. -private final class BigtableRedirectRefusal: NSObject, URLSessionTaskDelegate, Sendable { - func urlSession( - _ session: URLSession, - task: URLSessionTask, - willPerformHTTPRedirection response: HTTPURLResponse, - newRequest request: URLRequest - ) async -> URLRequest? { - nil - } -} - -private final class BigtableTaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { - private weak var transport: URLSessionBigtableTransport? - private let lock = NSLock() - private var task: URLSessionTask? - - init(transport: URLSessionBigtableTransport) { - self.transport = transport - } - - func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { - lock.withLock { self.task = task } - transport?.register(task) - } - - func finish() { - guard let task = lock.withLock({ task }) else { return } - transport?.unregister(task) - } -} - -/// A 401 drops the cached token and retries once, since a token can expire between being cached -/// and being used. -struct BigtableRequester: Sendable { - let transport: any BigtableTransport - let credential: BigtableCredential - let timeout: @Sendable () -> TimeInterval - - func send(method: String, url: URL, json: Any? = nil) async throws -> BigtableHTTPResponse { - let body = try json.map { try JSONSerialization.data(withJSONObject: $0, options: [.sortedKeys]) } - var response = try await attempt(method, url, body) - if response.statusCode == 401 { - await credential.invalidate() - response = try await attempt(method, url, body) - } - return response - } - - private func attempt(_ method: String, _ url: URL, _ body: Data?) async throws -> BigtableHTTPResponse { - var headers = ["Authorization": "Bearer " + (try await credential.token())] - if let project = credential.quotaProject { - headers["x-goog-user-project"] = project - } - if body != nil { - headers["Content-Type"] = "application/json" - } - try Task.checkCancellation() - return try await transport.send(BigtableHTTPRequest( - method: method, url: url, headers: headers, body: body, timeoutInterval: timeout() - )) - } -} diff --git a/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift b/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift index 4a9d7b52be..d6962fdd63 100644 --- a/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift +++ b/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift @@ -45,6 +45,7 @@ public enum SQLDDLFallbackPolicy { "Beancount", "Firestore", "Firebase Realtime Database", + "Bigtable", ] /// True when `DROP`/`TRUNCATE` built by the app is something this engine could run. diff --git a/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift b/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift index efb4884497..c750649150 100644 --- a/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift +++ b/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift @@ -1,9 +1,10 @@ import Foundation +import TableProGoogleCloud public final class FirestoreClient: @unchecked Sendable { public let projectId: String private let baseURL: URL - private let requester: FirebaseRequester + private let requester: GoogleRESTRequester private let lock = NSLock() private var database: String @@ -11,14 +12,14 @@ public final class FirestoreClient: @unchecked Sendable { projectId: String, database: String, baseURL: URL, - transport: any FirebaseTransport, - credential: FirebaseCredential, + transport: any GoogleRESTTransport, + credential: GoogleRESTCredential, timeout: @escaping @Sendable () -> TimeInterval ) { self.projectId = projectId self.database = database self.baseURL = baseURL - self.requester = FirebaseRequester(transport: transport, credential: credential, timeout: timeout) + self.requester = GoogleRESTRequester(transport: transport, credential: credential, timeout: timeout) } public var currentDatabase: String { @@ -176,7 +177,7 @@ public final class FirestoreClient: @unchecked Sendable { guard var components = URLComponents(string: baseURL.absoluteString + path) else { throw FirebaseError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) } - components.percentEncodedQuery = FirebaseRequester.queryString(query) + components.percentEncodedQuery = FirebaseQueryString.encode(query) guard let url = components.url else { throw FirebaseError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) } diff --git a/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift b/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift index 18c553c61d..6f28e9214b 100644 --- a/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift +++ b/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift @@ -1,18 +1,19 @@ import Foundation +import TableProGoogleCloud public final class RealtimeDatabaseClient: @unchecked Sendable { - private let requester: FirebaseRequester + private let requester: GoogleRESTRequester private let lock = NSLock() private var instanceURL: URL public init( instanceURL: URL, - transport: any FirebaseTransport, - credential: FirebaseCredential, + transport: any GoogleRESTTransport, + credential: GoogleRESTCredential, timeout: @escaping @Sendable () -> TimeInterval ) { self.instanceURL = instanceURL - self.requester = FirebaseRequester(transport: transport, credential: credential, timeout: timeout) + self.requester = GoogleRESTRequester(transport: transport, credential: credential, timeout: timeout) } public var currentURL: URL { @@ -165,7 +166,7 @@ public final class RealtimeDatabaseClient: @unchecked Sendable { let joined = ([basePath].filter { !$0.isEmpty } + segments).joined(separator: "/") components.percentEncodedPath = "/" + joined + ".json" let namespace = components.queryItems?.filter { $0.name == "ns" } ?? [] - components.percentEncodedQuery = FirebaseRequester.queryString(namespace + query) + components.percentEncodedQuery = FirebaseQueryString.encode(namespace + query) guard let url = components.url else { throw FirebaseError.configuration(String(format: String(localized: "\"%@\" is not a valid path."), path)) } @@ -184,10 +185,10 @@ public final class RealtimeDatabaseClient: @unchecked Sendable { } public struct RealtimeDatabaseInstanceLister: Sendable { - private let requester: FirebaseRequester + private let requester: GoogleRESTRequester - public init(transport: any FirebaseTransport, credential: FirebaseCredential, timeout: @escaping @Sendable () -> TimeInterval) { - self.requester = FirebaseRequester(transport: transport, credential: credential, timeout: timeout) + public init(transport: any GoogleRESTTransport, credential: GoogleRESTCredential, timeout: @escaping @Sendable () -> TimeInterval) { + self.requester = GoogleRESTRequester(transport: transport, credential: credential, timeout: timeout) } public func instances(projectId: String) async throws -> [URL] { diff --git a/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift b/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift index f1c2825923..c60b7ab6b9 100644 --- a/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift +++ b/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift @@ -1,215 +1,33 @@ import Foundation +import TableProGoogleCloud -public struct FirebaseHTTPRequest: Sendable, Equatable { - public let method: String - public let url: URL - public let headers: [String: String] - public let body: Data? - public let timeoutInterval: TimeInterval - - public init(method: String, url: URL, headers: [String: String] = [:], body: Data? = nil, timeoutInterval: TimeInterval) { - self.method = method - self.url = url - self.headers = headers - self.body = body - self.timeoutInterval = timeoutInterval - } -} - -public struct FirebaseHTTPResponse: Sendable, Equatable { - public let statusCode: Int - public let headers: [String: String] - public let body: Data - - public init(statusCode: Int, headers: [String: String] = [:], body: Data) { - self.statusCode = statusCode - self.headers = headers - self.body = body - } - - public var isSuccess: Bool { - (200..<300).contains(statusCode) - } - - public func header(_ name: String) -> String? { - headers.first { $0.key.caseInsensitiveCompare(name) == .orderedSame }?.value - } - - public func json() throws -> FirebaseJSON { +public extension GoogleRESTResponse { + func json() throws -> FirebaseJSON { guard !body.isEmpty else { return .null } return try FirebaseJSON.parse(body) } +} +extension GoogleRESTResponse { func requireSuccess(_ operation: String) throws { guard !isSuccess else { return } throw FirebaseError.from(status: statusCode, body: body, operation: operation) } } -public protocol FirebaseTransport: Sendable { - func send(_ request: FirebaseHTTPRequest) async throws -> FirebaseHTTPResponse - func cancelAll() -} - -/// The emulators take the fixed token `owner`, which bypasses the security rules there. -public struct FirebaseCredential: Sendable { - public let token: @Sendable () async throws -> String? - public let invalidate: @Sendable () async -> Void - public let quotaProject: String? - - public init( - token: @escaping @Sendable () async throws -> String?, - invalidate: @escaping @Sendable () async -> Void = {}, - quotaProject: String? = nil - ) { - self.token = token - self.invalidate = invalidate - self.quotaProject = quotaProject - } - - public static let emulator = FirebaseCredential(token: { "owner" }) -} - -public final class URLSessionFirebaseTransport: FirebaseTransport, @unchecked Sendable { - private let session: URLSession - private let lock = NSLock() - private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] - - public init(resourceTimeout: TimeInterval) { - let configuration = URLSessionConfiguration.ephemeral - configuration.timeoutIntervalForResource = resourceTimeout - configuration.httpCookieStorage = nil - configuration.urlCache = nil - session = URLSession(configuration: configuration, delegate: FirebaseRedirectRefusal(), delegateQueue: nil) - } - - deinit { - session.invalidateAndCancel() - } - - public func cancelAll() { - let tasks = lock.withLock { Array(inFlight.values) } - tasks.forEach { $0.cancel() } - } - - public func send(_ request: FirebaseHTTPRequest) async throws -> FirebaseHTTPResponse { - var urlRequest = URLRequest(url: request.url) - urlRequest.httpMethod = request.method - urlRequest.httpBody = request.body - urlRequest.timeoutInterval = request.timeoutInterval - for (name, value) in request.headers { - urlRequest.setValue(value, forHTTPHeaderField: name) - } - let tracker = FirebaseTaskTracker(transport: self) - defer { tracker.finish() } - do { - let (data, response) = try await session.data(for: urlRequest, delegate: tracker) - guard let http = response as? HTTPURLResponse else { - throw FirebaseError.transport(String(localized: "Firebase answered with something other than HTTP.")) - } - var headers: [String: String] = [:] - for (key, value) in http.allHeaderFields { - if let key = key as? String, let value = value as? String { headers[key] = value } - } - return FirebaseHTTPResponse(statusCode: http.statusCode, headers: headers, body: data) - } catch let error as URLError where error.code == .cancelled { - throw CancellationError() - } catch let error as URLError { - throw FirebaseError.transport(error.localizedDescription) - } - } - - fileprivate func register(_ task: URLSessionTask) { - lock.withLock { inFlight[ObjectIdentifier(task)] = task } - } - - fileprivate func unregister(_ task: URLSessionTask) { - lock.withLock { inFlight[ObjectIdentifier(task)] = nil } - } -} - -/// A redirect would carry the bearer token to whatever host it names. -private final class FirebaseRedirectRefusal: NSObject, URLSessionTaskDelegate, Sendable { - func urlSession( - _ session: URLSession, - task: URLSessionTask, - willPerformHTTPRedirection response: HTTPURLResponse, - newRequest request: URLRequest - ) async -> URLRequest? { - nil - } +public extension GoogleRESTCredential { + /// The emulators take the fixed token `owner`, which bypasses the security rules there. + static let firebaseEmulator = GoogleRESTCredential(token: { "owner" }) } -private final class FirebaseTaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { - private weak var transport: URLSessionFirebaseTransport? - private let lock = NSLock() - private var task: URLSessionTask? - - init(transport: URLSessionFirebaseTransport) { - self.transport = transport - } - - func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { - lock.withLock { self.task = task } - transport?.register(task) - } - - func finish() { - guard let task = lock.withLock({ task }) else { return } - transport?.unregister(task) +public extension URLSessionGoogleRESTTransport { + static func firebase(resourceTimeout: TimeInterval) -> URLSessionGoogleRESTTransport { + URLSessionGoogleRESTTransport(resourceTimeout: resourceTimeout) { FirebaseError.transport($0) } } } -/// A 401 drops the cached token and retries once, since a token can expire between being cached -/// and being used. -struct FirebaseRequester: Sendable { - let transport: any FirebaseTransport - let credential: FirebaseCredential - let timeout: @Sendable () -> TimeInterval - - func send( - method: String, - url: URL, - body: Data? = nil, - headers: [String: String] = [:], - timeoutOverride: TimeInterval? = nil - ) async throws -> FirebaseHTTPResponse { - var response = try await attempt(method, url, body, headers, timeoutOverride) - if response.statusCode == 401 { - await credential.invalidate() - response = try await attempt(method, url, body, headers, timeoutOverride) - } - return response - } - - private func attempt( - _ method: String, - _ url: URL, - _ body: Data?, - _ extraHeaders: [String: String], - _ timeoutOverride: TimeInterval? - ) async throws -> FirebaseHTTPResponse { - var headers = extraHeaders - if let token = try await credential.token() { - headers["Authorization"] = "Bearer " + token - } - if let project = credential.quotaProject, headers["x-goog-user-project"] == nil { - headers["x-goog-user-project"] = project - } - if body != nil { - headers["Content-Type"] = "application/json" - } - try Task.checkCancellation() - return try await transport.send(FirebaseHTTPRequest( - method: method, - url: url, - headers: headers, - body: body, - timeoutInterval: timeoutOverride ?? timeout() - )) - } - - static func queryString(_ items: [URLQueryItem]) -> String? { +enum FirebaseQueryString { + static func encode(_ items: [URLQueryItem]) -> String? { guard !items.isEmpty else { return nil } var allowed = CharacterSet.urlQueryAllowed allowed.remove(charactersIn: "&=+?#") diff --git a/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTRequester.swift b/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTRequester.swift new file mode 100644 index 0000000000..62b177d372 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTRequester.swift @@ -0,0 +1,106 @@ +import Foundation + +public struct GoogleRESTCredential: Sendable { + public let token: @Sendable () async throws -> String? + public let invalidate: @Sendable () async -> Void + public let quotaProject: String? + + public init( + token: @escaping @Sendable () async throws -> String?, + invalidate: @escaping @Sendable () async -> Void = {}, + quotaProject: String? = nil + ) { + self.token = token + self.invalidate = invalidate + self.quotaProject = quotaProject + } + + public init(provider: any GoogleAccessTokenProviding, quotaProject: String?) { + self.init( + token: { try await provider.accessToken() }, + invalidate: { await provider.invalidateCachedToken() }, + quotaProject: quotaProject + ) + } + + /// `projectHint` is the project the key or the login names. Application Default Credentials + /// that name none fall back to `GOOGLE_CLOUD_PROJECT`, as the gcloud tools do. + public static func resolving( + _ source: GoogleCredentialSource, + scopes: [String], + environment: [String: String] = ProcessInfo.processInfo.environment, + readFile: (String) -> Data? = { FileManager.default.contents(atPath: $0) }, + http: any GoogleHTTPClient = URLSessionGoogleHTTPClient() + ) throws -> (credential: GoogleRESTCredential, projectHint: String?) { + let credentials = try GoogleTokenProviders.credentials( + from: source, + scopes: scopes, + readFile: readFile, + environment: environment, + http: http + ) + let environmentProject = source == .applicationDefault + ? GoogleCredentialDocument.trimmed(environment["GOOGLE_CLOUD_PROJECT"] ?? "") + : "" + return ( + GoogleRESTCredential(provider: credentials.tokenProvider, quotaProject: credentials.quotaProject), + credentials.projectHint ?? (environmentProject.isEmpty ? nil : environmentProject) + ) + } +} + +/// A 401 drops the cached token and retries once, since a token can expire between being cached +/// and being used. +public struct GoogleRESTRequester: Sendable { + public let transport: any GoogleRESTTransport + public let credential: GoogleRESTCredential + public let timeout: @Sendable () -> TimeInterval + + public init(transport: any GoogleRESTTransport, credential: GoogleRESTCredential, timeout: @escaping @Sendable () -> TimeInterval) { + self.transport = transport + self.credential = credential + self.timeout = timeout + } + + public func send( + method: String, + url: URL, + body: Data? = nil, + headers: [String: String] = [:], + timeoutOverride: TimeInterval? = nil + ) async throws -> GoogleRESTResponse { + var response = try await attempt(method, url, body, headers, timeoutOverride) + if response.statusCode == 401 { + await credential.invalidate() + response = try await attempt(method, url, body, headers, timeoutOverride) + } + return response + } + + private func attempt( + _ method: String, + _ url: URL, + _ body: Data?, + _ extraHeaders: [String: String], + _ timeoutOverride: TimeInterval? + ) async throws -> GoogleRESTResponse { + var headers = extraHeaders + if let token = try await credential.token() { + headers["Authorization"] = "Bearer " + token + } + if let project = credential.quotaProject, headers["x-goog-user-project"] == nil { + headers["x-goog-user-project"] = project + } + if body != nil { + headers["Content-Type"] = "application/json" + } + try Task.checkCancellation() + return try await transport.send(GoogleRESTRequest( + method: method, + url: url, + headers: headers, + body: body, + timeoutInterval: timeoutOverride ?? timeout() + )) + } +} diff --git a/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTTransport.swift b/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTTransport.swift new file mode 100644 index 0000000000..b398d77a0d --- /dev/null +++ b/Packages/TableProCore/Sources/TableProGoogleCloud/REST/GoogleRESTTransport.swift @@ -0,0 +1,122 @@ +import Foundation + +public struct GoogleRESTRequest: Sendable, Equatable { + public let method: String + public let url: URL + public let headers: [String: String] + public let body: Data? + public let timeoutInterval: TimeInterval + + public init(method: String, url: URL, headers: [String: String] = [:], body: Data? = nil, timeoutInterval: TimeInterval) { + self.method = method + self.url = url + self.headers = headers + self.body = body + self.timeoutInterval = timeoutInterval + } +} + +public struct GoogleRESTResponse: Sendable, Equatable { + public let statusCode: Int + public let headers: [String: String] + public let body: Data + + public init(statusCode: Int, headers: [String: String] = [:], body: Data) { + self.statusCode = statusCode + self.headers = headers + self.body = body + } + + public var isSuccess: Bool { + (200..<300).contains(statusCode) + } + + public func header(_ name: String) -> String? { + headers.first { $0.key.caseInsensitiveCompare(name) == .orderedSame }?.value + } +} + +public protocol GoogleRESTTransport: Sendable { + func send(_ request: GoogleRESTRequest) async throws -> GoogleRESTResponse + func cancelAll() +} + +public final class URLSessionGoogleRESTTransport: GoogleRESTTransport, @unchecked Sendable { + private let session: URLSession + private let transportError: @Sendable (String) -> any Error + private let lock = NSLock() + private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] + + public init(resourceTimeout: TimeInterval, transportError: @escaping @Sendable (String) -> any Error) { + let configuration = URLSessionConfiguration.ephemeral + configuration.timeoutIntervalForResource = resourceTimeout + configuration.httpCookieStorage = nil + configuration.urlCache = nil + session = URLSession(configuration: configuration, delegate: GoogleRedirectRefusingDelegate(), delegateQueue: nil) + self.transportError = transportError + } + + deinit { + session.invalidateAndCancel() + } + + public func cancelAll() { + let tasks = lock.withLock { Array(inFlight.values) } + tasks.forEach { $0.cancel() } + } + + public func send(_ request: GoogleRESTRequest) async throws -> GoogleRESTResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = request.timeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + let tracker = GoogleRESTTaskTracker(transport: self) + defer { tracker.finish() } + do { + let (data, response) = try await session.data(for: urlRequest, delegate: tracker) + guard let http = response as? HTTPURLResponse else { + throw transportError(String(localized: "The server answered with something other than HTTP.")) + } + var headers: [String: String] = [:] + for (key, value) in http.allHeaderFields { + if let key = key as? String, let value = value as? String { headers[key] = value } + } + return GoogleRESTResponse(statusCode: http.statusCode, headers: headers, body: data) + } catch let error as URLError where error.code == .cancelled { + throw CancellationError() + } catch let error as URLError { + throw transportError(error.localizedDescription) + } + } + + fileprivate func register(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = task } + } + + fileprivate func unregister(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = nil } + } +} + +private final class GoogleRESTTaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { + private weak var transport: URLSessionGoogleRESTTransport? + private let lock = NSLock() + private var task: URLSessionTask? + + init(transport: URLSessionGoogleRESTTransport) { + self.transport = transport + } + + func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { + lock.withLock { self.task = task } + transport?.register(task) + } + + func finish() { + guard let task = lock.withLock({ task }) else { return } + transport?.unregister(task) + } +} diff --git a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index d60c9bd3f9..26c31c4877 100644 --- a/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -285,6 +285,9 @@ public struct SQLLexicalProfile: Sendable, Hashable { "Typesense": commandLineFamily, "Weaviate": commandLineFamily, "Kafka": commandLineFamily, + "Firestore": commandLineFamily, + "Firebase Realtime Database": commandLineFamily, + "Bigtable": commandLineFamily, ] }() } diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableClientTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableClientTests.swift new file mode 100644 index 0000000000..e1f22a245e --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableClientTests.swift @@ -0,0 +1,113 @@ +import Foundation +@testable import TableProBigtableCore +import TableProGoogleCloud +import Testing + +struct BigtableConnectionSettingsTests { + @Test("Settings take the project from the credential and refuse IDs that would change the path") + func settings() throws { + let parsed = try BigtableConnectionSettings.parse(fields: ["btInstanceId": "ea-bt-01"], credentialProject: "my-proj") + #expect(parsed == BigtableConnectionSettings(projectId: "my-proj", instanceId: "ea-bt-01")) + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: [:]) } + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "p-123", "btInstanceId": "../x"]) } + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "Bad Project"]) } + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "p-123"], database: "a/b") } + } + + @Test("The database a connection opens on names its instance ahead of the Instance ID field") + func databaseNamesTheInstance() throws { + let fields = ["btProjectId": "my-proj", "btInstanceId": "first"] + #expect(try BigtableConnectionSettings.parse(fields: fields, database: "second").instanceId == "second") + #expect(try BigtableConnectionSettings.parse(fields: fields, database: " ").instanceId == "first") + #expect(try BigtableConnectionSettings.parse(fields: ["btProjectId": "my-proj"]).instanceId == nil) + } + + @Test("The credential comes from the key field, the password, or Application Default Credentials") + func credentialSource() throws { + #expect(try BigtableConnectionSettings.credentialSource(fields: ["btServiceAccountJson": "/k.json"], password: nil) == .serviceAccountKey("/k.json")) + #expect(try BigtableConnectionSettings.credentialSource(fields: [:], password: " {} ") == .serviceAccountKey("{}")) + #expect(try BigtableConnectionSettings.credentialSource(fields: ["btAuthMethod": "adc"], password: nil) == .applicationDefault) + #expect(throws: BigtableError.self) { try BigtableConnectionSettings.credentialSource(fields: [:], password: "") } + } + + /// `PluginMetadataRegistry+BigtableDefaults.swift` in the app spells these IDs out, since the app + /// links no engine core. Renaming one here strands the connections saved under the old one. + @Test("Field IDs keep the spelling saved connections and the app's defaults use") + func fieldIDs() { + #expect([ + BigtableFieldID.authMethod, BigtableFieldID.serviceAccountJson, BigtableFieldID.projectId, + BigtableFieldID.instanceId, BigtableFieldID.appProfileId + ] == ["btAuthMethod", "btServiceAccountJson", "btProjectId", "btInstanceId", "btAppProfileId"]) + #expect(BigtableAuthMethod.applicationDefault.rawValue == "adc") + } +} + +struct BigtableClientTests { + private static let committedRow = """ + [{"chunks": [{"rowKey": "\(b64("k"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "", "commitRow": true}]}] + """ + + private func client(_ transport: RecordingTransport) -> BigtableClient { + BigtableClient( + projectId: "proj", instanceId: "inst", transport: transport, + credential: GoogleRESTCredential(token: { "token" }), timeout: { 30 } + ) + } + + private static func reply(_ body: String, status: Int = 200) -> GoogleRESTResponse { + GoogleRESTResponse(statusCode: status, body: Data(body.utf8)) + } + + @Test("An insert onto a key that already has a row is refused without writing") + func insertRefusesExistingKey() async throws { + let transport = RecordingTransport { _ in Self.reply(Self.committedRow) } + let column = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + + await #expect(throws: BigtableError.self) { + try await client(transport).insertRow(table: "t", key: Data("k".utf8), mutations: [.setCell(column, Data("1".utf8))]) + } + #expect(transport.requests.map { $0.path } == ["/v2/projects/proj/instances/inst/tables/t:readRows"]) + } + + @Test("An insert onto a free key looks it up, then writes the row") + func insertWritesFreeKey() async throws { + let transport = RecordingTransport { request in + Self.reply(request.path.hasSuffix(":readRows") ? "[]" : "{}") + } + let column = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + + try await client(transport).insertRow(table: "t", key: Data("k".utf8), mutations: [.setCell(column, Data("1".utf8))]) + + let requests = transport.requests + #expect(requests.map { $0.path } == [ + "/v2/projects/proj/instances/inst/tables/t:readRows", + "/v2/projects/proj/instances/inst/tables/t:mutateRow" + ]) + #expect(jsonText(requests[0].jsonBody["rows"] ?? [:]) == #"{"rowKeys":["\#(b64("k"))"]}"#) + #expect(requests[1].jsonBody["rowKey"] as? String == b64("k")) + } + + @Test("A read the filters ruled out sends no request") + func emptyReadSendsNothing() async throws { + let transport = RecordingTransport { _ in Self.reply("[]") } + #expect(try await client(transport).readRows(table: "t", body: nil).isEmpty) + #expect(transport.requests.isEmpty) + } + + @Test("A health check asks for one table name, not every page of them") + func pingReadsOnePage() async throws { + let transport = RecordingTransport { _ in Self.reply(#"{"tables": [{"name": "x/tables/a"}], "nextPageToken": "more"}"#) } + try await client(transport).ping() + let request = try #require(transport.requests.first) + #expect(transport.requests.count == 1) + #expect(URLComponents(url: request.url, resolvingAgainstBaseURL: false)?.queryItems?.first { $0.name == "pageSize" }?.value == "1") + } + + @Test("A denied read is a permission error naming the table, not an empty page") + func deniedRead() async { + let transport = RecordingTransport { _ in Self.reply(#"[{"error": {"code": 403, "message": "bigtable.tables.readRows denied"}}]"#, status: 403) } + await #expect(throws: BigtableError.permissionDenied(operation: "read t", detail: "bigtable.tables.readRows denied")) { + try await client(transport).readRows(table: "t", body: [:]) + } + } +} diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableCoreTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableCoreTests.swift deleted file mode 100644 index 0fe0ef0549..0000000000 --- a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableCoreTests.swift +++ /dev/null @@ -1,234 +0,0 @@ -import Foundation -@testable import TableProBigtableCore -import Testing - -private func b64(_ text: String) -> String { Data(text.utf8).base64EncodedString() } -private func json(_ text: String) -> Any { (try? JSONSerialization.jsonObject(with: Data(text.utf8))) ?? [:] } - -@Suite("Bigtable chunk assembly") -struct BigtableChunkAssemblerTests { - @Test("Family and qualifier carry over, split values join, and only committed rows count") - func assembly() throws { - let response = json(""" - [{"chunks": [ - {"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "10", "value": "\(b64("he"))", "valueSize": 5}, - {"value": "\(b64("llo"))"}, - {"qualifier": "\(b64("b"))", "timestampMicros": "10", "value": "\(b64("x"))", "commitRow": true} - ]}, - {"chunks": [ - {"rowKey": "\(b64("r2"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "\(b64("old"))"}, - {"resetRow": true}, - {"rowKey": "\(b64("r2"))", "familyName": {"value": "cf"}, "qualifier": "\(b64("a"))", "timestampMicros": "2", "value": "\(b64("new"))", "commitRow": true} - ]}] - """) - let rows = try BigtableChunkAssembler.rows(fromResponse: response) - let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) - #expect(rows.map(\.key) == [Data("r1".utf8), Data("r2".utf8)]) - #expect(rows[0].cells[a]?.value == Data("hello".utf8)) - #expect(rows[0].cells[BigtableColumn(family: "cf", qualifier: Data("b".utf8))]?.value == Data("x".utf8)) - #expect(rows[1].cells[a] == BigtableCell(value: Data("new".utf8), timestampMicros: 2)) - } - - @Test("A response that stops inside a row is refused") - func truncated() { - let response = json(""" - [{"chunks": [{"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "\(b64("v"))"}]}] - """) - #expect(throws: BigtableError.self) { try BigtableChunkAssembler.rows(fromResponse: response) } - } - - @Test("Rows lay out with _key first, columns in order, and missing cells absent") - func rowSet() { - let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) - let z = BigtableColumn(family: "meta", qualifier: Data([0xFF, 0x00])) - let set = BigtableRowSet(rows: [ - BigtableRow(key: Data("k1".utf8), cells: [a: BigtableCell(value: Data("v".utf8), timestampMicros: 1)]), - BigtableRow(key: Data([0x00, 0x01]), cells: [z: BigtableCell(value: Data([0xC3]), timestampMicros: 1)]) - ]) - #expect(set.columns == ["_key", "cf:a", "meta:b64:/wA="]) - #expect(set.rows[0] == [.text("k1"), .text("v"), .absent]) - #expect(set.rows[1] == [.text("b64:AAE="), .absent, .bytes(Data([0xC3]))]) - #expect(set.absentCells == [0: [2], 1: [1]]) - #expect(set.locators == [b64("k1"), Data([0x00, 0x01]).base64EncodedString()]) - } -} - -@Suite("Bigtable bytes and columns") -struct BigtableBytesTests { - @Test("Printable text shows as itself, anything else as b64 that reads back") - func displayRoundTrip() throws { - for data in [Data("user#42".utf8), Data([0, 1, 255]), Data("b64:looks-tagged".utf8), Data("line\u{07}bell".utf8)] { - #expect(try BigtableBytes.parse(BigtableBytes.display(data)) == data) - } - #expect(BigtableBytes.display(Data("user#42".utf8)) == "user#42") - #expect(throws: BigtableError.self) { try BigtableBytes.parse("b64:***") } - } - - @Test("Prefix end increments the last byte that can grow") - func prefixEnd() { - #expect(BigtableBytes.prefixEnd(Data("ab".utf8)) == Data("ac".utf8)) - #expect(BigtableBytes.prefixEnd(Data([0x61, 0xFF])) == Data([0x62])) - #expect(BigtableBytes.prefixEnd(Data([0xFF, 0xFF])) == nil) - } - - @Test("Regex literals escape operators and non-printable bytes") - func regexLiteral() { - #expect(BigtableBytes.regexText(Data("a.b*(c)".utf8)) == #"a\.b\*\(c\)"#) - #expect(BigtableBytes.regexText(Data([0x00, 0x41])) == #"\x00A"#) - } - - @Test("A column name splits at its first colon") - func columnNames() throws { - let column = try BigtableColumn(name: "cf:a:b") - #expect(column.family == "cf" && column.qualifier == Data("a:b".utf8)) - #expect(try BigtableColumn(name: "cf:b64:/wA=").qualifier == Data([0xFF, 0x00])) - #expect(throws: BigtableError.self) { try BigtableColumn(name: "nocolon") } - #expect(throws: BigtableError.self) { try BigtableColumn(name: ":q") } - } -} - -@Suite("Bigtable read requests") -struct BigtableReadRequestTests { - private func body(_ filters: [BigtableFilterSpec], or: Bool = false, descending: Bool = false, after: String? = nil) throws -> [String: Any] { - let spec = BigtableBrowseSpec(table: "t", filters: filters, logicMode: or ? "OR" : "AND", descending: descending, limit: 10, offset: 0) - return try BigtableReadRequest.body(spec: spec, limit: 10, after: after.map { Data($0.utf8) }) - } - - private func text(_ object: Any) -> String { - (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys])).flatMap { String(bytes: $0, encoding: .utf8) } ?? "" - } - - @Test("A plain page reads the newest cells, with the limit as a string") - func plain() throws { - #expect(text(try body([])) == #"{"filter":{"cellsPerColumnLimitFilter":1},"rowsLimit":"10"}"#) - } - - @Test("Key filters become row ranges, intersected under AND") - func keyRanges() throws { - let prefix = try body([BigtableFilterSpec(column: "_key", op: "STARTS WITH", value: "user#")]) - #expect(text(prefix["rows"] ?? [:]) == #"{"rowRanges":[{"endKeyOpen":"\#(b64("user$"))","startKeyClosed":"\#(b64("user#"))"}]}"#) - let both = try body([ - BigtableFilterSpec(column: "_key", op: ">=", value: "b"), - BigtableFilterSpec(column: "_key", op: "<", value: "m"), - BigtableFilterSpec(column: "_key", op: ">", value: "c") - ]) - #expect(text(both["rows"] ?? [:]) == #"{"rowRanges":[{"endKeyOpen":"\#(b64("m"))","startKeyOpen":"\#(b64("c"))"}]}"#) - let exact = try body([BigtableFilterSpec(column: "_key", op: "IN", value: "a, b, z"), BigtableFilterSpec(column: "_key", op: "<", value: "c")]) - #expect(text(exact["rows"] ?? [:]) == #"{"rowKeys":["\#(b64("a"))","\#(b64("b"))"]}"#) - } - - @Test("A cursor continues after the last key, and a descending read is refused") - func cursors() throws { - #expect(text(try body([], after: "k9")["rows"] ?? [:]) == #"{"rowRanges":[{"startKeyOpen":"\#(b64("k9"))"}]}"#) - #expect(throws: BigtableError.self) { try body([], descending: true) } - } - - @Test("Key filters ORed together become several ranges, cut by the cursor") - func orRanges() throws { - let rows = try body([ - BigtableFilterSpec(column: "_key", op: "STARTS WITH", value: "a"), - BigtableFilterSpec(column: "_key", op: "=", value: "q") - ], or: true, after: "a5")["rows"] as? [String: Any] - #expect(text(rows ?? [:]) == #"{"rowKeys":["\#(b64("q"))"],"rowRanges":[{"endKeyOpen":"\#(b64("b"))","startKeyOpen":"\#(b64("a5"))"}]}"#) - } - - @Test("Column filters keep or drop whole rows through conditions") - func columnConditions() throws { - let equal = try body([BigtableFilterSpec(column: "cf:name", op: "=", value: "A.n")]) - let filter = text(equal["filter"] ?? [:]) - #expect(filter.contains(#""familyNameRegexFilter":"^cf$""#)) - #expect(filter.contains(b64(#"(?s)^A\.n$"#))) - #expect(filter.contains(#""trueFilter":{"passAllFilter":true}"#)) - let missing = text(try body([BigtableFilterSpec(column: "cf:name", op: "IS NULL", value: "")])["filter"] ?? [:]) - #expect(missing.contains(#""falseFilter":{"passAllFilter":true}"#) && !missing.contains("valueRegexFilter")) - let range = text(try body([BigtableFilterSpec(column: "cf:n", op: "BETWEEN", value: "1", secondValue: "9")])["filter"] ?? [:]) - #expect(range.contains(#""startValueClosed":"\#(b64("1"))""#) && range.contains(#""endValueClosed":"\#(b64("9"))""#)) - } - - @Test("Several column filters chain under AND and interleave under OR") - func combinations() throws { - let filters = [BigtableFilterSpec(column: "cf:a", op: "=", value: "1"), BigtableFilterSpec(column: "cf:b", op: "CONTAINS", value: "x")] - #expect(text(try body(filters)["filter"] ?? [:]).hasPrefix(#"{"chain":{"filters":[{"chain""#)) - #expect(text(try body(filters, or: true)["filter"] ?? [:]).hasPrefix(#"{"chain":{"filters":[{"interleave""#)) - } - - @Test("What Bigtable cannot express is refused with the reason") - func refusals() { - #expect(throws: BigtableError.self) { - try body([BigtableFilterSpec(column: "_key", op: ">", value: "a"), BigtableFilterSpec(column: "cf:a", op: "=", value: "1")], or: true) - } - #expect(throws: BigtableError.self) { try body([BigtableFilterSpec(column: "nocolon", op: "=", value: "1")]) } - #expect(throws: BigtableError.self) { try body([BigtableFilterSpec(column: "__RAW__", op: "", value: "not json")]) } - let sorted = BigtableBrowseSpec(table: "t", sortColumn: "cf:a", limit: 1, offset: 0) - #expect(throws: BigtableError.self) { try BigtableReadRequest.body(spec: sorted, limit: 1, after: nil) } - } - - @Test("An empty key list asks for nothing and is skipped") - func asksForNothing() throws { - let impossible = try body([BigtableFilterSpec(column: "_key", op: "=", value: "a"), BigtableFilterSpec(column: "_key", op: "=", value: "b")]) - #expect(BigtableReadRequest.asksForNothing(impossible)) - #expect(!BigtableReadRequest.asksForNothing(try body([]))) - } - - @Test("A keys-only read strips values and keeps one cell per row") - func keysOnly() throws { - let spec = BigtableBrowseSpec(table: "t", limit: 0, offset: 0) - let object = try BigtableReadRequest.body(spec: spec, limit: 500, after: nil, keysOnly: true) - #expect(text(object["filter"] ?? [:]) == #"{"chain":{"filters":[{"cellsPerRowLimitFilter":1},{"stripValueTransformer":true}]}}"#) - } -} - -@Suite("Bigtable statements and settings") -struct BigtableStatementTests { - @Test("Request lines read table, key, column, query and body") - func requests() throws { - let get = try BigtableRequest.parse("get events/user#42") - #expect(get.verb == .get && get.table == "events" && get.key == Data("user#42".utf8)) - let quoted = try BigtableRequest.parse(#"DELETE events/"a b/c" profile:name"#) - #expect(quoted.key == Data("a b/c".utf8) && quoted.column == "profile:name") - let scan = try BigtableRequest.parse("SCAN events/user#?limit=20") - #expect(scan.key == Data("user#".utf8) && scan.limit == 20) - let all = try BigtableRequest.parse("SCAN events/") - #expect(all.table == "events" && all.key == nil) - let put = try BigtableRequest.parse(#"PUT events/b64:AAE= {"cf:a": "1"}"#) - #expect(put.key == Data([0, 1]) && put.body == #"{"cf:a": "1"}"#) - let read = try BigtableRequest.parse(#"READ events {"rowsLimit": "5"}"#) - #expect(read.table == "events" && read.body == #"{"rowsLimit": "5"}"#) - } - - @Test("Malformed requests explain the syntax") - func malformed() { - for text in ["SELECT * FROM t", "GET events/", "PUT events/k", #"GET events/"open"#, "READ events"] { - #expect(throws: BigtableError.self) { try BigtableRequest.parse(text) } - } - } - - @Test("A request reads back from its own text, quoting keys that need it") - func roundTrip() throws { - for key in [Data("user#42".utf8), Data("a b".utf8), Data([0xFF]), Data("x/y".utf8)] { - let request = BigtableRequest(verb: .delete, table: "events", key: key, column: "cf:a") - #expect(try BigtableRequest.parse(request.text(defaultTable: nil)) == request) - } - let spec = BigtableBrowseSpec(table: "t", filters: [BigtableFilterSpec(column: "_key", op: "=", value: "k")], limit: 5, offset: 10) - #expect(try BigtableStatement.parse(BigtableStatement.encode(spec)) == .browse(spec)) - } - - @Test("The read-only gate sees reads, writes and deletes") - func classify() { - #expect(BigtableStatementClassifier.classify(BigtableStatement.encode(BigtableBrowseSpec(table: "t", limit: 1, offset: 0))) == .read) - #expect(BigtableStatementClassifier.classify("SCAN events/a") == .read) - #expect(BigtableStatementClassifier.classify(#"READ events {}"#) == .read) - #expect(BigtableStatementClassifier.classify(#"PUT events/k {"cf:a": "1"}"#) == .write) - #expect(BigtableStatementClassifier.classify("DELETE events/k") == .destructive) - #expect(BigtableStatementClassifier.classify("DROP TABLE events") == .write) - } - - @Test("Settings take the project from the credential and refuse IDs that would change the path") - func settings() throws { - let parsed = try BigtableConnectionSettings.parse(fields: ["btInstanceId": "ea-bt-01"], credentialProject: "my-proj") - #expect(parsed.projectId == "my-proj" && parsed.instanceId == "ea-bt-01" && parsed.authMethod == .serviceAccount) - #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: [:]) } - #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "p-123", "btInstanceId": "../x"]) } - #expect(throws: BigtableError.self) { try BigtableConnectionSettings.parse(fields: ["btProjectId": "Bad Project"]) } - } -} diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableReadRequestTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableReadRequestTests.swift new file mode 100644 index 0000000000..4fc77e55fc --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableReadRequestTests.swift @@ -0,0 +1,148 @@ +import Foundation +@testable import TableProBigtableCore +import Testing + +struct BigtableReadRequestTests { + private func body( + _ filters: [BigtableFilterSpec], + or: Bool = false, + descending: Bool = false, + after: String? = nil + ) throws -> [String: Any]? { + let spec = BigtableBrowseSpec(table: "t", filters: filters, logicMode: or ? "OR" : "AND", descending: descending, limit: 10, offset: 0) + return try BigtableReadRequest.body(spec: spec, limit: 10, after: after.map { Data($0.utf8) }) + } + + private func rows(_ filters: [BigtableFilterSpec], or: Bool = false, after: String? = nil) throws -> String { + jsonText(try #require(try body(filters, or: or, after: after))["rows"] ?? [String: Any]()) + } + + private func filter(_ filters: [BigtableFilterSpec], or: Bool = false) throws -> String { + jsonText(try #require(try body(filters, or: or))["filter"] ?? [String: Any]()) + } + + private func key(_ op: String, _ value: String, _ second: String? = nil, caseSensitive: Bool = true) -> BigtableFilterSpec { + BigtableFilterSpec(column: "_key", op: op, value: value, secondValue: second, isCaseSensitive: caseSensitive) + } + + @Test("A plain page reads the newest cells, with the limit as a string") + func plain() throws { + #expect(jsonText(try #require(try body([]))) == #"{"filter":{"cellsPerColumnLimitFilter":1},"rowsLimit":"10"}"#) + } + + @Test("Key filters become row ranges, intersected under AND") + func keyRanges() throws { + #expect(try rows([key("STARTS WITH", "user#")]) + == #"{"rowRanges":[{"endKeyOpen":"\#(b64("user$"))","startKeyClosed":"\#(b64("user#"))"}]}"#) + #expect(try rows([key(">=", "b"), key("<", "m"), key(">", "c")]) + == #"{"rowRanges":[{"endKeyOpen":"\#(b64("m"))","startKeyOpen":"\#(b64("c"))"}]}"#) + #expect(try rows([key("IN", "a, b, z"), key("<", "c")]) == #"{"rowKeys":["\#(b64("a"))","\#(b64("b"))"]}"#) + } + + @Test("A stricter bound keeps its own open or closed end") + func stricterBoundKeepsItsEnd() throws { + #expect(try rows([key("STARTS WITH", "user#"), key("<=", "user#5")]) + == #"{"rowRanges":[{"endKeyClosed":"\#(b64("user#5"))","startKeyClosed":"\#(b64("user#"))"}]}"#) + #expect(try rows([key(">", "a"), key(">=", "b")]) == #"{"rowRanges":[{"startKeyClosed":"\#(b64("b"))"}]}"#) + #expect(try rows([key(">=", "b"), key(">", "b")]) == #"{"rowRanges":[{"startKeyOpen":"\#(b64("b"))"}]}"#) + } + + @Test("Filters that admit no key read nothing instead of sending an empty or inverted range") + func emptySelections() throws { + #expect(try body([key("<=", "k")], after: "k") == nil) + #expect(try body([key(">", "m"), key("<", "c")]) == nil) + #expect(try body([key("=", "a"), key("=", "b")]) == nil) + #expect(try body([key("=", "a")], after: "a") == nil) + } + + @Test("A cursor continues after the last key, and a descending read is refused") + func cursors() throws { + #expect(try rows([], after: "k9") == #"{"rowRanges":[{"startKeyOpen":"\#(b64("k9"))"}]}"#) + #expect(throws: BigtableError.self) { try body([], descending: true) } + } + + @Test("Key filters ORed together become several ranges, and a range the cursor passed is dropped") + func orRanges() throws { + #expect(try rows([key("STARTS WITH", "a"), key("=", "q")], or: true, after: "a5") + == #"{"rowKeys":["\#(b64("q"))"],"rowRanges":[{"endKeyOpen":"\#(b64("b"))","startKeyOpen":"\#(b64("a5"))"}]}"#) + #expect(try rows([key("STARTS WITH", "a"), key("=", "q")], or: true, after: "b5") == #"{"rowKeys":["\#(b64("q"))"]}"#) + #expect(try body([key("STARTS WITH", "a"), key("=", "q")], or: true, after: "r") == nil) + } + + @Test("Base64 items in an IN list each read back as bytes") + func inListOfBytes() throws { + #expect(try rows([key("IN", "b64:AAE=, b64:AAI=")]) == #"{"rowKeys":["AAE=","AAI="]}"#) + let column = try filter([BigtableFilterSpec(column: "cf:a", op: "IN", value: "b64:AAE=, b64:AAI=")]) + #expect(column.contains(b64(#"(?s)^(\x00\x01|\x00\x02)$"#))) + } + + @Test("A case-insensitive key match runs as a key regex, since a row range is exact") + func caseInsensitiveKey() throws { + let object = try #require(try body([key("=", "Alice", caseSensitive: false)])) + #expect(object["rows"] == nil) + #expect(jsonText(object["filter"] ?? [:]).contains(#""rowKeyRegexFilter":"\#(b64("(?is)^Alice$"))""#)) + } + + @Test("Case-insensitive text filters fold case, case-sensitive ones do not") + func caseFolding() throws { + let insensitive = try filter([BigtableFilterSpec(column: "cf:name", op: "CONTAINS", value: "alice", isCaseSensitive: false)]) + #expect(insensitive.contains(b64("(?is).*alice.*"))) + let sensitive = try filter([BigtableFilterSpec(column: "cf:name", op: "CONTAINS", value: "alice")]) + #expect(sensitive.contains(b64("(?s).*alice.*"))) + let regex = try filter([BigtableFilterSpec(column: "cf:name", op: "REGEX", value: "^a", isCaseSensitive: false)]) + #expect(regex.contains(b64("(?i)^a"))) + } + + @Test("Column filters keep or drop whole rows through conditions") + func columnConditions() throws { + let equal = try filter([BigtableFilterSpec(column: "cf:name", op: "=", value: "A.n")]) + #expect(equal.contains(#""familyNameRegexFilter":"^cf$""#)) + #expect(equal.contains(b64(#"(?s)^A\.n$"#))) + #expect(equal.contains(#""trueFilter":{"passAllFilter":true}"#)) + let missing = try filter([BigtableFilterSpec(column: "cf:name", op: "IS NULL", value: "")]) + #expect(missing.contains(#""falseFilter":{"passAllFilter":true}"#) && !missing.contains("valueRegexFilter")) + let range = try filter([BigtableFilterSpec(column: "cf:n", op: "BETWEEN", value: "1", secondValue: "9")]) + #expect(range.contains(#""startValueClosed":"\#(b64("1"))""#) && range.contains(#""endValueClosed":"\#(b64("9"))""#)) + } + + @Test("A negative test on a column also needs the column, as in SQL a missing value matches neither side") + func negativeFiltersNeedTheColumn() throws { + for op in ["!=", "NOT CONTAINS", "NOT IN", "IS NOT EMPTY"] { + let text = try filter([BigtableFilterSpec(column: "cf:name", op: op, value: "x")]) + let present = try #require(text.range(of: #""trueFilter":{"passAllFilter":true}"#), "\(op)") + let excluded = try #require(text.range(of: #""falseFilter":{"passAllFilter":true}"#), "\(op)") + #expect(present.lowerBound < excluded.lowerBound, "\(op)") + } + } + + @Test("Several column filters chain under AND and interleave under OR") + func combinations() throws { + let filters = [BigtableFilterSpec(column: "cf:a", op: "=", value: "1"), BigtableFilterSpec(column: "cf:b", op: "CONTAINS", value: "x")] + #expect(try filter(filters).hasPrefix(#"{"chain":{"filters":[{"chain""#)) + #expect(try filter(filters, or: true).hasPrefix(#"{"chain":{"filters":[{"interleave""#)) + } + + @Test("What Bigtable cannot express is refused with the reason") + func refusals() { + #expect(throws: BigtableError.self) { + try body([key(">", "a"), BigtableFilterSpec(column: "cf:a", op: "=", value: "1")], or: true) + } + #expect(throws: BigtableError.self) { try body([BigtableFilterSpec(column: "nocolon", op: "=", value: "1")]) } + #expect(throws: BigtableError.self) { try body([BigtableFilterSpec(column: "__RAW__", op: "", value: "not json")]) } + let sorted = BigtableBrowseSpec(table: "t", sortColumn: "cf:a", limit: 1, offset: 0) + #expect(throws: BigtableError.self) { try BigtableReadRequest.body(spec: sorted, limit: 1, after: nil) } + } + + @Test("A keys-only read strips values and keeps one cell per row") + func keysOnly() throws { + let spec = BigtableBrowseSpec(table: "t", limit: 0, offset: 0) + let object = try #require(try BigtableReadRequest.body(spec: spec, limit: 500, after: nil, keysOnly: true)) + #expect(jsonText(object["filter"] ?? [:]) == #"{"chain":{"filters":[{"cellsPerRowLimitFilter":1},{"stripValueTransformer":true}]}}"#) + } + + @Test("A column sample strips values, since only the column names are wanted") + func columnSample() { + #expect(jsonText(BigtableReadRequest.columnSample(limit: 50)) + == #"{"filter":{"chain":{"filters":[{"cellsPerColumnLimitFilter":1},{"stripValueTransformer":true}]}},"rowsLimit":"50"}"#) + } +} diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift new file mode 100644 index 0000000000..36757a8d44 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift @@ -0,0 +1,122 @@ +import Foundation +@testable import TableProBigtableCore +import Testing + +struct BigtableChunkAssemblerTests { + @Test("Family and qualifier carry over, split values join, and only committed rows count") + func assembly() throws { + let response = json(""" + [{"chunks": [ + {"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "10", "value": "\(b64("he"))", "valueSize": 5}, + {"value": "\(b64("llo"))"}, + {"qualifier": "\(b64("b"))", "timestampMicros": "10", "value": "\(b64("x"))", "commitRow": true} + ]}, + {"chunks": [ + {"rowKey": "\(b64("r2"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "\(b64("old"))"}, + {"resetRow": true}, + {"rowKey": "\(b64("r2"))", "familyName": {"value": "cf"}, "qualifier": "\(b64("a"))", "timestampMicros": "2", "value": "\(b64("new"))", "commitRow": true} + ]}] + """) + let rows = try BigtableChunkAssembler.rows(fromResponse: response, operation: "read t") + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + #expect(rows.map(\.key) == [Data("r1".utf8), Data("r2".utf8)]) + #expect(rows[0].cells[a]?.value == Data("hello".utf8)) + #expect(rows[0].cells[BigtableColumn(family: "cf", qualifier: Data("b".utf8))]?.value == Data("x".utf8)) + #expect(rows[1].cells[a] == BigtableCell(value: Data("new".utf8), timestampMicros: 2)) + } + + @Test("A response that stops inside a row is refused") + func truncated() { + let response = json(""" + [{"chunks": [{"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "\(b64("v"))"}]}] + """) + #expect(throws: BigtableError.self) { try BigtableChunkAssembler.rows(fromResponse: response, operation: "read t") } + } + + @Test("An error that ends a stream after its first rows fails the read instead of shortening it") + func streamError() { + let response = json(""" + [{"chunks": [{"rowKey": "\(b64("r1"))", "familyName": "cf", "qualifier": "\(b64("a"))", "timestampMicros": "1", "value": "", "commitRow": true}]}, + {"error": {"code": 503, "message": "The service is unavailable.", "status": "UNAVAILABLE"}}] + """) + #expect(throws: BigtableError.api(status: 503, message: "The service is unavailable.")) { + try BigtableChunkAssembler.rows(fromResponse: response, operation: "read t") + } + } +} + +struct BigtableRowSetTests { + @Test("Rows lay out with _key first, columns in order, and missing cells absent") + func layout() { + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + let z = BigtableColumn(family: "meta", qualifier: Data([0xFF, 0x00])) + let set = BigtableRowSet(rows: [ + BigtableRow(key: Data("k1".utf8), cells: [a: BigtableCell(value: Data("v".utf8), timestampMicros: 1)]), + BigtableRow(key: Data([0x00, 0x01]), cells: [z: BigtableCell(value: Data([0xC3]), timestampMicros: 1)]) + ]) + #expect(set.columns == ["_key", "cf:a", "meta:b64:/wA="]) + #expect(set.rows[0] == [.text("k1"), .text("v"), .absent]) + #expect(set.rows[1] == [.text("b64:AAE="), .absent, .bytes(Data([0xC3]))]) + #expect(set.absentCells == [0: [2], 1: [1]]) + #expect(set.locators == [b64("k1"), Data([0x00, 0x01]).base64EncodedString()]) + } + + @Test("Text that starts with b64: reaches the grid as bytes, so writing it back keeps it as text") + func taggedTextRoundTrips() throws { + let stored = Data("b64:AAE=".utf8) + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + let set = BigtableRowSet(rows: [BigtableRow(key: Data("k".utf8), cells: [a: BigtableCell(value: stored, timestampMicros: 1)])]) + #expect(set.rows[0][1] == .bytes(stored)) + + let write = try #require(try BigtableRowWrites.update(table: "t", key: Data("k".utf8), changes: [("cf:a", set.rows[0][1])])) + #expect(try write.mutations() == [.setCell(a, stored)]) + } + + @Test("An export page lines up under the first page's columns and keeps the rest in __extra__") + func exportAlignment() { + let a = BigtableColumn(family: "cf", qualifier: Data("a".utf8)) + let b = BigtableColumn(family: "cf", qualifier: Data("b".utf8)) + let page = BigtableRowSet(rows: [ + BigtableRow(key: Data("k1".utf8), cells: [a: BigtableCell(value: Data("1".utf8), timestampMicros: 1)]), + BigtableRow(key: Data("k2".utf8), cells: [b: BigtableCell(value: Data("x/y".utf8), timestampMicros: 1)]) + ]) + let header = ["_key", "cf:a", BigtableRowSet.extraColumn] + #expect(page.rows(alignedTo: header) == [ + [.text("k1"), .text("1"), .absent], + [.text("k2"), .absent, .text(#"{"cf:b":"x/y"}"#)] + ]) + } +} + +struct BigtableBytesTests { + @Test("Printable text shows as itself, anything else as b64 that reads back") + func displayRoundTrip() throws { + for data in [Data("user#42".utf8), Data([0, 1, 255]), Data("b64:looks-tagged".utf8), Data("line\u{07}bell".utf8)] { + #expect(try BigtableBytes.parse(BigtableBytes.display(data)) == data) + } + #expect(BigtableBytes.display(Data("user#42".utf8)) == "user#42") + #expect(throws: BigtableError.self) { try BigtableBytes.parse("b64:***") } + } + + @Test("Prefix end increments the last byte that can grow") + func prefixEnd() { + #expect(BigtableBytes.prefixEnd(Data("ab".utf8)) == Data("ac".utf8)) + #expect(BigtableBytes.prefixEnd(Data([0x61, 0xFF])) == Data([0x62])) + #expect(BigtableBytes.prefixEnd(Data([0xFF, 0xFF])) == nil) + } + + @Test("Regex literals escape operators and non-printable bytes") + func regexLiteral() { + #expect(BigtableBytes.regexText(Data("a.b*(c)".utf8)) == #"a\.b\*\(c\)"#) + #expect(BigtableBytes.regexText(Data([0x00, 0x41])) == #"\x00A"#) + } + + @Test("A column name splits at its first colon") + func columnNames() throws { + let column = try BigtableColumn(name: "cf:a:b") + #expect(column.family == "cf" && column.qualifier == Data("a:b".utf8)) + #expect(try BigtableColumn(name: "cf:b64:/wA=").qualifier == Data([0xFF, 0x00])) + #expect(throws: BigtableError.self) { try BigtableColumn(name: "nocolon") } + #expect(throws: BigtableError.self) { try BigtableColumn(name: ":q") } + } +} diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift new file mode 100644 index 0000000000..f8bc8ba3f0 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift @@ -0,0 +1,135 @@ +import Foundation +@testable import TableProBigtableCore +import Testing + +struct BigtableStatementTests { + private let nameColumn = BigtableColumn(family: "cf", qualifier: Data("name".utf8)) + + @Test("Request lines read table, key, column, query and body") + func requests() throws { + let get = try BigtableRequest.parse("get events/user#42") + #expect(get.verb == .get && get.table == "events" && get.key == Data("user#42".utf8)) + let quoted = try BigtableRequest.parse(#"DELETE events/"a b/c" profile:name"#) + #expect(quoted.key == Data("a b/c".utf8) && quoted.column == "profile:name") + let scan = try BigtableRequest.parse("SCAN events/user#?limit=20") + #expect(scan.key == Data("user#".utf8) && scan.limit == 20) + let put = try BigtableRequest.parse(#"PUT events/b64:AAE= {"cf:a": "1"}"#) + #expect(put.key == Data([0, 1]) && put.body == #"{"cf:a": "1"}"#) + let insert = try BigtableRequest.parse(#"INSERT events/k {"cf:a": "1"}"#) + #expect(insert.verb == .insert && insert.key == Data("k".utf8)) + let read = try BigtableRequest.parse(#"READ events {"rowsLimit": "5"}"#) + #expect(read.table == "events" && read.body == #"{"rowsLimit": "5"}"#) + } + + @Test("A scan names a whole table with or without a slash") + func scanWholeTable() throws { + for text in ["SCAN events", "SCAN events/"] { + let scan = try BigtableRequest.parse(text) + #expect(scan.table == "events" && scan.key == nil && scan.limit == nil) + } + #expect(try BigtableRequest.parse("SCAN events?limit=5").limit == 5) + } + + @Test("A scan limit is a whole number above zero, and no other query is taken") + func scanLimits() { + for text in ["SCAN events/?limit=0", "SCAN events/?limit=-5", "SCAN events/?limit=x", "SCAN events/?rows=5"] { + #expect(throws: BigtableError.self, "\(text)") { try BigtableRequest.parse(text) } + } + } + + @Test("A column after the key may hold a space, and nothing after a request is ignored") + func columnsAndTrailingText() throws { + let spaced = try BigtableRequest.parse("DELETE events/k cf:first name") + #expect(spaced.column == "cf:first name") + #expect(try spaced.mutations() == [.deleteColumn(BigtableColumn(family: "cf", qualifier: Data("first name".utf8)))]) + for text in ["GET events/k extra", #"DELETE events/k "cf:a" extra"#, "SCAN events/a?limit=5 more"] { + #expect(throws: BigtableError.self, "\(text)") { try BigtableRequest.parse(text) } + } + } + + @Test("Malformed requests explain the syntax") + func malformed() { + for text in [ + "SELECT * FROM t", "GET events/", "PUT events/k", #"GET events/"open"#, "READ events", + #"READ {"rowsLimit": "5"}"#, "GET user#42" + ] { + #expect(throws: BigtableError.self, "\(text)") { try BigtableRequest.parse(text) } + } + } + + @Test("A request reads back from its own text, quoting keys and columns that need it") + func roundTrip() throws { + for key in [Data("user#42".utf8), Data("a b".utf8), Data([0xFF]), Data("x/y".utf8), Data("\"q".utf8)] { + for column in ["cf:a", "cf:first name"] { + let request = BigtableRequest(verb: .delete, table: "events", key: key, column: column) + #expect(try BigtableRequest.parse(request.text) == request) + } + } + let scan = BigtableRequest(verb: .scan, table: "events", key: Data("user#".utf8), limit: 20) + #expect(scan.text == "SCAN events/user#?limit=20") + #expect(try BigtableRequest.parse(scan.text) == scan) + let spec = BigtableBrowseSpec(table: "t", filters: [BigtableFilterSpec(column: "_key", op: "=", value: "k")], limit: 5, offset: 10) + #expect(try BigtableStatement.parse(BigtableStatement.encode(spec)) == .browse(spec)) + } + + @Test("A write body sets cells, deletes nulled ones under PUT, and writes numbers and booleans as text") + func mutations() throws { + let put = BigtableRequest(verb: .put, table: "t", key: Data("k".utf8), body: #"{"cf:name": null, "cf:n": 42, "cf:on": true, "cf:raw": "b64:AAE="}"#) + #expect(try put.mutations() == [ + .setCell(BigtableColumn(family: "cf", qualifier: Data("n".utf8)), Data("42".utf8)), + .deleteColumn(nameColumn), + .setCell(BigtableColumn(family: "cf", qualifier: Data("on".utf8)), Data("true".utf8)), + .setCell(BigtableColumn(family: "cf", qualifier: Data("raw".utf8)), Data([0, 1])) + ]) + #expect(try BigtableRequest(verb: .delete, table: "t", key: Data("k".utf8)).mutations() == [.deleteRow]) + } + + @Test("INSERT only sets cells, and no write sends an empty mutation") + func writeRefusals() { + let key = Data("k".utf8) + #expect(throws: BigtableError.self) { try BigtableRequest(verb: .insert, table: "t", key: key, body: #"{"cf:a": null}"#).mutations() } + #expect(throws: BigtableError.self) { try BigtableRequest(verb: .put, table: "t", key: key, body: "{}").mutations() } + #expect(throws: BigtableError.self) { try BigtableRequest(verb: .put, table: "t", key: key, body: #"{"cf:a": [1]}"#).mutations() } + } +} + +struct BigtableRowWritesTests { + private let key = Data("k".utf8) + + @Test("A new row is an INSERT of its non-empty cells under its _key") + func insert() throws { + let request = try BigtableRowWrites.insert( + table: "events", columns: ["_key", "cf:a", "cf:b", "cf:c"], values: [.text("k"), .text("1"), .absent, .bytes(Data([0xFF]))] + ) + #expect(request.text == #"INSERT events/k {"cf:a":"1","cf:c":"b64:/w=="}"#) + } + + @Test("A new row without a _key, or with nothing but one, is refused") + func insertRefusals() { + #expect(throws: BigtableError.self) { + try BigtableRowWrites.insert(table: "events", columns: ["_key", "cf:a"], values: [.absent, .text("1")]) + } + #expect(throws: BigtableError.self) { + try BigtableRowWrites.insert(table: "events", columns: ["_key", "cf:a"], values: [.text("k"), .absent]) + } + } + + @Test("An edit sends only the changed cells, and a removed cell as null") + func update() throws { + let request = try #require(try BigtableRowWrites.update(table: "events", key: key, changes: [("cf:a", .text("2")), ("cf:b", .absent)])) + #expect(request.text == #"PUT events/k {"cf:a":"2","cf:b":null}"#) + #expect(try BigtableRowWrites.update(table: "events", key: key, changes: []) == nil) + } + + @Test("An edit to an existing row's key is refused, since Bigtable cannot rename a row") + func keyChangeRefused() { + #expect(throws: BigtableError.self) { + try BigtableRowWrites.update(table: "events", key: key, changes: [("_key", .text("k2"))]) + } + } + + @Test("A delete names the row's key") + func delete() { + #expect(BigtableRowWrites.delete(table: "events", key: Data("a b".utf8)).text == #"DELETE events/"a b""#) + } +} diff --git a/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableTestSupport.swift b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableTestSupport.swift new file mode 100644 index 0000000000..f4c7223de5 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableTestSupport.swift @@ -0,0 +1,45 @@ +import Foundation +import TableProGoogleCloud + +func b64(_ text: String) -> String { + Data(text.utf8).base64EncodedString() +} + +func json(_ text: String) -> Any { + (try? JSONSerialization.jsonObject(with: Data(text.utf8))) ?? [String: Any]() +} + +func jsonText(_ object: Any) -> String { + (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys, .withoutEscapingSlashes])).flatMap { String(bytes: $0, encoding: .utf8) } ?? "" +} + +final class RecordingTransport: GoogleRESTTransport, @unchecked Sendable { + private let lock = NSLock() + private var sent: [GoogleRESTRequest] = [] + private let reply: @Sendable (GoogleRESTRequest) -> GoogleRESTResponse + + init(reply: @escaping @Sendable (GoogleRESTRequest) -> GoogleRESTResponse) { + self.reply = reply + } + + var requests: [GoogleRESTRequest] { + lock.withLock { sent } + } + + func send(_ request: GoogleRESTRequest) async throws -> GoogleRESTResponse { + lock.withLock { sent.append(request) } + return reply(request) + } + + func cancelAll() {} +} + +extension GoogleRESTRequest { + var path: String { + url.path + } + + var jsonBody: [String: Any] { + body.flatMap { try? JSONSerialization.jsonObject(with: $0) as? [String: Any] } ?? [:] + } +} diff --git a/Packages/TableProCore/Tests/TableProGoogleCloudTests/GoogleRESTRequesterTests.swift b/Packages/TableProCore/Tests/TableProGoogleCloudTests/GoogleRESTRequesterTests.swift new file mode 100644 index 0000000000..eff1db09ee --- /dev/null +++ b/Packages/TableProCore/Tests/TableProGoogleCloudTests/GoogleRESTRequesterTests.swift @@ -0,0 +1,100 @@ +import Foundation +@testable import TableProGoogleCloud +import Testing + +private final class ScriptedTransport: GoogleRESTTransport, @unchecked Sendable { + private let lock = NSLock() + private var statuses: [Int] + private var sent: [GoogleRESTRequest] = [] + + init(statuses: [Int]) { + self.statuses = statuses + } + + var requests: [GoogleRESTRequest] { + lock.withLock { sent } + } + + func send(_ request: GoogleRESTRequest) async throws -> GoogleRESTResponse { + let status = lock.withLock { + sent.append(request) + return statuses.isEmpty ? 200 : statuses.removeFirst() + } + return GoogleRESTResponse(statusCode: status, body: Data()) + } + + func cancelAll() {} +} + +struct GoogleRESTRequesterTests { + private let url = URL(string: "https://bigtable.googleapis.com/v2/x") ?? URL(fileURLWithPath: "/") + + @Test("A 401 drops the cached token and retries once with a fresh one") + func unauthorizedRetriesOnce() async throws { + let transport = ScriptedTransport(statuses: [401, 401]) + let tokens = LockedBox(["stale", "fresh"]) + let credential = GoogleRESTCredential( + token: { tokens.value.first }, + invalidate: { tokens.mutate { $0.removeFirst() } } + ) + let response = try await GoogleRESTRequester(transport: transport, credential: credential, timeout: { 5 }) + .send(method: "GET", url: url) + + #expect(response.statusCode == 401) + #expect(transport.requests.map { $0.headers["Authorization"] } == ["Bearer stale", "Bearer fresh"]) + } + + @Test("A body is sent as JSON, a login's quota project is billed, and the caller's own quota header wins") + func headers() async throws { + let transport = ScriptedTransport(statuses: []) + let requester = GoogleRESTRequester( + transport: transport, credential: GoogleRESTCredential(token: { "t" }, quotaProject: "quota"), timeout: { 7 } + ) + _ = try await requester.send(method: "POST", url: url, body: Data("{}".utf8)) + _ = try await requester.send(method: "GET", url: url, headers: ["x-goog-user-project": "browsed"], timeoutOverride: 2) + + let first = try #require(transport.requests.first) + #expect(first.headers["Content-Type"] == "application/json") + #expect(first.headers["x-goog-user-project"] == "quota") + #expect(first.timeoutInterval == 7) + let second = try #require(transport.requests.last) + #expect(second.headers["Content-Type"] == nil) + #expect(second.headers["x-goog-user-project"] == "browsed") + #expect(second.timeoutInterval == 2) + } + + @Test("A credential with no token sends no Authorization header") + func tokenlessCredential() async throws { + let transport = ScriptedTransport(statuses: []) + _ = try await GoogleRESTRequester(transport: transport, credential: GoogleRESTCredential(token: { nil }), timeout: { 5 }) + .send(method: "GET", url: url) + #expect(transport.requests.first?.headers["Authorization"] == nil) + } + + @Test("Application Default Credentials that name no project fall back to GOOGLE_CLOUD_PROJECT") + func projectHintFromEnvironment() throws { + let login = #"{"type":"authorized_user","client_id":"cid","client_secret":"cs","refresh_token":"1//rt"}"# + let resolved = try GoogleRESTCredential.resolving( + .applicationDefault, + scopes: [GoogleOAuthClient.cloudPlatformScope], + environment: ["GOOGLE_CLOUD_PROJECT": " env-proj "], + readFile: { $0 == GoogleApplicationDefaultCredentials.defaultPath ? Data(login.utf8) : nil }, + http: StubGoogleHTTPClient(json: "{}") + ) + #expect(resolved.projectHint == "env-proj") + #expect(resolved.credential.quotaProject == nil) + } + + @Test("A service account key names its own project, whatever the environment says") + func projectHintFromKey() throws { + let rsa = try #require(TestRSAKey.shared) + let resolved = try GoogleRESTCredential.resolving( + .serviceAccountKey(rsa.serviceAccountJSON(projectId: "key-project")), + scopes: [GoogleOAuthClient.cloudPlatformScope], + environment: ["GOOGLE_CLOUD_PROJECT": "env-proj"], + readFile: { _ in nil }, + http: StubGoogleHTTPClient(json: "{}") + ) + #expect(resolved.projectHint == "key-project") + } +} diff --git a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift index 966b805fe9..8d194f955e 100644 --- a/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift +++ b/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLLexicalReadingsTests.swift @@ -11,6 +11,7 @@ struct SQLLexicalReadingsTests { "SQLite", "libSQL", "Turso", "Cloudflare D1", "DuckDB", "Oracle", "Dameng", "SQL Server", "ClickHouse", "Snowflake", "BigQuery", "Spanner", "Trino", "Teradata", "SAP HANA", "Cassandra", "ScyllaDB", "DynamoDB", "SurrealDB", "Redis", "MongoDB", "etcd", "Elasticsearch", "Typesense", "Weaviate", "Kafka", + "Firestore", "Firebase Realtime Database", "Bigtable", ] let missing = shipped.filter { SQLLexicalProfile.curated(forDatabaseTypeId: $0) == nil } #expect(missing.isEmpty) diff --git a/Plugins/BigtableDriverPlugin/BigtableCredentialFactory.swift b/Plugins/BigtableDriverPlugin/BigtableCredentialFactory.swift deleted file mode 100644 index d7078eee59..0000000000 --- a/Plugins/BigtableDriverPlugin/BigtableCredentialFactory.swift +++ /dev/null @@ -1,51 +0,0 @@ -import Foundation -import TableProBigtableCore -import TableProGoogleCloud -import TableProPluginKit - -internal enum BigtableCredentialFactory { - static let scopes = [GoogleOAuthClient.cloudPlatformScope] - - struct Resolved { - let credential: BigtableCredential - let projectHint: String? - } - - static func resolve(config: DriverConnectionConfig) throws -> Resolved { - let fields = config.additionalFields - let method = trimmed(fields[BigtableFieldID.authMethod]).flatMap(BigtableAuthMethod.init(rawValue:)) ?? .serviceAccount - let source: GoogleCredentialSource - switch method { - case .serviceAccount: - guard let value = trimmed(fields[BigtableFieldID.serviceAccountJson]) ?? trimmed(config.password) else { - throw BigtableError.configuration(String(localized: "Choose a service account key file, or paste its JSON.")) - } - source = .serviceAccountKey(value) - case .applicationDefault: - source = .applicationDefault - } - let environment = ProcessInfo.processInfo.environment - let credentials = try GoogleTokenProviders.credentials( - from: source, - scopes: scopes, - readFile: { FileManager.default.contents(atPath: $0) }, - environment: environment, - http: URLSessionGoogleHTTPClient() - ) - let provider = credentials.tokenProvider - let environmentProject = source == .applicationDefault ? trimmed(environment["GOOGLE_CLOUD_PROJECT"]) : nil - return Resolved( - credential: BigtableCredential( - token: { try await provider.accessToken() }, - invalidate: { await provider.invalidateCachedToken() }, - quotaProject: credentials.quotaProject - ), - projectHint: credentials.projectHint ?? environmentProject - ) - } - - private static func trimmed(_ value: String?) -> String? { - guard let value = value?.trimmingCharacters(in: .whitespacesAndNewlines), !value.isEmpty else { return nil } - return value - } -} diff --git a/Plugins/BigtableDriverPlugin/BigtablePlugin.swift b/Plugins/BigtableDriverPlugin/BigtablePlugin.swift index ffe8a49a3b..78e221f1d5 100644 --- a/Plugins/BigtableDriverPlugin/BigtablePlugin.swift +++ b/Plugins/BigtableDriverPlugin/BigtablePlugin.swift @@ -42,8 +42,7 @@ final class BigtablePlugin: NSObject, TableProPlugin, DriverPlugin { static let tableEntityName = "Tables" static let containerEntityName = "Instance" static let defaultSchemaName = "" - static let immutableColumns: [String] = ["_key"] - static let defaultPrimaryKeyColumn: String? = "_key" + static let defaultPrimaryKeyColumn: String? = BigtableRowSet.keyColumn static let structureColumnFields: [StructureColumnField] = [.name, .type, .nullable] static let sqlDialect: SQLDialectDescriptor? = nil @@ -103,6 +102,7 @@ let bigtableCompletions: [CompletionEntry] = [ CompletionEntry(label: "GET row", insertText: "GET events/user#42"), CompletionEntry(label: "SCAN prefix", insertText: "SCAN events/user#?limit=20"), CompletionEntry(label: "READ", insertText: "READ events {\"rowsLimit\": \"20\", \"filter\": {\"cellsPerColumnLimitFilter\": 1}}"), + CompletionEntry(label: "INSERT row", insertText: "INSERT events/user#42 {\"profile:name\": \"Ann\"}"), CompletionEntry(label: "PUT cells", insertText: "PUT events/user#42 {\"profile:name\": \"Ann\"}"), CompletionEntry(label: "DELETE row", insertText: "DELETE events/user#42"), CompletionEntry(label: "DELETE column", insertText: "DELETE events/user#42 profile:name") diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift index 6dc838d62d..6f92b8c0a4 100644 --- a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift @@ -3,8 +3,8 @@ import TableProBigtableCore import TableProPluginKit extension BigtablePluginDriver { - /// Each change becomes one request line, the text the editor takes, so the history shows what - /// was sent. All cells of a row change in one atomic `mutateRow`. + private static let defaultMarker = "__DEFAULT__" + func generateRowWrites( table: String, schema: String?, @@ -18,23 +18,24 @@ extension BigtablePluginDriver { try changes.compactMap { change in do { return try request(for: change, table: table, columns: columns, insertedRowData: insertedRowData) - .map { PluginRowWrite(statement: $0.text(defaultTable: table), rowIndices: [change.rowIndex]) } + .map { PluginRowWrite(statement: $0.text, rowIndices: [change.rowIndex]) } } catch { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: error.localizedDescription) } } } + /// Restoring a deleted row writes it back under its own key, refused if that key is taken again. func generateIdentityPreservingInsert( table: String, schema: String?, columns: [String], primaryKeyColumns: [String], - rows: [[PluginCellValue]] + rows: [[PluginCellValue]], + absentCells: [Int: Set] ) -> [(statement: String, parameters: [PluginCellValue])]? { - let statements = rows.enumerated().compactMap { index, row -> String? in - let change = PluginRowChange(rowIndex: index, type: .insert, cellChanges: [], originalRow: nil) - return (try? request(for: change, table: table, columns: columns, insertedRowData: [index: row]))??.text(defaultTable: table) + let statements = rows.enumerated().compactMap { index, row in + try? BigtableRowWrites.insert(table: table, columns: columns, values: cells(row, absent: absentCells[index] ?? [])).text } return statements.count == rows.count ? statements.map { ($0, []) } : nil } @@ -45,72 +46,47 @@ extension BigtablePluginDriver { columns: [String], insertedRowData: [Int: [PluginCellValue]] ) throws -> BigtableRequest? { - let keyColumn = BigtableRowSet.keyColumn + let absent = change.absentColumns ?? [] switch change.type { case .insert: - let row = insertedRowData[change.rowIndex] ?? [] - let absent = change.absentColumns ?? [] - var cells: [String: Any] = [:] - var key: Data? - for (index, column) in columns.enumerated() where index < row.count && !absent.contains(index) { - if column == keyColumn { - key = try keyBytes(row[index]) - continue - } - guard let value = Self.text(row[index]) else { continue } - cells[column] = value - } - guard let key, !key.isEmpty else { - throw BigtableError.configuration(String(localized: "A new row needs a _key. Bigtable has no generated keys.")) - } - guard !cells.isEmpty else { - throw BigtableError.configuration(String(localized: "A new row needs at least one cell. Bigtable does not store empty rows.")) - } - return BigtableRequest(verb: .put, key: key, body: try Self.body(cells)) + return try BigtableRowWrites.insert( + table: table, columns: columns, values: cells(insertedRowData[change.rowIndex] ?? [], absent: absent) + ) case .update: - let key = try originalKey(change, columns: columns) - let absent = change.absentColumns ?? [] - var cells: [String: Any] = [:] - for cell in change.cellChanges where cell.columnName != keyColumn { - cells[cell.columnName] = absent.contains(cell.columnIndex) ? NSNull() : (Self.text(cell.newValue) ?? NSNull()) - } - for index in absent where index < columns.count && columns[index] != keyColumn && cells[columns[index]] == nil { - cells[columns[index]] = NSNull() + let changes = try change.cellChanges.map { cell in + guard cell.newValue.asText != Self.defaultMarker else { + throw BigtableError.unsupported(String(localized: "Bigtable has no default values. Delete the cell, or enter a value.")) + } + return (column: cell.columnName, value: absent.contains(cell.columnIndex) ? BigtableRowSet.Cell.absent : Self.cell(cell.newValue)) } - guard !cells.isEmpty else { return nil } - return BigtableRequest(verb: .put, key: key, body: try Self.body(cells)) + return try BigtableRowWrites.update(table: table, key: try originalKey(change, columns: columns), changes: changes) case .delete: - return BigtableRequest(verb: .delete, key: try originalKey(change, columns: columns)) + return BigtableRowWrites.delete(table: table, key: try originalKey(change, columns: columns)) } } private func originalKey(_ change: PluginRowChange, columns: [String]) throws -> Data { guard let index = columns.firstIndex(of: BigtableRowSet.keyColumn), - let row = change.originalRow, index < row.count, let key = try keyBytes(row[index]), !key.isEmpty else { + let row = change.originalRow, index < row.count, + let key = try BigtableRowWrites.key(Self.cell(row[index])) else { throw BigtableError.configuration(String(localized: "The row has no _key, so the row it came from is unknown. Reload the table.")) } return key } - private func keyBytes(_ cell: PluginCellValue) throws -> Data? { - switch cell { - case .null: return nil - case .bytes(let data): return data - case .text(let text): return text.isEmpty ? nil : try BigtableBytes.parse(text) + /// A new row holds the host's Default marker where it has no value yet. Bigtable generates + /// nothing, so that cell is left out, and a `_key` left that way is refused. + private func cells(_ row: [PluginCellValue], absent: Set) -> [BigtableRowSet.Cell] { + row.enumerated().map { index, value in + absent.contains(index) || value.asText == Self.defaultMarker ? .absent : Self.cell(value) } } - /// Bytes go back as `b64:` text so a binary value survives the JSON body unchanged. - private static func text(_ cell: PluginCellValue) -> String? { - switch cell { - case .null: return nil - case .text(let text): return text - case .bytes(let data): return BigtableBytes.base64Prefix + data.base64EncodedString() + private static func cell(_ value: PluginCellValue) -> BigtableRowSet.Cell { + switch value { + case .null: return .absent + case .text(let text): return .text(text) + case .bytes(let data): return .bytes(data) } } - - private static func body(_ cells: [String: Any]) throws -> String { - let data = try JSONSerialization.data(withJSONObject: cells, options: [.sortedKeys, .withoutEscapingSlashes]) - return String(bytes: data, encoding: .utf8) ?? "{}" - } } diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift index 4f73dddd7a..1b3c890c81 100644 --- a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift @@ -5,50 +5,65 @@ import TableProPluginKit extension BigtablePluginDriver { static let scanPageSize = 100 static let exportPageSize = 500 - /// Rows skipped per keys-only read when a page has no cursor yet. static let skipBatchSize = 5_000 - /// Export fixes its columns from the first page; a column only later rows have goes here as - /// JSON rather than being dropped. - static let extraColumn = "__extra__" func execute(query: String) async throws -> PluginQueryResult { + try await run(query, rowCap: PluginRowLimits.emergencyMax) + } + + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { + try await run(query, rowCap: rowCap ?? PluginRowLimits.emergencyMax) + } + + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + try await run(query, rowCap: rowCap) + } + + /// Every read asks the server for one row past `rowCap`, which is how a truncated result is + /// told apart from one that ended there, without buffering the rest of the table. + private func run(_ query: String, rowCap: Int) async throws -> PluginQueryResult { let started = Date() let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.lowercased() == "select 1" { try await ping() return PluginQueryResult(columns: ["ok"], columnTypeNames: ["integer"], rows: [[.text("1")]], - rowsAffected: 0, executionTime: elapsed(since: started)) + rowsAffected: 0, timing: PluginQueryTiming(total: Date().timeIntervalSince(started))) } + let cap = min(max(rowCap, 1), PluginRowLimits.emergencyMax) + let rows: [BigtableRow] switch try BigtableStatement.parse(trimmed) { case .browse(let spec): - return try await browse(spec, started: started) + rows = try await browse(spec, rowCap: cap) case .request(let request): - return try await execute(request, started: started) + if let written = try await write(request, started: started) { + return written + } + rows = try await read(request, rowCap: cap) } + return result(BigtableRowSet(rows: Array(rows.prefix(cap))), isTruncated: rows.count > cap, started: started) } // MARK: - Browse - func browse(_ spec: BigtableBrowseSpec, started: Date = Date()) async throws -> PluginQueryResult { + func browse(_ spec: BigtableBrowseSpec, rowCap: Int) async throws -> [BigtableRow] { let client = try requireClient() let after = try await startKey(for: spec, client: client) if spec.offset > 0, after == nil { - return result(BigtableRowSet(rows: []), started: started) + return [] } - let body = try BigtableReadRequest.body(spec: spec, limit: spec.limit, after: after?.key) - let rows = try await client.readRows(table: spec.table, body: body) + let limit = spec.limit > 0 ? min(spec.limit, rowCap + 1) : rowCap + 1 + let rows = try await client.readRows(table: spec.table, body: try BigtableReadRequest.body(spec: spec, limit: limit, after: after)) if let last = rows.last?.key, spec.limit > 0, rows.count == spec.limit { rememberCursor(view: spec.viewKey, offset: spec.offset + rows.count, key: last) } - return result(BigtableRowSet(rows: rows), started: started) + return rows } - /// The key a page starts after. Page 1 needs none; a later page starts from the nearest cursor an - /// earlier page left and skips the rest by reading keys alone, which costs no values. - private func startKey(for spec: BigtableBrowseSpec, client: BigtableClient) async throws -> (key: Data, offset: Int)? { + /// Page 1 needs no start key; a later page starts from the nearest cursor an earlier page left + /// and skips the rest by reading keys alone. + private func startKey(for spec: BigtableBrowseSpec, client: BigtableClient) async throws -> Data? { guard spec.offset > 0 else { return nil } var position = cursor(view: spec.viewKey, offset: spec.offset) - if position?.offset == spec.offset { return position.map { ($0.key, $0.offset) } } while (position?.offset ?? 0) < spec.offset { try Task.checkCancellation() let start = position?.offset ?? 0 @@ -61,16 +76,17 @@ extension BigtablePluginDriver { if keys.count < step { break } } guard let position, position.offset == spec.offset else { return nil } - return (position.key, position.offset) + return position.key } - func result(_ rows: BigtableRowSet, started: Date) -> PluginQueryResult { + func result(_ rows: BigtableRowSet, isTruncated: Bool = false, started: Date) -> PluginQueryResult { var result = PluginQueryResult( columns: rows.columns, columnTypeNames: rows.columns.map { _ in "bytes" }, rows: rows.rows.map { $0.map(Self.cell) }, rowsAffected: 0, - executionTime: elapsed(since: started) + timing: PluginQueryTiming(total: Date().timeIntervalSince(started)), + isTruncated: isTruncated ) result.rowLocators = rows.locators result.absentCells = rows.absentCells @@ -87,73 +103,46 @@ extension BigtablePluginDriver { // MARK: - Request lines - private func execute(_ request: BigtableRequest, started: Date) async throws -> PluginQueryResult { + private func read(_ request: BigtableRequest, rowCap: Int) async throws -> [BigtableRow] { let client = try requireClient() - guard let table = request.table else { - throw BigtableError.configuration(String(localized: "Name the table before the key, as in GET events/user#42.")) - } switch request.verb { case .get: - let rows = try await client.readRows(table: table, body: [ - "rows": ["rowKeys": [request.key?.base64EncodedString() ?? ""]], - "filter": ["cellsPerColumnLimitFilter": 1] - ]) - return result(BigtableRowSet(rows: rows), started: started) + return try await client.readRows(table: request.table, body: BigtableReadRequest.get(request.key ?? Data())) case .scan: - var filters: [BigtableFilterSpec] = [] - if let key = request.key { - filters.append(BigtableFilterSpec(column: Self.keyColumn, op: "STARTS WITH", value: BigtableBytes.display(key))) - } - let spec = BigtableBrowseSpec(table: table, filters: filters, limit: request.limit ?? Self.scanPageSize, offset: 0) - let rows = try await client.readRows(table: table, body: try BigtableReadRequest.body(spec: spec, limit: spec.limit, after: nil)) - return result(BigtableRowSet(rows: rows), started: started) + let filters = request.key.map { + [BigtableFilterSpec(column: BigtableRowSet.keyColumn, op: "STARTS WITH", value: BigtableBytes.display($0))] + } ?? [] + let spec = BigtableBrowseSpec(table: request.table, filters: filters, limit: request.limit ?? Self.scanPageSize, offset: 0) + return try await browse(spec, rowCap: rowCap) case .read: guard let data = request.body?.data(using: .utf8), - let body = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + var body = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { throw BigtableError.configuration(String(localized: "READ needs a ReadRows JSON body, such as READ events {\"rowsLimit\": \"10\"}.")) } - return result(BigtableRowSet(rows: try await client.readRows(table: table, body: body)), started: started) - case .put: - let mutations = try Self.mutations(fromBody: request.body ?? "{}") - try await client.mutateRow(table: table, key: request.key ?? Data(), mutations: mutations) - forgetCursors(table: table) - return writeResult(table: table, key: request.key, started: started) - case .delete: - let mutation: BigtableMutation = try request.column.map { .deleteColumn(try BigtableColumn(name: $0)) } ?? .deleteRow - try await client.mutateRow(table: table, key: request.key ?? Data(), mutations: [mutation]) - forgetCursors(table: table) - return writeResult(table: table, key: request.key, started: started) + let asked = (body["rowsLimit"] as? String).flatMap(Int.init) ?? (body["rowsLimit"] as? NSNumber)?.intValue ?? 0 + body["rowsLimit"] = String(asked > 0 ? min(asked, rowCap + 1) : rowCap + 1) + return try await client.readRows(table: request.table, body: body) + case .insert, .put, .delete: + return [] } } - /// A `PUT` body maps `family:qualifier` to the new value as text, `b64:` text for bytes, or - /// null to delete that cell. All of it lands in one atomic row mutation. - static func mutations(fromBody body: String) throws -> [BigtableMutation] { - guard let data = body.data(using: .utf8), let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { - throw BigtableError.configuration(String(localized: "PUT needs a JSON object of cells, such as {\"profile:name\": \"Ann\"}.")) - } - return try object.keys.sorted().map { name in - let column = try BigtableColumn(name: name) - switch object[name] { - case is NSNull: - return .deleteColumn(column) - case let text as String: - return .setCell(column, try BigtableBytes.parse(text)) - case let number as NSNumber: - return .setCell(column, Data(number.stringValue.utf8)) - default: - throw BigtableError.configuration(String( - format: String(localized: "The value of %@ must be text or null. Bigtable stores bytes, so a number is written as its digits."), name - )) - } + private func write(_ request: BigtableRequest, started: Date) async throws -> PluginQueryResult? { + let client = try requireClient() + let key = request.key ?? Data() + switch request.verb { + case .get, .scan, .read: + return nil + case .insert: + try await client.insertRow(table: request.table, key: key, mutations: try request.mutations()) + case .put, .delete: + try await client.mutateRow(table: request.table, key: key, mutations: try request.mutations()) } - } - - private func writeResult(table: String, key: Data?, started: Date) -> PluginQueryResult { - PluginQueryResult( + forgetCursors(table: request.table) + return PluginQueryResult( columns: ["row"], columnTypeNames: ["bytes"], - rows: [[.text(table + "/" + (key.map(BigtableBytes.display) ?? ""))]], - rowsAffected: 1, executionTime: elapsed(since: started) + rows: [[.text(request.table + "/" + BigtableBytes.display(key))]], + rowsAffected: 1, timing: PluginQueryTiming(total: Date().timeIntervalSince(started)) ) } @@ -167,8 +156,9 @@ extension BigtablePluginDriver { try await self.export(spec, continuation: continuation) } else { let result = try await self.execute(query: query) - continuation.yield(.header(PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames, - estimatedRowCount: nil))) + continuation.yield(.header(PluginStreamHeader( + columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: nil + ))) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } } continuation.finish() @@ -180,6 +170,7 @@ extension BigtablePluginDriver { } } + /// The columns come from the first page; a column only later rows hold goes into `__extra__`. private func export(_ spec: BigtableBrowseSpec, continuation: AsyncThrowingStream.Continuation) async throws { let client = try requireClient() var header: [String]? @@ -189,32 +180,17 @@ extension BigtablePluginDriver { let body = try BigtableReadRequest.body(spec: spec, limit: Self.exportPageSize, after: after) let rows = try await client.readRows(table: spec.table, body: body) let page = BigtableRowSet(rows: rows) + let columns = header ?? page.columns + [BigtableRowSet.extraColumn] if header == nil { - header = page.columns + [Self.extraColumn] + header = columns continuation.yield(.header(PluginStreamHeader( - columns: header ?? [], columnTypeNames: (header ?? []).map { _ in "bytes" }, estimatedRowCount: nil + columns: columns, columnTypeNames: columns.map { _ in "bytes" }, estimatedRowCount: nil ))) } - let fixed = header ?? [] - continuation.yield(.rows(page.rows.map { Self.align($0.map(Self.cell), columns: page.columns, to: fixed) })) + if !rows.isEmpty { + continuation.yield(.rows(page.rows(alignedTo: columns).map { $0.map(Self.cell) })) + } after = rows.count == Self.exportPageSize ? rows.last?.key : nil } while after != nil } - - /// Places a page's cells under the export's columns and gathers the rest into `__extra__`. - static func align(_ row: [PluginCellValue], columns: [String], to header: [String]) -> [PluginCellValue] { - var byName: [String: PluginCellValue] = [:] - for (column, value) in zip(columns, row) { byName[column] = value } - var aligned = header.dropLast().map { byName.removeValue(forKey: $0) ?? .null } - let extra = byName.filter { !$0.value.isNull }.mapValues { value -> String in - value.asText ?? (value.asBytes.map { BigtableBytes.base64Prefix + $0.base64EncodedString() } ?? "") - } - if extra.isEmpty { - aligned.append(.null) - } else { - let data = (try? JSONSerialization.data(withJSONObject: extra, options: [.sortedKeys])) ?? Data() - aligned.append(String(bytes: data, encoding: .utf8).map(PluginCellValue.text) ?? .null) - } - return aligned - } } diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift index d27667cf4f..6a3fb17afc 100644 --- a/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Metadata.swift @@ -4,7 +4,7 @@ import TableProPluginKit extension BigtablePluginDriver { static let sampleSize = 50 - static let keyColumn = BigtableRowSet.keyColumn + static let countBatchSize = 10_000 func fetchTables(schema: String?) async throws -> [PluginTableInfo] { try await requireClient().listTables().map { PluginTableInfo(name: $0, type: "TABLE") } @@ -13,25 +13,24 @@ extension BigtablePluginDriver { /// Bigtable has no fixed columns, so these are the `family:qualifier` pairs a sample of rows /// holds. A family with no sampled cell has no column until a row writes one. func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - let sample = try await browse(BigtableBrowseSpec(table: table, limit: Self.sampleSize, offset: 0)) - return sample.columns.map { name in - PluginColumnInfo(name: name, dataType: "bytes", isNullable: name != Self.keyColumn, isPrimaryKey: name == Self.keyColumn) + try await sampledColumns(table: table).map { name in + let isKey = name == BigtableRowSet.keyColumn + return PluginColumnInfo(name: name, dataType: "bytes", isNullable: !isKey, isPrimaryKey: isKey) } } func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { - [PluginIndexInfo(name: Self.keyColumn, columns: [Self.keyColumn], isUnique: true, isPrimary: true, type: "PRIMARY KEY")] + let key = BigtableRowSet.keyColumn + return [PluginIndexInfo(name: key, columns: [key], isUnique: true, isPrimary: true, type: "PRIMARY KEY")] } func fetchTableDDL(table: String, schema: String?) async throws -> String { - let client = try requireClient() - let families = try await client.columnFamilies(table: table) - let sample = try await browse(BigtableBrowseSpec(table: table, limit: Self.sampleSize, offset: 0)) + let families = try await requireClient().columnFamilies(table: table) + let columns = try await sampledColumns(table: table) let description: [String: Any] = [ "table": table, "columnFamilies": families, - "sampledRows": sample.rows.count, - "sampledColumns": Array(sample.columns.dropFirst()) + "sampledColumns": Array(columns.dropFirst()) ] let data = (try? JSONSerialization.data(withJSONObject: description, options: [.prettyPrinted, .sortedKeys])) ?? Data() return String(bytes: data, encoding: .utf8) ?? "{}" @@ -41,6 +40,11 @@ extension BigtablePluginDriver { PluginTableMetadata(tableName: table, engine: "Bigtable") } + private func sampledColumns(table: String) async throws -> [String] { + let rows = try await requireClient().readRows(table: table, body: BigtableReadRequest.columnSample(limit: Self.sampleSize)) + return BigtableRowSet(rows: rows).columns + } + // MARK: - Browse statements func buildBrowseQuery( @@ -71,10 +75,9 @@ extension BigtablePluginDriver { guard sort.columnIndex >= 0, sort.columnIndex < columns.count else { return nil } return (columns[sort.columnIndex], sort.ascending) } - let specs = filters.map { BigtableFilterSpec(column: $0.column, op: $0.op, value: $0.value, secondValue: $0.secondValue) } return BigtableStatement.encode(BigtableBrowseSpec( table: table, - filters: specs, + filters: filters.map(Self.filterSpec), logicMode: logicMode, descending: sort.map { !$0.ascending } ?? false, sortColumn: sort?.column, @@ -87,8 +90,7 @@ extension BigtablePluginDriver { /// app runs it only when asked, since it is a full scan. func fetchExactRowCount(table: String, schema: String?, queryFilters filters: [PluginQueryFilter], logicMode: String) async throws -> Int? { let client = try requireClient() - let specs = filters.map { BigtableFilterSpec(column: $0.column, op: $0.op, value: $0.value, secondValue: $0.secondValue) } - let spec = BigtableBrowseSpec(table: table, filters: specs, logicMode: logicMode, limit: 0, offset: 0) + let spec = BigtableBrowseSpec(table: table, filters: filters.map(Self.filterSpec), logicMode: logicMode, limit: 0, offset: 0) var total = 0 var after: Data? repeat { @@ -101,5 +103,10 @@ extension BigtablePluginDriver { return total } - static let countBatchSize = 10_000 + private static func filterSpec(_ filter: PluginQueryFilter) -> BigtableFilterSpec { + BigtableFilterSpec( + column: filter.column, op: filter.op, value: filter.value, + secondValue: filter.secondValue, isCaseSensitive: filter.isCaseSensitive + ) + } } diff --git a/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift b/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift index 2a73f449de..6a6b3de9c0 100644 --- a/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift +++ b/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift @@ -1,20 +1,19 @@ import Foundation import os import TableProBigtableCore +import TableProGoogleCloud import TableProLogRedaction import TableProPluginKit internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Sendable { static let logger = Logger(subsystem: "com.TablePro", category: "BigtablePluginDriver") - - /// Page cursors of this many views are kept; a view is one table with one set of filters. static let cursorViewLimit = 64 let config: DriverConnectionConfig let queryTimeout = HttpQueryTimeoutBox() private let lock = NSLock() private var client: BigtableClient? - private var transport: URLSessionBigtableTransport? + private var transport: URLSessionGoogleRESTTransport? private var cursors: [String: [Int: Data]] = [:] private var cursorOrder: [String] = [] @@ -33,35 +32,36 @@ internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Send // MARK: - Lifecycle - /// The first calls run under the connect deadline; once they answer, the client switches to the - /// query timeout the user set. + /// The probe runs under the connect deadline and the client after it under the query timeout. func connect() async throws { - let resolved = try BigtableCredentialFactory.resolve(config: config) - let settings = try BigtableConnectionSettings.parse(fields: config.additionalFields, credentialProject: resolved.projectHint) - let connectMilliseconds = PluginConnectTimeout.milliseconds( + let resolved = try GoogleRESTCredential.resolving( + try BigtableConnectionSettings.credentialSource(fields: config.additionalFields, password: config.password), + scopes: BigtableClient.scopes + ) + let settings = try BigtableConnectionSettings.parse( + fields: config.additionalFields, database: config.database, credentialProject: resolved.projectHint + ) + let deadline = PluginConnectDeadline(milliseconds: PluginConnectTimeout.milliseconds( in: config.additionalFields, default: Int(HttpQueryTimeout().requestTimeoutInterval * 1_000) - ) - let deadline = PluginConnectDeadline(milliseconds: connectMilliseconds) - let transport = URLSessionBigtableTransport(resourceTimeout: HttpQueryTimeout.sessionResourceTimeout) + )) + let transport = URLSessionGoogleRESTTransport.bigtable(resourceTimeout: HttpQueryTimeout.sessionResourceTimeout) let timeoutBox = queryTimeout let probe = BigtableClient( projectId: settings.projectId, instanceId: settings.instanceId ?? "-", appProfileId: settings.appProfileId, transport: transport, credential: resolved.credential, timeout: { deadline.remainingSeconds() } ) - var instanceId = settings.instanceId - if instanceId == nil { + if settings.instanceId == nil { guard let first = try await probe.listInstances().first else { throw BigtableError.notFound(String(format: String(localized: "Project %@ has no Bigtable instance."), settings.projectId)) } - instanceId = first try probe.switchInstance(to: first) } try await probe.ping() let client = BigtableClient( - projectId: settings.projectId, instanceId: instanceId ?? probe.instanceId, appProfileId: settings.appProfileId, + projectId: settings.projectId, instanceId: probe.instanceId, appProfileId: settings.appProfileId, transport: transport, credential: resolved.credential, timeout: { timeoutBox.requestTimeoutInterval } ) lock.withLock { @@ -151,8 +151,8 @@ internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Send } func forgetCursors(table: String) { + let prefix = BigtableBrowseSpec.viewKeyPrefix(table: table) lock.withLock { - let prefix = table + "\u{3}" for view in cursors.keys where view.hasPrefix(prefix) { cursors[view] = nil } @@ -166,14 +166,6 @@ internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Send BigtableStatement.encode(BigtableBrowseSpec(table: table, limit: 0, offset: 0)) } - func dropObjectStatement(name: String, objectType: String, schema: String?, cascade: Bool) -> String? { - nil - } - - func truncateTableStatements(table: String, schema: String?, cascade: Bool) -> [String]? { - nil - } - func fetchViewDefinition(view: String, schema: String?) async throws -> String { throw BigtableError.unsupported(String(localized: "Bigtable has no views.")) } @@ -185,8 +177,4 @@ internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Send func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { nil } - - func elapsed(since start: Date) -> TimeInterval { - Date().timeIntervalSince(start) - } } diff --git a/Plugins/BigtableDriverPlugin/Info.plist b/Plugins/BigtableDriverPlugin/Info.plist index 2c5227ec99..2da24fdd81 100644 --- a/Plugins/BigtableDriverPlugin/Info.plist +++ b/Plugins/BigtableDriverPlugin/Info.plist @@ -3,7 +3,7 @@ TableProMinAppVersion - 0.80.0 + 0.81.0 TableProPluginKitVersion 38 TableProProvidesDatabaseTypeIds diff --git a/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift b/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift index 79b79c28f8..d7966f88a5 100644 --- a/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift +++ b/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift @@ -11,45 +11,21 @@ internal enum FirebaseCredentialFactory { "https://www.googleapis.com/auth/userinfo.email" ] - struct Resolved { - let credential: FirebaseCredential - let projectHint: String? - } - - static func resolve(config: DriverConnectionConfig) throws -> Resolved { + static func resolve(config: DriverConnectionConfig) throws -> (credential: GoogleRESTCredential, projectHint: String?) { let fields = config.additionalFields let method = trimmed(fields[FirebaseFieldID.authMethod]).flatMap(FirebaseAuthMethod.init(rawValue:)) ?? .serviceAccount - let source: GoogleCredentialSource switch method { case .emulator: - return Resolved(credential: .emulator, projectHint: nil) + return (.firebaseEmulator, nil) case .serviceAccount: guard let value = trimmed(fields[FirebaseFieldID.serviceAccountJson]) ?? trimmed(config.password) else { throw FirebaseError.configuration(String(localized: "Choose a service account key file, or paste its JSON.")) } - source = .serviceAccountKey(value) + return try GoogleRESTCredential.resolving(.serviceAccountKey(value), scopes: scopes) case .applicationDefault: - source = .applicationDefault + return try GoogleRESTCredential.resolving(.applicationDefault, scopes: scopes) } - let environment = ProcessInfo.processInfo.environment - let credentials = try GoogleTokenProviders.credentials( - from: source, - scopes: scopes, - readFile: { FileManager.default.contents(atPath: $0) }, - environment: environment, - http: URLSessionGoogleHTTPClient() - ) - let provider = credentials.tokenProvider - let environmentProject = source == .applicationDefault ? trimmed(environment["GOOGLE_CLOUD_PROJECT"]) : nil - return Resolved( - credential: FirebaseCredential( - token: { try await provider.accessToken() }, - invalidate: { await provider.invalidateCachedToken() }, - quotaProject: credentials.quotaProject - ), - projectHint: credentials.projectHint ?? environmentProject - ) } private static func trimmed(_ value: String?) -> String? { diff --git a/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift b/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift index ace0cbad58..abad5c8f93 100644 --- a/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift +++ b/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift @@ -1,6 +1,7 @@ import Foundation import os import TableProFirebaseCore +import TableProGoogleCloud import TableProLogRedaction import TableProPluginKit @@ -51,7 +52,7 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send default: Int(HttpQueryTimeout().requestTimeoutInterval * 1_000) ) let deadline = PluginConnectDeadline(milliseconds: connectMilliseconds) - let transport = URLSessionFirebaseTransport(resourceTimeout: HttpQueryTimeout.sessionResourceTimeout) + let transport = URLSessionGoogleRESTTransport.firebase(resourceTimeout: HttpQueryTimeout.sessionResourceTimeout) let timeoutBox = queryTimeout let timeout: @Sendable () -> TimeInterval = { timeoutBox.requestTimeoutInterval } let connectTimeout: @Sendable () -> TimeInterval = { deadline.remainingSeconds() } @@ -74,8 +75,8 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send /// The probe runs under the connect deadline and the client after it under the query timeout. private func connectFirestore( settings: FirebaseConnectionSettings, - transport: URLSessionFirebaseTransport, - credential: FirebaseCredential, + transport: URLSessionGoogleRESTTransport, + credential: GoogleRESTCredential, connectTimeout: @escaping @Sendable () -> TimeInterval, timeout: @escaping @Sendable () -> TimeInterval ) async throws -> FirebaseSession { @@ -93,8 +94,8 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send private func connectRealtimeDatabase( settings: FirebaseConnectionSettings, - transport: URLSessionFirebaseTransport, - credential: FirebaseCredential, + transport: URLSessionGoogleRESTTransport, + credential: GoogleRESTCredential, connectTimeout: @escaping @Sendable () -> TimeInterval, timeout: @escaping @Sendable () -> TimeInterval ) async throws -> FirebaseSession { @@ -298,8 +299,8 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send struct FirebaseSession: Sendable { let settings: FirebaseConnectionSettings - let transport: URLSessionFirebaseTransport - let credential: FirebaseCredential + let transport: URLSessionGoogleRESTTransport + let credential: GoogleRESTCredential let timeout: @Sendable () -> TimeInterval let firestore: FirestoreClient? let database: RealtimeDatabaseClient? diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift index 6e399ed175..3e10fedf55 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift @@ -3,7 +3,8 @@ import TableProPluginKit extension PluginMetadataRegistry { /// Rows come only in key order and Bigtable keeps no count, so a column sort is the grid's own - /// and a count reads every key, only when asked. + /// and a count reads every key, only when asked. `_key` stays writable because a new row has + /// no other way to get one; the driver refuses a change to an existing row's key. func bigtablePluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { [ ("Bigtable", PluginMetadataSnapshot( @@ -46,7 +47,7 @@ extension PluginMetadataRegistry { tableEntityName: "Tables", containerEntityName: "Instance", defaultPrimaryKeyColumn: "_key", - immutableColumns: ["_key"], + immutableColumns: [], systemDatabaseNames: [], systemSchemaNames: [], fileExtensions: [], @@ -74,6 +75,7 @@ private let bigtableCompletions: [CompletionEntry] = [ CompletionEntry(label: "GET row", insertText: "GET events/user#42"), CompletionEntry(label: "SCAN prefix", insertText: "SCAN events/user#?limit=20"), CompletionEntry(label: "READ", insertText: "READ events {\"rowsLimit\": \"20\", \"filter\": {\"cellsPerColumnLimitFilter\": 1}}"), + CompletionEntry(label: "INSERT row", insertText: "INSERT events/user#42 {\"profile:name\": \"Ann\"}"), CompletionEntry(label: "PUT cells", insertText: "PUT events/user#42 {\"profile:name\": \"Ann\"}"), CompletionEntry(label: "DELETE row", insertText: "DELETE events/user#42"), CompletionEntry(label: "DELETE column", insertText: "DELETE events/user#42 profile:name") diff --git a/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/TablePro/Core/Utilities/SQL/QueryClassifier.swift index ee6519262e..f44e6ab55d 100644 --- a/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -1094,23 +1094,23 @@ private extension QueryClassifier { } } - /// A grid page is a tagged spec and anything else a request line led by its verb. A PUT that sets - /// a cell to null deletes that cell, so it ranks with DELETE. + /// A PUT that sets a cell to null deletes that cell, so it ranks with DELETE. INSERT refuses a + /// key that already has a row, so it never overwrites one. static func bigtableClassification(_ trimmed: String) -> QueryClassification { if trimmed.hasPrefix("BIGTABLE_BROWSE:") { return .safe } + let write = QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) let destructive = QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) - let verb = trimmed.prefix { !$0.isWhitespace }.uppercased() - switch verb { + switch trimmed.prefix(while: { !$0.isWhitespace }).uppercased() { case "GET", "SCAN", "READ": return .safe case "DELETE": return destructive - case "PUT" where bigtableBodyDeletesCell(trimmed): - return destructive + case "PUT": + return bigtableBodyDeletesCell(trimmed) ? destructive : write default: - return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + return write } } diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 243bee744f..79f5e0814a 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -873,6 +873,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "\"%@\" is not a row limit. Use a whole number above 0." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "\"%@\" is not a URL. Enter an endpoint such as http://localhost:8000." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -15597,10 +15601,6 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "A new row needs a _key. Bigtable has no generated keys." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "A new row needs at least one cell. Bigtable does not store empty rows." : { "extractionState" : "manual", "generatesSymbol" : false @@ -15877,11 +15877,11 @@ } } }, - "A quoted row key is missing its closing quote." : { + "A quoted value is missing its closing quote." : { "extractionState" : "manual", "generatesSymbol" : false }, - "A quoted row key is not a valid JSON string." : { + "A quoted value is not a valid JSON string." : { "extractionState" : "manual", "generatesSymbol" : false }, @@ -15953,6 +15953,10 @@ } } }, + "A row key cannot change. Duplicate the row, give the copy a new _key, then delete this one." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A row needs a key. Bigtable has no empty row key." : { "extractionState" : "manual", "generatesSymbol" : false @@ -16824,6 +16828,10 @@ } } }, + "A write needs at least one cell. Bigtable does not store empty rows." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "about %@ row" : { "localizations" : { "fr" : { @@ -30479,10 +30487,6 @@ } } }, - "Bigtable answered with something other than HTTP." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "Bigtable cannot filter with %@." : { "extractionState" : "manual", "generatesSymbol" : false @@ -30495,6 +30499,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Bigtable has no default values. Delete the cell, or enter a value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Bigtable has no views." : { "extractionState" : "manual", "generatesSymbol" : false @@ -69932,6 +69940,10 @@ } } }, + "Enter a _key for the new row. Bigtable has no generated keys." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Enter a collection name." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -81280,10 +81292,6 @@ } } }, - "Firebase answered with something other than HTTP." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "Firebase has no views." : { "extractionState" : "manual", "generatesSymbol" : false @@ -92247,6 +92255,14 @@ } } }, + "INSERT needs a JSON object of cells, such as INSERT events/user#42 {\"profile:name\": \"Ann\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "INSERT only sets cells. Leave %@ out, or delete it with PUT." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Insert Row" : { "localizations" : { "fr" : { @@ -136170,10 +136186,6 @@ } } }, - "PUT needs a JSON object of cells, such as {\"profile:name\": \"Ann\"}." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "PUT needs a JSON object of cells, such as PUT events/user#42 {\"profile:name\": \"Ann\"}." : { "extractionState" : "manual", "generatesSymbol" : false @@ -148889,6 +148901,10 @@ } } }, + "Row %@ already exists. Edit that row, or give the new one another key." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Row %@ is not an object." : { "localizations" : { "fr" : { @@ -153587,6 +153603,10 @@ } } }, + "SCAN takes only ?limit=, as in SCAN events/user#?limit=20." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Scan with filters" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -186202,6 +186222,10 @@ } } }, + "The server answered with something other than HTTP." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The server chose this row's key." : { "localizations" : { "fr" : { @@ -210145,6 +210169,10 @@ } } }, + "Write the cells as a JSON object, such as {\"profile:name\": \"Ann\"}." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Writes %lld files, one for each database." : { "localizations" : { "fr" : { diff --git a/TableProTests/Core/ChangeTracking/DataChangeManagerTests.swift b/TableProTests/Core/ChangeTracking/DataChangeManagerTests.swift index 5d0232bd08..94410b44dc 100644 --- a/TableProTests/Core/ChangeTracking/DataChangeManagerTests.swift +++ b/TableProTests/Core/ChangeTracking/DataChangeManagerTests.swift @@ -684,4 +684,18 @@ struct DataChangeManagerImmutableColumnTests { #expect(!manager.isColumnWritable("total")) } + + @Test("Bigtable's _key is writable, since a new row has no other way to get a key") + func bigtableRowKeyWritable() { + let manager = DataChangeManager() + manager.configureForTable( + tableName: "events", + columns: ["_key", "cf:name"], + primaryKeyColumns: ["_key"], + databaseType: .bigtable, + generatedColumns: [] + ) + + #expect(manager.isColumnWritable("_key")) + } } diff --git a/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 980c62bffe..3a3b85a50d 100644 --- a/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -696,6 +696,7 @@ struct QueryClassifierNonSqlTests { #expect(!QueryClassifier.isWriteQuery("GET events/user#42", databaseType: .bigtable)) #expect(!QueryClassifier.isWriteQuery("scan events/user#?limit=5", databaseType: .bigtable)) #expect(!QueryClassifier.isWriteQuery("READ events {\"rowsLimit\": \"5\"}", databaseType: .bigtable)) + #expect(QueryClassifier.classifyTier("INSERT events/k {\"cf:a\": \"1\"}", databaseType: .bigtable) == .write) #expect(QueryClassifier.classifyTier("PUT events/k {\"cf:a\": \"1\"}", databaseType: .bigtable) == .write) #expect(QueryClassifier.classifyTier("PUT events/k {\"cf:a\": null}", databaseType: .bigtable) == .destructive) #expect(QueryClassifier.classifyTier("PUT events/\"a{b\" {\"cf:a\": null}", databaseType: .bigtable) == .destructive) diff --git a/TableProTests/Models/Database/SQLDDLFallbackPolicyTests.swift b/TableProTests/Models/Database/SQLDDLFallbackPolicyTests.swift index cfb685d6ec..1228d0f36d 100644 --- a/TableProTests/Models/Database/SQLDDLFallbackPolicyTests.swift +++ b/TableProTests/Models/Database/SQLDDLFallbackPolicyTests.swift @@ -24,7 +24,7 @@ struct SQLDDLFallbackPolicyTests { func nonSQLEnginesRefuse() { for type in [ DatabaseType.elasticsearch, .kafka, .weaviate, .etcd, - .redis, .mongodb, .typesense, .surrealdb, .dynamodb, .beancount, + .redis, .mongodb, .typesense, .surrealdb, .dynamodb, .beancount, .bigtable, ] { #expect(!SQLDDLFallbackPolicy.allowsGeneratedDDL(for: type), "\(type.rawValue) should refuse") } From 96c07960c3ba4150b8f088136bcc9b80207ef602 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 15 Oct 2026 10:45:27 +0700 Subject: [PATCH 3/3] fix(plugin-bigtable): let Match Case reach the driver --- Plugins/BigtableDriverPlugin/BigtablePlugin.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Plugins/BigtableDriverPlugin/BigtablePlugin.swift b/Plugins/BigtableDriverPlugin/BigtablePlugin.swift index 78e221f1d5..024d94cf5d 100644 --- a/Plugins/BigtableDriverPlugin/BigtablePlugin.swift +++ b/Plugins/BigtableDriverPlugin/BigtablePlugin.swift @@ -45,6 +45,7 @@ final class BigtablePlugin: NSObject, TableProPlugin, DriverPlugin { static let defaultPrimaryKeyColumn: String? = BigtableRowSet.keyColumn static let structureColumnFields: [StructureColumnField] = [.name, .type, .nullable] static let sqlDialect: SQLDialectDescriptor? = nil + static let caseSensitivityStyle: SQLDialectDescriptor.CaseSensitivityStyle = .driverManaged static let columnTypesByCategory: [String: [String]] = ["Binary": ["bytes"]] static let additionalConnectionFields: [ConnectionField] = bigtableConnectionFields()