diff --git a/CHANGELOG.md b/CHANGELOG.md index a1c23d154a..72eda2181c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Every ClickHouse query after connecting failing with `UNKNOWN_SETTING` on 23.8.0 to 23.8.8. (#3444) +- ClickHouse users with a read-only profile unable to connect. +- ClickHouse indexes, view definitions, functions, table comments and sizes failing to load on servers before 22.3. +- A ClickHouse query that failed partway showing its partial rows and the error text as a row instead of the error. +- A ClickHouse query with a `FORMAT` clause showing garbled rows in the grid and in an export. +- ClickHouse SQL export writing the current database's table structure for a table in another database. +- Opening a ClickHouse executable function's definition showing a syntax error. +- SQL Server 2016 RTM offered a `CREATE OR ALTER VIEW` template it cannot run, and Azure SQL a drop-and-create one. + ## [0.81.0] - 2026-10-16 Cloud Firestore, Firebase Realtime Database and Google Cloud Bigtable connect through new registry plugins. diff --git a/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLServerBanner.swift b/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLServerBanner.swift index ad666e50c4..a9d5f5e1b1 100644 --- a/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLServerBanner.swift +++ b/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLServerBanner.swift @@ -23,14 +23,37 @@ public enum MSSQLServerBanner { /// Matches against the whole banner rather than the first line alone, so a future layout that /// moves the build number cannot silently return nil the way the fixed prefix did. - public static func majorVersion(from banner: String?) -> Int? { + public static func server(from banner: String?) -> MSSQLServerBuild? { guard let banner else { return nil } - guard let regex = try? NSRegularExpression(pattern: #"(\d+)\.\d+\.\d+"#), - let match = regex.firstMatch(in: banner, range: NSRange(banner.startIndex..., in: banner)), - let range = Range(match.range(at: 1), in: banner) + guard let regex = try? NSRegularExpression(pattern: #"(\d+)\.(\d+)\.(\d+)"#), + let match = regex.firstMatch(in: banner, range: NSRange(banner.startIndex..., in: banner)) else { return nil } - return Int(banner[range]) + let parts = (1...3).compactMap { index in + Range(match.range(at: index), in: banner).flatMap { Int(banner[$0]) } + } + guard parts.count == 3 else { return nil } + return MSSQLServerBuild( + major: parts[0], + minor: parts[1], + build: parts[2], + isAzureSQL: banner.hasPrefix("Microsoft SQL Azure") + ) + } +} + +/// Azure SQL Database and Managed Instance report `12.0.2000.x` whatever engine they run, so their +/// features cannot be read off the number. +public struct MSSQLServerBuild: Sendable, Equatable { + public let major: Int + public let minor: Int + public let build: Int + public let isAzureSQL: Bool + + /// `CREATE OR ALTER` applies to "Azure SQL Database and SQL Server (starting with SQL Server + /// 2016 (13.x) SP1)" (CREATE VIEW, Microsoft Learn). 13.0.4001 is 2016 SP1; RTM is 13.0.1601. + public var hasCreateOrAlter: Bool { + isAzureSQL || (major, minor, build) >= (13, 0, 4_001) } } diff --git a/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSessionOptionsTests.swift b/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSessionOptionsTests.swift index b75f53c713..f363127a6e 100644 --- a/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSessionOptionsTests.swift +++ b/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSessionOptionsTests.swift @@ -68,12 +68,13 @@ struct MSSQLServerBannerTests { @Test("A patched server's major version survives the banner") func patchedServerParses() { - #expect(MSSQLServerBanner.majorVersion(from: Self.sqlServer2022) == 16) + #expect(MSSQLServerBanner.server(from: Self.sqlServer2022) + == MSSQLServerBuild(major: 16, minor: 0, build: 4_275, isAzureSQL: false)) } @Test("A fixed 50-character prefix would have lost it") func fixedPrefixLosesTheBuild() { - #expect(MSSQLServerBanner.majorVersion(from: String(Self.sqlServer2022.prefix(50))) == nil) + #expect(MSSQLServerBanner.server(from: String(Self.sqlServer2022.prefix(50))) == nil) } @Test("Display text is the product line alone") @@ -87,14 +88,29 @@ struct MSSQLServerBannerTests { @Test("Azure SQL Database reports its version on the first line too") func azureParses() { let banner = "Microsoft SQL Azure (RTM) - 12.0.2000.8 \n\tOct 18 2025 12:00:00 \n\tCopyright (C) 2025 Microsoft Corporation" - #expect(MSSQLServerBanner.majorVersion(from: banner) == 12) + #expect(MSSQLServerBanner.server(from: banner) + == MSSQLServerBuild(major: 12, minor: 0, build: 2_000, isAzureSQL: true)) #expect(MSSQLServerBanner.displayText(from: banner) == "Microsoft SQL Azure (RTM) - 12.0.2000.8") } @Test("A banner with no build number reports no version rather than a wrong one") func unparsableBanner() { - #expect(MSSQLServerBanner.majorVersion(from: "Microsoft SQL Server") == nil) - #expect(MSSQLServerBanner.majorVersion(from: nil) == nil) + #expect(MSSQLServerBanner.server(from: "Microsoft SQL Server") == nil) + #expect(MSSQLServerBanner.server(from: nil) == nil) + } + + @Test( + "CREATE OR ALTER needs SQL Server 2016 SP1 or Azure SQL", + arguments: [ + ("Microsoft SQL Server 2016 (RTM) - 13.0.1601.5 (X64)", false), + ("Microsoft SQL Server 2016 (SP1) (KB3182545) - 13.0.4001.0 (X64)", true), + ("Microsoft SQL Server 2014 (SP3) (KB4022619) - 12.0.6024.0 (X64)", false), + ("Microsoft SQL Server 2019 (RTM-CU18) (KB5017593) - 15.0.4261.1 (X64)", true), + ("Microsoft SQL Azure (RTM) - 12.0.2000.8", true), + ] + ) + func createOrAlterGate(banner: String, expected: Bool) { + #expect(MSSQLServerBanner.server(from: banner)?.hasCreateOrAlter == expected) } @Test("A single-line banner is returned whole") diff --git a/Plugins/ClickHouseDriverPlugin/ClickHouseCapabilities.swift b/Plugins/ClickHouseDriverPlugin/ClickHouseCapabilities.swift deleted file mode 100644 index 10b6a31149..0000000000 --- a/Plugins/ClickHouseDriverPlugin/ClickHouseCapabilities.swift +++ /dev/null @@ -1,34 +0,0 @@ -// -// ClickHouseCapabilities.swift -// ClickHouseDriverPlugin -// - -import Foundation - -struct ClickHouseCapabilities: Sendable, Equatable { - let major: Int - let minor: Int - - static let unknown = ClickHouseCapabilities(major: 0, minor: 0) - - var hasDataSkippingIndicesTable: Bool { - major > 19 || (major == 19 && minor >= 17) - } - - var hasWriteExceptionInOutputFormatSetting: Bool { - major > 23 || (major == 23 && minor >= 8) - } - - // https://clickhouse.com/docs/sql-reference/statements/alter/comment: "available since version 23.9" - var hasModifyComment: Bool { - major > 23 || (major == 23 && minor >= 9) - } - - static func parse(_ version: String?) -> ClickHouseCapabilities { - guard let version else { return .unknown } - let parts = version.split(separator: ".") - guard let major = parts.first.flatMap({ Int($0) }) else { return .unknown } - let minor = parts.count > 1 ? (Int(parts[1]) ?? 0) : 0 - return ClickHouseCapabilities(major: major, minor: minor) - } -} diff --git a/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift b/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift index c42fa492ce..9d370cb7ee 100644 --- a/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift +++ b/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift @@ -16,8 +16,9 @@ import TableProPluginKit /// rather than letting the rest of the result pile up in memory. That resume is why the body is /// read through `forEachChunk` and not by iterating a sequence: a caller cannot forget it. internal final class ClickHouseHTTPChunks: NSObject, URLSessionDataDelegate, @unchecked Sendable { - /// An error body is text the user reads, not a result, so it is held whole up to this and the - /// rest dropped. ClickHouse exception text runs to a few kilobytes at most. + /// An error body is text the user reads, not a result. Its last bytes are kept, because a + /// failure on 25.10 and later can arrive as HTTP 500 with the rows written so far ahead of + /// the text. ClickHouse exception text runs to a few kilobytes at most. private static let errorBodyByteCap = 65_536 private let stream: AsyncThrowingStream @@ -27,6 +28,7 @@ internal final class ClickHouseHTTPChunks: NSObject, URLSessionDataDelegate, @un private var isTransferSuspended = false private var failureStatusCode: Int? private var failureBody = Data() + private var responseHeaders: [String: String] = [:] internal init(session: URLSession, request: URLRequest) { (stream, continuation) = AsyncThrowingStream.makeStream() @@ -39,6 +41,15 @@ internal final class ClickHouseHTTPChunks: NSObject, URLSessionDataDelegate, @un dataTask.resume() } + /// Set before the first chunk is handed over, so a reader can look at it from there on. + internal var headers: [String: String] { + lock.withLock { responseHeaders } + } + + internal func headerValue(_ name: String) -> String? { + headers.first { $0.key.caseInsensitiveCompare(name) == .orderedSame }?.value + } + /// The transfer is torn down on every way out, a thrown error and a cancellation included. A /// reader that walks away leaves the task suspended mid-body with nothing left to resume it, so /// without this the connection stays open until the driver disconnects. @@ -56,8 +67,13 @@ internal final class ClickHouseHTTPChunks: NSObject, URLSessionDataDelegate, @un didReceive response: URLResponse, completionHandler: @escaping (URLSession.ResponseDisposition) -> Void ) { - if let statusCode = (response as? HTTPURLResponse)?.statusCode, statusCode >= 400 { - lock.withLock { failureStatusCode = statusCode } + let httpResponse = response as? HTTPURLResponse + let fields = ClickHousePluginDriver.headerFields(httpResponse) + lock.withLock { + responseHeaders = fields + if let httpResponse, ClickHousePluginDriver.isFailure(httpResponse) { + failureStatusCode = httpResponse.statusCode + } } completionHandler(.allow) } @@ -65,8 +81,10 @@ internal final class ClickHouseHTTPChunks: NSObject, URLSessionDataDelegate, @un internal func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { let isFailure = lock.withLock { () -> Bool in guard failureStatusCode != nil else { return false } - let room = Self.errorBodyByteCap - failureBody.count - if room > 0 { failureBody.append(data.prefix(room)) } + failureBody.append(data) + if failureBody.count > Self.errorBodyByteCap { + failureBody = Data(failureBody.suffix(Self.errorBodyByteCap)) + } return true } guard !isFailure else { return } diff --git a/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift b/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift index 321e3c7fa8..a5294d4a2f 100644 --- a/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift +++ b/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift @@ -147,7 +147,7 @@ struct ClickHouseError: Error, PluginDriverError { } static func httpFailure(statusCode: Int, body: String) -> ClickHouseError { - let serverText = body.trimmingCharacters(in: .whitespacesAndNewlines) + let serverText = ClickHouseServerException.message(inFailureBody: body) guard serverText.isEmpty else { return ClickHouseError(message: serverText) } return ClickHouseError(message: String( format: String(localized: "ClickHouse returned HTTP %lld."), @@ -174,7 +174,9 @@ struct CHQueryResult { final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { let config: DriverConnectionConfig + private let sessionConfiguration: URLSessionConfiguration private var _serverVersion: String? + private var _serverProfile = ClickHouseServerProfile.bootstrap let lock = NSLock() var session: URLSession? @@ -191,7 +193,12 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// column definition and `generateModifyColumnSQL` is synchronous, so it cannot ask again. var nonDefaultColumnKinds: Set = [] - var serverVersion: String? { _serverVersion } + var serverVersion: String? { lock.withLock { _serverVersion } } + + var serverProfile: ClickHouseServerProfile { + get { lock.withLock { _serverProfile } } + set { lock.withLock { _serverProfile = newValue } } + } var supportsSchemas: Bool { false } var supportsTransactions: Bool { false } @@ -232,8 +239,9 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { func rollbackTransaction() async throws {} var currentSchema: String? { nil } - init(config: DriverConnectionConfig) { + init(config: DriverConnectionConfig, sessionConfiguration: URLSessionConfiguration = .default) { self.config = config + self.sessionConfiguration = sessionConfiguration self._currentDatabase = config.database } @@ -251,7 +259,7 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) let deadline = PluginConnectDeadline(milliseconds: connectTimeout) - let urlConfig = URLSessionConfiguration.default + let urlConfig = sessionConfiguration urlConfig.timeoutIntervalForRequest = max( HttpQueryTimeout.sessionBootstrapRequestTimeout, deadline.remainingSeconds() @@ -266,8 +274,10 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } + let versionText: String? do { - _ = try await executeRaw("SELECT 1", requestTimeout: deadline.remainingSeconds()) + let result = try await executeRaw("SELECT version()", requestTimeout: deadline.remainingSeconds()) + versionText = result.rows.first?.first?.asText } catch { lock.withLock { session?.invalidateAndCancel() @@ -278,14 +288,40 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } report(.preparingSession) - if let result = try? await executeRaw("SELECT version()", requestTimeout: deadline.remainingSeconds()), - let versionStr = result.rows.first?.first?.asText { - _serverVersion = versionStr + let profile: ClickHouseServerProfile + do { + let settings = try await probe(ClickHouseServerProfile.settingsProbeQuery, deadline: deadline) + let catalog = try await probe(ClickHouseServerProfile.catalogProbeQuery, deadline: deadline) + profile = ClickHouseServerProfile( + versionText: versionText, + settings: settings.map { (columns: $0.columns, rows: $0.rows) }, + catalogRow: catalog?.rows.first + ) + } catch { + disconnect() + throw error + } + lock.withLock { + _serverVersion = versionText + _serverProfile = profile } Self.logger.debug("Connected to ClickHouse at \(self.config.host):\(self.config.port)") } + /// A probe that fails leaves its half of the profile at a safe default, so the session still + /// opens. A cancelled connect stops here rather than sending the next probe. + private func probe(_ query: String, deadline: PluginConnectDeadline) async throws -> CHQueryResult? { + try Task.checkCancellation() + do { + return try await executeRaw(query, requestTimeout: deadline.remainingSeconds()) + } catch { + try Task.checkCancellation() + Self.logger.error("ClickHouse capability probe failed, using defaults: \(error.localizedDescription, privacy: .private)") + return nil + } + } + func disconnect() { lock.lock() currentTask?.cancel() @@ -526,7 +562,10 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { let escapedId = queryId.replacingOccurrences(of: "'", with: "''") let request = try buildRequest( query: "KILL QUERY WHERE query_id = '\(escapedId)'", - database: "" + database: "", + queryId: nil, + executionTimeLimit: nil, + shape: .buffered(parameters: nil) ) let task = killSession.dataTask(with: request) { _, _, _ in killSession.invalidateAndCancel() @@ -544,12 +583,14 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// appended to the statement, which a query already carrying a `LIMIT` rejects outright. func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { let started = Date() + let isOutputCut = OSAllocatedUnfairLock(initialState: false) let stream = PluginRowStream.make { continuation, abort in let streamTask = Task { do { try await self.performBoundedStreamRows( query: query, rowCap: rowCap, + isOutputCut: isOutputCut, continuation: continuation ) } catch { @@ -558,27 +599,39 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } abort.onAbort { streamTask.cancel() } } - return try await PluginBoundedStream.collect(stream, rowCap: rowCap, startedAt: started) + let result = try await PluginBoundedStream.collect(stream, rowCap: rowCap, startedAt: started) + guard isOutputCut.withLock({ $0 }) else { return result } + return PluginQueryResult( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + rows: result.rows, + rowsAffected: result.rowsAffected, + timing: result.timing, + isTruncated: true + ) } private func performBoundedStreamRows( query: String, rowCap: Int, + isOutputCut: OSAllocatedUnfairLock, continuation: AsyncThrowingStream.Continuation ) async throws { let queryId = UUID().uuidString let (session, database, executionTimeLimit) = try streamingContext(queryId: queryId) - let request = try buildStreamRequest( - query: Self.withoutTrailingSemicolons(query), + var request = try buildRequest( + query: query, database: database, queryId: queryId, executionTimeLimit: executionTimeLimit, - rowCap: rowCap + shape: .streamed(rowCap: rowCap) ) + request.timeoutInterval = _queryTimeout.requestTimeoutInterval try await streamTabSeparatedRows( request: request, session: session, batchSize: min(5_000, rowCap + 1), + boundedOutputCut: isOutputCut, continuation: continuation ) } @@ -587,13 +640,21 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// the columns arrive with the rows and survive a result holding none. It is the format the /// non-streaming read asks for too, which is what keeps an exported value and the same value in /// the grid the same text, and unlike a JSON string it can carry a byte no encoding covers. + /// + /// A `FORMAT` clause in the query wins over `default_format`. A bounded read shows that body as + /// one value, as the buffered path does, and sets `boundedOutputCut` before the stream finishes + /// so the collector reads it after the last element. An unbounded read is an export or a copy, + /// which needs rows, so it refuses the body rather than cut it or decode it as rows. private func streamTabSeparatedRows( request: URLRequest, session: URLSession, batchSize: Int, + boundedOutputCut: OSAllocatedUnfairLock? = nil, continuation: AsyncThrowingStream.Continuation ) async throws { var decoder = ClickHouseTabSeparatedRowDecoder() + var tail = ClickHouseStreamTail() + var otherFormatBody: Data? var headerSent = false var batch: [PluginRow] = [] batch.reserveCapacity(batchSize) @@ -608,18 +669,62 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { ))) } - try await ClickHouseHTTPChunks(session: session, request: request).forEachChunk { chunk in - try Task.checkCancellation() - let rows = decoder.consume(chunk) - if let header = decoder.header { - sendHeader(header) + let chunks = ClickHouseHTTPChunks(session: session, request: request) + do { + try await chunks.forEachChunk { chunk in + try Task.checkCancellation() + if otherFormatBody == nil, + let format = chunks.headerValue("X-ClickHouse-Format"), + format != ClickHouseResponseClassifier.requestedFormat { + guard boundedOutputCut != nil else { + throw ClickHouseError(message: String( + format: String(localized: "The query names its own output format, %@. Remove its FORMAT clause to read it as rows."), + format + )) + } + otherFormatBody = Data() + } + if otherFormatBody != nil { + otherFormatBody?.append(chunk) + if let body = otherFormatBody, body.count > Self.otherFormatByteCap { + throw OtherFormatBodyFull() + } + return + } + let rows = decoder.consume(tail.release(chunk)) + if let header = decoder.header { + sendHeader(header) + } + batch.append(contentsOf: rows) + guard batch.count >= batchSize else { return } + continuation.yield(.rows(batch)) + batch.removeAll(keepingCapacity: true) } - batch.append(contentsOf: rows) - guard batch.count >= batchSize else { return } - continuation.yield(.rows(batch)) - batch.removeAll(keepingCapacity: true) + } catch is OtherFormatBodyFull {} + + if let otherFormatBody { + try ClickHouseServerException.throwIfTagged( + Array(otherFormatBody.suffix(ClickHouseStreamTail.heldByteCount)), + tag: chunks.headerValue("X-ClickHouse-Exception-Tag") + ) + let outcome = ClickHouseResponseClassifier.classify(headers: chunks.headers, body: otherFormatBody) + sendHeader(ClickHouseTabSeparatedRowDecoder.Header( + columns: outcome.columns, + columnTypeNames: outcome.columnTypeNames + )) + if !outcome.rows.isEmpty { + continuation.yield(.rows(outcome.rows)) + } + boundedOutputCut?.withLock { $0 = outcome.isTruncated } + continuation.finish() + return } + let rest = try tail.finish( + exceptionTag: chunks.headerValue("X-ClickHouse-Exception-Tag"), + header: decoder.header + ) + batch.append(contentsOf: decoder.consume(rest)) batch.append(contentsOf: decoder.finish()) sendHeader(decoder.header ?? ClickHouseTabSeparatedRowDecoder.Header(columns: [], columnTypeNames: [])) if !batch.isEmpty { @@ -628,13 +733,11 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.finish() } - private static func withoutTrailingSemicolons(_ query: String) -> String { - var trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) - while trimmed.hasSuffix(";") { - trimmed = String(trimmed.dropLast()).trimmingCharacters(in: .whitespacesAndNewlines) - } - return trimmed - } + /// The buffered path's cap for a body in another format. Past it the classifier marks the + /// value cut, so reading stops at the first chunk over it. + private static let otherFormatByteCap = 1_048_576 + + private struct OtherFormatBodyFull: Error {} func streamRows(query: String) -> AsyncThrowingStream { return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in @@ -657,12 +760,14 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) async throws { let queryId = UUID().uuidString let (session, database, executionTimeLimit) = try streamingContext(queryId: queryId) - let request = try buildStreamRequest( - query: Self.withoutTrailingSemicolons(query), + var request = try buildRequest( + query: query, database: database, queryId: queryId, - executionTimeLimit: executionTimeLimit + executionTimeLimit: executionTimeLimit, + shape: .streamed(rowCap: nil) ) + request.timeoutInterval = _queryTimeout.requestTimeoutInterval try await streamTabSeparatedRows( request: request, session: session, @@ -681,59 +786,6 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - private func buildStreamRequest( - query: String, - database: String, - queryId: String, - executionTimeLimit: Int?, - rowCap: Int? = nil - ) throws -> URLRequest { - let useTLS = config.ssl.isEnabled - - var components = URLComponents() - components.scheme = useTLS ? "https" : "http" - components.host = config.host - components.port = config.port - components.path = "/" - - var queryItems = [URLQueryItem]() - if !database.isEmpty { - queryItems.append(URLQueryItem(name: "database", value: database)) - } - queryItems.append(URLQueryItem(name: "query_id", value: queryId)) - queryItems.append(URLQueryItem( - name: "default_format", - value: ClickHouseResponseClassifier.requestedFormat - )) - queryItems.append(contentsOf: Self.executionTimeLimitItems(executionTimeLimit)) - if let rowCap { - /// The bound rides as an HTTP setting so the SQL in the body stays exactly what the - /// user wrote. One row past the cap, so a full page can be told from a truncated one. - queryItems.append(URLQueryItem(name: "max_result_rows", value: String(rowCap + 1))) - queryItems.append(URLQueryItem(name: "result_overflow_mode", value: "break")) - queryItems.append(URLQueryItem(name: "cancel_http_readonly_queries_on_client_close", value: "1")) - } - components.queryItems = queryItems - - guard let url = components.url else { - throw ClickHouseError(message: "Failed to construct request URL") - } - - var request = URLRequest(url: url) - request.httpMethod = "POST" - request.timeoutInterval = _queryTimeout.requestTimeoutInterval - - if let authorization = ClickHouseCredentials.basicAuthorizationHeader( - username: config.username, - password: config.password - ) { - request.setValue(authorization, forHTTPHeaderField: "Authorization") - } - - request.httpBody = query.data(using: .utf8) - return request - } - // MARK: - Create Table DDL func generateCreateTableSQL(definition: PluginCreateTableDefinition) -> String? { @@ -849,11 +901,10 @@ final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } private func fetchStoredColumnTypes(table: String) async throws -> [(name: String, type: String)] { - let escapedTable = table.replacingOccurrences(of: "'", with: "''") let result = try await execute(query: """ SELECT name, type FROM system.columns - WHERE database = currentDatabase() AND table = '\(escapedTable)' + WHERE database = currentDatabase() AND table = \(Self.stringLiteral(table)) ORDER BY position """) return result.rows.compactMap { row in diff --git a/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift b/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift index e5d2f7a361..37d986cb4d 100644 --- a/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift +++ b/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift @@ -52,8 +52,8 @@ extension ClickHousePluginDriver { query: query, database: database, queryId: queryId, - params: params, - executionTimeLimit: executionTimeLimit + executionTimeLimit: executionTimeLimit, + shape: .buffered(parameters: params) ) request.timeoutInterval = requestTimeout ?? _queryTimeout.requestTimeoutInterval return try await perform(request: request, session: session) @@ -70,9 +70,9 @@ extension ClickHousePluginDriver { lock.withLock { currentTask = nil } let httpResponse = response as? HTTPURLResponse - if let httpResponse, httpResponse.statusCode >= 400 { + if let httpResponse, Self.isFailure(httpResponse) { let body = String(decoding: data, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion - let exceptionCode = httpResponse.value(forHTTPHeaderField: "X-ClickHouse-Exception-Code") ?? "none" + let exceptionCode = httpResponse.value(forHTTPHeaderField: Self.exceptionCodeHeader) ?? "none" Self.logger.error("ClickHouse HTTP \(httpResponse.statusCode) exception \(exceptionCode): \(body)") throw ClickHouseError.httpFailure(statusCode: httpResponse.statusCode, body: body) } @@ -121,7 +121,15 @@ extension ClickHousePluginDriver { } } - private static func headerFields(_ response: HTTPURLResponse?) -> [String: String] { + static let exceptionCodeHeader = "X-ClickHouse-Exception-Code" + + /// With `send_progress_in_http_headers` the status goes out with the first progress header, so + /// a query that fails after it answers HTTP 200 and names the failure in this header. + static func isFailure(_ response: HTTPURLResponse) -> Bool { + response.statusCode >= 400 || response.value(forHTTPHeaderField: exceptionCodeHeader) != nil + } + + static func headerFields(_ response: HTTPURLResponse?) -> [String: String] { guard let response else { return [:] } var fields: [String: String] = [:] for (key, value) in response.allHeaderFields { @@ -131,39 +139,27 @@ extension ClickHousePluginDriver { return fields } + /// Every request goes through here, so a URL setting is sent the same way on every path and + /// only when the server profile says this user may change it. func buildRequest( query: String, database: String, - queryId: String? = nil, - params: [String: String?]? = nil, - executionTimeLimit: Int? = nil + queryId: String?, + executionTimeLimit: Int?, + shape: ClickHouseResponseShape ) throws -> URLRequest { - let useTLS = config.ssl.isEnabled - var components = URLComponents() - components.scheme = useTLS ? "https" : "http" + components.scheme = config.ssl.isEnabled ? "https" : "http" components.host = config.host components.port = config.port components.path = "/" - - var queryItems = [URLQueryItem]() - if !database.isEmpty { - queryItems.append(URLQueryItem(name: "database", value: database)) - } - if let queryId { - queryItems.append(URLQueryItem(name: "query_id", value: queryId)) - } - queryItems.append(contentsOf: Self.executionTimeLimitItems(executionTimeLimit)) - queryItems.append(URLQueryItem(name: "send_progress_in_http_headers", value: "1")) - queryItems.append(contentsOf: ClickHouseResponseClassifier.transportQueryItems( - supportsWriteExceptionSetting: ClickHouseCapabilities.parse(serverVersion).hasWriteExceptionInOutputFormatSetting - )) - if let params { - for (key, value) in params.sorted(by: { $0.key < $1.key }) { - queryItems.append(URLQueryItem(name: "param_\(key)", value: value)) - } - } - components.queryItems = queryItems + components.queryItems = Self.queryItems( + database: database, + queryId: queryId, + executionTimeLimit: executionTimeLimit, + shape: shape, + profile: serverProfile + ) guard let url = components.url else { throw ClickHouseError(message: "Failed to construct request URL") @@ -178,11 +174,83 @@ extension ClickHousePluginDriver { ) { request.setValue(authorization, forHTTPHeaderField: "Authorization") } + /// 25.10 compresses by default, and a compressed stream that fails partway is cut mid-row + /// with the exception text gone. URLSession asks for gzip unless told otherwise. + if case .streamed = shape { + request.setValue("identity", forHTTPHeaderField: "Accept-Encoding") + } - let trimmedQuery = query.trimmingCharacters(in: .whitespacesAndNewlines) - .replacingOccurrences(of: ";+$", with: "", options: .regularExpression) - request.httpBody = trimmedQuery.data(using: .utf8) - + request.httpBody = Self.withoutTrailingSemicolons(query).data(using: .utf8) return request } + + static func queryItems( + database: String, + queryId: String?, + executionTimeLimit: Int?, + shape: ClickHouseResponseShape, + profile: ClickHouseServerProfile + ) -> [URLQueryItem] { + var items = [URLQueryItem]() + if !database.isEmpty { + items.append(URLQueryItem(name: "database", value: database)) + } + if let queryId { + items.append(URLQueryItem(name: "query_id", value: queryId)) + } + items.append(URLQueryItem(name: "default_format", value: ClickHouseResponseClassifier.requestedFormat)) + items.append(contentsOf: executionTimeLimitItems(executionTimeLimit)) + /// A user-written `FORMAT JSON` or `XML` that fails is otherwise wrapped in that format, + /// or sent as an empty body on 23.9. + if let item = profile.queryItem(.httpWriteExceptionInOutputFormat, "0") { + items.append(item) + } + + switch shape { + case .buffered(let parameters): + items.append(URLQueryItem(name: "wait_end_of_query", value: "1")) + if let item = profile.queryItem(.sendProgressInHTTPHeaders, "1") { + items.append(item) + } + for (key, value) in (parameters ?? [:]).sorted(by: { $0.key < $1.key }) { + items.append(URLQueryItem(name: "param_\(key)", value: value)) + } + case .streamed(let rowCap): + guard let rowCap else { break } + /// The bound rides as an HTTP setting so the SQL in the body stays exactly what the + /// user wrote. One row past the cap, so a full page can be told from a truncated one. + /// `max_result_rows` alone makes an overflow throw, so it goes out only with `break`. + /// Held on the server, a capped read costs nothing, since the collector waits for the + /// whole page anyway, and a failure then arrives as a status on every version. Without + /// the cap the server would hold the whole result, so the reader stops at the cap and + /// the body is checked for an exception instead. + if let rows = profile.queryItem(.maxResultRows, String(rowCap + 1)), + let mode = profile.queryItem(.resultOverflowMode, "break") { + items.append(contentsOf: [rows, mode, URLQueryItem(name: "wait_end_of_query", value: "1")]) + } + if let item = profile.queryItem(.cancelHTTPReadonlyQueriesOnClientClose, "1") { + items.append(item) + } + } + return items + } + + static func withoutTrailingSemicolons(_ query: String) -> String { + var trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + while trimmed.hasSuffix(";") { + trimmed = String(trimmed.dropLast()).trimmingCharacters(in: .whitespacesAndNewlines) + } + return trimmed + } +} + +/// How the server hands a result back. +enum ClickHouseResponseShape: Equatable { + /// The server holds the whole result until the query ends, so a failure is an HTTP status and + /// `X-ClickHouse-Summary` carries the totals. + case buffered(parameters: [String: String?]?) + /// Rows arrive through the chunk reader. Unbounded, they come as the server produces them, so + /// a failure after the first block arrives in the body under HTTP 200: `wait_end_of_query` + /// would hold a whole export on the server. + case streamed(rowCap: Int?) } diff --git a/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Routines.swift b/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Routines.swift index 62b3965c51..d9d69c47a7 100644 --- a/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Routines.swift +++ b/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Routines.swift @@ -28,16 +28,33 @@ public enum ClickHouseObjectQueries { ORDER BY name """ - /// system.functions.create_query is documented as obsolete and is empty on several versions, - /// so it is the fallback rather than the source. + /// 21.10 has `CREATE FUNCTION` and `create_query` but no `origin`. A built-in has no + /// `create_query`, so the SQL functions a user created are the ones that do. + public static let functionListWithoutOrigin = """ + SELECT name, '' AS origin, is_aggregate, create_query + FROM system.functions + WHERE create_query != '' + ORDER BY name + """ + + /// ClickHouse has no `SHOW CREATE FUNCTION`: every version answers it with a syntax error. + /// `create_query` is the definition of a SQL function and empty for an executable one, which + /// lives in the server's configuration. public static func functionDefinition(name: String) -> String { - "SHOW CREATE FUNCTION \(quoteIdentifier(name))" + "SELECT create_query FROM system.functions WHERE name = '\(escapeLiteral(name))'" } } extension ClickHousePluginDriver { + /// A server without `system.functions.create_query` (21.8 and older) predates + /// `CREATE FUNCTION`, so it has no user-defined function to list. func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { - let result = try await execute(query: ClickHouseObjectQueries.functionList) + let profile = serverProfile + guard profile.hasFunctionCreateQuery else { return [] } + let query = profile.hasFunctionOrigin + ? ClickHouseObjectQueries.functionList + : ClickHouseObjectQueries.functionListWithoutOrigin + let result = try await execute(query: query) return result.rows.compactMap { row -> PluginRoutineInfo? in guard let name = row[safe: 0]?.asText, !name.isEmpty else { return nil } var attributes: [PluginObjectAttribute] = [] diff --git a/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Schema.swift b/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Schema.swift index 634386ee23..be762f379e 100644 --- a/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Schema.swift +++ b/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Schema.swift @@ -21,13 +21,21 @@ extension ClickHousePluginDriver { /// `Code 62 SYNTAX_ERROR` under quote-only escaping, and one created as ``q\\' OR 1=1 -- `` /// closed the literal early and listed every table on the server. static func tableListSQL(schema: String?) -> String { - let database = schema.flatMap { $0.isEmpty ? nil : $0 } - .map { "'\(Self.escapeStringLiteral($0))'" } ?? "currentDatabase()" - return """ - SELECT name, engine FROM system.tables - WHERE database = \(database) AND name NOT LIKE '.%' - ORDER BY name - """ + """ + SELECT name, engine FROM system.tables + WHERE database = \(databaseExpression(schema)) AND name NOT LIKE '.%' + ORDER BY name + """ + } + + /// Every metadata read names the database it was asked about, for the same reason as + /// `tableListSQL`: the export tree reads tables of other databases over one session. + static func databaseExpression(_ schema: String?) -> String { + schema.flatMap { $0.isEmpty ? nil : stringLiteral($0) } ?? "currentDatabase()" + } + + static func stringLiteral(_ value: String) -> String { + "'\(escapeStringLiteral(value))'" } func fetchTables(schema: String?) async throws -> [PluginTableInfo] { @@ -41,11 +49,12 @@ extension ClickHousePluginDriver { } func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - let escapedTable = table.replacingOccurrences(of: "'", with: "''") + let database = Self.databaseExpression(schema) + let tableLiteral = Self.stringLiteral(table) let pkSql = """ SELECT primary_key, sorting_key FROM system.tables - WHERE database = currentDatabase() AND name = '\(escapedTable)' + WHERE database = \(database) AND name = \(tableLiteral) """ let pkResult = try await execute(query: pkSql) let primaryKey = pkResult.rows.first.flatMap { $0[safe: 0]?.asText } ?? "" @@ -56,7 +65,7 @@ extension ClickHousePluginDriver { let sql = """ SELECT name, type, default_kind, default_expression, comment FROM system.columns - WHERE database = currentDatabase() AND table = '\(escapedTable)' + WHERE database = \(database) AND table = \(tableLiteral) ORDER BY position """ let result = try await execute(query: sql) @@ -99,9 +108,10 @@ extension ClickHousePluginDriver { // Pre-fetch PK columns for all tables. Falls back to sorting_key when // primary_key is empty (MergeTree without explicit PRIMARY KEY clause). // Note: expression-based keys like toDate(col) won't match bare column names. + let database = Self.databaseExpression(schema) let pkSql = """ SELECT name, primary_key, sorting_key FROM system.tables - WHERE database = currentDatabase() + WHERE database = \(database) """ let pkResult = try await execute(query: pkSql) var pkLookup: [String: Set] = [:] @@ -118,7 +128,7 @@ extension ClickHousePluginDriver { let sql = """ SELECT table, name, type, default_kind, default_expression, comment FROM system.columns - WHERE database = currentDatabase() + WHERE database = \(database) ORDER BY table, position """ let result = try await execute(query: sql) @@ -168,12 +178,13 @@ extension ClickHousePluginDriver { } func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { - let escapedTable = table.replacingOccurrences(of: "'", with: "''") + let database = Self.databaseExpression(schema) + let tableLiteral = Self.stringLiteral(table) var indexes: [PluginIndexInfo] = [] let sortingKeySql = """ SELECT sorting_key FROM system.tables - WHERE database = currentDatabase() AND name = '\(escapedTable)' + WHERE database = \(database) AND name = \(tableLiteral) """ let sortingResult = try await execute(query: sortingKeySql) if let row = sortingResult.rows.first, @@ -190,11 +201,10 @@ extension ClickHousePluginDriver { )) } - let caps = ClickHouseCapabilities.parse(serverVersion) - guard caps.hasDataSkippingIndicesTable else { return indexes } + guard serverProfile.hasDataSkippingIndices else { return indexes } let skippingSql = """ SELECT name, expr FROM system.data_skipping_indices - WHERE database = currentDatabase() AND table = '\(escapedTable)' + WHERE database = \(database) AND table = \(tableLiteral) """ let skippingResult = try await execute(query: skippingSql) for row in skippingResult.rows { @@ -220,10 +230,9 @@ extension ClickHousePluginDriver { } func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { - let escapedTable = table.replacingOccurrences(of: "'", with: "''") let sql = """ SELECT sum(rows) FROM system.parts - WHERE database = currentDatabase() AND table = '\(escapedTable)' AND active = 1 + WHERE database = \(Self.databaseExpression(schema)) AND table = \(Self.stringLiteral(table)) AND active = 1 """ let result = try await execute(query: sql) if let row = result.rows.first, let cell = row.first, let str = cell.asText { @@ -233,7 +242,7 @@ extension ClickHousePluginDriver { } func fetchTableDDL(table: String, schema: String?) async throws -> String { - let sql = "SHOW CREATE TABLE \(clickHouseQuotedIdentifier(table))" + let sql = "SHOW CREATE TABLE \(qualified(database: schema, name: table))" let result = try await execute(query: sql) return result.rows.first?.first?.asText ?? "" } @@ -242,10 +251,14 @@ extension ClickHousePluginDriver { /// and engine clause. `as_select` is the `SELECT` alone and would describe it as an ordinary /// view, so the restore would build one and lose where the rows are actually kept. func fetchViewDefinition(view: String, schema: String?) async throws -> String { - let escapedView = view.replacingOccurrences(of: "'", with: "''") + /// `system.tables` has no `as_select` on 21.8, and `SHOW CREATE TABLE` answers for a view + /// with its whole `CREATE VIEW` statement. + guard serverProfile.hasViewSelectText else { + return try await fetchTableDDL(table: view, schema: schema) + } let sql = """ SELECT engine, as_select FROM system.tables - WHERE database = currentDatabase() AND name = '\(escapedView)' + WHERE database = \(Self.databaseExpression(schema)) AND name = \(Self.stringLiteral(view)) """ let result = try await execute(query: sql) guard let row = result.rows.first else { @@ -268,11 +281,12 @@ extension ClickHousePluginDriver { } func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { - let escapedTable = table.replacingOccurrences(of: "'", with: "''") + let database = Self.databaseExpression(schema) + let tableLiteral = Self.stringLiteral(table) let engineSql = """ - SELECT engine, comment FROM system.tables - WHERE database = currentDatabase() AND name = '\(escapedTable)' + SELECT engine, \(serverProfile.hasTableComments ? "comment" : "''") FROM system.tables + WHERE database = \(database) AND name = \(tableLiteral) """ let engineResult = try await execute(query: engineSql) let engine = engineResult.rows.first.flatMap { $0[safe: 0]?.asText } @@ -281,7 +295,7 @@ extension ClickHousePluginDriver { let partsSql = """ SELECT sum(rows), sum(bytes_on_disk) FROM system.parts - WHERE database = currentDatabase() AND table = '\(escapedTable)' AND active = 1 + WHERE database = \(database) AND table = \(tableLiteral) AND active = 1 """ let partsResult = try await execute(query: partsSql) if let row = partsResult.rows.first { @@ -306,10 +320,9 @@ extension ClickHousePluginDriver { } func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { - let escapedDb = database.replacingOccurrences(of: "'", with: "''") let sql = """ - SELECT count() AS table_count, sum(total_bytes) AS size_bytes - FROM system.tables WHERE database = '\(escapedDb)' + SELECT count() AS table_count, \(totalBytesExpression) AS size_bytes + FROM system.tables WHERE database = \(Self.stringLiteral(database)) """ let result = try await execute(query: sql) if let row = result.rows.first { @@ -325,16 +338,23 @@ extension ClickHousePluginDriver { } func fetchAllDatabaseMetadata() async throws -> [PluginDatabaseMetadata] { - let aggregate = try await execute(query: Self.databaseTableAggregateQuery) + let aggregate = try await execute(query: databaseTableAggregateQuery) let names = try await fetchDatabases() return Self.databaseMetadata(names: names, aggregateRows: aggregate.rows) } - static let databaseTableAggregateQuery = """ - SELECT database, count() AS table_count, sum(total_bytes) AS size_bytes + var databaseTableAggregateQuery: String { + """ + SELECT database, count() AS table_count, \(totalBytesExpression) AS size_bytes FROM system.tables GROUP BY database """ + } + + /// 19.16 and 20.3 have no `total_bytes`, and a size the server cannot give is unknown, not 0. + private var totalBytesExpression: String { + serverProfile.hasTableTotals ? "sum(total_bytes)" : "NULL" + } /// `SHOW DATABASES` is the list and `system.tables` only supplies the numbers. The aggregate has no row for a /// database that holds no tables, so a list read from it dropped every empty database. @@ -394,16 +414,17 @@ extension ClickHousePluginDriver { // MARK: - All Tables Metadata func allTablesMetadataSQL(schema: String?) -> String? { - """ + let profile = serverProfile + return """ SELECT database as `schema`, name, engine as kind, - total_rows as estimated_rows, - formatReadableSize(total_bytes) as total_size, - comment + \(profile.hasTableTotals ? "total_rows" : "NULL") as estimated_rows, + \(profile.hasTableTotals ? "formatReadableSize(total_bytes)" : "NULL") as total_size, + \(profile.hasTableComments ? "comment" : "'' as comment") FROM system.tables - WHERE database = currentDatabase() + WHERE database = \(Self.databaseExpression(schema)) ORDER BY name """ } diff --git a/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+TableOperations.swift b/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+TableOperations.swift index 34589562e5..b25ae5821f 100644 --- a/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+TableOperations.swift +++ b/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+TableOperations.swift @@ -17,7 +17,7 @@ extension ClickHousePluginDriver { database: schema, objectType: objectType, comment: comment, - capabilities: ClickHouseCapabilities.parse(serverVersion) + profile: serverProfile ) } } diff --git a/Plugins/ClickHouseDriverPlugin/ClickHouseServerException.swift b/Plugins/ClickHouseDriverPlugin/ClickHouseServerException.swift new file mode 100644 index 0000000000..677c10539e --- /dev/null +++ b/Plugins/ClickHouseDriverPlugin/ClickHouseServerException.swift @@ -0,0 +1,201 @@ +// +// ClickHouseServerException.swift +// ClickHouseDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// A streamed read has already had HTTP 200 when a query fails partway, so the server writes the +/// exception into the body and URLSession reports a clean end. What it writes depends on the +/// version, measured from 23.8 to 26.3: +/// - before 24.11, the raw text, `Code: 395. DB::Exception: ... (version 23.8.8.20 (official build))`; +/// - 24.11 to 25.10, `__exception__\r\n` and then the text (ClickHouse#68800); +/// - 25.11 on, a block framed by the `X-ClickHouse-Exception-Tag` header's tag (ClickHouse#88818). +/// +/// So the last bytes of the body are held back until it ends, and then read for one of these. +struct ClickHouseStreamTail { + /// Exception text runs to a few kilobytes, and the tagged block caps it at 16 KiB. + static let heldByteCount = 65_536 + + private var held: [UInt8] = [] + private var headerLinesToPass = 2 + + /// The bytes that can be decoded now: the two header lines, and every whole line older than + /// the held tail. + mutating func release(_ data: Data) -> Data { + held.append(contentsOf: data) + var end = 0 + while headerLinesToPass > 0, let newline = held[end...].firstIndex(of: Self.lineFeed) { + end = newline + 1 + headerLinesToPass -= 1 + } + if headerLinesToPass == 0, held.count - end > Self.heldByteCount, + let newline = held[end..<(held.count - Self.heldByteCount)].lastIndex(of: Self.lineFeed) { + end = newline + 1 + } + let released = Data(held[.. Data { + let tail = held + held = [] + if let message = ClickHouseServerException.framedMessage(in: tail, tag: exceptionTag, header: header) { + throw ClickHouseError(message: message) + } + if let message = ClickHouseServerException.rawMessage(in: tail, header: header) { + throw ClickHouseError(message: message) + } + guard tail.isEmpty || tail.last == Self.lineFeed else { + throw ClickHouseError(message: String( + localized: "ClickHouse ended the result partway through a row, so the rows that arrived are incomplete." + )) + } + return Data(tail) + } + + fileprivate static let lineFeed = UInt8(ascii: "\n") +} + +enum ClickHouseServerException { + private static let marker = Array("__exception__\r\n".utf8) + + /// The tagged block is exact in any output format. The untagged marker is not: a quoted CSV + /// value can hold a raw carriage return. + static func throwIfTagged(_ bytes: [UInt8], tag: String?) throws { + guard let tag, !tag.isEmpty, let message = framedMessage(in: bytes, tag: tag, header: nil) else { return } + throw ClickHouseError(message: message) + } + + /// A failure reported by status or by `X-ClickHouse-Exception-Code` can still carry rows + /// ahead of the text: 25.10 and later compress by default, and a failure before the first + /// flush comes back as HTTP 500 with the rows written so far. The text is the end of the body. + static func message(inFailureBody body: String) -> String { + let bytes = Array(body.utf8) + let text = framedMessage(in: bytes, tag: nil, header: nil) + ?? rawExceptionStart(in: bytes).map { text(of: bytes[$0...]) } + ?? body + return text.trimmingCharacters(in: .whitespacesAndNewlines) + } + + /// The tagged block (25.11 on) or the untagged marker (24.11 to 25.10). A raw carriage return + /// never appears in TabSeparated data, so the marker cannot be a value, unless the user's + /// profile turned on CRLF line ends, which the header lines then show. + fileprivate static func framedMessage( + in bytes: [UInt8], + tag: String?, + header: ClickHouseTabSeparatedRowDecoder.Header? + ) -> String? { + if let tag, !tag.isEmpty { + let opening = marker + Array("\(tag)\r\n".utf8) + guard let start = firstRange(of: opening, in: bytes) else { return nil } + var message = bytes[start.upperBound...] + if let closing = firstRange(of: Array(" \(tag)\r\n".utf8), in: Array(message)), + let lengthLine = message[..<(message.startIndex + closing.lowerBound)].lastIndex(of: ClickHouseStreamTail.lineFeed) { + message = message[.. String? { + guard let start = rawExceptionStart(in: bytes) else { return nil } + let block = bytes[start...] + let lines = block.split(separator: ClickHouseStreamTail.lineFeed, omittingEmptySubsequences: true) + guard let lastLine = lines.last, endsWithVersion(String(decoding: lastLine, as: UTF8.self)) else { + return nil + } + guard let header else { return text(of: block) } + + let columnCount = header.columns.count + let tab = UInt8(ascii: "\t") + let isWrongWidth = lines.contains { line in line.count(where: { $0 == tab }) + 1 != columnCount } + guard !isWrongWidth else { return text(of: block) } + guard columnCount == 1, let type = header.columnTypeNames.first else { return nil } + guard isStringFamily(type) else { return text(of: block) } + /// TabSeparated writes a quote inside a String as `\'`, and exception text quotes names. + let quote = UInt8(ascii: "'") + let backslash = UInt8(ascii: "\\") + let hasRawQuote = block.indices.contains { index in + block[index] == quote && (index == block.startIndex || block[index - 1] != backslash) + } + return hasRawQuote ? text(of: block) : nil + } + + private static let rawPrefixes: [String] = [ + #"^Code: \d+\. "#, + #"^Code: \d+, e\.displayText\(\) = "#, + #"^Poco::Exception\. Code: "#, + #"^std::exception\. Code: "#, + #"^Unknown exception\. Code: "#, + ] + + private static let rawPrefixPattern: NSRegularExpression? = try? NSRegularExpression( + pattern: rawPrefixes.map { "(?:\($0))" }.joined(separator: "|") + ) + + private static let versionSuffixPattern: NSRegularExpression? = try? NSRegularExpression( + pattern: #"\(version [0-9][^()]*(?: \([A-Za-z ]+\))?\)\s*$"# + ) + + /// Where the last line that opens an exception begins. + private static func rawExceptionStart(in bytes: [UInt8]) -> Int? { + guard let rawPrefixPattern else { return nil } + var lineStart = bytes.endIndex + while lineStart > bytes.startIndex { + let previous = bytes[.. Bool { + guard let versionSuffixPattern else { return false } + return versionSuffixPattern.firstMatch(in: line, range: NSRange(line.startIndex..., in: line)) != nil + } + + /// A profile with CRLF line ends leaves the carriage return on the last header field. + private static func isStringFamily(_ type: String) -> Bool { + let bare = ClickHousePluginDriver.unwrapTypeWrappers(type.trimmingCharacters(in: CharacterSet(charactersIn: "\r"))) + return bare == "String" || bare.hasPrefix("FixedString(") + } + + private static func firstRange(of needle: [UInt8], in haystack: [UInt8]) -> Range? { + guard !needle.isEmpty, haystack.count >= needle.count else { return nil } + let first = needle[0] + var index = haystack.startIndex + while let candidate = haystack[index...].firstIndex(of: first) { + guard candidate + needle.count <= haystack.endIndex else { return nil } + if haystack[candidate..<(candidate + needle.count)].elementsEqual(needle) { + return candidate..<(candidate + needle.count) + } + index = candidate + 1 + } + return nil + } + + private static func text(of bytes: ArraySlice) -> String { + String(decoding: bytes, as: UTF8.self).trimmingCharacters(in: .whitespacesAndNewlines) // swiftlint:disable:this optional_data_string_conversion + } +} diff --git a/Plugins/ClickHouseDriverPlugin/ClickHouseServerProfile.swift b/Plugins/ClickHouseDriverPlugin/ClickHouseServerProfile.swift new file mode 100644 index 0000000000..cfcede3b7d --- /dev/null +++ b/Plugins/ClickHouseDriverPlugin/ClickHouseServerProfile.swift @@ -0,0 +1,251 @@ +// +// ClickHouseServerProfile.swift +// ClickHouseDriverPlugin +// + +import Foundation +import TableProPluginKit + +/// What one server accepts and holds, asked of the server at connect instead of predicted from its +/// version. `http_write_exception_in_output_format` first shipped in 23.9 and reached the 23.8 line +/// only in 23.8.9, Altinity builds carry their own numbering, and a settings profile or a +/// constraint makes a setting read-only for one user: none of that shows in the version. A setting +/// the server does not know, or will not let this user change, fails the whole request. +struct ClickHouseServerProfile: Sendable, Equatable { + let version: ClickHouseServerVersion? + /// The settings this user may change, each with the bounds a settings constraint puts on it. + let settingLimits: [ClickHouseURLSetting: ClickHouseSettingLimits] + let catalog: Set + + /// Before the probes answer, a request carries nothing the server could refuse, so a + /// `readonly = 1` user can still connect. + static let bootstrap = ClickHouseServerProfile( + version: nil, + writableSettings: [], + catalog: Set(ClickHouseCatalogFeature.allCases) + ) + + /// `system.settings` is readable by every user, grants or not, and reports per user whether + /// a setting can be changed. Its `readonly` column arrived in 20.3, so the columns are read by + /// name, and the user's own `readonly` setting answers for 19.16. + static let settingsProbeQuery = """ + SELECT * FROM system.settings + WHERE name IN (\((ClickHouseURLSetting.allCases.map(\.rawValue) + ["readonly"]).map { "'\($0)'" }.joined(separator: ", "))) + """ + + /// One row, one column per feature. `system.columns` lists nothing to a user without grants + /// on the system database, which 25.8's shipped config requires, while `hasColumnInTable` and + /// `system.tables` still answer. + static let catalogProbeQuery = "SELECT " + ClickHouseCatalogFeature.allCases + .map { "\($0.probeExpression) AS \($0.rawValue)" } + .joined(separator: ", ") + + init( + version: ClickHouseServerVersion?, + settingLimits: [ClickHouseURLSetting: ClickHouseSettingLimits], + catalog: Set + ) { + self.version = version + self.settingLimits = settingLimits + self.catalog = catalog + } + + init( + version: ClickHouseServerVersion?, + writableSettings: Set, + catalog: Set + ) { + self.init( + version: version, + settingLimits: Dictionary(uniqueKeysWithValues: writableSettings.map { ($0, ClickHouseSettingLimits()) }), + catalog: catalog + ) + } + + /// A settings probe that could not be read sends nothing optional, which every user may do. + /// A catalog probe that could not be read asks for everything and lets the server answer, as + /// the driver did before it asked. + init( + versionText: String?, + settings: (columns: [String], rows: [[PluginCellValue]])?, + catalogRow: [PluginCellValue]? + ) { + self.init( + version: versionText.flatMap(ClickHouseServerVersion.init), + settingLimits: settings.map { Self.settingLimits(columns: $0.columns, rows: $0.rows) } ?? [:], + catalog: catalogRow.map(Self.catalog(row:)) ?? Set(ClickHouseCatalogFeature.allCases) + ) + } + + var writableSettings: Set { Set(settingLimits.keys) } + + /// The URL item for `value`, or nil when this user may not set it to that. + func queryItem(_ setting: ClickHouseURLSetting, _ value: String) -> URLQueryItem? { + guard settingLimits[setting]?.allows(value) == true else { return nil } + return URLQueryItem(name: setting.rawValue, value: value) + } + + + var hasDataSkippingIndices: Bool { catalog.contains(.dataSkippingIndices) } + var hasTableComments: Bool { catalog.contains(.tablesComment) } + var hasViewSelectText: Bool { catalog.contains(.tablesAsSelect) } + var hasTableTotals: Bool { catalog.isSuperset(of: [.tablesTotalRows, .tablesTotalBytes]) } + var hasFunctionOrigin: Bool { catalog.contains(.functionsOrigin) } + var hasFunctionCreateQuery: Bool { catalog.contains(.functionsCreateQuery) } + + /// The statement parses from 21.11 but does nothing on ReplicatedMergeTree before 23.9 while + /// reporting success (ClickHouse#51304), so it is offered from 23.9. No catalog says so. + var hasModifyComment: Bool { version?.isAtLeast(major: 23, minor: 9) ?? false } + + static func writableSettings(columns: [String], rows: [[PluginCellValue]]) -> Set { + Set(settingLimits(columns: columns, rows: rows).keys) + } + + static func settingLimits( + columns: [String], + rows: [[PluginCellValue]] + ) -> [ClickHouseURLSetting: ClickHouseSettingLimits] { + guard let nameIndex = columns.firstIndex(of: "name") else { return [:] } + /// `readonly = 1` forbids changing any setting; 2 still allows it. + if let valueIndex = columns.firstIndex(of: "value"), + rows.contains(where: { $0[safe: nameIndex]?.asText == "readonly" && $0[safe: valueIndex]?.asText == "1" }) { + return [:] + } + let readonlyIndex = columns.firstIndex(of: "readonly") + let cell = { (row: [PluginCellValue], column: String) -> String? in + columns.firstIndex(of: column).flatMap { row[safe: $0]?.asText } + } + var limits: [ClickHouseURLSetting: ClickHouseSettingLimits] = [:] + for row in rows { + guard let setting = row[safe: nameIndex]?.asText.flatMap(ClickHouseURLSetting.init(rawValue:)) else { + continue + } + if let readonlyIndex, row[safe: readonlyIndex]?.asText != "0" { continue } + limits[setting] = ClickHouseSettingLimits( + min: cell(row, "min"), + max: cell(row, "max"), + disallowedValues: cell(row, "disallowed_values") + ) + } + return limits + } + + static func catalog(row: [PluginCellValue]) -> Set { + Set(ClickHouseCatalogFeature.allCases.enumerated().compactMap { index, feature in + let count = row[safe: index]?.asText.flatMap { Int($0) } ?? 0 + return count > 0 ? feature : nil + }) + } +} + +/// The bounds a settings constraint puts on one setting (`system.settings` `min`, `max` and, +/// from 25.x, `disallowed_values`). A writable setting can still refuse a value outside them. +struct ClickHouseSettingLimits: Sendable, Equatable { + var min: Double? + var max: Double? + var disallowed: [String] = [] + + init(min: Double? = nil, max: Double? = nil, disallowed: [String] = []) { + self.min = min + self.max = max + self.disallowed = disallowed + } + + init(min: String?, max: String?, disallowedValues: String?) { + self.init( + min: Self.number(min), + max: Self.number(max), + disallowed: Self.arrayItems(disallowedValues) + ) + } + + func allows(_ value: String) -> Bool { + let number = Self.number(value) + let isDisallowed = disallowed.contains { item in + item == value || (number != nil && Self.number(item) == number) + } + guard !isDisallowed else { return false } + guard let number else { return true } + if let min, number < min { return false } + if let max, number > max { return false } + return true + } + + private static func number(_ text: String?) -> Double? { + switch text { + case "true": 1 + case "false": 0 + default: text.flatMap(Double.init) + } + } + + /// `['break', 'throw']` as TabSeparated writes an `Array(String)`. + private static func arrayItems(_ text: String?) -> [String] { + guard let text, text.hasPrefix("["), text.hasSuffix("]") else { return [] } + return text.dropFirst().dropLast() + .split(separator: ",") + .map { $0.trimmingCharacters(in: CharacterSet(charactersIn: " '")) } + .filter { !$0.isEmpty } + } +} + +/// The URL settings the driver sends only when the server lets this user change them. +/// `max_execution_time` is not here: `applyQueryTimeout` sends it once and keeps it only when the +/// server accepts it. +enum ClickHouseURLSetting: String, CaseIterable, Sendable { + case sendProgressInHTTPHeaders = "send_progress_in_http_headers" + case httpWriteExceptionInOutputFormat = "http_write_exception_in_output_format" + case maxResultRows = "max_result_rows" + case resultOverflowMode = "result_overflow_mode" + case cancelHTTPReadonlyQueriesOnClientClose = "cancel_http_readonly_queries_on_client_close" +} + +/// System catalog parts the metadata reads use that older servers lack, measured: 19.16 and 20.3 +/// have no `system.tables.total_rows` or `total_bytes`, 21.3 no `system.tables.comment`, 21.8 no +/// `as_select`, 21.10 no `system.functions.origin`, and `system.data_skipping_indices` first +/// ships in 21.8. On 25.8's shipped config a user without system grants cannot read the +/// skipping-index table, and `system.tables` does not list it to them either. +enum ClickHouseCatalogFeature: String, CaseIterable, Sendable { + case tablesComment = "tables_comment" + case tablesAsSelect = "tables_as_select" + case tablesTotalRows = "tables_total_rows" + case tablesTotalBytes = "tables_total_bytes" + case functionsOrigin = "functions_origin" + case functionsCreateQuery = "functions_create_query" + case dataSkippingIndices = "data_skipping_indices" + + /// `hasColumnInTable` on a table the server lacks throws, so the skipping-index table is + /// looked up in `system.tables` instead. + var probeExpression: String { + switch self { + case .tablesComment: "hasColumnInTable('system', 'tables', 'comment')" + case .tablesAsSelect: "hasColumnInTable('system', 'tables', 'as_select')" + case .tablesTotalRows: "hasColumnInTable('system', 'tables', 'total_rows')" + case .tablesTotalBytes: "hasColumnInTable('system', 'tables', 'total_bytes')" + case .functionsOrigin: "hasColumnInTable('system', 'functions', 'origin')" + case .functionsCreateQuery: "hasColumnInTable('system', 'functions', 'create_query')" + case .dataSkippingIndices: + "(SELECT count() FROM system.tables WHERE database = 'system' AND name = 'data_skipping_indices')" + } + } +} + +/// `version()` is four numbers on official and Cloud builds, and Altinity appends a fifth, +/// non-numeric part (`24.8.14.10548.altinitystable`). +struct ClickHouseServerVersion: Sendable, Equatable { + let components: [Int] + + init?(_ text: String) { + let numbers = text.trimmingCharacters(in: .whitespacesAndNewlines) + .split(separator: ".") + .prefix { Int($0) != nil } + .compactMap { Int($0) } + guard !numbers.isEmpty else { return nil } + components = Array(numbers.prefix(4)) + } + + func isAtLeast(major: Int, minor: Int) -> Bool { + let own = (components[0], components.count > 1 ? components[1] : 0) + return own >= (major, minor) + } +} diff --git a/Plugins/ClickHouseDriverPlugin/ClickHouseTableOperations.swift b/Plugins/ClickHouseDriverPlugin/ClickHouseTableOperations.swift index c9c0327101..8ed4ba7aad 100644 --- a/Plugins/ClickHouseDriverPlugin/ClickHouseTableOperations.swift +++ b/Plugins/ClickHouseDriverPlugin/ClickHouseTableOperations.swift @@ -36,9 +36,9 @@ func clickHouseCommentStatement( database: String?, objectType: String, comment: String?, - capabilities: ClickHouseCapabilities + profile: ClickHouseServerProfile ) -> String? { - guard capabilities.hasModifyComment, objectType.uppercased() == "TABLE" else { return nil } + guard profile.hasModifyComment, objectType.uppercased() == "TABLE" else { return nil } let table = database.flatMap { $0.isEmpty ? nil : "\(clickHouseQuotedIdentifier($0)).\(clickHouseQuotedIdentifier(name))" } ?? clickHouseQuotedIdentifier(name) diff --git a/Plugins/ClickHouseDriverTests/ClickHouseCapabilitiesTests.swift b/Plugins/ClickHouseDriverTests/ClickHouseCapabilitiesTests.swift deleted file mode 100644 index 3726bd908b..0000000000 --- a/Plugins/ClickHouseDriverTests/ClickHouseCapabilitiesTests.swift +++ /dev/null @@ -1,37 +0,0 @@ -// -// ClickHouseCapabilitiesTests.swift -// TableProTests -// - -import Foundation -import Testing - -struct ClickHouseCapabilitiesTests { - @Test("The write-exception setting needs ClickHouse 23.8 or later") - func writeExceptionSettingGate() { - #expect(!ClickHouseCapabilities.parse("23.7").hasWriteExceptionInOutputFormatSetting) - #expect(ClickHouseCapabilities.parse("23.8").hasWriteExceptionInOutputFormatSetting) - #expect(ClickHouseCapabilities.parse("23.8.1.94").hasWriteExceptionInOutputFormatSetting) - #expect(ClickHouseCapabilities.parse("24.1").hasWriteExceptionInOutputFormatSetting) - #expect(!ClickHouseCapabilities.parse("19.17").hasWriteExceptionInOutputFormatSetting) - } - - @Test("An unknown server version is treated as unsupported") - func unknownVersionIsUnsupported() { - #expect(!ClickHouseCapabilities.parse(nil).hasWriteExceptionInOutputFormatSetting) - #expect(!ClickHouseCapabilities.parse("garbage").hasWriteExceptionInOutputFormatSetting) - #expect(!ClickHouseCapabilities.parse(nil).hasModifyComment) - #expect(!ClickHouseCapabilities.parse("garbage").hasModifyComment) - } - - @Test("MODIFY COMMENT needs ClickHouse 23.9 or later") - func modifyCommentGate() { - #expect(!ClickHouseCapabilities.parse("23.8").hasModifyComment) - #expect(!ClickHouseCapabilities.parse("23.8.16.40").hasModifyComment) - #expect(!ClickHouseCapabilities.parse("22.12").hasModifyComment) - #expect(ClickHouseCapabilities.parse("23.9").hasModifyComment) - #expect(ClickHouseCapabilities.parse("23.9.1.1854").hasModifyComment) - #expect(ClickHouseCapabilities.parse("23.12").hasModifyComment) - #expect(ClickHouseCapabilities.parse("24.3.2.23").hasModifyComment) - } -} diff --git a/Plugins/ClickHouseDriverTests/ClickHouseConnectProfileTests.swift b/Plugins/ClickHouseDriverTests/ClickHouseConnectProfileTests.swift new file mode 100644 index 0000000000..58904f4789 --- /dev/null +++ b/Plugins/ClickHouseDriverTests/ClickHouseConnectProfileTests.swift @@ -0,0 +1,166 @@ +// +// ClickHouseConnectProfileTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +/// Drives `connect()` against a stub that answers the version and the two probes the way the +/// measured servers did, then checks what the requests after it carry. +struct ClickHouseConnectProfileTests { + private static let optionalSettingNames = Set(ClickHouseURLSetting.allCases.map(\.rawValue)) + + private static func tsv(_ columns: [String], _ rows: [[String]]) -> ClickHouseStubReply { + let lines = [columns, columns.map { _ in "String" }] + rows + return ClickHouseStubReply(statusCode: 200, body: lines.map { $0.joined(separator: "\t") }.joined(separator: "\n") + "\n") + } + + private static func answer( + _ request: ClickHouseStubRequest, + version: String, + settings: [ClickHouseURLSetting], + readonly: String = "0", + catalog: Set = Set(ClickHouseCatalogFeature.allCases) + ) -> ClickHouseStubReply { + if request.body == "SELECT version()" { + return tsv(["version()"], [[version]]) + } + if request.body.hasPrefix("SELECT * FROM system.settings") { + return tsv( + ClickHouseSettingsRows.columns20, + settings.map { [$0.rawValue, "0", "0", "", "\\N", "\\N", readonly] } + ) + } + if request.body.hasPrefix("SELECT hasColumnInTable(") { + let features = ClickHouseCatalogFeature.allCases + return tsv(features.map(\.rawValue), [features.map { catalog.contains($0) ? "1" : "0" }]) + } + return .oneRow + } + + private static func carriesOptionalSetting(_ request: ClickHouseStubRequest) -> Bool { + !optionalSettingNames.isDisjoint(with: request.queryItems.keys) + } + + @Test("Connected to 23.8.8.20, a query carries no write-exception setting (#3444)") + func server2388() async throws { + let server = ClickHouseStubServer { request in + if request.item("http_write_exception_in_output_format") != nil { + return ClickHouseStubReply(statusCode: 404, body: "Code: 115. DB::Exception: Setting http_write_exception_in_output_format is neither a builtin setting nor started with the prefix 'SQL_' registered for user-defined settings. (UNKNOWN_SETTING)") + } + return Self.answer( + request, + version: "23.8.8.20", + settings: ClickHouseURLSetting.allCases.filter { $0 != .httpWriteExceptionInOutputFormat } + ) + } + let driver = server.unconnectedDriver() + defer { driver.disconnect() } + + try await driver.connect() + let result = try await driver.execute(query: "SELECT 42") + + #expect(result.rows.first?.first?.asText == "1") + let query = try #require(server.requests.first { $0.body == "SELECT 42" }) + #expect(query.item("http_write_exception_in_output_format") == nil) + #expect(query.item("send_progress_in_http_headers") == "1") + #expect(driver.serverVersion == "23.8.8.20") + } + + @Test("Connected to 25.8, the write-exception setting goes out") + func server258() async throws { + let server = ClickHouseStubServer { Self.answer($0, version: "25.8.33.6", settings: ClickHouseURLSetting.allCases) } + let driver = server.unconnectedDriver() + defer { driver.disconnect() } + + try await driver.connect() + _ = try await driver.execute(query: "SELECT 42") + + let query = try #require(server.requests.first { $0.body == "SELECT 42" }) + #expect(query.item("http_write_exception_in_output_format") == "0") + } + + @Test("Connect and its probes send no setting a server could refuse") + func connectRequestsAreMinimal() async throws { + let server = ClickHouseStubServer { Self.answer($0, version: "25.8.33.6", settings: ClickHouseURLSetting.allCases) } + let driver = server.unconnectedDriver() + defer { driver.disconnect() } + + try await driver.connect() + + #expect(server.requests.count == 3) + #expect(!server.requests.contains(where: Self.carriesOptionalSetting)) + } + + @Test("A read-only user connects, queries and reads a capped result") + func readonlyUser() async throws { + let server = ClickHouseStubServer { request in + if Self.carriesOptionalSetting(request) { + return ClickHouseStubReply(statusCode: 500, body: "Code: 164. DB::Exception: Cannot modify setting in readonly mode. (READONLY)") + } + return Self.answer(request, version: "25.8.33.6", settings: ClickHouseURLSetting.allCases, readonly: "1") + } + let driver = server.unconnectedDriver() + defer { driver.disconnect() } + + try await driver.connect() + _ = try await driver.execute(query: "SELECT 42") + let bounded = try await driver.executeBoundedQuery(query: "SELECT 43", rowCap: 10) + + #expect(bounded?.rows.count == 1) + #expect(!server.requests.contains(where: Self.carriesOptionalSetting)) + } + + @Test("A settings probe that fails sends nothing optional afterwards") + func failedSettingsProbe() async throws { + let server = ClickHouseStubServer { request in + if request.body.hasPrefix("SELECT * FROM system.settings") { + return ClickHouseStubReply(statusCode: 500, body: "Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED)") + } + return Self.answer(request, version: "25.8.33.6", settings: ClickHouseURLSetting.allCases) + } + let driver = server.unconnectedDriver() + defer { driver.disconnect() } + + try await driver.connect() + _ = try await driver.execute(query: "SELECT 42") + + let query = try #require(server.requests.first { $0.body == "SELECT 42" }) + #expect(!Self.carriesOptionalSetting(query)) + } + + /// On 25.8's shipped config a user with no system grants is not shown the skipping-index + /// table, while comments, view text and functions still read. + @Test("A user without system grants keeps comments and functions and skips the skipping indices") + func userWithoutSystemGrants() async throws { + let server = ClickHouseStubServer { request in + Self.answer( + request, + version: "25.8.33.6", + settings: ClickHouseURLSetting.allCases, + catalog: Set(ClickHouseCatalogFeature.allCases).subtracting([.dataSkippingIndices]) + ) + } + let driver = server.unconnectedDriver() + defer { driver.disconnect() } + + try await driver.connect() + + #expect(driver.serverProfile.hasTableComments) + #expect(driver.serverProfile.hasFunctionOrigin) + #expect(!driver.serverProfile.hasDataSkippingIndices) + } + + @Test("A server that refuses the version query fails the connect with its own answer") + func refusedVersionQueryFailsConnect() async throws { + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 516, body: "Code: 516. DB::Exception: default: Authentication failed. (AUTHENTICATION_FAILED)") + } + let driver = server.unconnectedDriver() + + await #expect(throws: ClickHouseError.self) { try await driver.connect() } + #expect(server.requests.count == 1) + } +} diff --git a/Plugins/ClickHouseDriverTests/ClickHouseDatabaseMetadataTests.swift b/Plugins/ClickHouseDriverTests/ClickHouseDatabaseMetadataTests.swift index fd5ab45f91..64cc5e7aef 100644 --- a/Plugins/ClickHouseDriverTests/ClickHouseDatabaseMetadataTests.swift +++ b/Plugins/ClickHouseDriverTests/ClickHouseDatabaseMetadataTests.swift @@ -60,9 +60,21 @@ struct ClickHouseDatabaseMetadataTests { @Test("The aggregate reads system.tables once, grouped by database") func aggregateQueryGroupsSystemTables() { - let query = ClickHousePluginDriver.databaseTableAggregateQuery + let query = ClickHouseStubServer().connectedDriver().databaseTableAggregateQuery #expect(query.contains("FROM system.tables")) #expect(query.contains("GROUP BY database")) + #expect(query.contains("sum(total_bytes)")) + } + + @Test("A server without system.tables.total_bytes reports the size as unknown") + func aggregateQueryWithoutTotals() throws { + let driver = ClickHouseStubServer().connectedDriver() + driver.serverProfile = .testing(catalog: [.tablesComment]) + + #expect(!driver.databaseTableAggregateQuery.contains("total_bytes")) + let allTables = try #require(driver.allTablesMetadataSQL(schema: nil)) + #expect(!allTables.contains("total_rows")) + #expect(!allTables.contains("total_bytes")) } } diff --git a/Plugins/ClickHouseDriverTests/ClickHouseHTTPStub.swift b/Plugins/ClickHouseDriverTests/ClickHouseHTTPStub.swift index b70868c081..c026a3ab9f 100644 --- a/Plugins/ClickHouseDriverTests/ClickHouseHTTPStub.swift +++ b/Plugins/ClickHouseDriverTests/ClickHouseHTTPStub.swift @@ -14,6 +14,7 @@ struct ClickHouseStubRequest: Sendable { struct ClickHouseStubReply: Sendable { let statusCode: Int let body: String + var headers: [String: String] = [:] static let oneRow = ClickHouseStubReply(statusCode: 200, body: "n\nUInt8\n1\n") static let empty = ClickHouseStubReply(statusCode: 200, body: "") @@ -51,7 +52,12 @@ final class ClickHouseHTTPStubProtocol: URLProtocol, @unchecked Sendable { } guard let responder, let reply = responder(stubRequest), - let response = HTTPURLResponse(url: url, statusCode: reply.statusCode, httpVersion: nil, headerFields: nil) + let response = HTTPURLResponse( + url: url, + statusCode: reply.statusCode, + httpVersion: nil, + headerFields: reply.headers + ) else { return } client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) client?.urlProtocol(self, didLoad: Data(reply.body.utf8)) @@ -89,17 +95,29 @@ struct ClickHouseStubServer { } func connectedDriver() -> ClickHousePluginDriver { - let driver = ClickHousePluginDriver(config: DriverConnectionConfig( - host: host, - port: 8_123, - username: "default", - password: "", - database: "default" - )) + let driver = unconnectedDriver() + driver.session = URLSession(configuration: Self.stubConfiguration()) + return driver + } + + /// A driver whose own `connect()` reaches this stub. + func unconnectedDriver() -> ClickHousePluginDriver { + ClickHousePluginDriver( + config: DriverConnectionConfig( + host: host, + port: 8_123, + username: "default", + password: "", + database: "default" + ), + sessionConfiguration: Self.stubConfiguration() + ) + } + + private static func stubConfiguration() -> URLSessionConfiguration { let configuration = URLSessionConfiguration.ephemeral configuration.protocolClasses = [ClickHouseHTTPStubProtocol.self] - driver.session = URLSession(configuration: configuration) - return driver + return configuration } func firstRequest(where matches: (ClickHouseStubRequest) -> Bool) async -> ClickHouseStubRequest? { diff --git a/Plugins/ClickHouseDriverTests/ClickHouseMetadataScopeTests.swift b/Plugins/ClickHouseDriverTests/ClickHouseMetadataScopeTests.swift new file mode 100644 index 0000000000..78a19ed739 --- /dev/null +++ b/Plugins/ClickHouseDriverTests/ClickHouseMetadataScopeTests.swift @@ -0,0 +1,144 @@ +// +// ClickHouseMetadataScopeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +/// The export tree reads tables of other databases over one session, so every metadata read names +/// the database it was asked about. Measured on 23.8.8.20 before this: `SHOW CREATE TABLE events` +/// on a session in `va` answered for `va.events` when the caller asked about `vb.events`. +struct ClickHouseMetadataScopeTests { + @Test("Column, index, row count, metadata and DDL reads name the database asked about") + func readsNameTheDatabase() async throws { + let server = ClickHouseStubServer() + let driver = server.connectedDriver() + + _ = try await driver.fetchColumns(table: "events", schema: "vb") + _ = try await driver.fetchAllColumns(schema: "vb") + _ = try await driver.fetchIndexes(table: "events", schema: "vb") + _ = try await driver.fetchApproximateRowCount(table: "events", schema: "vb") + _ = try await driver.fetchTableMetadata(table: "events", schema: "vb") + _ = try await driver.fetchTableDDL(table: "events", schema: "vb") + + let bodies = server.requests.map(\.body) + #expect(!bodies.contains { $0.contains("currentDatabase()") }) + #expect(bodies.filter { $0.contains("database = 'vb'") }.count == bodies.count - 1) + #expect(bodies.contains("SHOW CREATE TABLE `vb`.`events`")) + } + + @Test("No database named reads the session's") + func noDatabaseReadsTheSession() async throws { + let server = ClickHouseStubServer() + let driver = server.connectedDriver() + + _ = try await driver.fetchColumns(table: "events", schema: nil) + _ = try await driver.fetchTableDDL(table: "events", schema: nil) + + let bodies = server.requests.map(\.body) + #expect(bodies.filter { $0.contains("database = currentDatabase()") }.count == 2) + #expect(bodies.contains("SHOW CREATE TABLE `events`")) + } + + @Test("A backslash and a quote in a table name cannot end the literal early") + func tableNameIsEscaped() async throws { + let server = ClickHouseStubServer() + let driver = server.connectedDriver() + + _ = try await driver.fetchColumns(table: #"a\'b"#, schema: nil) + + #expect(server.requests.allSatisfy { $0.body.contains(#"= 'a\\''b'"#) }) + } + + @Test("All-tables metadata names the database asked about") + func allTablesMetadataNamesTheDatabase() { + let driver = ClickHouseStubServer().connectedDriver() + #expect(driver.allTablesMetadataSQL(schema: "vb")?.contains("WHERE database = 'vb'") == true) + } + + @Test("A server without the data-skipping table reads only the sorting key") + func noDataSkippingTable() async throws { + let server = ClickHouseStubServer() + let driver = server.connectedDriver() + driver.serverProfile = .testing(catalog: []) + + _ = try await driver.fetchIndexes(table: "events", schema: nil) + + #expect(server.requests.count == 1) + #expect(!server.requests.contains { $0.body.contains("data_skipping_indices") }) + } + + @Test("A server without system.tables.comment is not asked for it") + func noTableCommentColumn() async throws { + let server = ClickHouseStubServer() + let driver = server.connectedDriver() + driver.serverProfile = .testing(catalog: []) + + _ = try await driver.fetchTableMetadata(table: "events", schema: nil) + + #expect(!server.requests.contains { $0.body.contains("comment") }) + #expect(driver.allTablesMetadataSQL(schema: nil)?.contains("'' as comment") == true) + } + + /// 21.8 answers with a qualified name and a column list, the shape a materialized view + /// already gets. + @Test("A server without as_select answers a view definition with SHOW CREATE TABLE") + func noAsSelectColumn() async throws { + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 200, body: "statement\nString\nCREATE VIEW default.v AS SELECT 1\n") + } + let driver = server.connectedDriver() + driver.serverProfile = .testing(catalog: []) + + let definition = try await driver.fetchViewDefinition(view: "v", schema: nil) + + #expect(definition == "CREATE VIEW default.v AS SELECT 1") + #expect(server.requests.map(\.body) == ["SHOW CREATE TABLE `v`"]) + } + + @Test("A server without system.functions.create_query lists no functions and is not asked") + func noFunctionCatalog() async throws { + let server = ClickHouseStubServer() + let driver = server.connectedDriver() + driver.serverProfile = .testing(catalog: [.tablesComment]) + + #expect(try await driver.fetchRoutines(schema: nil).isEmpty) + #expect(server.requests.isEmpty) + } + + @Test("21.10 has create_query but no origin, so its SQL functions are listed by create_query") + func functionsWithoutOrigin() async throws { + let server = ClickHouseStubServer() + let driver = server.connectedDriver() + driver.serverProfile = .testing(catalog: [.functionsCreateQuery]) + + _ = try await driver.fetchRoutines(schema: nil) + + #expect(server.requests.map(\.body) == [ClickHouseObjectQueries.functionListWithoutOrigin]) + } + + /// `SHOW CREATE FUNCTION` answered `Code: 62 SYNTAX_ERROR` on 21.3, 21.8, 23.8.8 and 25.8.33. + @Test("A function with no definition is read from create_query and reported as not found") + func executableFunctionHasNoSource() async throws { + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 200, body: "create_query\nString\n\n") + } + let driver = server.connectedDriver() + let routine = PluginRoutineInfo( + name: "exec_fn", + kind: .function, + schema: nil, + returnType: nil, + language: "SQL", + argumentSignature: nil, + identity: nil, + definition: "", + attributes: [] + ) + + await #expect(throws: PluginObjectSourceError.self) { try await driver.fetchRoutineDDL(routine) } + #expect(server.requests.map(\.body) == ["SELECT create_query FROM system.functions WHERE name = 'exec_fn'"]) + } +} diff --git a/Plugins/ClickHouseDriverTests/ClickHouseRequestTests.swift b/Plugins/ClickHouseDriverTests/ClickHouseRequestTests.swift new file mode 100644 index 0000000000..914a528047 --- /dev/null +++ b/Plugins/ClickHouseDriverTests/ClickHouseRequestTests.swift @@ -0,0 +1,145 @@ +// +// ClickHouseRequestTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct ClickHouseRequestTests { + private static let server2388 = ClickHouseServerProfile.testing( + version: "23.8.8.20", + writableSettings: ClickHouseServerProfile.writableSettings( + columns: ClickHouseSettingsRows.columns20, + rows: ClickHouseSettingsRows.server2388 + ) + ) + private static let server258 = ClickHouseServerProfile.testing( + version: "25.8.33.6", + writableSettings: Set(ClickHouseURLSetting.allCases) + ) + private static let readonlyUser = ClickHouseServerProfile.testing(version: "25.8.33.6") + + private static func items( + _ shape: ClickHouseResponseShape, + _ profile: ClickHouseServerProfile, + executionTimeLimit: Int? = nil + ) -> [String: String] { + let items = ClickHousePluginDriver.queryItems( + database: "default", + queryId: "q1", + executionTimeLimit: executionTimeLimit, + shape: shape, + profile: profile + ) + return Dictionary(items.map { ($0.name, $0.value ?? "") }, uniquingKeysWith: { _, last in last }) + } + + @Test("Before the probes answer, a request carries only what every user may send") + func bootstrapRequestIsMinimal() { + let items = Self.items(.buffered(parameters: nil), .bootstrap) + #expect(items == [ + "database": "default", + "query_id": "q1", + "default_format": "TabSeparatedWithNamesAndTypes", + "wait_end_of_query": "1", + ]) + } + + @Test("23.8.8.20 is never sent http_write_exception_in_output_format (#3444)") + func server2388GetsNoWriteExceptionSetting() { + let buffered = Self.items(.buffered(parameters: nil), Self.server2388) + let streamed = Self.items(.streamed(rowCap: 100), Self.server2388) + #expect(buffered["http_write_exception_in_output_format"] == nil) + #expect(streamed["http_write_exception_in_output_format"] == nil) + #expect(buffered["send_progress_in_http_headers"] == "1") + } + + @Test("A server that knows the setting gets it on both shapes") + func server258GetsWriteExceptionSetting() { + #expect(Self.items(.buffered(parameters: nil), Self.server258)["http_write_exception_in_output_format"] == "0") + #expect(Self.items(.streamed(rowCap: nil), Self.server258)["http_write_exception_in_output_format"] == "0") + } + + /// A bounded read is collected whole anyway, and `wait_end_of_query` turns a failure into a + /// status on every version. An unbounded one is an export and must not be held on the server. + @Test("A bounded stream caps rows one past the cap and waits for the end; an unbounded one does neither") + func shapesCarryTheirOwnItems() { + let bounded = Self.items(.streamed(rowCap: 100), Self.server258) + #expect(bounded["max_result_rows"] == "101") + #expect(bounded["result_overflow_mode"] == "break") + #expect(bounded["cancel_http_readonly_queries_on_client_close"] == "1") + #expect(bounded["wait_end_of_query"] == "1") + #expect(bounded["send_progress_in_http_headers"] == nil) + + let unbounded = Self.items(.streamed(rowCap: nil), Self.server258) + #expect(unbounded["max_result_rows"] == nil) + #expect(unbounded["cancel_http_readonly_queries_on_client_close"] == nil) + #expect(unbounded["wait_end_of_query"] == nil) + } + + @Test("A stream asks for an uncompressed body, a buffered read keeps compression") + func streamsAskForIdentityEncoding() throws { + let driver = ClickHouseStubServer().connectedDriver() + let stream = try driver.buildRequest( + query: "SELECT 1", database: "", queryId: nil, executionTimeLimit: nil, shape: .streamed(rowCap: nil) + ) + let buffered = try driver.buildRequest( + query: "SELECT 1", database: "", queryId: nil, executionTimeLimit: nil, shape: .buffered(parameters: nil) + ) + #expect(stream.value(forHTTPHeaderField: "Accept-Encoding") == "identity") + #expect(buffered.value(forHTTPHeaderField: "Accept-Encoding") == nil) + } + + @Test("A read-only user gets no setting it cannot change, on either shape") + func readonlyUserGetsNoSettings() { + let buffered = Self.items(.buffered(parameters: nil), Self.readonlyUser) + let streamed = Self.items(.streamed(rowCap: 100), Self.readonlyUser) + for name in ClickHouseURLSetting.allCases.map(\.rawValue) { + #expect(buffered[name] == nil, "\(name)") + #expect(streamed[name] == nil, "\(name)") + } + } + + @Test("Without the server-side cap a bounded read is not held on the server") + func uncappedBoundedReadIsNotHeld() { + #expect(Self.items(.streamed(rowCap: 100), Self.readonlyUser)["wait_end_of_query"] == nil) + } + + @Test("max_result_rows goes out only with result_overflow_mode, or an overflow would throw") + func rowCapNeedsOverflowMode() { + let rowsOnly = ClickHouseServerProfile.testing(writableSettings: [.maxResultRows]) + #expect(Self.items(.streamed(rowCap: 100), rowsOnly)["max_result_rows"] == nil) + } + + @Test("A cap above the constraint's max leaves the bound to the reader") + func rowCapAboveConstraint() { + let limited = ClickHouseServerProfile( + version: nil, + settingLimits: [.maxResultRows: ClickHouseSettingLimits(max: 50), .resultOverflowMode: ClickHouseSettingLimits()], + catalog: [] + ) + #expect(Self.items(.streamed(rowCap: 100), limited)["max_result_rows"] == nil) + #expect(Self.items(.streamed(rowCap: 10), limited)["max_result_rows"] == "11") + } + + @Test("Query parameters ride as param_ items on a buffered request") + func parametersRideAsParamItems() { + let items = Self.items(.buffered(parameters: ["b": "2", "a": nil]), Self.server258) + #expect(items["param_a"] == "") + #expect(items["param_b"] == "2") + } + + @Test("An accepted execution time limit rides on both shapes") + func executionTimeLimitRidesOnBothShapes() { + #expect(Self.items(.buffered(parameters: nil), .bootstrap, executionTimeLimit: 30)["max_execution_time"] == "30") + #expect(Self.items(.streamed(rowCap: 5), .bootstrap, executionTimeLimit: 30)["max_execution_time"] == "30") + } + + @Test("Trailing semicolons and the space between them are trimmed") + func trailingSemicolonsAreTrimmed() { + #expect(ClickHousePluginDriver.withoutTrailingSemicolons("SELECT 1 ; ;\n") == "SELECT 1") + #expect(ClickHousePluginDriver.withoutTrailingSemicolons("SELECT ';'") == "SELECT ';'") + } +} diff --git a/Plugins/ClickHouseDriverTests/ClickHouseServerProfileTests.swift b/Plugins/ClickHouseDriverTests/ClickHouseServerProfileTests.swift new file mode 100644 index 0000000000..b2e897dd42 --- /dev/null +++ b/Plugins/ClickHouseDriverTests/ClickHouseServerProfileTests.swift @@ -0,0 +1,209 @@ +// +// ClickHouseServerProfileTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +extension ClickHouseServerProfile { + static func testing( + version: String? = nil, + writableSettings: Set = [], + catalog: Set = Set(ClickHouseCatalogFeature.allCases) + ) -> ClickHouseServerProfile { + ClickHouseServerProfile( + version: version.flatMap(ClickHouseServerVersion.init), + writableSettings: writableSettings, + catalog: catalog + ) + } +} + +/// `system.settings` rows as the servers answered them, the columns cut to the ones the profile +/// reads plus a neighbour, in the order each version lists them. +enum ClickHouseSettingsRows { + static let columns20 = ["name", "value", "changed", "description", "min", "max", "readonly"] + + static func rows(_ names: [ClickHouseURLSetting], readonly: String = "0") -> [[PluginCellValue]] { + names.map { [.text($0.rawValue), .text("0"), .text("0"), .text(""), .null, .null, .text(readonly)] } + } + + /// 23.8.8.20 and Altinity 23.8.8.21: no `http_write_exception_in_output_format` row. + static let server2388 = rows(ClickHouseURLSetting.allCases.filter { $0 != .httpWriteExceptionInOutputFormat }) + static let server258 = rows(ClickHouseURLSetting.allCases) + static let readonlyUser = rows(ClickHouseURLSetting.allCases, readonly: "1") +} + +struct ClickHouseServerProfileTests { + @Test( + "version() parses on official, Cloud and Altinity builds", + arguments: [ + ("23.8.8.20", [23, 8, 8, 20]), + ("24.6.1.4472", [24, 6, 1, 4_472]), + ("24.8.14.10548.altinitystable", [24, 8, 14, 10_548]), + ("25.8.33.20002.altinityantalya", [25, 8, 33, 20_002]), + ("23.9\n", [23, 9]), + ] + ) + func versionParses(text: String, components: [Int]) { + #expect(ClickHouseServerVersion(text)?.components == components) + } + + @Test("A version string with no leading number is no version", arguments: ["", "garbage", "v23.8"]) + func garbageVersionIsNil(text: String) { + #expect(ClickHouseServerVersion(text) == nil) + } + + @Test( + "MODIFY COMMENT is offered from 23.9, where it works on ReplicatedMergeTree", + arguments: [ + ("21.11.11.1", false), + ("23.8.16.16", false), + ("23.8.8.21.altinitystable", false), + ("23.9.6.20", true), + ("24.8.14.10548.altinitystable", true), + ("26.3.52.9", true), + ] + ) + func modifyCommentGate(version: String, expected: Bool) { + #expect(ClickHouseServerProfile.testing(version: version).hasModifyComment == expected) + } + + @Test("An unknown version gets no MODIFY COMMENT") + func unknownVersionHasNoModifyComment() { + #expect(!ClickHouseServerProfile.testing().hasModifyComment) + #expect(!ClickHouseServerProfile.bootstrap.hasModifyComment) + } + + @Test("23.8.8.20 lists no write-exception setting, so the driver never sends it there") + func server2388HasNoWriteExceptionSetting() { + let settings = ClickHouseServerProfile.writableSettings( + columns: ClickHouseSettingsRows.columns20, + rows: ClickHouseSettingsRows.server2388 + ) + let profile = ClickHouseServerProfile.testing(version: "23.8.8.20", writableSettings: settings) + #expect(profile.queryItem(.httpWriteExceptionInOutputFormat, "0") == nil) + #expect(profile.queryItem(.sendProgressInHTTPHeaders, "1") != nil) + } + + @Test("A server that lists the setting gets it") + func server258SendsWriteExceptionSetting() { + let settings = ClickHouseServerProfile.writableSettings( + columns: ClickHouseSettingsRows.columns20, + rows: ClickHouseSettingsRows.server258 + ) + #expect(settings == Set(ClickHouseURLSetting.allCases)) + } + + @Test("A setting read-only for this user is never sent") + func readonlyUserGetsNoOptionalSetting() { + let settings = ClickHouseServerProfile.writableSettings( + columns: ClickHouseSettingsRows.columns20, + rows: ClickHouseSettingsRows.readonlyUser + ) + #expect(settings.isEmpty) + } + + @Test("19.16 has no readonly column, so the user's readonly setting decides", arguments: ["0", "2"]) + func serverWithoutReadonlyColumn(readonly: String) { + let columns = ["name", "value", "changed", "description"] + let rows: [[PluginCellValue]] = [ + [.text("send_progress_in_http_headers"), .text("0"), .text("0"), .text("")], + [.text("max_result_rows"), .text("0"), .text("0"), .text("")], + [.text("readonly"), .text(readonly), .text("0"), .text("")], + ] + #expect(ClickHouseServerProfile.writableSettings(columns: columns, rows: rows) + == [.sendProgressInHTTPHeaders, .maxResultRows]) + } + + @Test("readonly = 1 on 19.16 makes every setting unchangeable") + func readonlyUserWithoutReadonlyColumn() { + let columns = ["name", "value", "changed", "description"] + let rows: [[PluginCellValue]] = [ + [.text("send_progress_in_http_headers"), .text("0"), .text("0"), .text("")], + [.text("readonly"), .text("1"), .text("1"), .text("")], + ] + #expect(ClickHouseServerProfile.writableSettings(columns: columns, rows: rows).isEmpty) + } + + @Test("A settings answer with no name column yields nothing") + func settingsWithoutNameColumn() { + #expect(ClickHouseServerProfile.writableSettings(columns: ["value"], rows: [[.text("0")]]).isEmpty) + } + + @Test("A settings constraint's min, max and disallowed values are read from the row") + func constraintsAreRead() { + let columns = ["name", "value", "changed", "description", "min", "max", "disallowed_values", "readonly"] + let rows: [[PluginCellValue]] = [ + [.text("http_write_exception_in_output_format"), .text("1"), .text("0"), .text(""), .text("1"), .null, .text("[]"), .text("0")], + [.text("max_result_rows"), .text("0"), .text("0"), .text(""), .null, .text("50"), .text("[]"), .text("0")], + [.text("result_overflow_mode"), .text("throw"), .text("0"), .text(""), .null, .null, .text("['break']"), .text("0")], + [.text("send_progress_in_http_headers"), .text("0"), .text("0"), .text(""), .null, .null, .text("[]"), .text("0")], + ] + let profile = ClickHouseServerProfile( + version: nil, + settingLimits: ClickHouseServerProfile.settingLimits(columns: columns, rows: rows), + catalog: [] + ) + #expect(profile.queryItem(.httpWriteExceptionInOutputFormat, "0") == nil) + #expect(profile.queryItem(.maxResultRows, "51") == nil) + #expect(profile.queryItem(.maxResultRows, "50") != nil) + #expect(profile.queryItem(.resultOverflowMode, "break") == nil) + #expect(profile.queryItem(.sendProgressInHTTPHeaders, "1") != nil) + } + + @Test("A Boolean bound written as true or false compares with 0 and 1") + func booleanBounds() { + #expect(!ClickHouseSettingLimits(min: "true", max: nil, disallowedValues: nil).allows("0")) + #expect(ClickHouseSettingLimits(min: "true", max: nil, disallowedValues: nil).allows("1")) + #expect(!ClickHouseSettingLimits(min: nil, max: nil, disallowedValues: "['false']").allows("0")) + } + + /// The catalog probe's one row as each server answered it, in `ClickHouseCatalogFeature` order: + /// comment, as_select, total_rows, total_bytes, origin, create_query, skipping-index table. + @Test( + "The catalog probe row maps to features as the servers answered", + arguments: [ + (["0", "0", "0", "0", "0", "0", "0"], Set()), + (["1", "0", "1", "1", "0", "0", "1"], [.tablesComment, .tablesTotalRows, .tablesTotalBytes, .dataSkippingIndices]), + (["1", "1", "1", "1", "1", "1", "1"], Set(ClickHouseCatalogFeature.allCases)), + (["1", "1", "1", "1", "1", "1", "0"], Set(ClickHouseCatalogFeature.allCases).subtracting([.dataSkippingIndices])), + ] as [([String], Set)] + ) + func catalogRow(answer: [String], expected: Set) { + #expect(ClickHouseServerProfile.catalog(row: answer.map { .text($0) }) == expected) + } + + @Test("A null count reads as absent") + func nullCatalogCountIsAbsent() { + let row: [PluginCellValue] = [.text("1"), .text("1"), .text("1"), .text("1"), .text("1"), .text("1"), .null] + #expect(!ClickHouseServerProfile.testing(catalog: ClickHouseServerProfile.catalog(row: row)).hasDataSkippingIndices) + } + + @Test("A settings probe that cannot be read sends nothing optional; a catalog one asks for everything") + func failedProbesFallBack() { + let profile = ClickHouseServerProfile(versionText: "23.8.8.20", settings: nil, catalogRow: nil) + #expect(profile.writableSettings.isEmpty) + #expect(profile.catalog == Set(ClickHouseCatalogFeature.allCases)) + #expect(profile.version?.components == [23, 8, 8, 20]) + } + + @Test("Before the probes answer nothing optional is sent") + func bootstrapSendsNothingOptional() { + #expect(ClickHouseServerProfile.bootstrap.writableSettings.isEmpty) + } + + @Test("The probes ask for every setting and catalog feature the profile reads, in order") + func probeQueriesNameEverything() { + for setting in ClickHouseURLSetting.allCases { + #expect(ClickHouseServerProfile.settingsProbeQuery.contains("'\(setting.rawValue)'")) + } + #expect(ClickHouseServerProfile.settingsProbeQuery.contains("'readonly'")) + let aliases = ClickHouseCatalogFeature.allCases.map { " AS \($0.rawValue)" } + let positions = aliases.compactMap { ClickHouseServerProfile.catalogProbeQuery.range(of: $0)?.lowerBound } + #expect(positions.count == aliases.count) + #expect(positions == positions.sorted()) + } +} diff --git a/Plugins/ClickHouseDriverTests/ClickHouseStreamFailureTests.swift b/Plugins/ClickHouseDriverTests/ClickHouseStreamFailureTests.swift new file mode 100644 index 0000000000..2f1774f053 --- /dev/null +++ b/Plugins/ClickHouseDriverTests/ClickHouseStreamFailureTests.swift @@ -0,0 +1,251 @@ +// +// ClickHouseStreamFailureTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +/// Bodies as the servers wrote them when `SELECT number, throwIf(number = 100000) FROM +/// numbers(200000)` failed after HTTP 200, measured on 23.8.8.20, 25.8.33.6 and 26.3.52.9. +struct ClickHouseStreamFailureTests { + private static let twoColumnHeader = "number\tthrowIf(equals(number, 100000))\nUInt64\tUInt8\n" + private static let oneColumnHeader = "toString(number)\nString\n" + + private static func rows(_ count: Int, columns: Int = 2) -> String { + (0.. throwIf(equals(number, 100000)) UInt8 : 1'. " + + "(FUNCTION_THROW_IF_VALUE_IS_NON_ZERO)" + + /// Runs a body through the tail and the decoder in chunks, the way `streamTabSeparatedRows` does. + private static func decode( + _ body: String, + tag: String? = nil, + chunkSize: Int = 8_192 + ) throws -> [[PluginCellValue]] { + var decoder = ClickHouseTabSeparatedRowDecoder() + var tail = ClickHouseStreamTail() + var rows: [[PluginCellValue]] = [] + let bytes = Data(body.utf8) + var offset = 0 + while offset < bytes.count { + let end = min(offset + chunkSize, bytes.count) + rows += decoder.consume(tail.release(bytes.subdata(in: offset.. String? { + do { + _ = try decode(body, tag: tag) + return nil + } catch let error as ClickHouseError { + return error.message + } catch { + return "\(error)" + } + } + + @Test("23.8: the raw line after the last row is the error") + func rawLineBefore2411() { + let message = "\(Self.throwIfMessage) (version 23.8.8.20 (official build))" + let body = Self.twoColumnHeader + Self.rows(3_000) + message + "\n" + #expect(Self.failure(body) == message) + } + + @Test("23.8: a multi-line message still has the wrong width for the result") + func multiLineRawMessage() { + let message = "Code: 395. DB::Exception: line1\nline2\ttabbed: while executing. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " + + "(version 23.8.8.20 (official build))" + let body = Self.twoColumnHeader + Self.rows(10) + message + "\n" + #expect(Self.failure(body) == message) + } + + @Test("23.8: a one-column String result with a raw quote in the tail is the error") + func rawLineInOneColumnResult() { + let message = "\(Self.throwIfMessage) (version 23.8.8.20 (official build))" + let body = Self.oneColumnHeader + Self.rows(10, columns: 1) + message + "\n" + #expect(Self.failure(body) == message) + } + + /// `SELECT exception FROM system.query_log` returns exception text as values, and + /// TabSeparated escapes their quotes. + @Test("Exception text stored as a String value stays a row") + func storedExceptionTextIsARow() throws { + let value = #"Code: 60. DB::Exception: Table default.x doesn\'t exist. (UNKNOWN_TABLE) (version 23.8.8.20 (official build))"# + let rows = try Self.decode("exception\nString\n" + value + "\n") + #expect(rows == [[.text("Code: 60. DB::Exception: Table default.x doesn't exist. (UNKNOWN_TABLE) (version 23.8.8.20 (official build))")]]) + } + + @Test("Exception text stored as a String value stays a row with CRLF line ends too") + func storedExceptionTextIsARowInCRLFMode() throws { + let value = #"Code: 60. DB::Exception: Table default.x doesn\'t exist. (UNKNOWN_TABLE) (version 23.8.8.20 (official build))"# + let rows = try Self.decode("exception\r\nString\r\n" + value + "\r\n") + #expect(rows.count == 1) + } + + @Test("A row that reads Code: without a version suffix stays a row") + func codeTextWithoutVersionIsARow() throws { + let rows = try Self.decode("s\nString\nCode: 1. DB::Exception: x\n") + #expect(rows == [[.text("Code: 1. DB::Exception: x")]]) + } + + @Test("24.11 to 25.10: the marker and the text after it are the error") + func untaggedMarker() { + let message = "\(Self.throwIfMessage) (version 25.8.33.6 (official build))" + let body = Self.twoColumnHeader + Self.rows(3_000) + "__exception__\r\n" + message + "\n" + #expect(Self.failure(body) == message) + } + + @Test("The value __exception__ stays a row: TabSeparated never writes a raw carriage return") + func markerTextAsValueIsARow() throws { + let rows = try Self.decode("s\nString\n__exception__\n") + #expect(rows == [[.text("__exception__")]]) + } + + @Test("With CRLF line ends the marker line is a row, not the error") + func crlfModeKeepsTheValue() throws { + let rows = try Self.decode("s\r\nString\r\n__exception__\r\n") + #expect(rows.count == 1) + } + + @Test("25.11 on: the tagged block is the error, its length line and closing marker are not") + func taggedBlock() { + let tag = "abkxtrlzqztxfthi" + let message = "Code: 395. DB::Exception: line1\nline2\ttabbed: while executing. (version 26.3.52.9 (official build))" + let body = Self.twoColumnHeader + Self.rows(3_000) + + "\r\n__exception__\r\n\(tag)\r\n\(message)\n\(message.utf8.count + 1) \(tag)\r\n__exception__\r\n" + #expect(Self.failure(body, tag: tag) == message) + } + + @Test("A tag on a body with no block is a normal result") + func tagWithoutBlock() throws { + let rows = try Self.decode(Self.twoColumnHeader + Self.rows(5), tag: "abkxtrlzqztxfthi") + #expect(rows.count == 5) + } + + @Test("A body that ends inside a row was cut off and is reported, not decoded as a row") + func cutBody() { + let body = Self.twoColumnHeader + Self.rows(10) + "286719\tRJ7Ml" + #expect(Self.failure(body)?.contains("partway") == true) + } + + @Test("A long result is released as it arrives and still ends in the error") + func longBodyReleasesRows() throws { + var decoder = ClickHouseTabSeparatedRowDecoder() + var tail = ClickHouseStreamTail() + let body = Data((Self.twoColumnHeader + Self.rows(50_000)).utf8) + let released = decoder.consume(tail.release(body)) + #expect(released.count > 30_000) + let rest = try tail.finish(exceptionTag: nil, header: decoder.header) + #expect(released.count + decoder.consume(rest).count == 50_000) + } + + @Test("A failure body keeps the text after any rows the server wrote ahead of it") + func failureBodyMessage() { + let message = "\(Self.throwIfMessage) (version 25.10.7.6 (official build))" + #expect(ClickHouseServerException.message(inFailureBody: Self.twoColumnHeader + Self.rows(20) + message + "\n") == message) + #expect(ClickHouseServerException.message(inFailureBody: "Bad Gateway\n") == "Bad Gateway") + } + + @Test("A stream that ends in the marker throws instead of returning rows") + func streamThrowsThroughTheDriver() async throws { + let message = "\(Self.throwIfMessage) (version 25.8.33.6 (official build))" + let server = ClickHouseStubServer { _ in + ClickHouseStubReply( + statusCode: 200, + body: Self.twoColumnHeader + Self.rows(10) + "__exception__\r\n" + message + "\n", + headers: ["X-ClickHouse-Format": "TabSeparatedWithNamesAndTypes"] + ) + } + let driver = server.connectedDriver() + + await #expect(throws: ClickHouseError.self) { + for try await _ in driver.streamRows(query: "SELECT number FROM numbers(200000)") {} + } + } + + @Test("A capped read whose query names another format comes back as one Output value") + func otherFormatIsOneValue() async throws { + let json = #"{"meta": [{"name": "a", "type": "UInt8"}], "data": [{"a": 1}], "rows": 1}"# + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 200, body: json, headers: ["X-ClickHouse-Format": "JSON"]) + } + let driver = server.connectedDriver() + + let result = try #require(try await driver.executeBoundedQuery(query: "SELECT 1 AS a FORMAT JSON", rowCap: 100)) + + #expect(result.columns.count == 1) + #expect(result.rows == [[.text(json)]]) + #expect(!result.isTruncated) + } + + @Test("An export of a query that names another format is refused, not cut or decoded as rows") + func unboundedOtherFormatIsRefused() async throws { + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 200, body: "1,2\n3,4\n", headers: ["X-ClickHouse-Format": "CSV"]) + } + let driver = server.connectedDriver() + + await #expect(throws: ClickHouseError.self) { + for try await _ in driver.streamRows(query: "SELECT 1, 2 FORMAT CSV") {} + } + } + + @Test("A tagged block in another format's body is the error") + func taggedBlockInOtherFormat() async throws { + let tag = "abkxtrlzqztxfthi" + let message = "\(Self.throwIfMessage) (version 26.3.52.9 (official build))" + let body = "1,0\n2,0\n\r\n__exception__\r\n\(tag)\r\n\(message)\n\(message.utf8.count + 1) \(tag)\r\n__exception__\r\n" + let server = ClickHouseStubServer { _ in + ClickHouseStubReply( + statusCode: 200, + body: body, + headers: ["X-ClickHouse-Format": "CSV", "X-ClickHouse-Exception-Tag": tag] + ) + } + let driver = server.connectedDriver() + + await #expect(throws: ClickHouseError.self) { + _ = try await driver.executeBoundedQuery(query: "SELECT 1 FORMAT CSV", rowCap: 100) + } + } + + @Test("A body in another format past the cap is cut and marked truncated") + func otherFormatPastTheCap() async throws { + let big = String(repeating: "x", count: 1_100_000) + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 200, body: big, headers: ["X-ClickHouse-Format": "CSV"]) + } + let driver = server.connectedDriver() + + let result = try #require(try await driver.executeBoundedQuery(query: "SELECT 1 FORMAT CSV", rowCap: 100)) + + #expect(result.isTruncated) + } + + /// With `send_progress_in_http_headers` the status goes out early, so a query that fails after + /// the first progress header answers 200 and names the failure in a header. + @Test("A 200 that carries X-ClickHouse-Exception-Code is a failure") + func exceptionCodeHeaderOn200() async throws { + let message = "\(Self.throwIfMessage) (version 23.8.8.20 (official build))" + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 200, body: message + "\n", headers: ["X-ClickHouse-Exception-Code": "395"]) + } + let driver = server.connectedDriver() + + do { + _ = try await driver.execute(query: "SELECT throwIf(1)") + Issue.record("expected a failure") + } catch let error as ClickHouseError { + #expect(error.message == message) + } + } +} diff --git a/Plugins/ClickHouseDriverTests/ClickHouseTableOperationsTests.swift b/Plugins/ClickHouseDriverTests/ClickHouseTableOperationsTests.swift index 37f0262f95..cc97bf03ce 100644 --- a/Plugins/ClickHouseDriverTests/ClickHouseTableOperationsTests.swift +++ b/Plugins/ClickHouseDriverTests/ClickHouseTableOperationsTests.swift @@ -52,50 +52,50 @@ struct ClickHouseTableOperationsTests { // MARK: - Comments - private let commentCapable = ClickHouseCapabilities.parse("23.9") + private let commentCapable = ClickHouseServerProfile.testing(version: "23.9") @Test("A table comment uses MODIFY COMMENT") func tableCommentStatement() { #expect(clickHouseCommentStatement( - name: "orders", database: nil, objectType: "TABLE", comment: "Orders", capabilities: commentCapable + name: "orders", database: nil, objectType: "TABLE", comment: "Orders", profile: commentCapable ) == "ALTER TABLE `orders` MODIFY COMMENT 'Orders'") } @Test("A named database qualifies the table and backticks are doubled") func tableCommentQualifiesTheDatabase() { #expect(clickHouseCommentStatement( - name: "or`ders", database: "sh`op", objectType: "TABLE", comment: "x", capabilities: commentCapable + name: "or`ders", database: "sh`op", objectType: "TABLE", comment: "x", profile: commentCapable ) == "ALTER TABLE `sh``op`.`or``ders` MODIFY COMMENT 'x'") #expect(clickHouseCommentStatement( - name: "orders", database: "", objectType: "TABLE", comment: "x", capabilities: commentCapable + name: "orders", database: "", objectType: "TABLE", comment: "x", profile: commentCapable ) == "ALTER TABLE `orders` MODIFY COMMENT 'x'") } @Test("A nil or empty comment clears it with an empty literal", arguments: [nil, ""] as [String?]) func emptyCommentClears(comment: String?) { #expect(clickHouseCommentStatement( - name: "orders", database: nil, objectType: "TABLE", comment: comment, capabilities: commentCapable + name: "orders", database: nil, objectType: "TABLE", comment: comment, profile: commentCapable ) == "ALTER TABLE `orders` MODIFY COMMENT ''") } @Test("Quotes, backslashes and line breaks are escaped") func commentIsEscaped() { #expect(clickHouseCommentStatement( - name: "t", database: nil, objectType: "TABLE", comment: "it's C:\\tmp\nnext", capabilities: commentCapable + name: "t", database: nil, objectType: "TABLE", comment: "it's C:\\tmp\nnext", profile: commentCapable ) == #"ALTER TABLE `t` MODIFY COMMENT 'it''s C:\\tmp\nnext'"#) } @Test("Views and materialized views get no comment statement", arguments: ["VIEW", "MATERIALIZED VIEW", "SYSTEM TABLE"]) func nonTablesReturnNil(objectType: String) { #expect(clickHouseCommentStatement( - name: "v", database: nil, objectType: objectType, comment: "x", capabilities: commentCapable + name: "v", database: nil, objectType: objectType, comment: "x", profile: commentCapable ) == nil) } @Test("A server before 23.9, or of unknown version, gets no comment statement", arguments: ["23.8", "21.3.20.1", nil] as [String?]) func olderServersReturnNil(version: String?) { #expect(clickHouseCommentStatement( - name: "t", database: nil, objectType: "TABLE", comment: "x", capabilities: ClickHouseCapabilities.parse(version) + name: "t", database: nil, objectType: "TABLE", comment: "x", profile: ClickHouseServerProfile.testing(version: version) ) == nil) } @@ -116,7 +116,7 @@ struct ClickHouseTableOperationsTests { func commentStatementQuotesBackslashNames() { let statement = clickHouseCommentStatement( name: "x\\`y", database: "db", objectType: "TABLE", comment: "c", - capabilities: ClickHouseCapabilities.parse("24.3.1") + profile: ClickHouseServerProfile.testing(version: "24.3.1") ) #expect(statement == "ALTER TABLE `db`.`x\\\\``y` MODIFY COMMENT 'c'") } diff --git a/Plugins/MSSQLDriverPlugin/MSSQLCapabilities.swift b/Plugins/MSSQLDriverPlugin/MSSQLCapabilities.swift deleted file mode 100644 index 644fa1636a..0000000000 --- a/Plugins/MSSQLDriverPlugin/MSSQLCapabilities.swift +++ /dev/null @@ -1,20 +0,0 @@ -// -// MSSQLCapabilities.swift -// MSSQLDriverPlugin -// - -import Foundation -import TableProMSSQLCore - -struct MSSQLCapabilities: Sendable, Equatable { - let major: Int - - static let unknown = MSSQLCapabilities(major: 0) - - var hasCreateOrAlterView: Bool { major >= 13 } - - static func parse(_ versionString: String?) -> MSSQLCapabilities { - guard let major = MSSQLServerBanner.majorVersion(from: versionString) else { return .unknown } - return MSSQLCapabilities(major: major) - } -} diff --git a/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift b/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift index dab24af008..a5d3d840e0 100644 --- a/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift +++ b/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift @@ -312,8 +312,12 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - View Templates + private var hasCreateOrAlter: Bool { + MSSQLServerBanner.server(from: serverVersion)?.hasCreateOrAlter ?? false + } + func createViewTemplate() -> String? { - if MSSQLCapabilities.parse(serverVersion).hasCreateOrAlterView { + if hasCreateOrAlter { return "CREATE OR ALTER VIEW view_name AS\nSELECT column1, column2\nFROM table_name\nWHERE condition;" } return "CREATE VIEW view_name AS\nSELECT column1, column2\nFROM table_name\nWHERE condition;" @@ -323,7 +327,7 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { MSSQLObjectQueries.editViewTemplate( schema: effectiveSchema(nil), view: viewName, - createOrAlter: MSSQLCapabilities.parse(serverVersion).hasCreateOrAlterView + createOrAlter: hasCreateOrAlter ) } diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 70afedd759..0943acc3be 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -39160,6 +39160,10 @@ } } }, + "ClickHouse ended the result partway through a row, so the rows that arrived are incomplete." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "ClickHouse returned HTTP %lld." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -185229,6 +185233,10 @@ } } }, + "The query names its own output format, %@. Remove its FORMAT clause to read it as rows." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The query returned no rows." : { "localizations" : { "fr" : { diff --git a/scripts/probes/check-clickhouse-stream-failures.sh b/scripts/probes/check-clickhouse-stream-failures.sh new file mode 100755 index 0000000000..3e29a9bd73 --- /dev/null +++ b/scripts/probes/check-clickhouse-stream-failures.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# Check how a ClickHouse server reports a query that fails partway, against what the driver expects. +# +# A streamed read has had HTTP 200 by then, so the server writes the exception into the body, and +# ClickHouseServerException.swift reads it by shape: raw text before 24.11, an `__exception__` +# marker from 24.11, a block framed by the X-ClickHouse-Exception-Tag header from 25.11. A bounded +# read sends `wait_end_of_query=1`, which the driver relies on to turn the failure into a status. +# Neither shape is announced by the server, so a release that changes one would make the driver +# return the error text as rows. This sends the driver's own URL items and checks both. +# +# Usage: +# scripts/probes/check-clickhouse-stream-failures.sh [url] [user] +# +# Needs curl. The password, if any, comes from CLICKHOUSE_PASSWORD. Exits 0 when the server matches +# what the driver expects for its version, 1 when it does not, and 3 when the check could not run. + +set -uo pipefail + +URL="${1:-http://127.0.0.1:8123}" +USER_NAME="${2:-default}" +QUERY="SELECT number, throwIf(number = 100000) FROM numbers(200000) SETTINGS max_block_size = 1000" +STREAM_ITEMS="query_id=probe-stream&default_format=TabSeparatedWithNamesAndTypes" +BOUNDED_ITEMS="query_id=probe-bounded&default_format=TabSeparatedWithNamesAndTypes&wait_end_of_query=1&max_result_rows=200001&result_overflow_mode=break" + +# The credentials go through a config on stdin, so they never sit in the process list. A curl +# config string takes backslash escapes. +CREDENTIALS="$USER_NAME:${CLICKHOUSE_PASSWORD:-}" +CREDENTIALS="${CREDENTIALS//\\/\\\\}" +CREDENTIALS="${CREDENTIALS//\"/\\\"}" +ask() { + printf 'user = "%s"\n' "$CREDENTIALS" | curl -sS --max-time 30 --config - -H 'Accept-Encoding: identity' "$@" +} + +VERSION="$(ask "$URL/" --data-binary "SELECT version()")" || { + echo "no ClickHouse at $URL for $USER_NAME" >&2 + exit 3 +} +# The server's text reaches arithmetic only once it is digits: bash evaluates `$(...)` inside an +# array subscript in `$(( ))`. +MAJOR="${VERSION%%.*}" +MINOR="${VERSION#*.}" +MINOR="${MINOR%%.*}" +if [[ ! "$MAJOR" =~ ^[0-9]+$ || ! "$MINOR" =~ ^[0-9]+$ ]]; then + echo "unreadable version from $URL" >&2 + exit 3 +fi +LINE=$((10#$MAJOR * 100 + 10#$MINOR)) + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +ask -D "$WORK/stream.headers" -o "$WORK/stream.body" "$URL/?$STREAM_ITEMS" --data-binary "$QUERY" 2> /dev/null +TAG="$(tr -d '\r' < "$WORK/stream.headers" | awk -F': ' 'tolower($1) == "x-clickhouse-exception-tag" { print $2 }')" +if [ -n "$TAG" ] && grep -q "__exception__" "$WORK/stream.body" && grep -qF -- "$TAG" "$WORK/stream.body"; then + SHAPE="tagged" +elif grep -q $'__exception__\r' "$WORK/stream.body"; then + SHAPE="marker" +elif grep -q '^Code: [0-9]*[.,] .*(version ' <(tail -n 1 "$WORK/stream.body"); then + SHAPE="raw" +else + SHAPE="unknown" +fi + +if [ "$LINE" -ge 2511 ]; then + EXPECTED="tagged" +elif [ "$LINE" -ge 2411 ]; then + EXPECTED="marker" +else + EXPECTED="raw" +fi + +BOUNDED_STATUS="$(ask -o /dev/null -w '%{http_code}' "$URL/?$BOUNDED_ITEMS" --data-binary "$QUERY")" + +echo "server: $VERSION" +echo "stream failure shape: $SHAPE (driver expects $EXPECTED)" +echo "bounded failure status: $BOUNDED_STATUS (driver expects 500)" + +STATUS=0 +[ "$SHAPE" = "$EXPECTED" ] || STATUS=1 +[ "$BOUNDED_STATUS" = "500" ] || STATUS=1 +exit "$STATUS"