From f9cd7896b836291c18e3602774ce4466fb251bec Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 15 Oct 2026 13:57:52 +0700 Subject: [PATCH] fix(plugins): restore MySQL zero dates and keep session settings --- CHANGELOG.md | 11 + .../MySQLCreationSettings.swift | 50 ++ .../MySQLObjectQueries.swift | 35 ++ .../MySQLSessionStatements.swift | 35 ++ .../MySQLCreationSettingsTests.swift | 69 +++ .../MySQLObjectQueryTests.swift | 37 ++ .../MySQLSessionStatementsTests.swift | 73 +++ .../SQLExportCreationScopeTests.swift | 155 ++++++ .../SQLExportEncodingTests.swift | 153 ------ .../SQLExportReadScopeTests.swift | 375 +++++++++++++ .../SQLExportSessionDeclarationTests.swift | 271 +++++++++ .../MySQLPluginDriver+CreationScopes.swift | 50 ++ .../MySQLPluginDriver+SessionScope.swift | 112 ++++ .../MySQLDriverPlugin/MySQLPluginDriver.swift | 27 + .../MySQLSessionFootprint.swift | 24 + .../MySQLSessionScopePlan.swift | 127 +++++ .../MySQLStatementClassification.swift | 14 + .../MySQLSessionScopeTests.swift | 250 +++++++++ .../MySQLStatementClassificationTests.swift | 16 + .../LibPQDriverCore.swift | 11 + .../LibPQPluginConnection.swift | 38 ++ .../PostgreSQLPluginDriver+SessionScope.swift | 111 ++++ .../PostgreSQLSessionScopeSQL.swift | 134 +++++ .../PostgreSQLSessionScopeSQLTests.swift | 178 ++++++ .../SQLExportEncodingDeclaration.swift | 57 -- .../SQLExportPlugin/SQLExportFileWriter.swift | 16 +- Plugins/SQLExportPlugin/SQLExportPlugin.swift | 62 ++- .../SQLExportPlugin/SQLExportReadScope.swift | 86 +++ .../SQLExportSessionDeclaration.swift | 91 +++ .../SQLImportOptionsView.swift | 4 +- .../PluginDatabaseDriver.swift | 24 + .../PluginExportDataSource.swift | 13 + .../PluginSessionScope.swift | 64 +++ TablePro/Core/Database/DatabaseDriver.swift | 8 + TablePro/Core/ObjectCopy/ObjectCopyPlan.swift | 62 ++- .../ObjectCopyPlanner+SourceDefinitions.swift | 142 +++++ .../Core/ObjectCopy/ObjectCopyPlanner.swift | 58 +- .../Core/ObjectCopy/ObjectCopyRowCopier.swift | 31 ++ .../Core/ObjectCopy/ObjectCopyRunner.swift | 115 +++- .../Plugins/ExportDataSourceAdapter.swift | 75 ++- .../Core/Plugins/ImportDataSinkAdapter.swift | 110 +++- .../Core/Plugins/PluginDriverAdapter.swift | 8 + .../Core/Services/Export/ImportService.swift | 73 ++- .../Export/TableTransferService.swift | 32 +- .../Export/VerbatimTransferScopes.swift | 108 ++++ TablePro/Resources/Localizable.xcstrings | 12 + .../Components/TransferResultAlert.swift | 36 +- TablePro/Views/Import/ImportDialog.swift | 1 + TablePro/Views/Import/RowImportSheet.swift | 1 + .../TableTransferSessionScopeTests.swift | 259 +++++++++ .../Export/VerbatimTransferScopesTests.swift | 215 ++++++++ .../ObjectCopyCreationScopeReadTests.swift | 132 +++++ .../ObjectCopyDDLSessionTests.swift | 357 ++++++++++++ .../ObjectCopy/ObjectCopyRowCopierTests.swift | 29 +- .../ObjectCopySessionScopeTests.swift | 373 +++++++++++++ ...rtDataSourceAdapterSessionScopeTests.swift | 189 +++++++ .../ImportDataSinkAdapterRollbackTests.swift | 334 ++++++++++++ .../ImportServiceSessionScopeTests.swift | 378 +++++++++++++ .../Views/ImportCancelledTextTests.swift | 63 +++ scripts/probes/check-mysql-dump-session.sh | 516 ++++++++++++++++++ .../probes/check-postgresql-dump-session.sh | 259 +++++++++ 61 files changed, 6455 insertions(+), 294 deletions(-) create mode 100644 Packages/TableProCore/Sources/TableProMySQLCore/MySQLCreationSettings.swift create mode 100644 Packages/TableProCore/Sources/TableProMySQLCore/MySQLSessionStatements.swift create mode 100644 Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLCreationSettingsTests.swift create mode 100644 Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLSessionStatementsTests.swift create mode 100644 Plugins/FormatPluginTests/SQLExportCreationScopeTests.swift delete mode 100644 Plugins/FormatPluginTests/SQLExportEncodingTests.swift create mode 100644 Plugins/FormatPluginTests/SQLExportReadScopeTests.swift create mode 100644 Plugins/FormatPluginTests/SQLExportSessionDeclarationTests.swift create mode 100644 Plugins/MySQLDriverPlugin/MySQLPluginDriver+CreationScopes.swift create mode 100644 Plugins/MySQLDriverPlugin/MySQLPluginDriver+SessionScope.swift create mode 100644 Plugins/MySQLDriverPlugin/MySQLSessionScopePlan.swift create mode 100644 Plugins/MySQLDriverTests/MySQLSessionScopeTests.swift create mode 100644 Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+SessionScope.swift create mode 100644 Plugins/PostgreSQLDriverPlugin/PostgreSQLSessionScopeSQL.swift create mode 100644 Plugins/PostgreSQLDriverTests/PostgreSQLSessionScopeSQLTests.swift delete mode 100644 Plugins/SQLExportPlugin/SQLExportEncodingDeclaration.swift create mode 100644 Plugins/SQLExportPlugin/SQLExportReadScope.swift create mode 100644 Plugins/SQLExportPlugin/SQLExportSessionDeclaration.swift create mode 100644 Plugins/TableProPluginKit/PluginSessionScope.swift create mode 100644 TablePro/Core/Services/Export/VerbatimTransferScopes.swift create mode 100644 TableProTests/Core/Export/TableTransferSessionScopeTests.swift create mode 100644 TableProTests/Core/Export/VerbatimTransferScopesTests.swift create mode 100644 TableProTests/Core/ObjectCopy/ObjectCopyCreationScopeReadTests.swift create mode 100644 TableProTests/Core/ObjectCopy/ObjectCopyDDLSessionTests.swift create mode 100644 TableProTests/Core/ObjectCopy/ObjectCopySessionScopeTests.swift create mode 100644 TableProTests/Core/Plugins/ExportDataSourceAdapterSessionScopeTests.swift create mode 100644 TableProTests/Core/Plugins/ImportDataSinkAdapterRollbackTests.swift create mode 100644 TableProTests/Core/Services/ImportServiceSessionScopeTests.swift create mode 100644 TableProTests/Views/ImportCancelledTextTests.swift create mode 100755 scripts/probes/check-mysql-dump-session.sh create mode 100755 scripts/probes/check-postgresql-dump-session.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index bde337d5a6..453b8b3ac2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Edit > Data shown only in data file windows. - Smaller updates: about 6 MB from any of the last five releases and 21 MB from older ones, down from 34 MB. - Several statements run together on PostgreSQL from iPhone and iPad commit one at a time, and a failure keeps the ones before it. (#3399) +- MySQL dumps write TIMESTAMP values in UTC and turn off foreign key checks while they load, as mysqldump does. ### Removed @@ -85,6 +86,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Database type icon and checkmark hard to see on the selected row in Choose a Database. - Oracle connection timing out after 30 seconds when the password is about to expire. (#3401) - Oracle session left open on the server when a connect was canceled during login. +- MySQL dumps failing to restore on zero dates such as `0000-00-00`, or renumbering an AUTO_INCREMENT value of 0. (#3414) +- Strings with backslashes changing when a MySQL dump is restored on a server using NO_BACKSLASH_ESCAPES. +- TIMESTAMP values shifting when a MySQL dump or copy moves between servers in different time zones. +- MySQL dumps exported from an ANSI_QUOTES session failing to restore. +- Restored MySQL routines, triggers and events losing the SQL mode they were created with. +- MySQL dumps failing in the mysql client when a table references one created after it. +- PostgreSQL dumps changing dates, intervals and floats exported from a session with non-default output settings. +- Imported SQL files leaving their settings on the editor's connection, such as an empty search_path after a pg_dump file. +- MySQL Object Copy and Import from connection failing on zero dates and renumbering AUTO_INCREMENT values of 0. +- SQL import reporting a rollback after MySQL had already committed the file's schema changes. - Only the first statement running on iPhone and iPad when the query holds several. (#3399) - Query on iPhone and iPad appearing to do nothing after CREATE, DROP or a write that changed no rows, including every Oracle write. (#3399) - SQLite on iPhone and iPad reporting a CHECK or UNIQUE violation as a saved row in Add Row, the row editor and Shortcuts. (#3399) diff --git a/Packages/TableProCore/Sources/TableProMySQLCore/MySQLCreationSettings.swift b/Packages/TableProCore/Sources/TableProMySQLCore/MySQLCreationSettings.swift new file mode 100644 index 0000000000..23155d23a2 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProMySQLCore/MySQLCreationSettings.swift @@ -0,0 +1,50 @@ +import Foundation + +/// The `sql_mode`, and for an event the `time_zone`, the server recorded when a routine, trigger or event was created +/// and applies whenever it runs, as the statements a script writes around the object's `CREATE`. +/// +/// mysqldump's form (`switch_sql_mode`, `switch_time_zone` and their restores in `client/mysqldump.cc`): the object +/// saves the script's values in variables of its own, never the dump-wide `@OLD_SQL_MODE`, and the closing puts them +/// back in reverse order. No statement carries a terminator. +public struct MySQLCreationSettings: Equatable, Sendable { + public let sqlMode: String + public let timeZone: String? + + /// Nil when a value would close the version comment it is written in. + public init?(sqlMode: String, timeZone: String? = nil) { + guard !sqlMode.contains("*/"), timeZone?.contains("*/") != true else { return nil } + let portableMode = Self.portableMode(sqlMode) + self.sqlMode = portableMode + self.timeZone = timeZone + } + + /// MariaDB records `NO_AUTO_CREATE_USER` on every routine and trigger its default mode creates, and MySQL 8 + /// refuses the name with 1231, so a dump would stop at its first routine there. The mode only decides whether + /// `GRANT` may create an account, and mysqldump 5.7 drops it the same way (`remove_sql_mode`). + static func portableMode(_ mode: String) -> String { + mode.split(separator: ",", omittingEmptySubsequences: false) + .filter { $0 != "NO_AUTO_CREATE_USER" } + .joined(separator: ",") + } + + public var opening: [String] { + var statements = [ + "/*!50003 SET @saved_sql_mode = @@sql_mode */", + "/*!50003 SET sql_mode = '\(mysqlEscapeStringLiteral(sqlMode))' */", + ] + if let timeZone { + statements.append("/*!50003 SET @saved_time_zone = @@time_zone */") + statements.append("/*!50003 SET time_zone = '\(mysqlEscapeStringLiteral(timeZone))' */") + } + return statements + } + + public var closing: [String] { + var statements: [String] = [] + if timeZone != nil { + statements.append("/*!50003 SET time_zone = @saved_time_zone */") + } + statements.append("/*!50003 SET sql_mode = @saved_sql_mode */") + return statements + } +} diff --git a/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift b/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift index 6371ad33cc..b002609b1a 100644 --- a/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift +++ b/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift @@ -287,6 +287,41 @@ public enum MySQLObjectQueries { return header } + // The creation-settings reads select the name too: the catalog compares names case-insensitively on MySQL 5.7 + // and earlier and on MariaDB, while a trigger name is case-sensitive wherever table names are, so the driver + // keeps only the row named exactly. + + /// The `sql_mode` the server recorded for one routine. + public static func routineCreationSettings(schema: String, name: String, kind: String) -> String { + """ + SELECT ROUTINE_NAME, SQL_MODE + FROM information_schema.ROUTINES + WHERE ROUTINE_SCHEMA = '\(escapeLiteral(schema))' + AND ROUTINE_NAME = '\(escapeLiteral(name))' + AND ROUTINE_TYPE = '\(escapeLiteral(kind))' + """ + } + + /// The `sql_mode` the server recorded for one trigger. + public static func triggerCreationSettings(schema: String, name: String) -> String { + """ + SELECT TRIGGER_NAME, SQL_MODE + FROM information_schema.TRIGGERS + WHERE TRIGGER_SCHEMA = '\(escapeLiteral(schema))' + AND TRIGGER_NAME = '\(escapeLiteral(name))' + """ + } + + /// The `sql_mode` and `time_zone` the server recorded for one event. + public static func eventCreationSettings(schema: String, name: String) -> String { + """ + SELECT EVENT_NAME, SQL_MODE, TIME_ZONE + FROM information_schema.EVENTS + WHERE EVENT_SCHEMA = '\(escapeLiteral(schema))' + AND EVENT_NAME = '\(escapeLiteral(name))' + """ + } + /// A DEFINER arrives as `user@host` and both halves are identifiers, so quoting the whole /// string produces a name no server will accept. public static func quotedDefiner(_ definer: String) -> String { diff --git a/Packages/TableProCore/Sources/TableProMySQLCore/MySQLSessionStatements.swift b/Packages/TableProCore/Sources/TableProMySQLCore/MySQLSessionStatements.swift new file mode 100644 index 0000000000..19d2603e1d --- /dev/null +++ b/Packages/TableProCore/Sources/TableProMySQLCore/MySQLSessionStatements.swift @@ -0,0 +1,35 @@ +import Foundation + +/// The statements that read a session's settings and put them back, spelled with backslash escapes like every +/// statement the plugin writes. +public enum MySQLSessionStatements { + /// A system variable name the driver may interpolate into a statement: lowercase letters and underscores only. + public static func isSettingName(_ name: String) -> Bool { + !name.isEmpty && name.unicodeScalars.allSatisfy { ("a"..."z").contains($0) || $0 == "_" } + } + + /// `SHOW` rather than `SELECT @@name`, because `SHOW` reports a boolean as `ON` or `OFF`, which `SET` takes back + /// quoted; `SET SESSION foreign_key_checks = '1'` fails with 1231 (`scripts/probes/check-mysql-dump-session.sh`). + /// A name that is not one is left out, and nil means nothing was left to ask about. + public static func captureQuery(names: [String]) -> String? { + let asked = names.filter(isSettingName) + guard !asked.isEmpty else { return nil } + let list = asked.map { "'\($0)'" }.joined(separator: ", ") + return "SHOW SESSION VARIABLES WHERE Variable_name IN (\(list))" + } + + /// What a `SHOW` row's value stands for. `SHOW` prints a NULL `character_set_results` as an empty string, + /// measured on MariaDB 13.0.2, and no character set or collation is named by one, so there it means NULL. + public static func reportedValue(name: String, value: String?) -> String? { + guard let value, value.isEmpty else { return value } + let namesCharset = name.hasPrefix("character_set_") || name.hasPrefix("collation_") + return namesCharset ? nil : value + } + + /// `SET SESSION = `, with SQL `NULL` for a nil value, or nil for a name that is not one. + public static func assignment(name: String, value: String?) -> String? { + guard isSettingName(name) else { return nil } + let literal = value.map { "'\(mysqlEscapeStringLiteral($0))'" } ?? "NULL" + return "SET SESSION \(name) = \(literal)" + } +} diff --git a/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLCreationSettingsTests.swift b/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLCreationSettingsTests.swift new file mode 100644 index 0000000000..034c924471 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLCreationSettingsTests.swift @@ -0,0 +1,69 @@ +import Foundation +import Testing + +@testable import TableProMySQLCore + +struct MySQLCreationSettingsTests { + @Test("A routine or trigger switches the mode through its own variable and puts it back") + func modeOnly() throws { + let settings = try #require(MySQLCreationSettings(sqlMode: "STRICT_TRANS_TABLES,NO_ZERO_DATE")) + #expect(settings.opening == [ + "/*!50003 SET @saved_sql_mode = @@sql_mode */", + "/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES,NO_ZERO_DATE' */", + ]) + #expect(settings.closing == ["/*!50003 SET sql_mode = @saved_sql_mode */"]) + } + + @Test("An event also switches the time zone, and the closing undoes both in reverse order") + func modeAndTimeZone() throws { + let settings = try #require(MySQLCreationSettings(sqlMode: "", timeZone: "SYSTEM")) + #expect(settings.opening == [ + "/*!50003 SET @saved_sql_mode = @@sql_mode */", + "/*!50003 SET sql_mode = '' */", + "/*!50003 SET @saved_time_zone = @@time_zone */", + "/*!50003 SET time_zone = 'SYSTEM' */", + ]) + #expect(settings.closing == [ + "/*!50003 SET time_zone = @saved_time_zone */", + "/*!50003 SET sql_mode = @saved_sql_mode */", + ]) + } + + @Test("No statement carries a terminator, and none touches the dump-wide @OLD_SQL_MODE") + func leavesTheDumpAlone() throws { + let settings = try #require(MySQLCreationSettings(sqlMode: "ANSI_QUOTES", timeZone: "+02:00")) + for statement in settings.opening + settings.closing { + #expect(!statement.hasSuffix(";"), "\(statement)") + #expect(!statement.contains("@OLD_SQL_MODE"), "\(statement)") + } + } + + @Test("A quote in a value is escaped") + func quotesAreEscaped() throws { + let settings = try #require(MySQLCreationSettings(sqlMode: "A'B", timeZone: "C'D")) + #expect(settings.opening[1] == "/*!50003 SET sql_mode = 'A''B' */") + #expect(settings.opening[3] == "/*!50003 SET time_zone = 'C''D' */") + } + + /// MariaDB's default mode puts it on every routine and trigger, and MySQL 8 refuses the name with 1231. + @Test("NO_AUTO_CREATE_USER is dropped wherever it stands in the recorded mode", arguments: [ + ("STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION", + "STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION"), + ("NO_AUTO_CREATE_USER,STRICT_TRANS_TABLES", "STRICT_TRANS_TABLES"), + ("STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER", "STRICT_TRANS_TABLES"), + ("NO_AUTO_CREATE_USER", ""), + ("STRICT_TRANS_TABLES", "STRICT_TRANS_TABLES"), + ("", ""), + ]) + func noAutoCreateUserIsDropped(_ recorded: String, _ written: String) throws { + let settings = try #require(MySQLCreationSettings(sqlMode: recorded)) + #expect(settings.sqlMode == written) + #expect(settings.opening[1] == "/*!50003 SET sql_mode = '\(written)' */") + } + + @Test("A value that would close the version comment gives no settings") + func commentCloserIsRefused() { + #expect(MySQLCreationSettings(sqlMode: "x */ DROP TABLE t; /*") == nil) + #expect(MySQLCreationSettings(sqlMode: "", timeZone: "*/") == nil) + } +} diff --git a/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLObjectQueryTests.swift b/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLObjectQueryTests.swift index 303ce5e109..94944c1517 100644 --- a/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLObjectQueryTests.swift +++ b/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLObjectQueryTests.swift @@ -220,4 +220,41 @@ struct MySQLObjectQueryTests { #expect(legacy.contains("WHERE r.ROUTINE_SCHEMA = 'app'")) #expect(MySQLObjectQueries.routineList(schema: "app", includesParameters: true).contains("information_schema.PARAMETERS")) } + + @Test("A routine's recorded mode is read by schema, name and type, with the name selected for an exact match") + func routineCreationSettings() { + let sql = MySQLObjectQueries.routineCreationSettings(schema: "app", name: "add_course", kind: "PROCEDURE") + #expect(sql.contains("SELECT ROUTINE_NAME, SQL_MODE")) + #expect(sql.contains("FROM information_schema.ROUTINES")) + #expect(sql.contains("ROUTINE_SCHEMA = 'app'")) + #expect(sql.contains("ROUTINE_NAME = 'add_course'")) + #expect(sql.contains("ROUTINE_TYPE = 'PROCEDURE'")) + } + + @Test("A trigger's recorded mode is read by its own schema and name") + func triggerCreationSettings() { + let sql = MySQLObjectQueries.triggerCreationSettings(schema: "app", name: "audit") + #expect(sql.contains("SELECT TRIGGER_NAME, SQL_MODE")) + #expect(sql.contains("FROM information_schema.TRIGGERS")) + #expect(sql.contains("TRIGGER_SCHEMA = 'app'")) + #expect(sql.contains("TRIGGER_NAME = 'audit'")) + } + + @Test("An event's recorded mode comes with its time zone") + func eventCreationSettings() { + let sql = MySQLObjectQueries.eventCreationSettings(schema: "app", name: "nightly") + #expect(sql.contains("SELECT EVENT_NAME, SQL_MODE, TIME_ZONE")) + #expect(sql.contains("FROM information_schema.EVENTS")) + #expect(sql.contains("EVENT_SCHEMA = 'app'")) + #expect(sql.contains("EVENT_NAME = 'nightly'")) + } + + @Test("Names in the creation-settings reads are escaped as literals") + func creationSettingsEscapeNames() { + let routine = MySQLObjectQueries.routineCreationSettings(schema: #"a'b"#, name: #"c\d"#, kind: "FUNCTION") + #expect(routine.contains(#"ROUTINE_SCHEMA = 'a''b'"#)) + #expect(routine.contains(#"ROUTINE_NAME = 'c\\d'"#)) + #expect(MySQLObjectQueries.triggerCreationSettings(schema: "app", name: "it's").contains("TRIGGER_NAME = 'it''s'")) + #expect(MySQLObjectQueries.eventCreationSettings(schema: "app", name: "it's").contains("EVENT_NAME = 'it''s'")) + } } diff --git a/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLSessionStatementsTests.swift b/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLSessionStatementsTests.swift new file mode 100644 index 0000000000..ea0b06fffb --- /dev/null +++ b/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLSessionStatementsTests.swift @@ -0,0 +1,73 @@ +import Foundation +import Testing + +@testable import TableProMySQLCore + +struct MySQLSessionStatementsTests { + @Test("The capture query asks for the names in the order given") + func captureQueryListsNames() { + let query = MySQLSessionStatements.captureQuery(names: ["sql_mode", "time_zone", "foreign_key_checks"]) + #expect(query == "SHOW SESSION VARIABLES WHERE Variable_name IN ('sql_mode', 'time_zone', 'foreign_key_checks')") + } + + @Test("A name that is not a setting name is left out of the capture query") + func captureQueryDropsForeignNames() { + let query = MySQLSessionStatements.captureQuery(names: ["sql_mode", "x') OR ('1", "time_zone"]) + #expect(query == "SHOW SESSION VARIABLES WHERE Variable_name IN ('sql_mode', 'time_zone')") + #expect(MySQLSessionStatements.captureQuery(names: []) == nil) + #expect(MySQLSessionStatements.captureQuery(names: ["@@sql_mode"]) == nil) + } + + @Test("Only lowercase letters and underscores make a setting name", arguments: [ + ("sql_mode", true), + ("character_set_results", true), + ("", false), + ("SQL_MODE", false), + ("sql_mode2", false), + ("@@sql_mode", false), + ("sql_mode = ''", false), + ("sql_mode;", false), + ("time_zone/*", false), + ]) + func settingNames(_ name: String, _ isValid: Bool) { + #expect(MySQLSessionStatements.isSettingName(name) == isValid) + } + + @Test("A nil value is SQL NULL, any other value a quoted literal") + func assignmentSpellsNullAndLiterals() { + #expect(MySQLSessionStatements.assignment(name: "character_set_results", value: nil) + == "SET SESSION character_set_results = NULL") + #expect(MySQLSessionStatements.assignment(name: "sql_mode", value: "") == "SET SESSION sql_mode = ''") + #expect(MySQLSessionStatements.assignment(name: "time_zone", value: "+00:00") == "SET SESSION time_zone = '+00:00'") + } + + @Test("A value is escaped the way every plugin statement is, before the session respells it") + func assignmentEscapesValues() { + #expect(MySQLSessionStatements.assignment(name: "time_zone", value: #"it's\here"#) + == #"SET SESSION time_zone = 'it''s\\here'"#) + } + + /// `SHOW` reports a boolean as `ON` or `OFF`, which `SET` takes back quoted, where `'1'` is refused with 1231. + @Test("ON and OFF pass through as SHOW reported them") + func booleansPassThrough() { + #expect(MySQLSessionStatements.reportedValue(name: "foreign_key_checks", value: "OFF") == "OFF") + #expect(MySQLSessionStatements.assignment(name: "foreign_key_checks", value: "OFF") + == "SET SESSION foreign_key_checks = 'OFF'") + #expect(MySQLSessionStatements.assignment(name: "unique_checks", value: "ON") == "SET SESSION unique_checks = 'ON'") + } + + @Test("An empty character set or collation is NULL, and an empty sql_mode stays empty") + func emptyReportedValues() { + #expect(MySQLSessionStatements.reportedValue(name: "character_set_results", value: "") == nil) + #expect(MySQLSessionStatements.reportedValue(name: "collation_connection", value: "") == nil) + #expect(MySQLSessionStatements.reportedValue(name: "sql_mode", value: "") == "") + #expect(MySQLSessionStatements.reportedValue(name: "time_zone", value: nil) == nil) + #expect(MySQLSessionStatements.reportedValue(name: "character_set_results", value: "utf8mb4") == "utf8mb4") + } + + @Test("A name that is not a setting name gets no statement") + func assignmentRefusesForeignNames() { + #expect(MySQLSessionStatements.assignment(name: "sql_mode = '', time_zone", value: "x") == nil) + #expect(MySQLSessionStatements.assignment(name: "", value: nil) == nil) + } +} diff --git a/Plugins/FormatPluginTests/SQLExportCreationScopeTests.swift b/Plugins/FormatPluginTests/SQLExportCreationScopeTests.swift new file mode 100644 index 0000000000..a55f6b41dd --- /dev/null +++ b/Plugins/FormatPluginTests/SQLExportCreationScopeTests.swift @@ -0,0 +1,155 @@ +// +// SQLExportCreationScopeTests.swift +// FormatPluginTests +// +// MySQL stores the creating session's sql_mode with each routine, trigger and event, so the dump recreates each +// one under its own mode and then puts the dump's mode back. +// + +import Foundation +import TableProPluginKit +import Testing + +struct SQLExportCreationScopeTests { + private struct InjectedFailure: Error {} + + private static let opening = ["SET @saved_sql_mode = @@sql_mode", "SET sql_mode = 'STRICT_TRANS_TABLES'"] + private static let closing = ["SET sql_mode = @saved_sql_mode"] + + private final class DefinitionDataSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "MySQL" + private let definitions: [String: String] + private let failingScopes: Set + + private let lock = NSLock() + private var askedNames: [String] = [] + + init(definitions: [String: String], failingScopes: Set = []) { + self.definitions = definitions + self.failingScopes = failingScopes + } + + var scopeRequests: [String] { lock.withLock { askedNames } } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + definitions[object.name] ?? "" + } + + func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? { + lock.withLock { askedNames.append(object.name) } + if failingScopes.contains(object.name) { throw InjectedFailure() } + return PluginStatementScope(opening: SQLExportCreationScopeTests.opening, closing: SQLExportCreationScopeTests.closing) + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + AsyncThrowingStream { $0.finish() } + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { + "`\(identifier.replacingOccurrences(of: "`", with: "``"))`" + } + + func escapeStringLiteral(_ value: String) -> String { + value.replacingOccurrences(of: "'", with: "''") + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } + } + + private func object(_ name: String, kind: PluginExportObjectKind, parentTable: String? = nil) -> PluginExportTable { + PluginExportTable( + name: name, + databaseName: "", + tableType: kind.rawValue, + optionValues: [true, false, false], + schema: nil, + kind: kind, + parentTable: parentTable) + } + + /// The kit's default script text ends each statement with a `;`, and a MySQL dump ends it with a line break. + private func wrapped(_ ddl: String) -> String { + (Self.opening + [ddl] + Self.closing).map { "\($0);\n" }.joined() + } + + @Test("A routine, trigger and event are each written between the settings they were created with") + func definitionsAreWrapped() async throws { + let definitions = [ + "p": "CREATE PROCEDURE `p`() SELECT 1", + "t": "CREATE TRIGGER `t` BEFORE INSERT ON `orders` FOR EACH ROW SET NEW.n = 1", + "e": "CREATE EVENT `e` ON SCHEDULE EVERY 1 DAY DO DELETE FROM `orders`", + "v": "CREATE VIEW `v` AS SELECT 1", + ] + let source = DefinitionDataSource(definitions: definitions) + let output = try await SQLExportHarness.shared.dump( + tables: [ + object("p", kind: .routine), + object("t", kind: .trigger, parentTable: "orders"), + object("e", kind: .event), + object("v", kind: .view), + ], + dataSource: source) + + for name in ["p", "t", "e"] { + let ddl = try #require(definitions[name]) + #expect(output.text.contains(wrapped(ddl)), "\(name)") + } + #expect(output.text.contains("\nCREATE VIEW `v` AS SELECT 1;\n")) + #expect(!output.text.contains("'STRICT_TRANS_TABLES';\nCREATE VIEW")) + #expect(Set(source.scopeRequests) == ["p", "t", "e"]) + #expect(output.result.warnings.isEmpty) + } + + /// The second routine passes the cap. Written as separate statements, its opening settings fit in the first part + /// and its definition rotated into the second, which then ran under the dump's mode with nothing to put back. + @Test("A part rotation never separates a definition from its settings") + func rotationKeepsTheWrapInOnePart() async throws { + let big = "CREATE PROCEDURE `a_big`() SELECT '\(String(repeating: "x", count: 900 * 1_024))'" + let small = "CREATE PROCEDURE `b_small`() SELECT '\(String(repeating: "y", count: 200 * 1_024))'" + let source = DefinitionDataSource(definitions: ["a_big": big, "b_small": small]) + var options = SQLExportOptions() + options.splitSizeMegabytes = 1 + options.excludeDefiner = false + options.excludeAutoIncrementValue = false + + let output = try await SQLExportHarness.shared.dumpParts( + tables: [object("a_big", kind: .routine), object("b_small", kind: .routine)], + dataSource: source, + options: options) + + #expect(output.parts.count == 2) + for (index, part) in output.parts.enumerated() { + let openings = part.components(separatedBy: "SET @saved_sql_mode = @@sql_mode;").count - 1 + let closings = part.components(separatedBy: "SET sql_mode = @saved_sql_mode;").count - 1 + #expect(openings == 1, "part \(index + 1)") + #expect(closings == 1, "part \(index + 1)") + } + let secondPart = try #require(output.parts.last) + #expect(secondPart.contains(wrapped(small))) + } + + @Test("Settings that cannot be read still write the definition, and the summary names the object") + func failedFetchWritesTheDefinitionAlone() async throws { + let definitions = [ + "p": "CREATE PROCEDURE `p`() SELECT 1", + "q": "CREATE PROCEDURE `q`() SELECT 2", + ] + let source = DefinitionDataSource(definitions: definitions, failingScopes: ["p"]) + let output = try await SQLExportHarness.shared.dump( + tables: [object("p", kind: .routine), object("q", kind: .routine)], + dataSource: source) + + #expect(output.text.contains("\nCREATE PROCEDURE `p`() SELECT 1;\n")) + #expect(!output.text.contains(wrapped("CREATE PROCEDURE `p`() SELECT 1"))) + #expect(output.text.contains(wrapped("CREATE PROCEDURE `q`() SELECT 2"))) + #expect(output.result.warnings.count == 1) + let warning = try #require(output.result.warnings.first) + #expect(warning.contains("Could not read the settings p was created with")) + } +} diff --git a/Plugins/FormatPluginTests/SQLExportEncodingTests.swift b/Plugins/FormatPluginTests/SQLExportEncodingTests.swift deleted file mode 100644 index 6f74bba525..0000000000 --- a/Plugins/FormatPluginTests/SQLExportEncodingTests.swift +++ /dev/null @@ -1,153 +0,0 @@ -// -// SQLExportEncodingTests.swift -// TableProTests -// - -import Foundation -import TableProPluginKit -import Testing - -struct SQLExportEncodingTests { - private func temporaryDirectory() throws -> URL { - let directory = FileManager.default.temporaryDirectory - .appendingPathComponent(UUID().uuidString, isDirectory: true) - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - return directory - } - - @Test("A MySQL dump declares utf8mb4 the way mysqldump does, and puts the session back") - func mysqlDeclaresUTF8MB4() { - for typeId in ["MySQL", "MariaDB", "TiDB", "OceanBase"] { - let declaration = SQLExportEncodingDeclaration.forDatabaseType(typeId) - #expect(declaration.prologue.contains("/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;")) - #expect(declaration.prologue.contains("/*!40101 SET NAMES utf8 */;")) - #expect(declaration.prologue.contains("/*!50503 SET NAMES utf8mb4 */;")) - #expect(declaration.epilogue.contains("/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;")) - #expect(declaration.epilogue.contains("/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;")) - #expect(declaration.epilogue.contains("/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;")) - } - } - - @Test("A PostgreSQL dump declares UTF8 the way pg_dump does, on every engine that accepts the statement") - func postgresDeclaresUTF8() { - for typeId in ["PostgreSQL", "Greenplum", "AlloyDB", "Citus", "CockroachDB", "PGlite"] { - let declaration = SQLExportEncodingDeclaration.forDatabaseType(typeId) - #expect(declaration.prologue == "SET client_encoding = 'UTF8';\n\n", "\(typeId)") - #expect(declaration.epilogue.isEmpty, "\(typeId)") - } - } - - @Test("Redshift gets no declaration, because it does not document client_encoding as settable") - func redshiftDeclaresNothing() { - #expect(SQLExportEncodingDeclaration.forDatabaseType("Redshift") == .empty) - } - - @Test("Other engines write no declaration") - func otherEnginesDeclareNothing() { - for typeId in ["SQLite", "DuckDB", "Oracle", "MSSQL", "SomeFuturePlugin"] { - #expect(SQLExportEncodingDeclaration.forDatabaseType(typeId) == .empty, "\(typeId)") - } - } - - @Test("A PostgreSQL dump opens with the declaration and ends with the last statement") - func postgresDumpIsPrefixed() throws { - let directory = try temporaryDirectory() - defer { try? FileManager.default.removeItem(at: directory) } - let declaration = SQLExportEncodingDeclaration.forDatabaseType("PostgreSQL") - - let destination = directory.appendingPathComponent("dump.sql") - let writer = try SQLExportFileWriter( - destination: destination, splitSizeMegabytes: 0, encodingDeclaration: declaration - ) - try writer.write("INSERT INTO \"t\" VALUES ('メール');\n") - try writer.commit() - - let dump = try String(contentsOf: destination, encoding: .utf8) - #expect(dump == "SET client_encoding = 'UTF8';\n\nINSERT INTO \"t\" VALUES ('メール');\n") - } - - @Test("An unsplit dump opens with the declaration and closes by restoring the session") - func unsplitDumpIsWrapped() throws { - let directory = try temporaryDirectory() - defer { try? FileManager.default.removeItem(at: directory) } - let declaration = SQLExportEncodingDeclaration.forDatabaseType("MySQL") - - let destination = directory.appendingPathComponent("dump.sql") - let writer = try SQLExportFileWriter( - destination: destination, splitSizeMegabytes: 0, encodingDeclaration: declaration - ) - try writer.write("INSERT INTO `t` VALUES ('メール');\n") - try writer.commit() - - let dump = try String(contentsOf: destination, encoding: .utf8) - #expect(dump == declaration.prologue + "INSERT INTO `t` VALUES ('メール');\n" + declaration.epilogue) - } - - @Test( - "Every part of a split dump carries its own declaration, so each restores on its own", - arguments: ["MySQL", "PostgreSQL"] - ) - func everyPartIsWrapped(databaseTypeId: String) throws { - let directory = try temporaryDirectory() - defer { try? FileManager.default.removeItem(at: directory) } - let declaration = SQLExportEncodingDeclaration.forDatabaseType(databaseTypeId) - #expect(declaration != .empty) - - let destination = directory.appendingPathComponent("dump.sql") - let writer = try SQLExportFileWriter( - destination: destination, splitSizeMegabytes: 1, encodingDeclaration: declaration - ) - let chunk = String(repeating: "x", count: 700 * 1_024) - try writer.write("A\(chunk);\n") - try writer.write("B\(chunk);\n") - let parts = try writer.commit() - - #expect(parts.count == 2) - for (index, part) in parts.enumerated() { - let text = try String(contentsOf: part, encoding: .utf8) - #expect(text.hasPrefix(declaration.prologue), "part \(index + 1)") - #expect(text.hasSuffix(declaration.epilogue), "part \(index + 1)") - #expect(text.contains(index == 0 ? "A" : "B")) - } - } - - @Test("No part grows past the cap once its closing declaration is added") - func epilogueCountsTowardTheCap() throws { - let directory = try temporaryDirectory() - defer { try? FileManager.default.removeItem(at: directory) } - let declaration = SQLExportEncodingDeclaration.forDatabaseType("MySQL") - let cap = 1_024 * 1_024 - - let destination = directory.appendingPathComponent("dump.sql") - let writer = try SQLExportFileWriter( - destination: destination, splitSizeMegabytes: 1, encodingDeclaration: declaration - ) - let firstLength = cap - declaration.prologue.utf8.count - declaration.epilogue.utf8.count - 5 - try writer.write(String(repeating: "a", count: firstLength - 2) + ";\n") - try writer.write("SELECT 1;\n") - let parts = try writer.commit() - - #expect(parts.count == 2) - for part in parts { - let size = try FileManager.default.attributesOfItem(atPath: part.path)[.size] as? Int ?? 0 - #expect(size <= cap, "\(part.lastPathComponent) is \(size) bytes") - } - } - - @Test("A statement larger than the cap still lands in a part of its own, not after an empty one") - func oversizedStatementDoesNotLeaveAnEmptyPart() throws { - let directory = try temporaryDirectory() - defer { try? FileManager.default.removeItem(at: directory) } - - let destination = directory.appendingPathComponent("dump.sql") - let writer = try SQLExportFileWriter( - destination: destination, splitSizeMegabytes: 1, - encodingDeclaration: .forDatabaseType("MySQL") - ) - try writer.write(String(repeating: "y", count: 2 * 1_024 * 1_024) + ";\n") - let parts = try writer.commit() - - #expect(parts.count == 1) - #expect(!writer.didSplit) - } -} diff --git a/Plugins/FormatPluginTests/SQLExportReadScopeTests.swift b/Plugins/FormatPluginTests/SQLExportReadScopeTests.swift new file mode 100644 index 0000000000..9c41d5d950 --- /dev/null +++ b/Plugins/FormatPluginTests/SQLExportReadScopeTests.swift @@ -0,0 +1,375 @@ +// +// SQLExportReadScopeTests.swift +// FormatPluginTests +// +// The export reads on the user's own session, so the settings the driver pins for the read have to be put back +// on every exit, and the dump declares the time zone only when the read was pinned to it. +// + +import Foundation +import TableProPluginKit +import Testing + +struct SQLExportReadScopeTests { + private struct InjectedFailure: Error {} + + private enum OpenBehavior { + case returns(PluginSessionScope?) + case fails + } + + private static let utcScope = PluginSessionScope( + purpose: .verbatimRead, + saved: [ + PluginSessionSetting(name: "sql_mode", value: "ANSI_QUOTES"), + PluginSessionSetting(name: "time_zone", value: "SYSTEM"), + ], + applied: [ + PluginSessionSetting(name: "sql_mode", value: ""), + PluginSessionSetting(name: "time_zone", value: "+00:00"), + ]) + + private final class RecordingDataSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "MySQL" + private let readBehavior: OpenBehavior + private let preserveBehavior: OpenBehavior + private let failsStream: Bool + private let failsClose: Bool + private let onExecute: (@Sendable (String) -> Void)? + private let afterLastRow: (@Sendable () -> Void)? + + private let lock = NSLock() + private var recordedEvents: [String] = [] + private var recordedPurposes: [PluginSessionPurpose] = [] + private var recordedClosures: [PluginSessionScope] = [] + + init( + read readBehavior: OpenBehavior, + preserve preserveBehavior: OpenBehavior = .returns(nil), + failsStream: Bool = false, + failsClose: Bool = false, + onExecute: (@Sendable (String) -> Void)? = nil, + afterLastRow: (@Sendable () -> Void)? = nil + ) { + self.readBehavior = readBehavior + self.preserveBehavior = preserveBehavior + self.failsStream = failsStream + self.failsClose = failsClose + self.onExecute = onExecute + self.afterLastRow = afterLastRow + } + + var events: [String] { lock.withLock { recordedEvents } } + var openedPurposes: [PluginSessionPurpose] { lock.withLock { recordedPurposes } } + var closedScopes: [PluginSessionScope] { lock.withLock { recordedClosures } } + + private func record(_ event: String) { + lock.withLock { recordedEvents.append(event) } + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + lock.withLock { + recordedEvents.append("open") + recordedPurposes.append(purpose) + } + switch purpose == .preserve ? preserveBehavior : readBehavior { + case .returns(let scope): return scope + case .fails: throw InjectedFailure() + } + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + lock.withLock { + recordedEvents.append("close") + recordedClosures.append(scope) + } + if failsClose { throw InjectedFailure() } + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + record("rows") + let failsStream = failsStream + let afterLastRow = afterLastRow + return AsyncThrowingStream { continuation in + continuation.yield(.header(PluginStreamHeader(columns: ["id", "at"], columnTypeNames: ["INT", "TIMESTAMP"]))) + guard !failsStream else { + continuation.finish(throwing: InjectedFailure()) + return + } + continuation.yield(.rows([[.text("1"), .text("2024-06-01 12:00:00")]])) + afterLastRow?() + continuation.finish() + } + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + "CREATE TABLE `\(table)` (`id` int, `at` timestamp NULL)" + } + + func execute(query: String) async throws -> PluginQueryResult { + record(query) + onExecute?(query) + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { + "`\(identifier.replacingOccurrences(of: "`", with: "``"))`" + } + + func escapeStringLiteral(_ value: String) -> String { + value.replacingOccurrences(of: "'", with: "''") + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } + } + + private let table = PluginExportTable( + name: "events", databaseName: "", tableType: "table", optionValues: [true, false, true], schema: nil) + + private func options(consistentSnapshot: Bool = false) -> SQLExportOptions { + var options = SQLExportOptions() + options.consistentSnapshot = consistentSnapshot + return options + } + + @Test("A read pinned to UTC declares UTC in the dump and restores the zone after it") + func pinnedReadDeclaresItsZone() async throws { + let source = RecordingDataSource(read: .returns(Self.utcScope)) + let dump = try await SQLExportHarness.shared.dump(tables: [table], dataSource: source).text + + #expect(dump.contains("/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;\n/*!40103 SET TIME_ZONE='+00:00' */;\n")) + #expect(dump.contains("/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;")) + #expect(source.openedPurposes == [.verbatimRead]) + #expect(source.closedScopes == [Self.utcScope]) + } + + @Test("A driver with no read scope gets a dump that declares no zone") + func noScopeDeclaresNoZone() async throws { + let source = RecordingDataSource(read: .returns(nil)) + let dump = try await SQLExportHarness.shared.dump(tables: [table], dataSource: source).text + + #expect(!dump.contains("TIME_ZONE")) + #expect(dump.contains("SQL_MODE='NO_AUTO_VALUE_ON_ZERO'")) + #expect(source.closedScopes.isEmpty) + } + + @Test("A read scope that fails to open leaves the export as it was, with no zone declared") + func failedOpenExportsAsBefore() async throws { + let source = RecordingDataSource(read: .fails) + let output = try await SQLExportHarness.shared.dump(tables: [table], dataSource: source) + + #expect(!output.text.contains("TIME_ZONE")) + #expect(output.text.contains("INSERT INTO `events`")) + #expect(output.result.warnings.isEmpty) + #expect(source.closedScopes.isEmpty) + } + + private func scopedTable(_ name: String, rowScope: PluginExportRowScope) -> PluginExportTable { + PluginExportTable( + name: name, databaseName: "", tableType: "table", optionValues: [true, false, true], schema: nil, + kind: .table, rowScope: rowScope) + } + + /// The filter is the user's SQL for their own session: under UTC a `NOW()` picks other rows than the grid showed. + @Test("A row filter keeps the export on the user's session, so the dump declares no zone") + func filteredExportOpensNoScope() async throws { + let source = RecordingDataSource(read: .returns(Self.utcScope)) + let filtered = scopedTable("recent", rowScope: PluginExportRowScope(filter: "`at` > NOW() - INTERVAL 1 DAY")) + let dump = try await SQLExportHarness.shared.dump(tables: [table, filtered], dataSource: source).text + + #expect(!source.openedPurposes.contains(.verbatimRead)) + #expect(source.closedScopes.isEmpty) + #expect(!dump.contains("TIME_ZONE")) + #expect(dump.contains("SQL_MODE='NO_AUTO_VALUE_ON_ZERO'")) + } + + @Test( + "A scope that runs no filter still reads under the read scope", + arguments: [ + PluginExportRowScope(rowLimit: 10), + PluginExportRowScope(columns: ["id"]), + PluginExportRowScope(filter: " "), + PluginExportRowScope(filter: "id = 1; DROP TABLE events"), + ] + ) + func unfilteredNarrowingKeepsTheScope(rowScope: PluginExportRowScope) async throws { + let source = RecordingDataSource(read: .returns(Self.utcScope)) + let dump = try await SQLExportHarness.shared.dump( + tables: [scopedTable("events", rowScope: rowScope)], dataSource: source).text + + #expect(source.openedPurposes == [.verbatimRead]) + #expect(source.closedScopes == [Self.utcScope]) + #expect(dump.contains("/*!40103 SET TIME_ZONE='+00:00' */;")) + } + + private static let plainModeLine = "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;" + private static let ansiQuotesModeLine = + "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES' */;" + + private static func sessionProbe(sqlMode: String?) -> PluginSessionScope { + PluginSessionScope( + purpose: .preserve, + saved: [ + PluginSessionSetting(name: "sql_mode", value: sqlMode), + PluginSessionSetting(name: "time_zone", value: "SYSTEM"), + ], + applied: []) + } + + private var filteredTable: PluginExportTable { + scopedTable("recent", rowScope: PluginExportRowScope(filter: "`at` > NOW() - INTERVAL 1 DAY")) + } + + /// A filtered export reads its DDL under the session's mode, and `SHOW CREATE TABLE` under `ANSI_QUOTES` spells + /// identifiers with `"`, which the plain dump mode reads as strings. + @Test( + "A filtered export read under ANSI_QUOTES keeps ANSI_QUOTES in the dump's mode", + arguments: [ + "ANSI_QUOTES", + "REAL_AS_FLOAT,PIPES_AS_CONCAT,ANSI_QUOTES,IGNORE_SPACE,ONLY_FULL_GROUP_BY", + "strict_trans_tables, ansi_quotes", + ] + ) + func filteredExportUnderANSIQuotes(sqlMode: String) async throws { + let probe = Self.sessionProbe(sqlMode: sqlMode) + let source = RecordingDataSource(read: .returns(Self.utcScope), preserve: .returns(probe)) + let dump = try await SQLExportHarness.shared.dump(tables: [filteredTable], dataSource: source).text + + #expect(dump.contains(Self.ansiQuotesModeLine)) + #expect(!dump.contains(Self.plainModeLine)) + #expect(source.openedPurposes == [.preserve]) + #expect(source.closedScopes == [probe]) + #expect(Array(source.events.prefix(3)) == ["open", "close", "rows"]) + } + + @Test( + "A filtered export read without ANSI_QUOTES writes the plain mode", + arguments: [nil, "", "STRICT_TRANS_TABLES,NO_ZERO_DATE,NO_ENGINE_SUBSTITUTION"] as [String?] + ) + func filteredExportWithoutANSIQuotes(sqlMode: String?) async throws { + let probe = Self.sessionProbe(sqlMode: sqlMode) + let source = RecordingDataSource(read: .returns(Self.utcScope), preserve: .returns(probe)) + let dump = try await SQLExportHarness.shared.dump(tables: [filteredTable], dataSource: source).text + + #expect(dump.contains(Self.plainModeLine)) + #expect(source.openedPurposes == [.preserve]) + #expect(source.closedScopes == [probe]) + } + + @Test("A probe that fails or reads nothing writes the plain mode") + func failedProbeWritesThePlainMode() async throws { + let failing = RecordingDataSource(read: .returns(Self.utcScope), preserve: .fails) + let failingDump = try await SQLExportHarness.shared.dump(tables: [filteredTable], dataSource: failing).text + #expect(failingDump.contains(Self.plainModeLine)) + #expect(failing.closedScopes.isEmpty) + + let empty = RecordingDataSource(read: .returns(Self.utcScope), preserve: .returns(nil)) + let emptyDump = try await SQLExportHarness.shared.dump(tables: [filteredTable], dataSource: empty).text + #expect(emptyDump.contains(Self.plainModeLine)) + #expect(empty.closedScopes.isEmpty) + } + + /// The read scope pins `sql_mode` to '', so the DDL comes back with backticks whatever the session had. + @Test("An unfiltered export reads under the read scope and writes the plain mode, without probing") + func unfilteredExportWritesThePlainMode() async throws { + let probe = Self.sessionProbe(sqlMode: "ANSI_QUOTES") + let source = RecordingDataSource(read: .returns(Self.utcScope), preserve: .returns(probe)) + let dump = try await SQLExportHarness.shared.dump(tables: [table], dataSource: source).text + + #expect(dump.contains(Self.plainModeLine)) + #expect(source.openedPurposes == [.verbatimRead]) + #expect(source.closedScopes == [Self.utcScope]) + } + + @Test("A read scope that fails to open falls back to the session's own mode") + func failedReadScopeProbesTheSession() async throws { + let probe = Self.sessionProbe(sqlMode: "ANSI_QUOTES") + let source = RecordingDataSource(read: .fails, preserve: .returns(probe)) + let dump = try await SQLExportHarness.shared.dump(tables: [table], dataSource: source).text + + #expect(dump.contains(Self.ansiQuotesModeLine)) + #expect(!dump.contains("TIME_ZONE")) + #expect(source.openedPurposes == [.verbatimRead, .preserve]) + #expect(source.closedScopes == [probe]) + } + + @Test("The scope opens before the snapshot begins and closes after it ends") + func scopeEnclosesTheSnapshot() async throws { + let source = RecordingDataSource(read: .returns(Self.utcScope)) + _ = try await SQLExportHarness.shared.dump( + tables: [table], dataSource: source, options: options(consistentSnapshot: true)) + + #expect(source.events == ["open", "START TRANSACTION WITH CONSISTENT SNAPSHOT", "rows", "COMMIT", "close"]) + } + + @Test("The scope is closed once when the data phase fails") + func closedOnceWhenTheDataPhaseFails() async throws { + let source = RecordingDataSource(read: .returns(Self.utcScope), failsStream: true) + + await #expect(throws: InjectedFailure.self) { + _ = try await SQLExportHarness.shared.dump(tables: [self.table], dataSource: source) + } + #expect(source.closedScopes == [Self.utcScope]) + } + + @Test("The scope is closed once when the export is stopped while it reads") + func closedOnceOnCancellation() async throws { + let progress = PluginExportProgress(progress: Progress(totalUnitCount: 1)) + let source = RecordingDataSource(read: .returns(Self.utcScope), afterLastRow: { progress.cancel() }) + + await #expect(throws: (any Error).self) { + _ = try await SQLExportHarness.shared.dump( + tables: [self.table], dataSource: source, progress: progress) + } + #expect(source.closedScopes == [Self.utcScope]) + } + + /// The success path closes the scope before the publication point checks for a stop, and the stop then runs + /// the failure path, which closes again unless the first close is remembered. + @Test("The scope is closed once when the stop lands at the publication point") + func closedOnceWhenStoppedAtThePublicationPoint() async throws { + let progress = PluginExportProgress(progress: Progress(totalUnitCount: 1)) + let source = RecordingDataSource( + read: .returns(Self.utcScope), + onExecute: { query in + if query == "COMMIT" { progress.cancel() } + }) + + await #expect(throws: (any Error).self) { + _ = try await SQLExportHarness.shared.dump( + tables: [self.table], dataSource: source, + options: self.options(consistentSnapshot: true), progress: progress) + } + #expect(source.closedScopes == [Self.utcScope]) + } + + @Test("The scope is closed when the dump file cannot be created") + func closedWhenTheWriterCannotStart() async throws { + let source = RecordingDataSource(read: .returns(Self.utcScope)) + let tables = [table] + let destination = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + .appendingPathComponent("dump.sql") + + await #expect(throws: (any Error).self) { + try await SQLExportHarness.shared.withPlugin { plugin in + _ = try await plugin.export( + tables: tables, + dataSource: source, + destination: destination, + progress: PluginExportProgress(progress: Progress(totalUnitCount: 1))) + } + } + #expect(source.closedScopes == [Self.utcScope]) + } + + @Test("A scope that fails to close does not fail an export that was written") + func failedCloseIsNotAnExportFailure() async throws { + let source = RecordingDataSource(read: .returns(Self.utcScope), failsClose: true) + let output = try await SQLExportHarness.shared.dump(tables: [table], dataSource: source) + + #expect(output.text.contains("INSERT INTO `events`")) + #expect(output.result.warnings.isEmpty) + #expect(source.closedScopes == [Self.utcScope]) + } +} diff --git a/Plugins/FormatPluginTests/SQLExportSessionDeclarationTests.swift b/Plugins/FormatPluginTests/SQLExportSessionDeclarationTests.swift new file mode 100644 index 0000000000..2bb3b6da92 --- /dev/null +++ b/Plugins/FormatPluginTests/SQLExportSessionDeclarationTests.swift @@ -0,0 +1,271 @@ +// +// SQLExportSessionDeclarationTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct SQLExportSessionDeclarationTests { + private static let mysqlFamily = ["MySQL", "MariaDB", "TiDB", "OceanBase"] + + private func temporaryDirectory() throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory + } + + @Test("A MySQL dump declares utf8mb4 the way mysqldump does, and puts the session back") + func mysqlDeclaresUTF8MB4() { + for typeId in Self.mysqlFamily { + let declaration = SQLExportSessionDeclaration.forDatabaseType(typeId) + #expect(declaration.prologue.contains("/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;")) + #expect(declaration.prologue.contains("/*!40101 SET NAMES utf8 */;")) + #expect(declaration.prologue.contains("/*!50503 SET NAMES utf8mb4 */;")) + #expect(declaration.epilogue.contains("/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;")) + #expect(declaration.epilogue.contains("/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;")) + #expect(declaration.epilogue.contains("/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;")) + } + } + + /// #3414: without the mode line a MySQL 8 server refuses the dump's zero dates with 1292 and renumbers an + /// AUTO_INCREMENT value of 0. + @Test("A MySQL-family dump pins mysqldump's SQL_MODE and turns foreign key checks off, and restores both") + func mysqlFamilyPinsTheSQLModeAndForeignKeyChecks() { + for typeId in Self.mysqlFamily { + let declaration = SQLExportSessionDeclaration.forDatabaseType(typeId) + #expect(declaration.prologue.contains( + "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;"), "\(typeId)") + #expect(declaration.prologue.contains( + "/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;"), "\(typeId)") + #expect(declaration.epilogue.contains("/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;"), "\(typeId)") + #expect(declaration.epilogue.contains( + "/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;"), "\(typeId)") + } + } + + @Test("Without a read time zone the MySQL declaration is mysqldump's text minus the zone pair") + func mysqlDeclarationWithoutTimeZone() { + let declaration = SQLExportSessionDeclaration.forDatabaseType("MySQL") + #expect(declaration.prologue == """ + /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; + /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; + /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; + /*!40101 SET NAMES utf8 */; + /*!50503 SET NAMES utf8mb4 */; + /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; + /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; + + + """) + #expect(declaration.epilogue == """ + + /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; + /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; + /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; + /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; + /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; + + """) + #expect(!declaration.prologue.contains("TIME_ZONE")) + #expect(!declaration.epilogue.contains("TIME_ZONE")) + } + + @Test("The zone the rows were read in is declared before the mode and restored first") + func mysqlDeclarationWithTimeZone() { + for typeId in Self.mysqlFamily { + let declaration = SQLExportSessionDeclaration.forDatabaseType(typeId, readTimeZone: "+00:00") + #expect(declaration.prologue == """ + /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; + /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; + /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; + /*!40101 SET NAMES utf8 */; + /*!50503 SET NAMES utf8mb4 */; + /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; + /*!40103 SET TIME_ZONE='+00:00' */; + /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; + /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; + + + """, "\(typeId)") + #expect(declaration.epilogue == """ + + /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; + /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; + /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; + /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; + /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; + + """, "\(typeId)") + } + } + + @Test("A dump whose DDL was read under ANSI_QUOTES keeps it in the mode it pins, and restores the same way") + func ansiQuotesJoinsTheDumpMode() { + for typeId in Self.mysqlFamily { + let declaration = SQLExportSessionDeclaration.forDatabaseType(typeId, readsWithANSIQuotes: true) + let plain = SQLExportSessionDeclaration.forDatabaseType(typeId) + #expect(declaration.prologue == plain.prologue.replacingOccurrences( + of: "SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;", + with: "SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES' */;"), "\(typeId)") + #expect(declaration.prologue != plain.prologue, "\(typeId)") + #expect(declaration.epilogue == plain.epilogue, "\(typeId)") + } + #expect(SQLExportSessionDeclaration.forDatabaseType("PostgreSQL", readsWithANSIQuotes: true) + == SQLExportSessionDeclaration.forDatabaseType("PostgreSQL")) + #expect(SQLExportSessionDeclaration.forDatabaseType("SQLite", readsWithANSIQuotes: true) == .empty) + } + + @Test( + "A plain zone spelling is declared as written", + arguments: ["+00:00", "-05:30", "SYSTEM", "UTC", "Europe/Paris", "America/Argentina/Buenos_Aires", "Etc/GMT+5"] + ) + func plainTimeZoneIsDeclared(zone: String) { + let declaration = SQLExportSessionDeclaration.forDatabaseType("MySQL", readTimeZone: zone) + #expect(declaration.prologue.contains("/*!40103 SET TIME_ZONE='\(zone)' */;")) + #expect(declaration.epilogue.contains("/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;")) + } + + /// The zone sits inside `/*! */` and a string literal, so a `*/` or a quote would end one of them early. + @Test( + "A zone that is not a plain spelling is dropped, and the dump declares none", + arguments: [ + "+00:00' */; DROP TABLE t; /*", "*/", "it's", "Europe\\Paris", "UTC ", "", "Zürich", + String(repeating: "A", count: 65), + ] + ) + func unusualTimeZoneIsDropped(zone: String) { + let declaration = SQLExportSessionDeclaration.forDatabaseType("MySQL", readTimeZone: zone) + #expect(declaration == .forDatabaseType("MySQL")) + #expect(!declaration.prologue.contains("TIME_ZONE")) + #expect(!declaration.epilogue.contains("TIME_ZONE")) + } + + @Test("A PostgreSQL dump declares UTF8 the way pg_dump does, on every engine that accepts the statement") + func postgresDeclaresUTF8() { + for typeId in ["PostgreSQL", "Greenplum", "AlloyDB", "Citus", "CockroachDB", "PGlite"] { + let declaration = SQLExportSessionDeclaration.forDatabaseType(typeId) + #expect(declaration.prologue == "SET client_encoding = 'UTF8';\n\n", "\(typeId)") + #expect(declaration.epilogue.isEmpty, "\(typeId)") + #expect(SQLExportSessionDeclaration.forDatabaseType(typeId, readTimeZone: "+00:00") == declaration, "\(typeId)") + } + } + + @Test("Redshift gets no declaration, because it does not document client_encoding as settable") + func redshiftDeclaresNothing() { + #expect(SQLExportSessionDeclaration.forDatabaseType("Redshift") == .empty) + } + + @Test("Other engines write no declaration, with or without a read time zone") + func otherEnginesDeclareNothing() { + for typeId in ["SQLite", "DuckDB", "Oracle", "MSSQL", "Databend", "SomeFuturePlugin"] { + #expect(SQLExportSessionDeclaration.forDatabaseType(typeId) == .empty, "\(typeId)") + #expect(SQLExportSessionDeclaration.forDatabaseType(typeId, readTimeZone: "+00:00") == .empty, "\(typeId)") + } + } + + @Test("A PostgreSQL dump opens with the declaration and ends with the last statement") + func postgresDumpIsPrefixed() throws { + let directory = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: directory) } + let declaration = SQLExportSessionDeclaration.forDatabaseType("PostgreSQL") + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter( + destination: destination, splitSizeMegabytes: 0, sessionDeclaration: declaration + ) + try writer.write("INSERT INTO \"t\" VALUES ('メール');\n") + try writer.commit() + + let dump = try String(contentsOf: destination, encoding: .utf8) + #expect(dump == "SET client_encoding = 'UTF8';\n\nINSERT INTO \"t\" VALUES ('メール');\n") + } + + @Test("An unsplit dump opens with the declaration and closes by restoring the session") + func unsplitDumpIsWrapped() throws { + let directory = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: directory) } + let declaration = SQLExportSessionDeclaration.forDatabaseType("MySQL", readTimeZone: "+00:00") + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter( + destination: destination, splitSizeMegabytes: 0, sessionDeclaration: declaration + ) + try writer.write("INSERT INTO `t` VALUES ('メール');\n") + try writer.commit() + + let dump = try String(contentsOf: destination, encoding: .utf8) + #expect(dump == declaration.prologue + "INSERT INTO `t` VALUES ('メール');\n" + declaration.epilogue) + } + + @Test( + "Every part of a split dump carries its own declaration, so each restores on its own", + arguments: ["MySQL", "PostgreSQL"] + ) + func everyPartIsWrapped(databaseTypeId: String) throws { + let directory = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: directory) } + let declaration = SQLExportSessionDeclaration.forDatabaseType(databaseTypeId, readTimeZone: "+00:00") + #expect(declaration != .empty) + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter( + destination: destination, splitSizeMegabytes: 1, sessionDeclaration: declaration + ) + let chunk = String(repeating: "x", count: 700 * 1_024) + try writer.write("A\(chunk);\n") + try writer.write("B\(chunk);\n") + let parts = try writer.commit() + + #expect(parts.count == 2) + for (index, part) in parts.enumerated() { + let text = try String(contentsOf: part, encoding: .utf8) + #expect(text.hasPrefix(declaration.prologue), "part \(index + 1)") + #expect(text.hasSuffix(declaration.epilogue), "part \(index + 1)") + #expect(text.contains(index == 0 ? "A" : "B")) + } + } + + @Test("No part grows past the cap once its closing declaration is added") + func epilogueCountsTowardTheCap() throws { + let directory = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: directory) } + let declaration = SQLExportSessionDeclaration.forDatabaseType("MySQL", readTimeZone: "+00:00") + let cap = 1_024 * 1_024 + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter( + destination: destination, splitSizeMegabytes: 1, sessionDeclaration: declaration + ) + let firstLength = cap - declaration.prologue.utf8.count - declaration.epilogue.utf8.count - 5 + try writer.write(String(repeating: "a", count: firstLength - 2) + ";\n") + try writer.write("SELECT 1;\n") + let parts = try writer.commit() + + #expect(parts.count == 2) + for part in parts { + let size = try FileManager.default.attributesOfItem(atPath: part.path)[.size] as? Int ?? 0 + #expect(size <= cap, "\(part.lastPathComponent) is \(size) bytes") + } + } + + @Test("A statement larger than the cap still lands in a part of its own, not after an empty one") + func oversizedStatementDoesNotLeaveAnEmptyPart() throws { + let directory = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: directory) } + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter( + destination: destination, splitSizeMegabytes: 1, + sessionDeclaration: .forDatabaseType("MySQL") + ) + try writer.write(String(repeating: "y", count: 2 * 1_024 * 1_024) + ";\n") + let parts = try writer.commit() + + #expect(parts.count == 1) + #expect(!writer.didSplit) + } +} diff --git a/Plugins/MySQLDriverPlugin/MySQLPluginDriver+CreationScopes.swift b/Plugins/MySQLDriverPlugin/MySQLPluginDriver+CreationScopes.swift new file mode 100644 index 0000000000..ebbd663cff --- /dev/null +++ b/Plugins/MySQLDriverPlugin/MySQLPluginDriver+CreationScopes.swift @@ -0,0 +1,50 @@ +// +// MySQLPluginDriver+CreationScopes.swift +// MySQLDriverPlugin +// + +import Foundation +import TableProMySQLCore +import TableProPluginKit + +internal extension MySQLPluginDriver { + func fetchRoutineCreationScope(_ routine: PluginRoutineInfo) async throws -> PluginStatementScope? { + guard !flavor.isDatabend, serverHasInformationSchema else { return nil } + let query = MySQLObjectQueries.routineCreationSettings( + schema: routineSchema(routine.schema), + name: routine.name, + kind: routine.kind == .procedure ? "PROCEDURE" : "FUNCTION" + ) + let row = try await creationSettingsRow(query, named: routine.name) + return Self.creationScope(sqlMode: row?[safe: 1]?.asText, timeZone: nil) + } + + func fetchTriggerCreationScope(_ trigger: PluginTriggerInfo) async throws -> PluginStatementScope? { + guard !flavor.isDatabend, holdsForServer(MySQLServerVersion.hasTriggerCatalog(banner:flavor:)) else { + return nil + } + let query = MySQLObjectQueries.triggerCreationSettings(schema: routineSchema(trigger.schema), name: trigger.name) + let row = try await creationSettingsRow(query, named: trigger.name) + return Self.creationScope(sqlMode: row?[safe: 1]?.asText, timeZone: nil) + } + + func fetchEventCreationScope(_ event: PluginEventInfo) async throws -> PluginStatementScope? { + guard !flavor.isDatabend, holdsForServer(MySQLServerVersion.hasPartitionAndEventCatalogs(banner:flavor:)) else { + return nil + } + let query = MySQLObjectQueries.eventCreationSettings(schema: effectiveSchema(event.schema), name: event.name) + let row = try await creationSettingsRow(query, named: event.name) + return Self.creationScope(sqlMode: row?[safe: 1]?.asText, timeZone: row?[safe: 2]?.asText) + } + + /// Nil when the catalog has no row for the object, or no recorded mode. + static func creationScope(sqlMode: String?, timeZone: String?) -> PluginStatementScope? { + guard let sqlMode, let settings = MySQLCreationSettings(sqlMode: sqlMode, timeZone: timeZone) else { return nil } + return PluginStatementScope(opening: settings.opening, closing: settings.closing) + } + + private func creationSettingsRow(_ query: String, named name: String) async throws -> [PluginCellValue]? { + let result = try await execute(ownStatement: query) + return result.rows.first { $0[safe: 0]?.asText == name } + } +} diff --git a/Plugins/MySQLDriverPlugin/MySQLPluginDriver+SessionScope.swift b/Plugins/MySQLDriverPlugin/MySQLPluginDriver+SessionScope.swift new file mode 100644 index 0000000000..3478a2ec44 --- /dev/null +++ b/Plugins/MySQLDriverPlugin/MySQLPluginDriver+SessionScope.swift @@ -0,0 +1,112 @@ +// +// MySQLPluginDriver+SessionScope.swift +// MySQLDriverPlugin +// + +import Foundation +import os +import TableProMySQLCore +import TableProPluginKit + +internal extension MySQLPluginDriver { + /// Databend has no `sql_mode` to hold, and a server before 5.0.3 cannot filter `SHOW` with `WHERE`. + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + let names = MySQLSessionScopePlan.savedNames(for: purpose) + guard !flavor.isDatabend, + holdsForServer(MySQLServerVersion.showAcceptsWhere(banner:flavor:)), + !names.isEmpty + else { return nil } + let saved = try await readSessionSettings(names) + guard let applied = MySQLSessionScopePlan.applied(for: purpose, saved: saved) else { return nil } + guard !applied.isEmpty else { + return PluginSessionScope(purpose: purpose, saved: saved, applied: []) + } + let statements = applied.compactMap { MySQLSessionStatements.assignment(name: $0.name, value: $0.value) } + let savedNames = Set(saved.map(\.name)) + /// A setting with no saved value could not be put back, so the scope sets none of them. + guard statements.count == applied.count, applied.allSatisfy({ savedNames.contains($0.name) }) else { + Self.logger.warning("The server did not report every setting a session scope sets; running without one") + return nil + } + /// Held before the first `SET`, so the idle release cannot hand the session back between two of them. + adjustFootprint { $0.holdSessionScope() } + do { + for statement in statements { + _ = try await executeSetup(statement) + } + } catch { + let isRestored = await restoreAfterFailedOpen(saved) + adjustFootprint { $0.releaseSessionScope(isRestored: isRestored) } + throw error + } + return PluginSessionScope(purpose: purpose, saved: saved, applied: applied) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + guard !flavor.isDatabend else { return } + let heldSession = !scope.applied.isEmpty + let isRestored: Bool + do { + isRestored = try await restoreSessionSettings(scope.saved) + } catch { + noteClosedScope(heldSession: heldSession, isRestored: false) + throw error + } + noteClosedScope(heldSession: heldSession, isRestored: isRestored) + } + + private func noteClosedScope(heldSession: Bool, isRestored: Bool) { + adjustFootprint { footprint in + if heldSession { + footprint.releaseSessionScope(isRestored: isRestored) + } else if !isRestored { + footprint.noteUnrestoredSessionSettings() + } + } + } + + private func readSessionSettings(_ names: [String]) async throws -> [PluginSessionSetting] { + guard let query = MySQLSessionStatements.captureQuery(names: names) else { return [] } + let result = try await executeSetup(query) + return MySQLSessionScopePlan.captured(rows: result.rows, names: names) + } + + /// Every statement is tried, so one refused value does not leave the others changed, and the first refusal is what + /// the caller hears. False when a setting had to stay changed. + /// + /// The transaction state comes from the status flags of the `SHOW` just read, and anything but `.idle`, `.unknown` + /// included, keeps `autocommit` from committing a transaction that may be the user's. + private func restoreSessionSettings(_ saved: [PluginSessionSetting]) async throws -> Bool { + guard !saved.isEmpty else { return true } + let current = try await readSessionSettings(saved.map(\.name)) + let isTransactionOpen = await sessionTransactionState() != .idle + let restore = MySQLSessionScopePlan.restore(saved: saved, current: current, isTransactionOpen: isTransactionOpen) + if !restore.isComplete { + Self.logger.info("Left autocommit as the imported file set it, because the session holds an open transaction") + } + var firstFailure: Error? + for statement in restore.statements { + do { + _ = try await executeSetup(statement) + } catch { + Self.logger.error( + "Could not put a session setting back with \(statement, privacy: .public): \(error.localizedDescription)" + ) + firstFailure = firstFailure ?? error + } + } + if let firstFailure { + throw firstFailure + } + return restore.isComplete + } + + private func restoreAfterFailedOpen(_ saved: [PluginSessionSetting]) async -> Bool { + do { + return try await restoreSessionSettings(saved) + } catch { + Self.logger.error("A session scope that failed to open could not put its settings back") + return false + } + } +} diff --git a/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift b/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift index a90bf88595..2c58123048 100644 --- a/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift +++ b/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift @@ -364,6 +364,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { result = try await conn.executeParameterizedQuery(query, parameters: parameters, rowCap: rowCap) } catch { noteFailure(query) + await refreshTransactionFlag(after: error, on: conn) throw error } return PluginQueryResult( @@ -448,10 +449,25 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { if countsAsActivity { noteFailure(query) } + await refreshTransactionFlag(after: error, on: conn) throw error } } + /// One `DO 0` on the same connection, sent as driver setup, so the reply's status flags replace the ones the + /// failed statement's ERR packet left in place. Its own failure changes nothing the caller hears. + private func refreshTransactionFlag(after error: Error, on conn: MariaDBPluginConnection) async { + guard let failure = error as? MariaDBPluginError, + !failure.wasNotSentAfterSessionEnded, + mysqlTransactionFlagMayBeStale( + afterFailureCode: failure.code, + isFlagOpen: conn.isInTransaction, + hasStatusFlags: flavor.reportsSessionStatusFlags + ) + else { return } + _ = try? await conn.executeQuery("DO 0", rowCap: nil) + } + // MARK: - Idle connection release private func noteActivity(_ sql: String) { @@ -475,6 +491,17 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { sessionLock.withLock { mysqlMayReplay(query, on: footprint) } } + /// A statement the driver runs on its own behalf, like the query timeout's `SET`: spelled for this session, and + /// left out of the footprint and the idle clock. + func executeSetup(_ statement: String) async throws -> PluginQueryResult { + try await executeWithReconnect(query: literalSpelling.respelled(statement), isRetry: false, countsAsActivity: false) + } + + /// For what `executeSetup` changed, which the footprint cannot read from the statements. + func adjustFootprint(_ change: (inout MySQLSessionFootprint) -> Void) { + sessionLock.withLock { change(&footprint) } + } + /// Takes a server connection again if the last one was handed back. A connection that was /// never released costs a few flag reads. private func requireConnection() async throws -> MariaDBPluginConnection { diff --git a/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift b/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift index 4982b44d32..e1fb17c9a8 100644 --- a/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift +++ b/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift @@ -49,12 +49,19 @@ struct MySQLSessionFootprint: Equatable { /// table, take a lock or open a transaction. Opaque is the only honest reading. private(set) var ranOpaqueRoutine = false + /// The driver's own session scopes still open. They set `sql_mode` and `time_zone` as driver setup, which never + /// passes through `observe`, and a reconnect would carry on without them. + private(set) var heldSessionScopes = 0 + var isClean: Bool { self == MySQLSessionFootprint() } /// Why the session cannot be released, in the user's language, or nil when it can. var blockingReason: String? { + if heldSessionScopes > 0 { + return String(localized: "TablePro is reading or writing rows on this connection with settings of its own.") + } if hasOpenTransaction { return String(localized: "This connection has an open transaction. Commit or roll it back first.") } @@ -155,6 +162,23 @@ struct MySQLSessionFootprint: Equatable { self = MySQLSessionFootprint() } + mutating func holdSessionScope() { + heldSessionScopes += 1 + } + + /// A scope whose settings could not all be put back leaves the session carrying changed settings, so neither a + /// replay nor the idle release treats it as clean. + mutating func releaseSessionScope(isRestored: Bool) { + heldSessionScopes = max(0, heldSessionScopes - 1) + if !isRestored { + hasSessionSettings = true + } + } + + mutating func noteUnrestoredSessionSettings() { + hasSessionSettings = true + } + private mutating func observeStatement(_ statement: String) { let normalized = statement.uppercased() let head = Self.collapsedHead(of: normalized) diff --git a/Plugins/MySQLDriverPlugin/MySQLSessionScopePlan.swift b/Plugins/MySQLDriverPlugin/MySQLSessionScopePlan.swift new file mode 100644 index 0000000000..291959a2dc --- /dev/null +++ b/Plugins/MySQLDriverPlugin/MySQLSessionScopePlan.swift @@ -0,0 +1,127 @@ +// +// MySQLSessionScopePlan.swift +// MySQLDriverPlugin +// + +import Foundation +import TableProMySQLCore +import TableProPluginKit + +/// Which session settings each purpose saves and sets on MySQL, and what putting them back takes. +internal enum MySQLSessionScopePlan { + /// What a mysqldump or mariadb-dump file sets and puts back only at its end, which an imported file leaves changed + /// when it stops early. `SHOW` reports only the ones the server has: `note_verbosity` is MariaDB's. + /// + /// `autocommit` is last, and closing restores in this order: `SET autocommit = 1` commits whatever the session + /// still holds open, so every other setting is back before it. + static let preservedNames = [ + "sql_mode", + "time_zone", + "character_set_client", + "character_set_results", + "collation_connection", + "foreign_key_checks", + "unique_checks", + "sql_notes", + "note_verbosity", + "autocommit", + ] + + static let verbatimNames = ["sql_mode", "time_zone"] + + static func savedNames(for purpose: PluginSessionPurpose) -> [String] { + switch purpose { + case .preserve: + return preservedNames + case .verbatimRead, .verbatimWrite: + return verbatimNames + @unknown default: + return [] + } + } + + /// What `purpose` sets, in order, given what was saved, or nil for a purpose this driver does not know. Reading + /// runs as mysqldump's own connection does (`SET @@SQL_MODE = ''` and `TIME_ZONE = '+00:00'`), so no + /// `ANSI_QUOTES` reaches the DDL and every TIMESTAMP comes out in UTC. + static func applied(for purpose: PluginSessionPurpose, saved: [PluginSessionSetting]) -> [PluginSessionSetting]? { + switch purpose { + case .preserve: + return [] + case .verbatimRead: + return [ + PluginSessionSetting(name: "sql_mode", value: ""), + PluginSessionSetting(name: "time_zone", value: "+00:00"), + ] + case .verbatimWrite: + return [ + PluginSessionSetting(name: "sql_mode", value: writeMode(saved: saved)), + PluginSessionSetting(name: "time_zone", value: "+00:00"), + ] + @unknown default: + return nil + } + } + + /// Writing accepts what the user's session accepts, plus zero dates and an AUTO_INCREMENT value of 0: the saved + /// mode loses `NO_ZERO_DATE` and `NO_ZERO_IN_DATE` and gains `NO_AUTO_VALUE_ON_ZERO`, and every other flag stays. + /// Strict flags stay because a copy can write into an existing table its rows do not fit, `NO_BACKSLASH_ESCAPES` + /// because Object Copy spells its table DDL's literals for the session's mode, and the rest, `ALLOW_INVALID_DATES` + /// among them, so nothing that copied before is refused now (`scripts/probes/check-mysql-dump-session.sh`, step e). + /// + /// `TRADITIONAL` goes too: `SHOW` reports it beside the flags it stands for, and setting it again brings both + /// zero-date flags back. On MariaDB 13.0.2 and MySQL 8.4.11 it is the only combination mode `SHOW` reports that + /// implies either one, and its other flags are reported alongside it, so they stay. + static func writeMode(saved: [PluginSessionSetting]) -> String { + let savedMode = saved.first { $0.name == "sql_mode" }?.value ?? "" + var flags = savedMode + .split(separator: ",") + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty && !zeroDateModes.contains($0.uppercased()) } + if !flags.contains(where: { $0.uppercased() == "NO_AUTO_VALUE_ON_ZERO" }) { + flags.append("NO_AUTO_VALUE_ON_ZERO") + } + return flags.joined(separator: ",") + } + + private static let zeroDateModes: Set = ["NO_ZERO_DATE", "NO_ZERO_IN_DATE", "TRADITIONAL"] + + /// The settings a `SHOW SESSION VARIABLES` reply reports, in the order `names` asks for them. A name the server + /// did not report is left out. + static func captured(rows: [[PluginCellValue]], names: [String]) -> [PluginSessionSetting] { + var reported: [String: String?] = [:] + for row in rows { + guard let name = row[safe: 0]?.asText?.lowercased() else { continue } + reported.updateValue(row[safe: 1]?.asText, forKey: name) + } + return names.compactMap { name -> PluginSessionSetting? in + guard let value = reported[name] else { return nil } + return PluginSessionSetting(name: name, value: MySQLSessionStatements.reportedValue(name: name, value: value)) + } + } + + /// One `SET SESSION` for each saved setting the session no longer holds, in the order they were saved, and whether + /// every one of them is among them. + /// + /// `SET autocommit = 1` commits whatever transaction is open, and it may be the user's own, begun in the editor + /// before the import ran, so while one is open `autocommit` stays as the file left it. + static func restore( + saved: [PluginSessionSetting], + current: [PluginSessionSetting], + isTransactionOpen: Bool + ) -> (statements: [String], isComplete: Bool) { + var statements: [String] = [] + var isComplete = true + for setting in saved { + if let now = current.first(where: { $0.name == setting.name }), now.value == setting.value { continue } + if isTransactionOpen, setting.name == "autocommit" { + isComplete = false + continue + } + guard let statement = MySQLSessionStatements.assignment(name: setting.name, value: setting.value) else { + continue + } + statements.append(statement) + } + return (statements, isComplete) + } +} diff --git a/Plugins/MySQLDriverPlugin/MySQLStatementClassification.swift b/Plugins/MySQLDriverPlugin/MySQLStatementClassification.swift index 5bc95623d2..e47c701a3e 100644 --- a/Plugins/MySQLDriverPlugin/MySQLStatementClassification.swift +++ b/Plugins/MySQLDriverPlugin/MySQLStatementClassification.swift @@ -58,6 +58,20 @@ internal func mysqlMayReplay(_ query: String, on footprint: MySQLSessionFootprin /// The codes libmariadb reports when the connection under a statement is gone. internal let mysqlConnectionLossCodes: Set = [2_006, 2_013, 2_055] +/// libmariadb's own errors, `CR_MIN_ERROR` to `CR_MAX_ERROR`. The server never sends one. +internal let mysqlClientErrorCodes: ClosedRange = 2_000...2_999 + +/// Whether a failed statement leaves the connection's transaction flag in doubt, so the server should be asked again. +/// +/// An ERR packet carries no status flags, so after a failure the flag is still the last OK's. A failed `CREATE TABLE` +/// inside a transaction has already committed it (`scripts/probes/check-mysql-dump-session.sh`, step f), and the +/// flag goes on saying it is open. Only the server's own refusal counts: a client error leaves the session itself in +/// doubt, and code 0 is the driver's own error, which never reached the server. +internal func mysqlTransactionFlagMayBeStale(afterFailureCode code: UInt32, isFlagOpen: Bool, hasStatusFlags: Bool) -> Bool { + guard hasStatusFlags, isFlagOpen, code != 0 else { return false } + return !mysqlClientErrorCodes.contains(code) +} + /// Whether a lost connection may be retaken and the statement run again on the session that /// replaces it. /// diff --git a/Plugins/MySQLDriverTests/MySQLSessionScopeTests.swift b/Plugins/MySQLDriverTests/MySQLSessionScopeTests.swift new file mode 100644 index 0000000000..67741e4379 --- /dev/null +++ b/Plugins/MySQLDriverTests/MySQLSessionScopeTests.swift @@ -0,0 +1,250 @@ +// +// MySQLSessionScopeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct MySQLSessionScopeTests { + @Test("Preserving saves what a mysqldump or mariadb-dump file changes, autocommit last, and sets nothing") + func preserveSavesTheDumpSettings() { + #expect(MySQLSessionScopePlan.savedNames(for: .preserve) == [ + "sql_mode", "time_zone", "character_set_client", "character_set_results", + "collation_connection", "foreign_key_checks", "unique_checks", "sql_notes", "note_verbosity", "autocommit", + ]) + #expect(MySQLSessionScopePlan.applied(for: .preserve, saved: []) == []) + } + + private static let beforeImport = MySQLSessionScopePlan.captured(rows: [ + ["autocommit", "ON"], ["foreign_key_checks", "ON"], ["sql_mode", "STRICT_TRANS_TABLES"], ["sql_notes", "ON"], + ], names: MySQLSessionScopePlan.preservedNames) + + private static let leftByStoppedFile = MySQLSessionScopePlan.captured(rows: [ + ["autocommit", "OFF"], ["foreign_key_checks", "OFF"], ["sql_mode", "NO_AUTO_VALUE_ON_ZERO"], ["sql_notes", "OFF"], + ], names: MySQLSessionScopePlan.preservedNames) + + /// `SET autocommit = 1` commits what the session holds open, so it goes after every other restore. + @Test("With no transaction open, closing a preserve scope restores autocommit after everything else") + func autocommitIsRestoredLast() { + let restore = MySQLSessionScopePlan.restore( + saved: Self.beforeImport, current: Self.leftByStoppedFile, isTransactionOpen: false + ) + #expect(restore.statements == [ + "SET SESSION sql_mode = 'STRICT_TRANS_TABLES'", + "SET SESSION foreign_key_checks = 'ON'", + "SET SESSION sql_notes = 'ON'", + "SET SESSION autocommit = 'ON'", + ]) + #expect(restore.isComplete) + } + + /// The open transaction may be the user's, begun in the editor before the import, and `SET autocommit = 1` would + /// commit it along with whatever the file left pending. + @Test("With a transaction open, autocommit stays as the file left it and the restore reports itself incomplete") + func autocommitWaitsForOpenTransaction() { + let restore = MySQLSessionScopePlan.restore( + saved: Self.beforeImport, current: Self.leftByStoppedFile, isTransactionOpen: true + ) + #expect(restore.statements == [ + "SET SESSION sql_mode = 'STRICT_TRANS_TABLES'", + "SET SESSION foreign_key_checks = 'ON'", + "SET SESSION sql_notes = 'ON'", + ]) + #expect(!restore.isComplete) + } + + @Test("An open transaction holds nothing back when autocommit never changed") + func unchangedAutocommitIsComplete() { + let restore = MySQLSessionScopePlan.restore( + saved: Self.beforeImport, current: Self.beforeImport, isTransactionOpen: true + ) + #expect(restore.statements.isEmpty) + #expect(restore.isComplete) + } + + @Test("Reading runs as mysqldump's own connection does, and writing derives its mode from the session's") + func verbatimPurposesPinModeAndZone() { + let saved = [ + PluginSessionSetting(name: "sql_mode", value: "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_DATE"), + PluginSessionSetting(name: "time_zone", value: "SYSTEM"), + ] + #expect(MySQLSessionScopePlan.savedNames(for: .verbatimRead) == ["sql_mode", "time_zone"]) + #expect(MySQLSessionScopePlan.savedNames(for: .verbatimWrite) == ["sql_mode", "time_zone"]) + #expect(MySQLSessionScopePlan.applied(for: .verbatimRead, saved: saved) == [ + PluginSessionSetting(name: "sql_mode", value: ""), + PluginSessionSetting(name: "time_zone", value: "+00:00"), + ]) + #expect(MySQLSessionScopePlan.applied(for: .verbatimWrite, saved: saved) == [ + PluginSessionSetting(name: "sql_mode", value: "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_AUTO_VALUE_ON_ZERO"), + PluginSessionSetting(name: "time_zone", value: "+00:00"), + ]) + } + + /// A copy accepts what the user's session accepts plus zero dates and AUTO_INCREMENT 0. The TRADITIONAL rows are + /// what MySQL 8.4.11 and MariaDB 13.0.2 report for `SET sql_mode = 'TRADITIONAL'`. + @Test("The write mode drops only the zero-date flags and TRADITIONAL, and adds NO_AUTO_VALUE_ON_ZERO", arguments: [ + ("ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO"), + ("STRICT_ALL_TABLES,NO_ZERO_DATE", "STRICT_ALL_TABLES,NO_AUTO_VALUE_ON_ZERO"), + ("STRICT_TRANS_TABLES,ALLOW_INVALID_DATES", "STRICT_TRANS_TABLES,ALLOW_INVALID_DATES,NO_AUTO_VALUE_ON_ZERO"), + ("", "NO_AUTO_VALUE_ON_ZERO"), + ("NO_ENGINE_SUBSTITUTION", "NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO"), + ("STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_ENGINE_SUBSTITUTION", + "STRICT_TRANS_TABLES,STRICT_ALL_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO"), + ("STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION", + "STRICT_TRANS_TABLES,STRICT_ALL_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO"), + ("ANSI_QUOTES,NO_BACKSLASH_ESCAPES", "ANSI_QUOTES,NO_BACKSLASH_ESCAPES,NO_AUTO_VALUE_ON_ZERO"), + ("REAL_AS_FLOAT,PIPES_AS_CONCAT,ANSI_QUOTES,IGNORE_SPACE,ONLY_FULL_GROUP_BY,ANSI", + "REAL_AS_FLOAT,PIPES_AS_CONCAT,ANSI_QUOTES,IGNORE_SPACE,ONLY_FULL_GROUP_BY,ANSI,NO_AUTO_VALUE_ON_ZERO"), + ("NO_AUTO_VALUE_ON_ZERO,STRICT_TRANS_TABLES", "NO_AUTO_VALUE_ON_ZERO,STRICT_TRANS_TABLES"), + ]) + func writeModeFollowsTheSession(_ sessionMode: String, _ writeMode: String) { + let saved = [PluginSessionSetting(name: "sql_mode", value: sessionMode)] + #expect(MySQLSessionScopePlan.writeMode(saved: saved) == writeMode) + } + + @Test("A write scope with no saved mode adds only NO_AUTO_VALUE_ON_ZERO, and reading ignores the session's mode") + func writeModeWithoutSavedMode() { + #expect(MySQLSessionScopePlan.writeMode(saved: []) == "NO_AUTO_VALUE_ON_ZERO") + let saved = [PluginSessionSetting(name: "sql_mode", value: "ANSI_QUOTES,NO_BACKSLASH_ESCAPES")] + #expect(MySQLSessionScopePlan.applied(for: .verbatimRead, saved: saved)?.first?.value == "") + } + + @Test("Captured settings follow the names asked, read an empty character set as NULL and keep ON and OFF") + func capturedSettings() { + let rows: [[PluginCellValue]] = [ + ["UNIQUE_CHECKS", "ON"], + ["sql_mode", ""], + ["character_set_results", ""], + ["foreign_key_checks", "OFF"], + ["time_zone", "SYSTEM"], + ] + let captured = MySQLSessionScopePlan.captured(rows: rows, names: MySQLSessionScopePlan.preservedNames) + #expect(captured == [ + PluginSessionSetting(name: "sql_mode", value: ""), + PluginSessionSetting(name: "time_zone", value: "SYSTEM"), + PluginSessionSetting(name: "character_set_results", value: nil), + PluginSessionSetting(name: "foreign_key_checks", value: "OFF"), + PluginSessionSetting(name: "unique_checks", value: "ON"), + ]) + } + + @Test("Closing puts back only what changed, spelling NULL and ON or OFF the way SET takes them") + func restoresOnlyWhatChanged() { + let saved = [ + PluginSessionSetting(name: "sql_mode", value: "STRICT_TRANS_TABLES"), + PluginSessionSetting(name: "time_zone", value: "SYSTEM"), + PluginSessionSetting(name: "character_set_results", value: nil), + PluginSessionSetting(name: "foreign_key_checks", value: "ON"), + ] + let current = [ + PluginSessionSetting(name: "sql_mode", value: "NO_AUTO_VALUE_ON_ZERO"), + PluginSessionSetting(name: "time_zone", value: "SYSTEM"), + PluginSessionSetting(name: "character_set_results", value: "utf8mb4"), + PluginSessionSetting(name: "foreign_key_checks", value: "OFF"), + ] + #expect(MySQLSessionScopePlan.restore(saved: saved, current: current, isTransactionOpen: false).statements == [ + "SET SESSION sql_mode = 'STRICT_TRANS_TABLES'", + "SET SESSION character_set_results = NULL", + "SET SESSION foreign_key_checks = 'ON'", + ]) + #expect(MySQLSessionScopePlan.restore(saved: saved, current: saved, isTransactionOpen: false).statements.isEmpty) + } + + @Test("A saved setting the server no longer reports is still put back") + func restoresUnreportedSetting() { + let saved = [PluginSessionSetting(name: "unique_checks", value: "ON")] + #expect(MySQLSessionScopePlan.restore(saved: saved, current: [], isTransactionOpen: false).statements == [ + "SET SESSION unique_checks = 'ON'", + ]) + } + + @Test("A saved name that is not a setting name is never interpolated") + func refusesForeignName() { + let saved = [PluginSessionSetting(name: "sql_mode = ''; DROP TABLE t; --", value: "x")] + #expect(MySQLSessionScopePlan.restore(saved: saved, current: [], isTransactionOpen: false).statements.isEmpty) + } + + @Test("A held session scope blocks replay and the idle release until it closes") + func heldScopeBlocksReplayAndRelease() { + var footprint = MySQLSessionFootprint() + footprint.holdSessionScope() + #expect(!footprint.isClean) + #expect(footprint.blockingReason != nil) + #expect(!mysqlMayReplay("SELECT 1", on: footprint)) + + footprint.releaseSessionScope(isRestored: true) + #expect(footprint.isClean) + #expect(footprint.blockingReason == nil) + #expect(mysqlMayReplay("SELECT 1", on: footprint)) + } + + @Test("Every held scope has to close before the session is clean, and an extra close changes nothing") + func nestedScopes() { + var footprint = MySQLSessionFootprint() + footprint.holdSessionScope() + footprint.holdSessionScope() + footprint.releaseSessionScope(isRestored: true) + #expect(!footprint.isClean) + + footprint.releaseSessionScope(isRestored: true) + footprint.releaseSessionScope(isRestored: true) + #expect(footprint.isClean) + } + + @Test("Settings a scope could not put back leave the session marked as changed") + func unrestoredSettingsStayMarked() { + var held = MySQLSessionFootprint() + held.holdSessionScope() + held.releaseSessionScope(isRestored: false) + #expect(held.heldSessionScopes == 0) + #expect(held.hasSessionSettings) + #expect(held.blockingReason != nil) + #expect(!mysqlMayReplay("SELECT 1", on: held)) + + var preserved = MySQLSessionFootprint() + preserved.noteUnrestoredSessionSettings() + #expect(preserved.hasSessionSettings) + #expect(!mysqlMayReplay("SELECT 1", on: preserved)) + } + + @Test("Databend opens no session scope and records no creation settings") + func databendHasNoScopes() async throws { + let driver = MySQLPluginDriver(config: DriverConnectionConfig( + host: "", port: 0, username: "", password: "", database: "", + additionalFields: ["driverVariant": MySQLServerFlavor.databendVariant] + )) + for purpose in [PluginSessionPurpose.preserve, .verbatimRead, .verbatimWrite] { + #expect(try await driver.openSessionScope(for: purpose) == nil) + } + let routine = PluginRoutineInfo(name: "cleanup", kind: .procedure) + let trigger = PluginTriggerInfo( + name: "audit", table: "orders", timing: "BEFORE", event: "INSERT", statement: "BEGIN END" + ) + #expect(try await driver.fetchRoutineCreationScope(routine) == nil) + #expect(try await driver.fetchTriggerCreationScope(trigger) == nil) + #expect(try await driver.fetchEventCreationScope(PluginEventInfo(name: "nightly")) == nil) + } + + @Test("A creation scope needs a recorded mode, and carries the zone only when there is one") + func creationScopeNeedsRecordedMode() { + #expect(MySQLPluginDriver.creationScope(sqlMode: nil, timeZone: "SYSTEM") == nil) + let routine = MySQLPluginDriver.creationScope(sqlMode: "STRICT_TRANS_TABLES", timeZone: nil) + #expect(routine?.opening == [ + "/*!50003 SET @saved_sql_mode = @@sql_mode */", + "/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES' */", + ]) + #expect(routine?.closing == ["/*!50003 SET sql_mode = @saved_sql_mode */"]) + let event = MySQLPluginDriver.creationScope(sqlMode: "", timeZone: "SYSTEM") + #expect(event?.opening.count == 4) + #expect(event?.closing.first == "/*!50003 SET time_zone = @saved_time_zone */") + let mariaDBRoutine = MySQLPluginDriver.creationScope( + sqlMode: "STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION", + timeZone: nil + ) + #expect(mariaDBRoutine?.opening.last + == "/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' */") + } +} diff --git a/Plugins/MySQLDriverTests/MySQLStatementClassificationTests.swift b/Plugins/MySQLDriverTests/MySQLStatementClassificationTests.swift index 93149ccc16..0b2a6c6b62 100644 --- a/Plugins/MySQLDriverTests/MySQLStatementClassificationTests.swift +++ b/Plugins/MySQLDriverTests/MySQLStatementClassificationTests.swift @@ -187,4 +187,20 @@ struct MySQLReplaySafetyTests { #expect(!mysqlConnectionLossMayReplay(code: code, outlastedSocketTimeout: false), "\(code)") } } + + @Test("A transaction flag set before the server refused a statement is asked about again", arguments: [ + UInt32(1_050), 1_062, 1_213, 1_317, 3_024, 4_025, + ]) + func serverRefusalRefreshesTransactionFlag(_ code: UInt32) { + #expect(mysqlTransactionFlagMayBeStale(afterFailureCode: code, isFlagOpen: true, hasStatusFlags: true)) + } + + @Test("A client error, the driver's own error, a closed flag or a flavor without flags asks nothing") + func otherFailuresLeaveTransactionFlag() { + for code in [UInt32(2_006), 2_013, 2_055, 2_026, 0] { + #expect(!mysqlTransactionFlagMayBeStale(afterFailureCode: code, isFlagOpen: true, hasStatusFlags: true), "\(code)") + } + #expect(!mysqlTransactionFlagMayBeStale(afterFailureCode: 1_050, isFlagOpen: false, hasStatusFlags: true)) + #expect(!mysqlTransactionFlagMayBeStale(afterFailureCode: 1_050, isFlagOpen: true, hasStatusFlags: false)) + } } diff --git a/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift b/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift index 8b541a66c3..ae5a6b0da2 100644 --- a/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift +++ b/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift @@ -253,6 +253,17 @@ final class LibPQDriverCore: @unchecked Sendable { _ = try await execute(query: "SET statement_timeout = \(ms)") } + func executeSessionSetting( + _ statement: PostgreSQLBoundStatement, + requiresIdleSession: Bool = false + ) async throws -> [[PluginCellValue]] { + try await connection().executeSessionSetting( + statement.sql, + parameters: statement.parameters, + requiresIdleSession: requiresIdleSession + ).rows + } + /// A connection that has gone away answers `.unknown` rather than `.idle`: a caller that reads /// "nothing open" opens a transaction of its own, and this is the one answer that must never be /// a guess. diff --git a/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index 11ec16ecee..c05b82abbc 100644 --- a/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -630,6 +630,44 @@ final class LibPQPluginConnection: @unchecked Sendable { private static let scopedReadSavepoint = "SAVEPOINT tablepro_scoped_read" private static let scopedReadRollback = "ROLLBACK TO SAVEPOINT tablepro_scoped_read; RELEASE SAVEPOINT tablepro_scoped_read" + /// Runs one of the driver's own session-setting statements so a failure cannot abort a transaction block the + /// user has open. Inside one, a savepoint holds the statement: rolled back when it fails, released when it + /// succeeds, which keeps what it set in the block. One dispatch, for the reason `executeTransactionScopedRead` + /// gives. In an aborted block the statement goes as it is and the server answers `25P02`. `requiresIdleSession` + /// refuses in the same dispatch, so no transaction can open between the check and the statement. + func executeSessionSetting( + _ statement: String, + parameters: [PluginCellValue], + requiresIdleSession: Bool + ) async throws -> LibPQPluginQueryResult { + let statementToRun = String(statement) + let params = parameters + + return try await pluginDispatchAsync(on: queue) { [self] in + guard !isShuttingDown else { throw LibPQPluginError.notConnected } + let state = transactionStateOnQueue() + if requiresIdleSession, state != .idle { throw PostgreSQLSessionScopeError.sessionNotIdle } + guard state == .inTransaction else { + return try executeParameterizedQuerySync(statementToRun, parameters: params) + } + _ = try executeQuerySync(Self.sessionSettingSavepoint) + let result: LibPQPluginQueryResult + do { + result = try executeParameterizedQuerySync(statementToRun, parameters: params) + } catch { + _ = try? executeQuerySync(Self.sessionSettingRollback) + throw error + } + _ = try executeQuerySync(Self.sessionSettingRelease) + return result + } + } + + private static let sessionSettingSavepoint = "SAVEPOINT tablepro_session_setting" + private static let sessionSettingRelease = "RELEASE SAVEPOINT tablepro_session_setting" + private static let sessionSettingRollback = + "ROLLBACK TO SAVEPOINT tablepro_session_setting; RELEASE SAVEPOINT tablepro_session_setting" + /// Whether the session is inside a transaction block, including one a failed statement has /// aborted, so a statement sent now joins it. Called on the connection's queue only: one `PGconn` /// may not be used from two threads at once, and the lock guards the pointer rather than the call. diff --git a/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+SessionScope.swift b/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+SessionScope.swift new file mode 100644 index 0000000000..2d30cbd3b3 --- /dev/null +++ b/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+SessionScope.swift @@ -0,0 +1,111 @@ +// +// PostgreSQLPluginDriver+SessionScope.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import os +import TableProLogRedaction +import TableProPluginKit + +/// On this class rather than `LibPQBackedDriver`, so CockroachDB and Redshift keep PluginKit's nil scopes: this rests +/// on PostgreSQL's `pg_settings`, `set_config` and savepoints, measured on 17.11 only. +internal extension PostgreSQLPluginDriver { + private static let scopeLogger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "PostgreSQLSessionScope") + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + switch purpose { + case .preserve: + guard let saved = await captureSessionSettings(PostgreSQLSessionScopeSQL.preservedNames), + !saved.isEmpty else { return nil } + return PluginSessionScope(purpose: .preserve, saved: saved, applied: []) + case .verbatimRead: + return await openVerbatimReadScope() + case .verbatimWrite: + // What the read pins reads back the same under any settings; nil would make the host drop the read too. + return PluginSessionScope(purpose: .verbatimWrite, saved: [], applied: []) + @unknown default: + return nil + } + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + guard !scope.saved.isEmpty else { return } + try await restoreChangedSessionSettings(scope.saved) + } + + private func openVerbatimReadScope() async -> PluginSessionScope? { + guard PostgreSQLSessionScopeSQL.canOpenReadScope(in: await core.sessionTransactionState()) else { + Self.scopeLogger.info("Read scope not opened: the session has a transaction open") + return nil + } + let requested = PostgreSQLSessionScopeSQL.verbatimReadSettings + let names = PostgreSQLSessionScopeSQL.verbatimReadNames + guard let captured = await captureSessionSettings(names), + let saved = PostgreSQLSessionScopeSQL.completeSettings(captured, names: names), + let statement = PostgreSQLSessionScopeSQL.setStatement(requested) + else { + Self.scopeLogger.info("Read scope not opened: the server lacks one of its settings or the capture failed") + return nil + } + do { + let rows = try await core.executeSessionSetting(statement, requiresIdleSession: true) + let applied = PostgreSQLSessionScopeSQL.appliedSettings( + requested, + reportedRow: rows.first?.map(\.asText) ?? [] + ) + return PluginSessionScope(purpose: .verbatimRead, saved: saved, applied: applied) + } catch PostgreSQLSessionScopeError.sessionNotIdle { + Self.scopeLogger.info("Read scope not opened: a transaction opened before its settings were applied") + return nil + } catch { + // One statement outside a transaction: a refusal rolls back all of it, so nothing needs putting back. + Self.scopeLogger.error( + "Read scope not opened, its settings were refused: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + return nil + } + } + + private func captureSessionSettings(_ names: [String]) async -> [PluginSessionSetting]? { + guard PostgreSQLSessionScopeSQL.canRestore(in: await core.sessionTransactionState()) else { + Self.scopeLogger.info("Session scope not opened: the session is in an aborted transaction") + return nil + } + do { + let rows = try await core.executeSessionSetting(PostgreSQLSessionScopeSQL.captureStatement(names: names)) + return PostgreSQLSessionScopeSQL.settings(fromRows: rows.map { $0.map(\.asText) }, names: names) + } catch { + Self.scopeLogger.error( + "Session settings capture failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + return nil + } + } + + /// Nothing is sent in an aborted transaction: the server would refuse it, and the rollback the session still + /// needs reverts only what the transaction itself set, so the caller is told rather than left to assume. + private func restoreChangedSessionSettings(_ saved: [PluginSessionSetting]) async throws { + guard PostgreSQLSessionScopeSQL.canRestore(in: await core.sessionTransactionState()) else { + Self.scopeLogger.error("Session settings not restored: the session is in an aborted transaction") + throw PostgreSQLSessionScopeError.abortedTransaction + } + let names = saved.map(\.name) + let rows = try await core.executeSessionSetting(PostgreSQLSessionScopeSQL.captureStatement(names: names)) + let current = PostgreSQLSessionScopeSQL.settings(fromRows: rows.map { $0.map(\.asText) }, names: names) + + var refused: [String] = [] + for setting in PostgreSQLSessionScopeSQL.changedSettings(saved: saved, current: current) { + guard let statement = PostgreSQLSessionScopeSQL.setStatement([setting]) else { continue } + do { + _ = try await core.executeSessionSetting(statement) + } catch { + refused.append(setting.name) + Self.scopeLogger.error( + "Restoring \(setting.name, privacy: .public) failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + } + } + guard refused.isEmpty else { throw PostgreSQLSessionScopeError.restoreRefused(refused) } + } +} diff --git a/Plugins/PostgreSQLDriverPlugin/PostgreSQLSessionScopeSQL.swift b/Plugins/PostgreSQLDriverPlugin/PostgreSQLSessionScopeSQL.swift new file mode 100644 index 0000000000..65d30ab1f2 --- /dev/null +++ b/Plugins/PostgreSQLDriverPlugin/PostgreSQLSessionScopeSQL.swift @@ -0,0 +1,134 @@ +// +// PostgreSQLSessionScopeSQL.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import TableProPluginKit + +internal struct PostgreSQLBoundStatement: Equatable, Sendable { + let sql: String + let parameters: [PluginCellValue] +} + +internal enum PostgreSQLSessionScopeError: Error, Equatable { + case abortedTransaction + case sessionNotIdle + case restoreRefused([String]) +} + +internal enum PostgreSQLSessionScopeSQL { + /// Every `SET` a pg_dump 17.11 plain file runs and never puts back, the settings a value's text form depends on, + /// and `session_replication_role`, which the import's foreign key switch sets. + static let preservedNames = [ + "statement_timeout", + "lock_timeout", + "idle_in_transaction_session_timeout", + "transaction_timeout", + "client_encoding", + "standard_conforming_strings", + "search_path", + "check_function_bodies", + "xmloption", + "client_min_messages", + "row_security", + "default_tablespace", + "default_table_access_method", + "DateStyle", + "IntervalStyle", + "extra_float_digits", + "TimeZone", + "session_replication_role" + ] + + /// What pg_dump's own reading session sets. `ISO` alone changes DateStyle's output part and keeps the session's + /// field order, as pg_dump's `SET DATESTYLE = ISO` does. + static let verbatimReadSettings = [ + PluginSessionSetting(name: "DateStyle", value: "ISO"), + PluginSessionSetting(name: "IntervalStyle", value: "postgres"), + PluginSessionSetting(name: "extra_float_digits", value: "3") + ] + + static var verbatimReadNames: [String] { + verbatimReadSettings.map(\.name) + } + + /// `pg_settings` spells `DateStyle`, `IntervalStyle` and `TimeZone` in mixed case, so the match ignores case. + static func captureStatement(names: [String]) -> PostgreSQLBoundStatement { + let filter = PostgreSQLObjectQueries.membership("pg_catalog.lower(name)", in: names.map { $0.lowercased() }) + return PostgreSQLBoundStatement( + sql: "SELECT name, setting FROM pg_catalog.pg_settings WHERE \(filter)", + parameters: [] + ) + } + + static func settings(fromRows rows: [[String?]], names: [String]) -> [PluginSessionSetting] { + var reported: [String: String?] = [:] + for row in rows { + guard let first = row.first, let name = first else { continue } + let value: String? = row.count > 1 ? row[1] : nil + reported.updateValue(value, forKey: name.lowercased()) + } + return names.compactMap { name in + guard let value = reported[name.lowercased()] else { return nil } + return PluginSessionSetting(name: name, value: value) + } + } + + /// Nil unless the server reported every one of `names`: a purpose applies all its settings or none. + static func completeSettings(_ settings: [PluginSessionSetting], names: [String]) -> [PluginSessionSetting]? { + let reported = Set(settings.map { $0.name.lowercased() }) + return names.allSatisfy { reported.contains($0.lowercased()) } ? settings : nil + } + + /// One statement, so the server applies every setting or none: a refused one rolls back the statement's + /// implicit transaction, or the savepoint the connection holds around it inside a block. A nil value binds SQL + /// `NULL`, which `set_config` takes as a reset. + static func setStatement(_ settings: [PluginSessionSetting]) -> PostgreSQLBoundStatement? { + guard !settings.isEmpty else { return nil } + var calls: [String] = [] + var parameters: [PluginCellValue] = [] + for (index, setting) in settings.enumerated() { + calls.append("pg_catalog.set_config($\(index * 2 + 1), $\(index * 2 + 2), false)") + parameters.append(.text(setting.name)) + parameters.append(setting.value.map { PluginCellValue.text($0) } ?? .null) + } + return PostgreSQLBoundStatement(sql: "SELECT \(calls.joined(separator: ", "))", parameters: parameters) + } + + /// What each setting holds after `setStatement`, from the row it returned: `ISO` reads back as `ISO, MDY`. + static func appliedSettings(_ requested: [PluginSessionSetting], reportedRow: [String?]) -> [PluginSessionSetting] { + requested.enumerated().map { index, setting in + let reported = index < reportedRow.count ? reportedRow[index] : nil + return PluginSessionSetting(name: setting.name, value: reported ?? setting.value) + } + } + + /// The saved settings the session no longer holds. Setting `session_replication_role` takes a privilege even + /// when the value does not change, so an unchanged setting is never written. + static func changedSettings( + saved: [PluginSessionSetting], + current: [PluginSessionSetting] + ) -> [PluginSessionSetting] { + var currentValues: [String: String?] = [:] + for setting in current { + currentValues.updateValue(setting.value, forKey: setting.name.lowercased()) + } + return saved.filter { setting in + guard let value = currentValues[setting.name.lowercased()] else { return false } + return value != setting.value + } + } + + /// In an aborted transaction the server refuses every statement but a rollback, and that rollback reverts only + /// what the transaction itself set. + static func canRestore(in state: PluginSessionTransactionState) -> Bool { + state != .abortedTransaction + } + + /// Inside a transaction the capture can read a `SET LOCAL` value, and the session-level restore of it would then + /// outlive the commit in place of the session's own value, so a read scope opens on an idle session only. + static func canOpenReadScope(in state: PluginSessionTransactionState) -> Bool { + state == .idle + } +} diff --git a/Plugins/PostgreSQLDriverTests/PostgreSQLSessionScopeSQLTests.swift b/Plugins/PostgreSQLDriverTests/PostgreSQLSessionScopeSQLTests.swift new file mode 100644 index 0000000000..1829120d69 --- /dev/null +++ b/Plugins/PostgreSQLDriverTests/PostgreSQLSessionScopeSQLTests.swift @@ -0,0 +1,178 @@ +// +// PostgreSQLSessionScopeSQLTests.swift +// TableProTests +// + +import TableProPluginKit +import Testing + +struct PostgreSQLSessionScopeSQLTests { + private static func setting(_ name: String, _ value: String?) -> PluginSessionSetting { + PluginSessionSetting(name: name, value: value) + } + + @Test("Preserve covers every SET a pg_dump 17.11 plain file leaves on the session") + func preserveCoversPgDumpPrologue() { + let pgDumpSets = [ + "statement_timeout", "lock_timeout", "idle_in_transaction_session_timeout", "transaction_timeout", + "client_encoding", "standard_conforming_strings", "search_path", "check_function_bodies", "xmloption", + "client_min_messages", "row_security", "default_tablespace", "default_table_access_method" + ] + let preserved = Set(PostgreSQLSessionScopeSQL.preservedNames.map { $0.lowercased() }) + for name in pgDumpSets { + #expect(preserved.contains(name), "\(name)") + } + } + + @Test("Preserve also covers the settings a value's text depends on and the foreign key switch") + func preserveCoversTextAndReplicationRole() { + let preserved = Set(PostgreSQLSessionScopeSQL.preservedNames.map { $0.lowercased() }) + for name in ["datestyle", "intervalstyle", "extra_float_digits", "timezone", "session_replication_role"] { + #expect(preserved.contains(name), "\(name)") + } + } + + @Test("The read scope sets what pg_dump's reading session sets") + func verbatimReadMatchesPgDump() { + #expect(PostgreSQLSessionScopeSQL.verbatimReadSettings == [ + Self.setting("DateStyle", "ISO"), + Self.setting("IntervalStyle", "postgres"), + Self.setting("extra_float_digits", "3") + ]) + } + + @Test("The capture matches names ignoring case, since pg_settings spells DateStyle and TimeZone in mixed case") + func captureIgnoresCase() { + let statement = PostgreSQLSessionScopeSQL.captureStatement(names: ["DateStyle", "search_path", "TimeZone"]) + #expect(statement.sql == """ + SELECT name, setting FROM pg_catalog.pg_settings \ + WHERE pg_catalog.lower(name) IN ('datestyle', 'search_path', 'timezone') + """) + #expect(statement.parameters.isEmpty) + } + + @Test("Captured settings come back in the order asked and spelled as asked, whatever the server sent") + func settingsFollowTheAskedNames() { + let rows: [[String?]] = [["TimeZone", "UTC"], ["unrelated", "x"], ["DateStyle", "SQL, DMY"]] + let settings = PostgreSQLSessionScopeSQL.settings( + fromRows: rows, + names: ["datestyle", "intervalstyle", "timezone"] + ) + #expect(settings == [Self.setting("datestyle", "SQL, DMY"), Self.setting("timezone", "UTC")]) + } + + @Test("A reported NULL is kept as a nil value, not dropped") + func reportedNullIsKept() { + let settings = PostgreSQLSessionScopeSQL.settings(fromRows: [["search_path", nil]], names: ["search_path"]) + #expect(settings == [Self.setting("search_path", nil)]) + } + + @Test("A read scope applies only when the server has every one of its settings") + func readScopeNeedsEverySetting() { + let names = PostgreSQLSessionScopeSQL.verbatimReadNames + let all = [Self.setting("DateStyle", "ISO, MDY"), Self.setting("IntervalStyle", "postgres"), + Self.setting("extra_float_digits", "1")] + #expect(PostgreSQLSessionScopeSQL.completeSettings(all, names: names) == all) + #expect(PostgreSQLSessionScopeSQL.completeSettings(Array(all.prefix(2)), names: names) == nil) + } + + @Test("Every setting goes in one statement, its names and values bound rather than written into the text") + func setStatementBindsEverything() throws { + let statement = try #require(PostgreSQLSessionScopeSQL.setStatement([ + Self.setting("search_path", "\"$user\", public"), + Self.setting("default_tablespace", "") + ])) + #expect(statement.sql == "SELECT pg_catalog.set_config($1, $2, false), pg_catalog.set_config($3, $4, false)") + #expect(statement.parameters == [ + .text("search_path"), .text("\"$user\", public"), .text("default_tablespace"), .text("") + ]) + } + + @Test("A quote or backslash in a value never reaches the statement text") + func hostileValueStaysBound() throws { + let value = #"it's \'; RESET ALL; --"# + let statement = try #require(PostgreSQLSessionScopeSQL.setStatement([Self.setting("search_path", value)])) + #expect(statement.sql == "SELECT pg_catalog.set_config($1, $2, false)") + #expect(statement.parameters == [.text("search_path"), .text(value)]) + } + + @Test("A nil value binds SQL NULL, which set_config takes as a reset") + func nilValueBindsNull() throws { + let statement = try #require(PostgreSQLSessionScopeSQL.setStatement([Self.setting("extra_float_digits", nil)])) + #expect(statement.parameters == [.text("extra_float_digits"), .null]) + } + + @Test("No settings, no statement") + func emptySetIsNil() { + #expect(PostgreSQLSessionScopeSQL.setStatement([]) == nil) + } + + @Test("The applied values are what set_config returned, falling back to what was asked") + func appliedValuesComeFromTheServer() { + let applied = PostgreSQLSessionScopeSQL.appliedSettings( + PostgreSQLSessionScopeSQL.verbatimReadSettings, + reportedRow: ["ISO, DMY", "postgres"] + ) + #expect(applied == [ + Self.setting("DateStyle", "ISO, DMY"), + Self.setting("IntervalStyle", "postgres"), + Self.setting("extra_float_digits", "3") + ]) + } + + @Test("Only saved settings the session no longer holds are restored") + func onlyChangedSettingsRestore() { + let saved = [ + Self.setting("search_path", "\"$user\", public"), + Self.setting("session_replication_role", "origin"), + Self.setting("statement_timeout", "30000"), + Self.setting("default_tablespace", "") + ] + let current = [ + Self.setting("search_path", ""), + Self.setting("session_replication_role", "origin"), + Self.setting("statement_timeout", "0"), + Self.setting("default_tablespace", "") + ] + #expect(PostgreSQLSessionScopeSQL.changedSettings(saved: saved, current: current) == [ + Self.setting("search_path", "\"$user\", public"), + Self.setting("statement_timeout", "30000") + ]) + } + + @Test("Names compare ignoring case, and a setting the re-read did not report is left alone") + func changedSettingsMatchNamesAndSkipMissing() { + let saved = [Self.setting("DateStyle", "SQL, DMY"), Self.setting("transaction_timeout", "0")] + let current = [Self.setting("datestyle", "ISO, DMY")] + #expect(PostgreSQLSessionScopeSQL.changedSettings(saved: saved, current: current) == [ + Self.setting("DateStyle", "SQL, DMY") + ]) + } + + @Test("Nothing is restored inside an aborted transaction") + func noRestoreInAbortedTransaction() { + #expect(!PostgreSQLSessionScopeSQL.canRestore(in: .abortedTransaction)) + } + + @Test("A restore runs with no transaction, inside an open one, or when the state is not known") + func restoreRunsOtherwise() { + #expect(PostgreSQLSessionScopeSQL.canRestore(in: .idle)) + #expect(PostgreSQLSessionScopeSQL.canRestore(in: .inTransaction)) + #expect(PostgreSQLSessionScopeSQL.canRestore(in: .unknown)) + } + + @Test("A read scope opens on an idle session") + func readScopeOpensWhenIdle() { + #expect(PostgreSQLSessionScopeSQL.canOpenReadScope(in: .idle)) + } + + /// Measured on PostgreSQL 17.11: after `SET LOCAL extra_float_digits = 0` in a transaction, a capture reads 0, + /// and a session-level set to 3 then back to 0 leaves 0 after `COMMIT` in place of the session's 1. + @Test( + "A read scope does not open inside a transaction, open or aborted, or when the state is not known", + arguments: [PluginSessionTransactionState.inTransaction, .abortedTransaction, .unknown, .holdsSessionLocks] + ) + func readScopeRefusesOtherwise(state: PluginSessionTransactionState) { + #expect(!PostgreSQLSessionScopeSQL.canOpenReadScope(in: state)) + } +} diff --git a/Plugins/SQLExportPlugin/SQLExportEncodingDeclaration.swift b/Plugins/SQLExportPlugin/SQLExportEncodingDeclaration.swift deleted file mode 100644 index ec0e4acaa9..0000000000 --- a/Plugins/SQLExportPlugin/SQLExportEncodingDeclaration.swift +++ /dev/null @@ -1,57 +0,0 @@ -// -// SQLExportEncodingDeclaration.swift -// SQLExportPlugin -// - -import Foundation -import TableProPluginKit - -internal struct SQLExportEncodingDeclaration: Equatable { - static let empty = SQLExportEncodingDeclaration(prologue: "", epilogue: "") - - let prologue: String - let epilogue: String - - private static let typesAcceptingSetClientEncoding: Set = [ - "PostgreSQL", "Greenplum", "AlloyDB", "Citus", "CockroachDB", "PGlite" - ] - - static func forDatabaseType(_ databaseTypeId: String) -> SQLExportEncodingDeclaration { - switch SqlDialect.from(databaseTypeId: databaseTypeId) { - case .mysql: - return mysql - case .postgres where typesAcceptingSetClientEncoding.contains(databaseTypeId): - return postgres - default: - return .empty - } - } - - private static let mysql = SQLExportEncodingDeclaration( - prologue: """ - /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; - /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; - /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; - /*!40101 SET NAMES utf8 */; - /*!50503 SET NAMES utf8mb4 */; - - - """, - epilogue: """ - - /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; - /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; - /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; - - """ - ) - - private static let postgres = SQLExportEncodingDeclaration( - prologue: """ - SET client_encoding = 'UTF8'; - - - """, - epilogue: "" - ) -} diff --git a/Plugins/SQLExportPlugin/SQLExportFileWriter.swift b/Plugins/SQLExportPlugin/SQLExportFileWriter.swift index 78679ad1b4..4d1ba322f5 100644 --- a/Plugins/SQLExportPlugin/SQLExportFileWriter.swift +++ b/Plugins/SQLExportPlugin/SQLExportFileWriter.swift @@ -34,7 +34,7 @@ internal final class SQLExportFileWriter { private let destination: URL private let splitSizeBytes: Int - private let encodingDeclaration: SQLExportEncodingDeclaration + private let sessionDeclaration: SQLExportSessionDeclaration private let publish: (URL, URL) throws -> Void private var handle: FileHandle @@ -59,12 +59,12 @@ internal final class SQLExportFileWriter { internal init( destination: URL, splitSizeMegabytes: Int, - encodingDeclaration: SQLExportEncodingDeclaration = .empty, + sessionDeclaration: SQLExportSessionDeclaration = .empty, publish: @escaping (URL, URL) throws -> Void = SQLExportFileWriter.publishAtomically ) throws { self.destination = destination self.splitSizeBytes = max(0, splitSizeMegabytes) * 1_024 * 1_024 - self.encodingDeclaration = encodingDeclaration + self.sessionDeclaration = sessionDeclaration self.publish = publish let (handle, tempURL) = try PluginExportUtilities.beginAtomicWrite(for: destination) self.handle = handle @@ -173,10 +173,10 @@ internal final class SQLExportFileWriter { try beginPart() } - /// Declares the encoding and re-opens every scope that is still open, so a part carries the - /// session state its own statements were written under. + /// Declares the session settings and re-opens every scope that is still open, so a part carries + /// the session state its own statements were written under. private func beginPart() throws { - try writeRaw(encodingDeclaration.prologue) + try writeRaw(sessionDeclaration.prologue) for scope in openScopes { try writeRaw(scope.opener) } @@ -190,13 +190,13 @@ internal final class SQLExportFileWriter { for scope in openScopes.reversed() { try writeRaw(scope.closer) } - try writeRaw(encodingDeclaration.epilogue) + try writeRaw(sessionDeclaration.epilogue) } /// What the part still owes: the closer of every open scope and the closing declaration. Every /// write counts it, because a part that fits only until its own tail is added has not fitted. private var partTailBytes: Int { - openScopes.reduce(encodingDeclaration.epilogue.utf8.count) { $0 + $1.closer.utf8.count } + openScopes.reduce(sessionDeclaration.epilogue.utf8.count) { $0 + $1.closer.utf8.count } } private func writeRaw(_ text: String) throws { diff --git a/Plugins/SQLExportPlugin/SQLExportPlugin.swift b/Plugins/SQLExportPlugin/SQLExportPlugin.swift index bb312dbbbc..cdf8f85f5d 100644 --- a/Plugins/SQLExportPlugin/SQLExportPlugin.swift +++ b/Plugins/SQLExportPlugin/SQLExportPlugin.swift @@ -161,11 +161,21 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi metadataWarnings.append(String(localized: "A compressed export is written as one file, so the split size was not applied.")) } - let writer = try SQLExportFileWriter( - destination: actualDestination, - splitSizeMegabytes: splitSize, - encodingDeclaration: .forDatabaseType(dataSource.databaseTypeId) - ) + var readScope = await SQLExportReadScope.open(on: dataSource, for: tables) + let writer: SQLExportFileWriter + do { + writer = try SQLExportFileWriter( + destination: actualDestination, + splitSizeMegabytes: splitSize, + sessionDeclaration: .forDatabaseType( + dataSource.databaseTypeId, + readTimeZone: readScope.readTimeZone, + readsWithANSIQuotes: readScope.readsWithANSIQuotes) + ) + } catch { + await readScope.close(on: dataSource) + throw error + } var committed = false defer { if !committed { writer.rollback() } @@ -224,6 +234,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi if let snapshot { await snapshot.end(on: dataSource) } + await readScope.close(on: dataSource) /// The publication point. Every phase above can suspend on a driver call, and the /// grant phase makes no cancellation check of its own, so a Stop arriving in one of /// them used to resume and replace the user's file anyway. Nothing is published past @@ -245,6 +256,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi if let snapshot { await snapshot.end(on: dataSource) } + await readScope.close(on: dataSource) throw error } @@ -676,7 +688,8 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw SQLExportObjectError.emptyDefinition } - try writer.write(dataSource.scriptText(for: ddl) + dataSource.dumpStatementEnd) + let scope = try await creationScope(for: object, dataSource: dataSource, to: writer) + try writer.write(creationScript(ddl: ddl, scope: scope, dataSource: dataSource)) try writer.write("\n") } catch { ddlFailures.append(sanitizedName) @@ -689,6 +702,43 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } } + /// The kinds a server stores the creating session's settings with, such as MySQL's `sql_mode`. + private static let creationScopedKinds: Set = [.routine, .trigger, .event] + + /// An unreadable scope still lets the definition be written, so it is recorded and commented into the file + /// rather than costing the object. + private func creationScope( + for object: PluginExportTable, + dataSource: any PluginExportDataSource, + to writer: SQLExportFileWriter + ) async throws -> PluginStatementScope? { + guard Self.creationScopedKinds.contains(object.kind) else { return nil } + do { + return try await dataSource.fetchCreationScope(object) + } catch { + let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(object.name) + Self.logger.warning("Failed to fetch the creation settings for \(sanitizedName): \(error)") + metadataWarnings.append(String( + format: String(localized: "Could not read the settings %@ was created with, so it restores under the dump's SQL mode."), + object.name)) + let warning = "Warning: failed to fetch the creation settings for \(sanitizedName): \(error)" + try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(warning))\n") + return nil + } + } + + /// One string, so the writer cannot rotate between the settings and the definition they apply to. + private func creationScript( + ddl: String, + scope: PluginStatementScope?, + dataSource: any PluginExportDataSource + ) -> String { + let statements = (scope?.opening ?? []) + [ddl] + (scope?.closing ?? []) + return statements + .map { dataSource.scriptText(for: $0) + dataSource.dumpStatementEnd } + .joined() + } + /// Grants come last, because every object they name has to exist first. private func writeGrantPhase( objects: [PluginExportTable], diff --git a/Plugins/SQLExportPlugin/SQLExportReadScope.swift b/Plugins/SQLExportPlugin/SQLExportReadScope.swift new file mode 100644 index 0000000000..a42419289e --- /dev/null +++ b/Plugins/SQLExportPlugin/SQLExportReadScope.swift @@ -0,0 +1,86 @@ +// +// SQLExportReadScope.swift +// SQLExportPlugin +// + +import Foundation +import os +import TableProPluginKit + +/// The driver pins the read session, such as MySQL's `time_zone`, so the dump's values read back the same on the +/// server that loads it. The export reads on the user's own session, so every exit puts it back. +internal struct SQLExportReadScope { + private static let logger = Logger(subsystem: "com.TablePro", category: "SQLExportReadScope") + + private var scope: PluginSessionScope? + + internal let readTimeZone: String? + internal let readsWithANSIQuotes: Bool + + private init(scope: PluginSessionScope?, sqlMode: String?) { + self.scope = scope + self.readTimeZone = scope?.appliedValue(of: "time_zone") + self.readsWithANSIQuotes = sqlMode? + .split(separator: ",") + .contains { $0.trimmingCharacters(in: .whitespaces).caseInsensitiveCompare("ANSI_QUOTES") == .orderedSame } + ?? false + } + + /// A failed open leaves the session as it was, so the export reads the way it did before scopes existed and + /// the dump declares nothing the read did not pin. + /// + /// A row filter is the user's SQL, written for their own session: under the pinned zone and mode a `NOW()` or a + /// TIMESTAMP literal selects other rows, and a double-quoted identifier becomes a string. An export with one + /// reads as the session is. A column list or a row limit means the same either way. + internal static func open( + on dataSource: any PluginExportDataSource, + for tables: [PluginExportTable] + ) async -> SQLExportReadScope { + if tables.contains(where: { !$0.rowScope.sanitizedFilter.isEmpty }) { + Self.logger.info("Export reads without a read scope, because a row filter runs on the user's session") + } else if let scope = await openPinnedRead(on: dataSource) { + return SQLExportReadScope(scope: scope, sqlMode: scope.appliedValue(of: "sql_mode")) + } + return SQLExportReadScope(scope: nil, sqlMode: await sessionSQLMode(on: dataSource)) + } + + private static func openPinnedRead(on dataSource: any PluginExportDataSource) async -> PluginSessionScope? { + do { + return try await dataSource.openSessionScope(for: .verbatimRead) + } catch { + Self.logger.warning("Failed to open the export read scope: \(error.localizedDescription)") + return nil + } + } + + /// The mode the server spells the DDL under when the read is not pinned. `.preserve` applies nothing and saves the + /// session's own values, so opening and closing it at once reads them. Nil when it cannot tell. + private static func sessionSQLMode(on dataSource: any PluginExportDataSource) async -> String? { + let probe: PluginSessionScope? + do { + probe = try await dataSource.openSessionScope(for: .preserve) + } catch { + Self.logger.warning("Failed to read the export session's settings: \(error.localizedDescription)") + return nil + } + guard let probe else { return nil } + do { + try await dataSource.closeSessionScope(probe) + } catch { + Self.logger.warning("Failed to close the export session probe: \(error.localizedDescription)") + } + return probe.saved.first { $0.name.caseInsensitiveCompare("sql_mode") == .orderedSame }?.value + } + + /// Closes once however many exits reach it. A failed close is logged rather than thrown, for the reason + /// `SQLExportSnapshot.end` gives. + internal mutating func close(on dataSource: any PluginExportDataSource) async { + guard let scope else { return } + self.scope = nil + do { + try await dataSource.closeSessionScope(scope) + } catch { + Self.logger.warning("Failed to close the export read scope: \(error.localizedDescription)") + } + } +} diff --git a/Plugins/SQLExportPlugin/SQLExportSessionDeclaration.swift b/Plugins/SQLExportPlugin/SQLExportSessionDeclaration.swift new file mode 100644 index 0000000000..32c977f5e8 --- /dev/null +++ b/Plugins/SQLExportPlugin/SQLExportSessionDeclaration.swift @@ -0,0 +1,91 @@ +// +// SQLExportSessionDeclaration.swift +// SQLExportPlugin +// + +import Foundation +import TableProPluginKit + +internal struct SQLExportSessionDeclaration: Equatable { + static let empty = SQLExportSessionDeclaration(prologue: "", epilogue: "") + + let prologue: String + let epilogue: String + + private static let typesAcceptingSetClientEncoding: Set = [ + "PostgreSQL", "Greenplum", "AlloyDB", "Citus", "CockroachDB", "PGlite" + ] + + /// `readTimeZone` is the zone the rows were read in, from the export's read scope. Only a MySQL dump declares + /// it, and only when given: declaring a zone the rows were not read in shifts every TIMESTAMP on reload. + /// + /// `readsWithANSIQuotes` means the server spelled the DDL under `ANSI_QUOTES`, with `"` around identifiers, so + /// the dump's mode keeps it or the CREATE does not parse on reload. What the dump spells itself uses backticks and + /// single quotes, which read the same either way. + static func forDatabaseType( + _ databaseTypeId: String, + readTimeZone: String? = nil, + readsWithANSIQuotes: Bool = false + ) -> SQLExportSessionDeclaration { + switch SqlDialect.from(databaseTypeId: databaseTypeId) { + case .mysql: + return mysql( + readTimeZone: readTimeZone.flatMap(plainTimeZone), + sqlMode: readsWithANSIQuotes ? "NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES" : "NO_AUTO_VALUE_ON_ZERO") + case .postgres where typesAcceptingSetClientEncoding.contains(databaseTypeId): + return postgres + default: + return .empty + } + } + + /// mysqldump's own lines. The whole `SQL_MODE` is replaced rather than edited, so the target's strict mode, + /// `NO_ZERO_DATE` and `NO_BACKSLASH_ESCAPES` cannot reject or reread a literal, and `NO_AUTO_VALUE_ON_ZERO` + /// keeps an AUTO_INCREMENT value of 0. + private static func mysql(readTimeZone: String?, sqlMode: String) -> SQLExportSessionDeclaration { + var prologue = [ + "/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;", + "/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;", + "/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;", + "/*!40101 SET NAMES utf8 */;", + "/*!50503 SET NAMES utf8mb4 */;", + ] + var epilogue: [String] = [] + if let readTimeZone { + prologue.append("/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;") + prologue.append("/*!40103 SET TIME_ZONE='\(readTimeZone)' */;") + epilogue.append("/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;") + } + prologue.append("/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;") + prologue.append("/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='\(sqlMode)' */;") + epilogue.append(contentsOf: [ + "/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;", + "/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;", + "/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;", + "/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;", + "/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;", + ]) + return SQLExportSessionDeclaration( + prologue: prologue.joined(separator: "\n") + "\n\n", + epilogue: "\n" + epilogue.joined(separator: "\n") + "\n" + ) + } + + /// The zone is written inside a `/*! */` comment and a string literal, so anything but a plain spelling such as + /// `+00:00`, `SYSTEM` or `Europe/Paris` is dropped: a `*/` ends the comment, a quote ends the literal, and a + /// backslash reads differently under `NO_BACKSLASH_ESCAPES`. 64 is the width of `mysql.time_zone_name.Name`. + private static func plainTimeZone(_ value: String) -> String? { + guard !value.isEmpty, value.count <= 64 else { return nil } + let isPlain = value.allSatisfy { $0.isASCII && ($0.isLetter || $0.isNumber || "/_+-:".contains($0)) } + return isPlain ? value : nil + } + + private static let postgres = SQLExportSessionDeclaration( + prologue: """ + SET client_encoding = 'UTF8'; + + + """, + epilogue: "" + ) +} diff --git a/Plugins/SQLImportPlugin/SQLImportOptionsView.swift b/Plugins/SQLImportPlugin/SQLImportOptionsView.swift index 527775e6fa..129421264e 100644 --- a/Plugins/SQLImportPlugin/SQLImportOptionsView.swift +++ b/Plugins/SQLImportPlugin/SQLImportOptionsView.swift @@ -24,7 +24,9 @@ struct SQLImportOptionsView: View { .disabled(plugin.settings.errorHandling == .skipAndContinue) .help(plugin.settings.errorHandling == .skipAndContinue ? String(localized: "Not available in skip-and-continue mode") - : String(localized: "Execute all statements in a single transaction. If any statement fails, all changes are rolled back.")) + : String( + localized: "Runs every statement in one transaction, so a failure rolls back what the import changed. A database that commits CREATE, ALTER and DROP as they run, such as MySQL, keeps those statements and everything before them." + )) Toggle("Disable foreign key checks", isOn: $plugin.settings.disableForeignKeyChecks) .font(.system(size: 13)) diff --git a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index ed70f1c3aa..37eb622f3e 100644 --- a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -183,6 +183,12 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { func fetchEvents(schema: String?) async throws -> [PluginEventInfo] func fetchEventDDL(_ event: PluginEventInfo) async throws -> String + /// The settings the server recorded for an object when it was created and applies whenever it runs, such as a + /// MySQL routine's `sql_mode`, as statements a script writes around the object's `CREATE`. Nil when there are none. + func fetchRoutineCreationScope(_ routine: PluginRoutineInfo) async throws -> PluginStatementScope? + func fetchTriggerCreationScope(_ trigger: PluginTriggerInfo) async throws -> PluginStatementScope? + func fetchEventCreationScope(_ event: PluginEventInfo) async throws -> PluginStatementScope? + /// Sequences that stand on their own rather than backing a column. `fetchDependentSequences` /// answers the ones a table owns; this answers the rest, which a dump would otherwise drop. func fetchSequences(schema: String?) async throws -> [PluginSequenceInfo] @@ -228,6 +234,14 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { func cancelQuery() throws func applyQueryTimeout(_ seconds: Int) async throws + /// Saves the session settings `purpose` depends on and sets what it needs, all of them or none. Nil when the + /// engine has nothing to hold, and the work then runs under the session as it is. Its statements are driver setup, + /// not the user's. + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? + + /// Puts back the settings `scope` saved that no longer hold their saved value. + func closeSessionScope(_ scope: PluginSessionScope) async throws + /// What the command that hands this connection's held resource back should be called, or nil /// when the driver holds nothing it can give up. /// @@ -720,6 +734,12 @@ public extension PluginDatabaseDriver { return definition } + func fetchRoutineCreationScope(_ routine: PluginRoutineInfo) async throws -> PluginStatementScope? { nil } + + func fetchTriggerCreationScope(_ trigger: PluginTriggerInfo) async throws -> PluginStatementScope? { nil } + + func fetchEventCreationScope(_ event: PluginEventInfo) async throws -> PluginStatementScope? { nil } + func fetchSequences(schema: String?) async throws -> [PluginSequenceInfo] { [] } func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { @@ -844,6 +864,10 @@ public extension PluginDatabaseDriver { func applyQueryTimeout(_ seconds: Int) async throws {} + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { nil } + + func closeSessionScope(_ scope: PluginSessionScope) async throws {} + var releasableResourceCommandTitle: String? { nil } func releaseIdleResource() async throws -> PluginResourceRelease { .nothingToRelease } diff --git a/Plugins/TableProPluginKit/PluginExportDataSource.swift b/Plugins/TableProPluginKit/PluginExportDataSource.swift index 2dc7424985..ba42af17e9 100644 --- a/Plugins/TableProPluginKit/PluginExportDataSource.swift +++ b/Plugins/TableProPluginKit/PluginExportDataSource.swift @@ -65,6 +65,13 @@ public protocol PluginExportDataSource: AnyObject, Sendable { /// The object's rows, narrowed to its `rowScope`. Reading the whole object is what the default /// does, so a format that has not adopted row scope keeps behaving as it did. func streamRows(for object: PluginExportTable) -> AsyncThrowingStream + + /// Mirrors `PluginDatabaseDriver.openSessionScope(for:)` on the session the export reads from. + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? + func closeSessionScope(_ scope: PluginSessionScope) async throws + + /// Mirrors the driver's creation scopes for a routine, trigger or event. Nil for every other kind. + func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? } public extension PluginExportDataSource { @@ -103,4 +110,10 @@ public extension PluginExportDataSource { func streamRows(for object: PluginExportTable) -> AsyncThrowingStream { streamRows(table: object.name, databaseName: object.databaseName) } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { nil } + + func closeSessionScope(_ scope: PluginSessionScope) async throws {} + + func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? { nil } } diff --git a/Plugins/TableProPluginKit/PluginSessionScope.swift b/Plugins/TableProPluginKit/PluginSessionScope.swift new file mode 100644 index 0000000000..de64f78517 --- /dev/null +++ b/Plugins/TableProPluginKit/PluginSessionScope.swift @@ -0,0 +1,64 @@ +// +// PluginSessionScope.swift +// TableProPluginKit +// + +import Foundation + +/// Why a caller holds a session's settings for a stretch of work. Not `@frozen`, so a purpose can be added. +public enum PluginSessionPurpose: Sendable, Equatable { + /// Statements the caller does not write run inside, such as an imported file's `SET`s. Closing puts back the + /// settings the driver saved, the ones dump files change; anything else the statements left stays. + case preserve + + /// Rows and definitions are read for a database of the same engine to take back unchanged: a dump, or the source + /// side of a copy. + case verbatimRead + + /// Rows read under `verbatimRead` are written back: the target side of a copy. A driver pins the same + /// representation for both purposes, so a value read under one is written under the other unchanged. + case verbatimWrite +} + +/// One session setting, named and spelled the way its driver reads and writes it. A nil value is SQL `NULL`. +public struct PluginSessionSetting: Sendable, Equatable { + public let name: String + public let value: String? + + public init(name: String, value: String?) { + self.name = name + self.value = value + } +} + +/// What `openSessionScope(for:)` saved and set on one session. Only the driver that opened it reads `saved`. +/// +/// Not `@frozen`, so it can gain a field; a new field arrives with its own initializer overload. +public struct PluginSessionScope: Sendable, Equatable { + public let purpose: PluginSessionPurpose + public let saved: [PluginSessionSetting] + public let applied: [PluginSessionSetting] + + public init(purpose: PluginSessionPurpose, saved: [PluginSessionSetting], applied: [PluginSessionSetting]) { + self.purpose = purpose + self.saved = saved + self.applied = applied + } + + /// The value this scope set for `name`, matched case-insensitively, or nil when it set nothing by that name. + public func appliedValue(of name: String) -> String? { + applied.first { $0.name.caseInsensitiveCompare(name) == .orderedSame }?.value + } +} + +/// Statements written around another one, such as the `SET`s that recreate a routine under the `sql_mode` the server +/// recorded for it and then put the script's own mode back. +public struct PluginStatementScope: Sendable, Equatable { + public let opening: [String] + public let closing: [String] + + public init(opening: [String], closing: [String]) { + self.opening = opening + self.closing = closing + } +} diff --git a/TablePro/Core/Database/DatabaseDriver.swift b/TablePro/Core/Database/DatabaseDriver.swift index 5f8bf9f3de..8134688c52 100644 --- a/TablePro/Core/Database/DatabaseDriver.swift +++ b/TablePro/Core/Database/DatabaseDriver.swift @@ -56,6 +56,10 @@ protocol DatabaseDriver: AnyObject, Sendable { /// Apply query execution timeout (seconds, 0 = no limit) func applyQueryTimeout(_ seconds: Int) async throws + /// Mirrors `PluginDatabaseDriver.openSessionScope(for:)` and `closeSessionScope(_:)`. + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? + func closeSessionScope(_ scope: PluginSessionScope) async throws + /// What the command that hands this connection's held resource back should be called, or nil /// when the driver holds nothing it can give up. A per-connection answer, not a per-engine one. var releasableResourceCommandTitle: String? { get } @@ -857,6 +861,10 @@ extension DatabaseDriver { // No-op: each plugin's PluginDatabaseDriver implements its own timeout command. // The PluginDriverAdapter bridges this call to the plugin. } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { nil } + + func closeSessionScope(_ scope: PluginSessionScope) async throws {} } /// Which of the app's connections a driver is, so a plugin can name it to the server. diff --git a/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift b/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift index aaa5b42c99..127c8315fd 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift @@ -55,6 +55,9 @@ internal struct ObjectCopyTableStep: Identifiable, Sendable { /// The one statement that copies this table's rows without them leaving the server. Set only /// when both sides are the same connection; nil means the rows are streamed through the app. internal let serverSideInsert: SyncStatement? + /// True when the user narrowed the rows with a `WHERE` of their own, which reads them on the + /// sessions as they are (`VerbatimTransferScopes.filtersRows`). + internal let hasRowFilter: Bool /// Set when this table is in the plan but part of it cannot run, so the sheet can say why /// before the user presses Copy. internal let note: String? @@ -79,6 +82,7 @@ internal struct ObjectCopyTableStep: Identifiable, Sendable { conversionNotes: [CrossEngineConversionNote] = [], coercer: CrossEngineValueCoercer? = nil, serverSideInsert: SyncStatement? = nil, + hasRowFilter: Bool = false, note: String? ) { self.selection = selection @@ -97,6 +101,7 @@ internal struct ObjectCopyTableStep: Identifiable, Sendable { self.conversionNotes = conversionNotes self.coercer = coercer self.serverSideInsert = serverSideInsert + self.hasRowFilter = hasRowFilter self.note = note } @@ -120,17 +125,28 @@ internal struct ObjectCopyDefinitionStep: Identifiable, Sendable { internal let selection: ObjectCopySelection internal let dropStatements: [SyncStatement] internal let createStatements: [SyncStatement] + /// The settings the source recorded for the object, such as a MySQL routine's `sql_mode`, set + /// around its `CREATE` so the target records the same ones rather than its own session's. + internal let creationOpening: [SyncStatement] + internal let creationClosing: [SyncStatement] internal let note: String? internal init( selection: ObjectCopySelection, dropStatements: [SyncStatement], createStatements: [SyncStatement], + creationScope: PluginStatementScope? = nil, note: String? = nil ) { self.selection = selection self.dropStatements = dropStatements self.createStatements = createStatements + let summary = createStatements.first?.summary ?? "" + let wrap = { (sql: String) in + SyncStatement(sql: sql, objectName: selection.qualifiedName, summary: summary) + } + self.creationOpening = (creationScope?.opening ?? []).map(wrap) + self.creationClosing = (creationScope?.closing ?? []).map(wrap) self.note = note } @@ -138,6 +154,10 @@ internal struct ObjectCopyDefinitionStep: Identifiable, Sendable { internal var statements: [SyncStatement] { dropStatements + createStatements } + internal var creationGroup: ObjectCopyStatementGroup { + ObjectCopyStatementGroup(selection, createStatements, opening: creationOpening, closing: creationClosing) + } + /// A trigger fires on the rows the copy is about to write, so installing one before the data /// phase makes the copy trip it: an audit trigger writes a second row for every row copied, /// and a validating one rejects rows the source already holds. @@ -150,10 +170,25 @@ internal struct ObjectCopyDefinitionStep: Identifiable, Sendable { internal struct ObjectCopyStatementGroup: Sendable { internal let selection: ObjectCopySelection internal let statements: [SyncStatement] + /// Run before `statements`, and `closing` after them even when one fails, so a setting the + /// opening changed never outlives the object it was set for. + internal let opening: [SyncStatement] + internal let closing: [SyncStatement] - internal init(_ selection: ObjectCopySelection, _ statements: [SyncStatement]) { + internal init( + _ selection: ObjectCopySelection, + _ statements: [SyncStatement], + opening: [SyncStatement] = [], + closing: [SyncStatement] = [] + ) { self.selection = selection self.statements = statements + self.opening = opening + self.closing = closing + } + + internal var scriptStatements: [SyncStatement] { + statements.isEmpty ? [] : opening + statements + closing } } @@ -185,6 +220,10 @@ internal struct ObjectCopyPlan: Sendable { /// the schema it goes in could not be. internal let schemaStatements: [SyncStatement] internal let skipped: [ObjectCopySkip] + /// True when every source table the DDL was built from was read under the source's read scope, + /// so the run may write that DDL under the target's write scope. A definition read as the + /// session is runs as the session is, the way it did before scopes. + internal let isStructureReadPinned: Bool internal init( request: ObjectCopyRequest, @@ -192,7 +231,8 @@ internal struct ObjectCopyPlan: Sendable { tableSteps: [ObjectCopyTableStep], definitionSteps: [ObjectCopyDefinitionStep], schemaStatements: [SyncStatement] = [], - skipped: [ObjectCopySkip] = [] + skipped: [ObjectCopySkip] = [], + isStructureReadPinned: Bool = false ) { self.request = request self.createsDatabase = createsDatabase @@ -200,6 +240,7 @@ internal struct ObjectCopyPlan: Sendable { self.definitionSteps = definitionSteps self.schemaStatements = schemaStatements self.skipped = skipped + self.isStructureReadPinned = isStructureReadPinned } /// Shown above the script, for the caveats the copy cannot do anything about: a key column the @@ -281,9 +322,15 @@ internal struct ObjectCopyPlan: Sendable { } internal var creationGroups: [ObjectCopyStatementGroup] { + tableCreationGroups + definitionCreationGroups + } + + internal var tableCreationGroups: [ObjectCopyStatementGroup] { tableSteps.map { ObjectCopyStatementGroup($0.selection, $0.sequenceStatements + $0.createStatements) } - + definitionSteps.filter { !$0.runsAfterData } - .map { ObjectCopyStatementGroup($0.selection, $0.createStatements) } + } + + internal var definitionCreationGroups: [ObjectCopyStatementGroup] { + definitionSteps.filter { !$0.runsAfterData }.map(\.creationGroup) } /// Emptying the tables a data-only replace appends to, children first so a foreign key holds. @@ -314,13 +361,12 @@ internal struct ObjectCopyPlan: Sendable { /// materialized view is filled at the moment it is created, so one built over an empty table /// stays empty. internal var afterDataGroups: [ObjectCopyStatementGroup] { - definitionSteps.filter(\.runsAfterData) - .map { ObjectCopyStatementGroup($0.selection, $0.createStatements) } + definitionSteps.filter(\.runsAfterData).map(\.creationGroup) } internal var cleanupStatements: [SyncStatement] { cleanupGroups.flatMap(\.statements) } - internal var creationStatements: [SyncStatement] { creationGroups.flatMap(\.statements) } - internal var afterDataStatements: [SyncStatement] { afterDataGroups.flatMap(\.statements) } + internal var creationStatements: [SyncStatement] { creationGroups.flatMap(\.scriptStatements) } + internal var afterDataStatements: [SyncStatement] { afterDataGroups.flatMap(\.scriptStatements) } internal var dataSteps: [ObjectCopyTableStep] { tableSteps.filter(\.copiesData) diff --git a/TablePro/Core/ObjectCopy/ObjectCopyPlanner+SourceDefinitions.swift b/TablePro/Core/ObjectCopy/ObjectCopyPlanner+SourceDefinitions.swift index 565d42065d..2a6879073b 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyPlanner+SourceDefinitions.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyPlanner+SourceDefinitions.swift @@ -4,6 +4,7 @@ // import Foundation +import os import TableProPluginKit internal enum ObjectCopyDefinitionOutcome: Equatable, Sendable { @@ -25,7 +26,16 @@ internal enum ObjectCopyDefinitionBuild: Sendable { case refused(String) } +/// The settings each routine and trigger was created under on the source, by selection id. An id in +/// `unreadable` is created under the target's own settings, as before these were read. +internal struct ObjectCopyCreationScopes: Sendable { + internal var scopes: [String: PluginStatementScope] = [:] + internal var unreadable: Set = [] +} + internal extension ObjectCopyPlanner { + nonisolated private static let creationLogger = Logger(subsystem: "com.TablePro", category: "ObjectCopyPlanner") + nonisolated static func definitionBuild( for input: ObjectCopyDefinitionInput, using builder: SourceObjectSyncBuilder @@ -129,6 +139,138 @@ internal extension ObjectCopyPlanner { return reads } + /// The source tables a same-engine copy builds its `CREATE TABLE` from, read under the source's + /// read scope inside the caller's lease: MariaDB renders a TIMESTAMP default in the session's + /// zone, so the DDL is only right when it is written in the zone it was read in. `isPinned` is + /// false when the driver has no scope, and the DDL then runs as the session is. + nonisolated static func pinnedTableReads( + names: Set, + schema: String?, + databaseType: DatabaseType, + using plugin: any PluginDatabaseDriver + ) async throws -> (reads: [TableStructureRead], isPinned: Bool) { + let wanted = Set(names.map { $0.lowercased() }) + let scopes = await VerbatimTransferScopes.open(reading: SessionScopeHandle(plugin)) + do { + let tables = try await plugin.fetchTables(schema: schema).filter { wanted.contains($0.name.lowercased()) } + let reads = try await CompareMetadataService.read( + tables: tables, + schema: schema, + profile: .structure, + narrowed: true, + databaseType: databaseType, + indexedKinds: SourceObjectIndexes.carriedKinds(on: databaseType), + using: plugin + ) + await scopes?.close() + return (reads, scopes != nil) + } catch { + await scopes?.close() + throw error + } + } + + nonisolated static func creationScopes( + for selections: [ObjectCopySelection], + triggerTables: [String], + schema: String?, + using plugin: any PluginDatabaseDriver + ) async throws -> ObjectCopyCreationScopes { + var result = ObjectCopyCreationScopes() + + let routines = selections.filter { $0.kind == .procedure || $0.kind == .function } + if !routines.isEmpty { + let listed = try await creationListing(of: routines, into: &result) { + try await plugin.fetchRoutines(schema: schema) + } + for selection in routines { + let routine = listed.first { info in + selection.kind == (info.kind == .procedure ? .procedure : .function) + && selection.name.lowercased() == info.name.lowercased() + && (selection.signature ?? "") == (info.argumentSignature ?? "") + } + try await record(selection, into: &result) { + guard let routine else { throw PluginObjectSourceError.notFound(selection.name) } + return try await plugin.fetchRoutineCreationScope(routine) + } + } + } + + let triggers = selections.filter { $0.kind == .trigger } + if !triggers.isEmpty { + let owners = triggers.compactMap(\.owner) + let tables = owners.count == triggers.count ? Array(Set(owners)) : triggerTables + let listed = try await creationListing(of: triggers, into: &result) { + try await listTriggers(tables: tables, schema: schema, using: plugin) + } + for selection in triggers { + let trigger = listed.first { listing in + selection.name.lowercased() == listing.trigger.name.lowercased() + && (selection.owner.map { + $0.lowercased() == (listing.trigger.table ?? listing.owningTable ?? "").lowercased() + } ?? true) + } + try await record(selection, into: &result) { + guard let trigger else { throw PluginObjectSourceError.notFound(selection.name) } + return try await plugin.fetchTriggerCreationScope(trigger.trigger) + } + } + } + return result + } + + nonisolated private static func listTriggers( + tables: [String], + schema: String?, + using plugin: any PluginDatabaseDriver + ) async throws -> [(trigger: PluginTriggerInfo, owningTable: String?)] { + guard !plugin.providesBulkTriggerFetch else { + return try await plugin.fetchAllTriggers(schema: schema).map { (trigger: $0, owningTable: nil) } + } + var listed: [(trigger: PluginTriggerInfo, owningTable: String?)] = [] + for table in tables { + try Task.checkCancellation() + listed += try await plugin.fetchTriggers(table: table, schema: schema).map { (trigger: $0, owningTable: table) } + } + return listed + } + + /// A listing that fails leaves every object it would have matched unreadable. + nonisolated private static func creationListing( + of selections: [ObjectCopySelection], + into result: inout ObjectCopyCreationScopes, + _ list: () async throws -> [Listed] + ) async throws -> [Listed] { + do { + return try await list() + } catch { + try rethrowCancellation(error) + result.unreadable.formUnion(selections.map(\.id)) + return [] + } + } + + nonisolated private static func record( + _ selection: ObjectCopySelection, + into result: inout ObjectCopyCreationScopes, + _ fetch: () async throws -> PluginStatementScope? + ) async throws { + guard !result.unreadable.contains(selection.id) else { return } + do { + result.scopes[selection.id] = try await fetch() + } catch { + try rethrowCancellation(error) + creationLogger.warning( + "Creation settings read failed for \(selection.kind.rawValue, privacy: .public): \(error.publicLogShape, privacy: .public)" + ) + result.unreadable.insert(selection.id) + } + } + + nonisolated private static func rethrowCancellation(_ error: Error) throws { + guard !(error is CancellationError), !Task.isCancelled else { throw CancellationError() } + } + nonisolated private static func listingFailure( _ error: Error, for selections: [ObjectCopySelection] diff --git a/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift b/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift index 00b535ad1d..86a6cd2252 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift @@ -55,6 +55,7 @@ internal struct ObjectCopyPlanner { /// Every scope is read and drafted before any index is named, because a table a later scope /// replaces frees its index names for an earlier one: every drop runs before any create. var passes: [ScopePass] = [] + var isStructureReadPinned = true for scope in Self.scopes(of: request) { let names = Set(scope.objects.map(\.name)) @@ -64,9 +65,10 @@ internal struct ObjectCopyPlanner { targetNamespaces.append(namespace) } - let sourceReads = try await metadata.tableReads( - for: sourceEndpoint, connection: connections.source, includeViews: true, names: names + let (sourceReads, isPinned) = try await sourceTableReads( + request, endpoint: sourceEndpoint, connection: connections.source, names: names ) + isStructureReadPinned = isStructureReadPinned && isPinned let targetReads = try await existingTargetReads( request, endpoint: targetEndpoint, connection: connections.target, names: names ) @@ -149,10 +151,37 @@ internal struct ObjectCopyPlanner { tableSteps: tableSteps, definitionSteps: definitionSteps, schemaStatements: try await buildSchemaStatements(request, namespaces: targetNamespaces), - skipped: skipped + skipped: skipped, + isStructureReadPinned: isStructureReadPinned ) } + private func sourceTableReads( + _ request: ObjectCopyRequest, + endpoint: DatabaseEndpoint, + connection: DatabaseConnection, + names: Set + ) async throws -> (reads: [TableStructureRead], isPinned: Bool) { + guard request.content.includesStructure, + VerbatimTransferScopes.isSameEngine( + source: request.source.databaseType, target: request.target.databaseType + ) else { + let reads = try await metadata.tableReads( + for: endpoint, connection: connection, includeViews: true, names: names + ) + return (reads, false) + } + try await manager.ensureConnected(connection) + let schema = endpoint.schema + let databaseType = endpoint.databaseType + return try await manager.withMetadataDriver(scope: endpoint.scope) { driver in + guard let plugin = CompareMetadataService.pluginDriver(from: driver) else { return ([], false) } + return try await Self.pinnedTableReads( + names: names, schema: schema, databaseType: databaseType, using: plugin + ) + } + } + /// The schemas a duplicated database needs before its first `CREATE TABLE` names one. /// /// `CREATE DATABASE` gives the new database whatever schema its engine gives it, and a @@ -470,6 +499,7 @@ internal struct ObjectCopyPlanner { conversionNotes: draft.conversionNotes, coercer: draft.coercer, serverSideInsert: draft.copiesData ? serverSide[draft.selection.id] : nil, + hasRowFilter: VerbatimTransferScopes.filtersRows(draft.rowScope), note: draft.note ) } @@ -871,7 +901,7 @@ internal struct ObjectCopyPlanner { return [] } - let definitionReads = try await sourceDefinitions( + let (definitionReads, creationScopes) = try await sourceDefinitions( sourceEndpoint: sourceEndpoint, selections: selections, sourceReads: sourceReads, @@ -960,11 +990,14 @@ internal struct ObjectCopyPlanner { case .refused(let reason)?: skipped.append(ObjectCopySkip(selection: item.selection, reason: reason)) case .built(let drop, let create, let note)? where !create.isEmpty: + let settingsNote = creationScopes.unreadable.contains(item.selection.id) + ? Self.creationSettingsUnreadable : nil steps.append(ObjectCopyDefinitionStep( selection: item.selection, dropStatements: drop, createStatements: create, - note: note + creationScope: creationScopes.scopes[item.selection.id], + note: [note, settingsNote].compactMap { $0 }.joined(separator: " ").nilIfEmpty )) case .built?, nil: skipped.append(ObjectCopySkip(selection: item.selection, reason: Self.noDefinition)) @@ -978,7 +1011,7 @@ internal struct ObjectCopyPlanner { selections: [ObjectCopySelection], sourceReads: [TableStructureRead], connection: DatabaseConnection - ) async throws -> [String: RoutineSourceRead] { + ) async throws -> (reads: [String: RoutineSourceRead], creationScopes: ObjectCopyCreationScopes) { let views = sourceReads.filter { read in selections.contains { selection in (selection.kind == .view || selection.kind == .materializedView) @@ -991,8 +1024,10 @@ internal struct ObjectCopyPlanner { let endpointName = sourceEndpoint.qualifiedDescription try await manager.ensureConnected(connection) return try await manager.withMetadataDriver(scope: sourceEndpoint.scope) { driver in - guard let plugin = CompareMetadataService.pluginDriver(from: driver) else { return [:] } - return try await Self.sourceDefinitionReads( + guard let plugin = CompareMetadataService.pluginDriver(from: driver) else { + return ([:], ObjectCopyCreationScopes()) + } + let reads = try await Self.sourceDefinitionReads( for: selections, views: views, triggerTables: Array(triggerTables), @@ -1000,6 +1035,10 @@ internal struct ObjectCopyPlanner { endpointName: endpointName, using: plugin ) + let creationScopes = try await Self.creationScopes( + for: selections, triggerTables: Array(triggerTables), schema: schema, using: plugin + ) + return (reads, creationScopes) } } @@ -1094,6 +1133,9 @@ internal struct ObjectCopyPlanner { localized: "The target has it, but its structure could not be read." ) nonisolated private static let alreadyThere = String(localized: "Already in the target.") + nonisolated private static let creationSettingsUnreadable = String( + localized: "The settings it was created with could not be read, so it gets the target's own." + ) nonisolated internal static let noDefinition = String( localized: "The source reports no definition for it." ) diff --git a/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift b/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift index 6f7bd9cbda..2277b22a2b 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift @@ -44,12 +44,43 @@ internal struct ObjectCopyRowCopier: Sendable { } internal let step: ObjectCopyTableStep + internal let sourceDatabaseType: DatabaseType internal let targetDatabaseType: DatabaseType internal func copy( from sourceDriver: any PluginDatabaseDriver, to targetDriver: any PluginDatabaseDriver, onProgress: @Sendable (Int) -> Void + ) async throws -> Outcome { + let scopes = await openSessionScopes(source: sourceDriver, target: targetDriver) + do { + let outcome = try await transfer(from: sourceDriver, to: targetDriver, onProgress: onProgress) + await scopes?.close() + return outcome + } catch { + await scopes?.close() + throw error + } + } + + private func openSessionScopes( + source: any PluginDatabaseDriver, + target: any PluginDatabaseDriver + ) async -> VerbatimTransferScopes? { + guard VerbatimTransferScopes.isSameEngine(source: sourceDatabaseType, target: targetDatabaseType), + !step.hasRowFilter else { return nil } + guard step.serverSideInsert == nil else { + return await VerbatimTransferScopes.open(writing: SessionScopeHandle(target)) + } + return await VerbatimTransferScopes.open( + reading: SessionScopeHandle(source), writing: SessionScopeHandle(target) + ) + } + + private func transfer( + from sourceDriver: any PluginDatabaseDriver, + to targetDriver: any PluginDatabaseDriver, + onProgress: @Sendable (Int) -> Void ) async throws -> Outcome { if let statement = step.serverSideInsert { return try await copyOnServer(statement, using: targetDriver, onProgress: onProgress) diff --git a/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift b/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift index f650e5d176..f9208a3a5b 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift @@ -244,11 +244,15 @@ internal struct ObjectCopyRunner { request: ObjectCopyRequest, progress: ObjectCopyProgress ) async throws -> DDLResult { - let groups = plan.cleanupGroups + plan.creationGroups + let tableGroups = plan.cleanupGroups + plan.tableCreationGroups + let definitionGroups = plan.definitionCreationGroups let errorHandling = request.errorHandling let scope = request.target.scope let hasCleanup = !plan.cleanupGroups.isEmpty let schemaStatements = plan.schemaStatements + let pinsSession = plan.isStructureReadPinned && VerbatimTransferScopes.isSameEngine( + source: request.source.databaseType, target: request.target.databaseType + ) return try await manager.withMetadataDriver(scope: scope) { driver in guard let plugin = CompareMetadataService.pluginDriver(from: driver) else { @@ -264,16 +268,24 @@ internal struct ObjectCopyRunner { } } - /// Ahead of everything, and it throws rather than being attributed to an object: no - /// table, view or routine can be created if the schema naming it is not there, so - /// carrying on would report every one of them as its own failure for one cause. - for statement in schemaStatements { - _ = try await plugin.execute(query: statement.sql) - } - - var result = await Self.execute( - groups, on: plugin, errorHandling: errorHandling, progress: progress + var result = try await Self.executeTableDDL( + schemaStatements: schemaStatements, + groups: tableGroups, + on: plugin, + pinsSession: pinsSession, + errorHandling: errorHandling, + progress: progress ) + /// Outside the write scope, because a routine or trigger records the session's + /// `sql_mode`: it gets its own creation settings, or else the target's. + if !result.stopped { + let definitions = await Self.execute( + definitionGroups, on: plugin, errorHandling: errorHandling, progress: progress + ) + result.outcomes += definitions.outcomes + result.cancelled = definitions.cancelled + result.stopped = definitions.stopped + } result.joinedSessionTransaction = owner == .session if relaxesForeignKeys { @@ -302,6 +314,37 @@ internal struct ObjectCopyRunner { return rolled } + /// The schemas, drops and tables, under the target's write scope on a same-engine copy: a + /// `CREATE TABLE` read from the source can carry a default, such as MySQL's `'0000-00-00'`, that + /// the target session's own mode refuses. The scope never outlives this lease. + nonisolated private static func executeTableDDL( + schemaStatements: [SyncStatement], + groups: [ObjectCopyStatementGroup], + on driver: any PluginDatabaseDriver, + pinsSession: Bool, + errorHandling: ImportErrorHandling, + progress: ObjectCopyProgress + ) async throws -> DDLResult { + let writesAnything = !schemaStatements.isEmpty || groups.contains { !$0.statements.isEmpty } + let scopes = pinsSession && writesAnything + ? await VerbatimTransferScopes.open(writing: SessionScopeHandle(driver)) + : nil + do { + /// Ahead of everything, and it throws rather than being attributed to an object: no + /// table, view or routine can be created if the schema naming it is not there, so + /// carrying on would report every one of them as its own failure for one cause. + for statement in schemaStatements { + _ = try await driver.execute(query: statement.sql) + } + let result = await execute(groups, on: driver, errorHandling: errorHandling, progress: progress) + await scopes?.close() + return result + } catch { + await scopes?.close() + throw error + } + } + nonisolated private static func execute( _ groups: [ObjectCopyStatementGroup], on driver: any PluginDatabaseDriver, @@ -316,14 +359,17 @@ internal struct ObjectCopyRunner { break } progress.startObject(group.selection.qualifiedName) + let opened = await openSettings(of: group, on: driver) do { for statement in group.statements { _ = try await driver.execute(query: statement.sql) } + if opened { await closeSettings(of: group, on: driver) } result.outcomes.append(ObjectCopyObjectOutcome( selection: group.selection, rowsCopied: 0, error: nil )) } catch { + if opened { await closeSettings(of: group, on: driver) } logger.error( "Copy DDL failed for \(group.selection.qualifiedName, privacy: .public): \(error.publicLogShape, privacy: .public)" ) @@ -339,6 +385,44 @@ internal struct ObjectCopyRunner { return result } + /// True once any opening statement ran, so the closing puts back what it changed. One that fails + /// leaves the object to be created under the session's own settings, as it was before these. + nonisolated private static func openSettings( + of group: ObjectCopyStatementGroup, + on driver: any PluginDatabaseDriver + ) async -> Bool { + var ranAny = false + for statement in group.opening { + do { + _ = try await driver.execute(query: statement.sql) + ranAny = true + } catch { + logger.warning( + "Creation settings for \(group.selection.qualifiedName, privacy: .public) were not applied: \(error.publicLogShape, privacy: .public)" + ) + break + } + } + return ranAny + } + + /// Each statement shielded like a rollback: the connection goes back to the pool after this + /// lease, and a setting left behind would apply to whatever leases it next. + nonisolated private static func closeSettings( + of group: ObjectCopyStatementGroup, + on driver: any PluginDatabaseDriver + ) async { + for statement in group.closing { + do { + try await TaskCancellationShield.run { _ = try await driver.execute(query: statement.sql) } + } catch { + logger.error( + "Restoring settings after \(group.selection.qualifiedName, privacy: .public) failed: \(error.publicLogShape, privacy: .public)" + ) + } + } + } + private func runDDL( _ groups: [ObjectCopyStatementGroup], request: ObjectCopyRequest, @@ -403,6 +487,7 @@ internal struct ObjectCopyRunner { let manager = self.manager let clearGroups = plan.clearGroups let steps = plan.dataSteps + let sourceType = request.source.databaseType let targetType = request.target.databaseType let sourceScope = request.source.scope let errorHandling = request.errorHandling @@ -435,6 +520,7 @@ internal struct ObjectCopyRunner { from: sourceScope, into: targetPlugin, manager: manager, + sourceType: sourceType, targetType: targetType, errorHandling: errorHandling, progress: progress @@ -474,6 +560,7 @@ internal struct ObjectCopyRunner { from sourceScope: DatabaseScope, into targetPlugin: any PluginDatabaseDriver, manager: any ScopedMetadataProviding, + sourceType: DatabaseType, targetType: DatabaseType, errorHandling: ImportErrorHandling, progress: ObjectCopyProgress @@ -486,7 +573,9 @@ internal struct ObjectCopyRunner { break } progress.startObject(step.qualifiedTargetName) - let copier = ObjectCopyRowCopier(step: step, targetDatabaseType: targetType) + let copier = ObjectCopyRowCopier( + step: step, sourceDatabaseType: sourceType, targetDatabaseType: targetType + ) let completedBefore = result.rowsCopied do { let outcome = try await manager.withMetadataDriver( @@ -622,7 +711,9 @@ internal struct ObjectCopyRunner { ) async throws -> CopiedRows { let sourceScope = request.source.scope let targetScope = request.target.scope - let copier = ObjectCopyRowCopier(step: step, targetDatabaseType: targetType) + let copier = ObjectCopyRowCopier( + step: step, sourceDatabaseType: request.source.databaseType, targetDatabaseType: targetType + ) let errorHandling = request.errorHandling /// The injected manager on both sides. Reaching for the singleton on the target bypassed diff --git a/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index 3e5f9a8a42..4db3175243 100644 --- a/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -276,25 +276,13 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable /// now says so and the export names the object it could not write. return try await pluginDriver.fetchViewDefinition(view: object.name, schema: schema) case .routine: - guard let routine = try await cachedRoutines(schema: schema, databaseName: object.databaseName) - .first(where: { $0.name == object.name && ($0.argumentSignature ?? "") == (object.identity ?? "") }) - else { - throw PluginObjectSourceError.unsupported(object.name) - } + let routine = try await matchedRoutine(object, schema: schema) return try await pluginDriver.fetchRoutineDDL(routine) case .trigger: - guard let trigger = try await cachedTriggers(schema: schema, databaseName: object.databaseName) - .first(where: { $0.name == object.name && $0.table == object.parentTable }) - else { - throw PluginObjectSourceError.unsupported(object.name) - } + let trigger = try await matchedTrigger(object, schema: schema) return try await pluginDriver.fetchTriggerDDL(trigger) case .event: - guard let event = try await cachedEvents(schema: schema, databaseName: object.databaseName) - .first(where: { $0.name == object.name }) - else { - throw PluginObjectSourceError.unsupported(object.name) - } + let event = try await matchedEvent(object, schema: schema) return try await pluginDriver.fetchEventDDL(event) case .sequence: guard let sequence = try await cachedSequences(schema: schema, databaseName: object.databaseName) @@ -319,6 +307,63 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable } } + /// Matched to its info object the way `fetchObjectDDL` matches it, so the settings wrap the definition the dump + /// writes. + func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? { + guard let pluginDriver else { return nil } + let schema = exportSchema(for: object.databaseName) + switch object.kind { + case .routine: + let routine = try await matchedRoutine(object, schema: schema) + return try await pluginDriver.fetchRoutineCreationScope(routine) + case .trigger: + let trigger = try await matchedTrigger(object, schema: schema) + return try await pluginDriver.fetchTriggerCreationScope(trigger) + case .event: + let event = try await matchedEvent(object, schema: schema) + return try await pluginDriver.fetchEventCreationScope(event) + default: + return nil + } + } + + private func matchedRoutine(_ object: PluginExportTable, schema: String?) async throws -> PluginRoutineInfo { + guard let routine = try await cachedRoutines(schema: schema, databaseName: object.databaseName) + .first(where: { $0.name == object.name && ($0.argumentSignature ?? "") == (object.identity ?? "") }) + else { + throw PluginObjectSourceError.unsupported(object.name) + } + return routine + } + + private func matchedTrigger(_ object: PluginExportTable, schema: String?) async throws -> PluginTriggerInfo { + guard let trigger = try await cachedTriggers(schema: schema, databaseName: object.databaseName) + .first(where: { $0.name == object.name && $0.table == object.parentTable }) + else { + throw PluginObjectSourceError.unsupported(object.name) + } + return trigger + } + + private func matchedEvent(_ object: PluginExportTable, schema: String?) async throws -> PluginEventInfo { + guard let event = try await cachedEvents(schema: schema, databaseName: object.databaseName) + .first(where: { $0.name == object.name }) + else { + throw PluginObjectSourceError.unsupported(object.name) + } + return event + } + + // MARK: - Session Scope + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + try await driver.openSessionScope(for: purpose) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + try await driver.closeSessionScope(scope) + } + /// The engine renders its own GRANT text, because only the driver knows how it spells a /// grantee and a privilege target. `principal` is the name and `host` the MySQL-style host /// part, which is what separates two principals that share a name. diff --git a/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index fc2c5ce8cc..4134d7c08a 100644 --- a/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -8,6 +8,22 @@ import os import TableProPluginKit import TableProSQLGrammar +/// What became of the transaction an import opened, as far as the import saw. +internal enum ImportTransactionOutcome: Equatable, Sendable { + /// No transaction of the import's own, or one it did not see end: what ran stays as it ran. + case untracked + case committed + case rolledBack + /// A statement that succeeded left no transaction open, as a `CREATE` or the file's own `COMMIT` does on MySQL, so + /// what ran up to then was not rolled back. + case committedBeforeRollback + /// The transaction had ended by the rollback with nothing before that naming how: a deadlock rolls it back, a + /// failing `CREATE` commits it, and the session reads the same after either. + case endedByServer + /// The file ran its own `ROLLBACK`, so what stays depends on what the file ran after it. + case endedByFile +} + final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { let databaseTypeId: String let targetTable: String? @@ -33,6 +49,19 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { /// the user's own: a `BEGIN` there would join it, and the import's `COMMIT` would commit work the /// user never asked to commit, so the import then runs inside it and leaves ending it to them. private let ownsTransaction = OSAllocatedUnfairLock(initialState: false) + private let transactionRecord = OSAllocatedUnfairLock(initialState: TransactionRecord()) + + private struct TransactionRecord { + enum Ending { + case committed + case rolledBack + case endedByServer + } + + var hasCommittedEarly = false + var isEndedByFile = false + var ending: Ending? + } private static let logger = Logger(subsystem: "com.TablePro", category: "ImportDataSinkAdapter") @@ -79,6 +108,25 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { try await execute(statement: statement, line: 1) } + var transactionOutcome: ImportTransactionOutcome { + transactionRecord.withLock { record in + switch record.ending { + case .committed: + return .committed + case _ where record.hasCommittedEarly: + return .committedBeforeRollback + case _ where record.isEndedByFile: + return .endedByFile + case .rolledBack: + return .rolledBack + case .endedByServer: + return .endedByServer + case nil: + return .untracked + } + } + } + /// What the file ran that changes where a name resolves for the rest of the session, such as a temporary table, so /// the catalog can stop following a later drop of that name onto the real table. var nameHazardStatements: [String] { @@ -98,12 +146,17 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { nameHazards.withLock { $0 += hazards } } + func execute(statement: String, line: Int) async throws { + noteNameHazards(in: statement) + try await send(statement, line: line) + await noteTransactionEnd(after: statement) + } + /// A SQL Server file arrives a batch at a time, as sqlcmd reads it, or a statement at a time when it holds no `GO` /// line, and each goes to the server whole: T-SQL scopes a variable, a table variable and a `TRY...CATCH` to one /// batch, and a routine's body runs to the end of its batch. A driver that cannot send a batch whole runs its /// statements one by one, as the editor does. - func execute(statement: String, line: Int) async throws { - noteNameHazards(in: statement) + private func send(_ statement: String, line: Int) async throws { guard grammar.contains(.batchSeparatorLines) else { _ = try await driver.execute(query: statement) return @@ -262,6 +315,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { sessionState: await driver.heldSessionTransactionState() ) ownsTransaction.withLock { $0 = owner.opensTransaction } + transactionRecord.withLock { $0 = TransactionRecord() } guard owner.opensTransaction else { return } try await driver.beginTransaction(mode: .readWrite) } @@ -270,12 +324,64 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { guard ownsTransaction.withLock({ $0 }) else { return } try await driver.commitTransaction() ownsTransaction.withLock { $0 = false } + transactionRecord.withLock { $0.ending = .committed } } func rollbackTransaction() async throws { guard ownsTransaction.withLock({ $0 }) else { return } + guard await !sessionHoldsNoTransaction() else { + ownsTransaction.withLock { $0 = false } + transactionRecord.withLock { $0.ending = .endedByServer } + Self.logger.info("Import transaction was already ended by the server; no rollback sent") + return + } try await driver.rollbackTransaction() ownsTransaction.withLock { $0 = false } + transactionRecord.withLock { $0.ending = .rolledBack } + } + + /// Only a statement that succeeded can show a commit: a failure that ends the transaction may have rolled it back. + /// The file's own `COMMIT` or `ROLLBACK` is read from its text on every engine, since the session reads the same + /// after either. The state is read after each other statement, because under `autocommit = 0` the statement + /// after a DDL opens a new transaction, and only where DDL commits: SQL Server reads its state with a query, which + /// per statement would double the round trips. + private func noteTransactionEnd(after statement: String) async { + guard ownsTransaction.withLock({ $0 }) else { return } + switch fileTransactionEnd(of: statement) { + case .commit: + transactionRecord.withLock { $0.hasCommittedEarly = true } + case .rollback: + transactionRecord.withLock { $0.isEndedByFile = true } + case nil: + guard !driver.supportsTransactionalDDL, await sessionHoldsNoTransaction() else { return } + transactionRecord.withLock { $0.hasCommittedEarly = true } + } + } + + private enum FileTransactionEnd { + case commit + case rollback + } + + /// `ROLLBACK TO SAVEPOINT` keeps the transaction, and the classifier already reads it as not ending one. + private func fileTransactionEnd(of statement: String) -> FileTransactionEnd? { + guard CatalogChangeClassifier.effect(of: statement, databaseType: databaseType).endsTransaction else { + return nil + } + return Self.rollbackKeywords.contains(QueryClassifier.leadingKeyword(of: statement)) ? .rollback : .commit + } + + private static let rollbackKeywords: Set = ["ROLLBACK", "ABORT"] + + private func sessionHoldsNoTransaction() async -> Bool { + switch await driver.heldSessionTransactionState() { + case .idle, .holdsSessionLocks: + return true + case .inTransaction, .abortedTransaction, .unknown: + return false + @unknown default: + return false + } } func disableForeignKeyChecks() async throws { diff --git a/TablePro/Core/Plugins/PluginDriverAdapter.swift b/TablePro/Core/Plugins/PluginDriverAdapter.swift index afcbb9cb5f..e334959cb0 100644 --- a/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -346,6 +346,14 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor try await pluginDriver.applyQueryTimeout(seconds) } + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + try await pluginDriver.openSessionScope(for: purpose) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + try await pluginDriver.closeSessionScope(scope) + } + var releasableResourceCommandTitle: String? { pluginDriver.releasableResourceCommandTitle } diff --git a/TablePro/Core/Services/Export/ImportService.swift b/TablePro/Core/Services/Export/ImportService.swift index 09ce34119d..501436f233 100644 --- a/TablePro/Core/Services/Export/ImportService.swift +++ b/TablePro/Core/Services/Export/ImportService.swift @@ -2,9 +2,6 @@ // ImportService.swift // TablePro // -// Plugin-driven import orchestrator. Resolves the import format plugin, -// creates the adapter/source objects, and wires progress to the UI. -// import Combine import Foundation @@ -21,6 +18,7 @@ struct ImportState { var estimatedTotalStatements: Int = 0 var statusMessage: String = "" var errorMessage: String? + var transactionOutcome: ImportTransactionOutcome = .untracked } // MARK: - Import Service @@ -256,8 +254,43 @@ final class ImportService: ObservableObject { sourceFields: sourceFields, isCancelled: { progress.isCancelled } ) - defer { reportNameHazards(sink.nameHazardStatements) } - return try await plugin.performImport(source: source, sink: sink, progress: progress) + defer { + reportNameHazards(sink.nameHazardStatements) + state.transactionOutcome = sink.transactionOutcome + } + let sessionScope = await openPreservingScope(on: driver) + let result: PluginImportResult + do { + result = try await plugin.performImport(source: source, sink: sink, progress: progress) + } catch { + await closePreservingScope(sessionScope, on: driver) + guard sink.transactionOutcome == .committedBeforeRollback else { throw error } + throw ImportCommittedTransactionError.noting(error) + } + await closePreservingScope(sessionScope, on: driver) + return result + } + + /// The file runs on the editor's own session, so whatever it sets is put back once the import ends, however it + /// ends. + private func openPreservingScope(on driver: DatabaseDriver) async -> PluginSessionScope? { + do { + return try await driver.openSessionScope(for: .preserve) + } catch { + Self.logger.warning("Import runs without saving the session settings: \(error.publicLogShape, privacy: .public)") + return nil + } + } + + private func closePreservingScope(_ scope: PluginSessionScope?, on driver: DatabaseDriver) async { + guard let scope else { return } + do { + try await driver.closeSessionScope(scope) + } catch { + Self.logger.warning( + "Could not restore the session settings after the import: \(error.publicLogShape, privacy: .public)" + ) + } } /// The file ran on the same session as the editor, so a temporary table it made shadows a real one there @@ -292,3 +325,33 @@ final class ImportService: ObservableObject { ) } } + +internal struct ImportCommittedTransactionError: LocalizedError { + internal let cause: any Error + + internal static var note: String { + String(localized: """ + The rollback did not undo everything. A statement in the file, such as COMMIT, CREATE, ALTER or DROP, \ + committed the import's transaction as it ran, so some of the statements before the failure were kept. + """) + } + + internal var errorDescription: String? { + "\(cause.localizedDescription)\n\(Self.note)" + } + + /// A statement failure keeps its shape, because the alert reads its line and statement from it. A cancel stays a + /// cancel: its own alert already says what ran stays committed. + internal static func noting(_ error: any Error) -> any Error { + guard !(error is PluginImportCancellationError), !(error is CancellationError) else { return error } + if let pluginError = error as? PluginImportError, + case .statementFailed(let statement, let line, let underlyingError) = pluginError { + return PluginImportError.statementFailed( + statement: statement, + line: line, + underlyingError: ImportCommittedTransactionError(cause: underlyingError) + ) + } + return ImportCommittedTransactionError(cause: error) + } +} diff --git a/TablePro/Core/Services/Export/TableTransferService.swift b/TablePro/Core/Services/Export/TableTransferService.swift index f3bad03961..5dd84aff27 100644 --- a/TablePro/Core/Services/Export/TableTransferService.swift +++ b/TablePro/Core/Services/Export/TableTransferService.swift @@ -205,7 +205,38 @@ final class TableTransferService: ObservableObject { } let source = ExportDataSourceAdapter(driver: sourceDriver, databaseType: request.sourceType) + /// The destination can be the user's own editor session, so the scopes close on every exit. + let scopes = await openSessionScopes( + for: request, objects: rowObjects, sourceDriver: sourceDriver, destinationDriver: destinationDriver) + do { + try await transferEach( + rowObjects, from: source, request: request, destinationDriver: destinationDriver) + } catch { + await scopes?.close() + throw error + } + await scopes?.close() + state.warnings.append(contentsOf: source.cappedTableWarnings) + } + + private func openSessionScopes( + for request: Request, + objects: [ExportObjectItem], + sourceDriver: DatabaseDriver, + destinationDriver: DatabaseDriver + ) async -> VerbatimTransferScopes? { + guard VerbatimTransferScopes.isSameEngine(source: request.sourceType, target: request.destinationType), + !objects.contains(where: { VerbatimTransferScopes.filtersRows($0.rowScope) }) else { return nil } + return await VerbatimTransferScopes.open( + reading: SessionScopeHandle(sourceDriver), writing: SessionScopeHandle(destinationDriver)) + } + private func transferEach( + _ rowObjects: [ExportObjectItem], + from source: ExportDataSourceAdapter, + request: Request, + destinationDriver: DatabaseDriver + ) async throws { for (index, object) in rowObjects.enumerated() { try checkCancellation() state.currentTable = object.name @@ -223,7 +254,6 @@ final class TableTransferService: ObservableObject { ) try await transferOne(object: object, from: source, into: sink, request: request) } - state.warnings.append(contentsOf: source.cappedTableWarnings) } /// Checked for every table before the first is written: the INSERT would name the contested diff --git a/TablePro/Core/Services/Export/VerbatimTransferScopes.swift b/TablePro/Core/Services/Export/VerbatimTransferScopes.swift new file mode 100644 index 0000000000..06b60b0425 --- /dev/null +++ b/TablePro/Core/Services/Export/VerbatimTransferScopes.swift @@ -0,0 +1,108 @@ +// +// VerbatimTransferScopes.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit + +/// A driver's session scope calls, for callers that hold it as a plugin driver or as a host driver. +internal struct SessionScopeHandle: Sendable { + fileprivate let open: @Sendable (PluginSessionPurpose) async throws -> PluginSessionScope? + fileprivate let close: @Sendable (PluginSessionScope) async throws -> Void + + internal init(_ driver: any PluginDatabaseDriver) { + open = { try await driver.openSessionScope(for: $0) } + close = { try await driver.closeSessionScope($0) } + } + + internal init(_ driver: DatabaseDriver) { + open = { try await driver.openSessionScope(for: $0) } + close = { try await driver.closeSessionScope($0) } + } +} + +/// The session scopes a same-engine row copy runs under, so the target takes back what the source +/// held: zero dates, an AUTO_INCREMENT 0, TIMESTAMPs in one zone. +internal struct VerbatimTransferScopes: Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "VerbatimTransferScopes") + + private struct Held: Sendable { + let handle: SessionScopeHandle + let scope: PluginSessionScope + } + + private let held: [Held] + + /// Object Copy's own test for whether a copy crosses engines, so both copy paths agree on it. + internal static func isSameEngine(source: DatabaseType, target: DatabaseType) -> Bool { + !SQLTypeFamily.needsTranslation(from: source, to: target) + } + + /// A row filter is the user's SQL, written for their own session: under pinned settings a `NOW()` + /// or a TIMESTAMP literal selects other rows, and a double-quoted identifier becomes a string. A + /// copy that reads one keeps both sessions as they are. A column list or a row limit means the + /// same either way. The SQL export follows the same rule. + internal static func filtersRows(_ rowScope: PluginExportRowScope?) -> Bool { + !(rowScope?.sanitizedFilter.isEmpty ?? true) + } + + /// Both or neither: a value read under one side's scope is only safe to write under the other's. + internal static func open( + reading source: SessionScopeHandle, + writing target: SessionScopeHandle + ) async -> VerbatimTransferScopes? { + guard let read = await open(.verbatimRead, on: source) else { return nil } + let reading = Held(handle: source, scope: read) + guard let write = await open(.verbatimWrite, on: target) else { + await close(reading) + return nil + } + return VerbatimTransferScopes(held: [reading, Held(handle: target, scope: write)]) + } + + /// For an `INSERT ... SELECT`, whose rows never leave the server that writes them, and for DDL + /// read earlier under `open(reading:)`. + internal static func open(writing target: SessionScopeHandle) async -> VerbatimTransferScopes? { + guard let write = await open(.verbatimWrite, on: target) else { return nil } + return VerbatimTransferScopes(held: [Held(handle: target, scope: write)]) + } + + /// For definitions read in one lease and written in a later one under `open(writing:)`. + internal static func open(reading source: SessionScopeHandle) async -> VerbatimTransferScopes? { + guard let read = await open(.verbatimRead, on: source) else { return nil } + return VerbatimTransferScopes(held: [Held(handle: source, scope: read)]) + } + + /// Last opened, first closed: one unpooled session can serve both sides, and each scope saved + /// what the one before it had set. + internal func close() async { + for entry in held.reversed() { + await Self.close(entry) + } + } + + private static func open(_ purpose: PluginSessionPurpose, on handle: SessionScopeHandle) async -> PluginSessionScope? { + do { + return try await handle.open(purpose) + } catch { + logger.warning( + "Opening the \(String(describing: purpose), privacy: .public) session scope failed: \(error.publicLogShape, privacy: .public)" + ) + return nil + } + } + + /// Shielded like a rollback: a Stop is often what ends the copy, and the session, sometimes the + /// user's own editor session, still needs its settings back. + private static func close(_ entry: Held) async { + do { + try await TaskCancellationShield.run { try await entry.handle.close(entry.scope) } + } catch { + logger.error( + "Closing the \(String(describing: entry.scope.purpose), privacy: .public) session scope failed: \(error.publicLogShape, privacy: .public)" + ) + } + } +} diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 62261146b7..012be8bfff 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -52416,6 +52416,10 @@ } } }, + "Could not read the settings %@ was created with, so it restores under the dump's SQL mode." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Could not read the SurrealDB response: %@" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -151332,6 +151336,10 @@ } } }, + "Runs every statement in one transaction, so a failure rolls back what the import changed. A database that commits CREATE, ALTER and DROP as they run, such as MySQL, keeps those statements and everything before them." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Runs on '%@'." : { "localizations" : { "fr" : { @@ -176174,6 +176182,10 @@ } } }, + "TablePro is reading or writing rows on this connection with settings of its own." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "TablePro is still trying to reconnect." : { "localizations" : { "fr" : { diff --git a/TablePro/Views/Components/TransferResultAlert.swift b/TablePro/Views/Components/TransferResultAlert.swift index a55489b57b..8d3a82c75c 100644 --- a/TablePro/Views/Components/TransferResultAlert.swift +++ b/TablePro/Views/Components/TransferResultAlert.swift @@ -91,24 +91,48 @@ internal enum TransferResultAlert { return String(format: template, Int64(rowCount), destinationName, Int64(tableCount)) } - /// A stopped import leaves whatever it already ran committed, and used to close its progress - /// sheet without saying so. Stopping looked the same as importing nothing. + /// A stopped import used to close its progress sheet without saying what it left behind, so stopping looked the + /// same as importing nothing. internal static func presentImportCancelled( executedStatements: Int, + transaction: ImportTransactionOutcome, window: NSWindow?, completion: @escaping @MainActor () -> Void ) { let alert = NSAlert() alert.messageText = String(localized: "Import stopped") alert.alertStyle = .warning - let template = executedStatements == 1 - ? String(localized: "%lld statement had already run and stays committed.") - : String(localized: "%lld statements had already run and stay committed.") - alert.informativeText = String(format: template, Int64(executedStatements)) + alert.informativeText = importCancelledText(executedStatements: executedStatements, transaction: transaction) AlertHelper.addCancelButton(to: alert, title: String(localized: "Done")) AlertHelper.present(alert, in: window) { _ in completion() } } + internal static func importCancelledText(executedStatements: Int, transaction: ImportTransactionOutcome) -> String { + let isOne = executedStatements == 1 + let template: String + switch transaction { + case .rolledBack: + template = isOne + ? String(localized: "%lld statement had already run and was rolled back.") + : String(localized: "%lld statements had already run and were rolled back.") + case .committedBeforeRollback: + template = isOne + ? String(localized: """ + %lld statement had already run and was kept, because a statement in the file, such as COMMIT, \ + CREATE, ALTER or DROP, committed the import's transaction as it ran. + """) + : String(localized: """ + %lld statements had already run. Some were kept, because a statement in the file, such as \ + COMMIT, CREATE, ALTER or DROP, committed the import's transaction as it ran. + """) + case .untracked, .committed, .endedByServer, .endedByFile: + template = isOne + ? String(localized: "%lld statement had already run and stays committed.") + : String(localized: "%lld statements had already run and stay committed.") + } + return String(format: template, Int64(executedStatements)) + } + internal static func presentImportSuccess( result: PluginImportResult?, window: NSWindow?, diff --git a/TablePro/Views/Import/ImportDialog.swift b/TablePro/Views/Import/ImportDialog.swift index 91240250d1..cbe7059196 100644 --- a/TablePro/Views/Import/ImportDialog.swift +++ b/TablePro/Views/Import/ImportDialog.swift @@ -525,6 +525,7 @@ struct ImportDialog: View { showProgressDialog = false TransferResultAlert.presentImportCancelled( executedStatements: service.state.processedStatements, + transaction: service.state.transactionOutcome, window: hostWindow ) {} } diff --git a/TablePro/Views/Import/RowImportSheet.swift b/TablePro/Views/Import/RowImportSheet.swift index 19dc30fc87..3f2ccb62ed 100644 --- a/TablePro/Views/Import/RowImportSheet.swift +++ b/TablePro/Views/Import/RowImportSheet.swift @@ -977,6 +977,7 @@ struct RowImportSheet: View { showProgressDialog = false TransferResultAlert.presentImportCancelled( executedStatements: service.state.processedStatements, + transaction: service.state.transactionOutcome, window: hostWindow ) {} } diff --git a/TableProTests/Core/Export/TableTransferSessionScopeTests.swift b/TableProTests/Core/Export/TableTransferSessionScopeTests.swift new file mode 100644 index 0000000000..43d5541fdd --- /dev/null +++ b/TableProTests/Core/Export/TableTransferSessionScopeTests.swift @@ -0,0 +1,259 @@ +// +// TableTransferSessionScopeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +private struct InsertFailure: Error {} + +private final class ScopedTransferDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var log: [String] = [] + private var isInTransaction = false + private let header: [String] + private let rows: [PluginRow] + private let opensScope: Bool + private let failsInserts: Bool + private let onInsert: (@Sendable () async -> Void)? + + init( + header: [String] = [], + rows: [PluginRow] = [], + opensScope: Bool = true, + failsInserts: Bool = false, + onInsert: (@Sendable () async -> Void)? = nil + ) { + self.header = header + self.rows = rows + self.opensScope = opensScope + self.failsInserts = failsInserts + self.onInsert = onInsert + } + + var events: [String] { lock.withLock { log } } + + var supportsTransactions: Bool { true } + + private func record(_ event: String) { + lock.withLock { log.append(event) } + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + record("open \(purpose)") + guard opensScope else { return nil } + return PluginSessionScope( + purpose: purpose, saved: [], applied: [PluginSessionSetting(name: "time_zone", value: "+00:00")]) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + record("close \(scope.purpose)") + } + + func sessionTransactionState() async -> PluginSessionTransactionState { + let isOpen = lock.withLock { isInTransaction } + return isOpen ? .inTransaction : .idle + } + + func beginTransaction() async throws { recordTransaction("begin", isOpen: true) } + func commitTransaction() async throws { recordTransaction("commit", isOpen: false) } + func rollbackTransaction() async throws { recordTransaction("rollback", isOpen: false) } + + private func recordTransaction(_ event: String, isOpen: Bool) { + lock.withLock { + log.append(event) + isInTransaction = isOpen + } + } + + func defaultExportQuery(table: String, schema: String?) -> String? { + "SELECT * FROM \(table)" + } + + func streamRows(query: String) -> AsyncThrowingStream { + record("stream") + let streamHeader = PluginStreamHeader(columns: header, columnTypeNames: header.map { _ in "TEXT" }) + let batch = rows + return AsyncThrowingStream { continuation in + continuation.yield(.header(streamHeader)) + continuation.yield(.rows(batch)) + continuation.finish() + } + } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + record("insert") + await onInsert?() + if failsInserts { throw InsertFailure() } + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } +} + +@MainActor +struct TableTransferSessionScopeTests { + private func adapter(_ driver: ScopedTransferDriver, type: DatabaseType) -> PluginDriverAdapter { + PluginDriverAdapter(connection: DatabaseConnection(name: "Test", type: type), pluginDriver: driver) + } + + private func makeSource() -> ScopedTransferDriver { + ScopedTransferDriver( + header: ["id", "born"], + rows: [[.text("0"), .text("0000-00-00")], [.text("1"), .text("2020-00-15")]] + ) + } + + private func makeRequest( + from sourceType: DatabaseType, + to destinationType: DatabaseType, + petsScope: PluginExportRowScope = .unrestricted + ) -> TableTransferService.Request { + TableTransferService.Request( + objects: [ + ExportObjectItem(name: "people", kind: .table, isSelected: true), + ExportObjectItem(name: "pets", kind: .table, isSelected: true, rowScope: petsScope) + ], + sourceType: sourceType, + destinationType: destinationType, + columnMapping: [ + "people": ["id": "id", "born": "born"], + "pets": ["id": "id", "born": "born"] + ] + ) + } + + /// Once around every table, so the destination's scope outlives each table's own transaction. + @Test("A same-engine transfer reads under the source's scope and writes under the destination's") + func sameEngineTransferPinsBothSessions() async throws { + let source = makeSource() + let destination = ScopedTransferDriver() + let service = TableTransferService() + + try await service.transfer( + request: makeRequest(from: .mysql, to: .mysql), + sourceDriver: adapter(source, type: .mysql), + destinationDriver: adapter(destination, type: .mysql) + ) + + #expect(service.state.transferredRows == 4) + #expect(source.events == ["open verbatimRead", "stream", "stream", "close verbatimRead"]) + #expect(destination.events == [ + "open verbatimWrite", + "begin", "insert", "commit", + "begin", "insert", "commit", + "close verbatimWrite" + ]) + } + + @Test("A failed transfer still closes both scopes") + func failedTransferClosesBothScopes() async { + let source = makeSource() + let destination = ScopedTransferDriver(failsInserts: true) + + await #expect(throws: TableTransferError.self) { + try await TableTransferService().transfer( + request: makeRequest(from: .mysql, to: .mysql), + sourceDriver: adapter(source, type: .mysql), + destinationDriver: adapter(destination, type: .mysql) + ) + } + + #expect(source.events.first == "open verbatimRead") + #expect(source.events.last == "close verbatimRead") + #expect(destination.events.first == "open verbatimWrite") + #expect(destination.events.last == "close verbatimWrite") + #expect(destination.events.contains("rollback")) + } + + @Test("A stopped transfer still closes both scopes") + func stoppedTransferClosesBothScopes() async { + let service = TableTransferService() + let source = makeSource() + let destination = ScopedTransferDriver(onInsert: { await service.cancel() }) + + await #expect(throws: TableTransferError.self) { + try await service.transfer( + request: makeRequest(from: .mysql, to: .mysql), + sourceDriver: adapter(source, type: .mysql), + destinationDriver: adapter(destination, type: .mysql) + ) + } + + #expect(source.events == ["open verbatimRead", "stream", "close verbatimRead"]) + #expect(destination.events == ["open verbatimWrite", "begin", "insert", "rollback", "close verbatimWrite"]) + } + + @Test("A transfer between engines opens no scope") + func crossEngineTransferOpensNoScope() async throws { + let source = makeSource() + let destination = ScopedTransferDriver() + + try await TableTransferService().transfer( + request: makeRequest(from: .postgresql, to: .mysql), + sourceDriver: adapter(source, type: .postgresql), + destinationDriver: adapter(destination, type: .mysql) + ) + + #expect(!source.events.contains { $0.hasPrefix("open") }) + #expect(!destination.events.contains { $0.hasPrefix("open") }) + #expect(destination.events.filter { $0 == "insert" }.count == 2) + } + + /// One filtered table is enough: the scopes span every table of the transfer. + @Test("A transfer with a row filter reads and writes on the sessions as they are") + func filteredTransferOpensNoScope() async throws { + let source = makeSource() + let destination = ScopedTransferDriver() + let service = TableTransferService() + + try await service.transfer( + request: makeRequest( + from: .mysql, to: .mysql, petsScope: PluginExportRowScope(filter: "born > NOW() - INTERVAL 1 DAY") + ), + sourceDriver: adapter(source, type: .mysql), + destinationDriver: adapter(destination, type: .mysql) + ) + + #expect(service.state.transferredRows == 4) + #expect(!source.events.contains { $0.hasPrefix("open") }) + #expect(!destination.events.contains { $0.hasPrefix("open") }) + } + + @Test("A transfer limited to some rows keeps the scopes") + func limitedTransferKeepsScopes() async throws { + let source = makeSource() + let destination = ScopedTransferDriver() + + try await TableTransferService().transfer( + request: makeRequest(from: .mysql, to: .mysql, petsScope: PluginExportRowScope(rowLimit: 1)), + sourceDriver: adapter(source, type: .mysql), + destinationDriver: adapter(destination, type: .mysql) + ) + + #expect(source.events.first == "open verbatimRead") + #expect(source.events.last == "close verbatimRead") + #expect(destination.events.first == "open verbatimWrite") + #expect(destination.events.last == "close verbatimWrite") + } + + @Test("A destination with no scope closes the source's and still transfers every row") + func destinationWithoutScopeStillTransfers() async throws { + let source = makeSource() + let destination = ScopedTransferDriver(opensScope: false) + let service = TableTransferService() + + try await service.transfer( + request: makeRequest(from: .mysql, to: .mysql), + sourceDriver: adapter(source, type: .mysql), + destinationDriver: adapter(destination, type: .mysql) + ) + + #expect(service.state.transferredRows == 4) + #expect(source.events == ["open verbatimRead", "close verbatimRead", "stream", "stream"]) + #expect(destination.events.first == "open verbatimWrite") + #expect(!destination.events.contains("close verbatimWrite")) + } +} diff --git a/TableProTests/Core/Export/VerbatimTransferScopesTests.swift b/TableProTests/Core/Export/VerbatimTransferScopesTests.swift new file mode 100644 index 0000000000..f3802d751b --- /dev/null +++ b/TableProTests/Core/Export/VerbatimTransferScopesTests.swift @@ -0,0 +1,215 @@ +// +// VerbatimTransferScopesTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +private struct ScopeFailure: Error {} + +private final class ScopeDriver: PluginDriverStub, @unchecked Sendable { + enum Answer { + case scope + case missing + case failure + } + + private let lock = NSLock() + private var log: [String] = [] + private var cancelledAtClose: [Bool] = [] + private let answer: Answer + private let failsClose: Bool + + init(answer: Answer = .scope, failsClose: Bool = false) { + self.answer = answer + self.failsClose = failsClose + } + + var events: [String] { lock.withLock { log } } + var closesSawCancellation: [Bool] { lock.withLock { cancelledAtClose } } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + lock.withLock { log.append("open \(purpose)") } + switch answer { + case .scope: + return PluginSessionScope( + purpose: purpose, saved: [], applied: [PluginSessionSetting(name: "time_zone", value: "+00:00")]) + case .missing: + return nil + case .failure: + throw ScopeFailure() + } + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + let isCancelled = Task.isCancelled + lock.withLock { + log.append("close \(scope.purpose)") + cancelledAtClose.append(isCancelled) + } + if failsClose { throw ScopeFailure() } + } +} + +/// One session's `sql_mode`, saved and set by each scope the way a driver does it. +private final class OneSessionDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var mode: String + + init(mode: String) { + self.mode = mode + } + + var currentMode: String { lock.withLock { mode } } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + let applied = "\(purpose)" + let saved = lock.withLock { + defer { mode = applied } + return mode + } + return PluginSessionScope( + purpose: purpose, + saved: [PluginSessionSetting(name: "sql_mode", value: saved)], + applied: [PluginSessionSetting(name: "sql_mode", value: applied)] + ) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + guard let saved = scope.saved.first?.value else { return } + lock.withLock { mode = saved } + } +} + +struct VerbatimTransferScopesTests { + private func openPair(_ source: ScopeDriver, _ target: ScopeDriver) async -> VerbatimTransferScopes? { + await VerbatimTransferScopes.open(reading: SessionScopeHandle(source), writing: SessionScopeHandle(target)) + } + + @Test("A pair opens read on the source and write on the target, and closes both") + func pairOpensAndClosesBothSides() async throws { + let source = ScopeDriver() + let target = ScopeDriver() + + let scopes = try #require(await openPair(source, target)) + #expect(source.events == ["open verbatimRead"]) + #expect(target.events == ["open verbatimWrite"]) + + await scopes.close() + #expect(source.events == ["open verbatimRead", "close verbatimRead"]) + #expect(target.events == ["open verbatimWrite", "close verbatimWrite"]) + } + + @Test("A target with no scope, or one that fails to open, leaves the source closed and no pair") + func targetWithoutScopeClosesSource() async { + for answer in [ScopeDriver.Answer.missing, .failure] { + let source = ScopeDriver() + let target = ScopeDriver(answer: answer) + + let scopes = await openPair(source, target) + + #expect(scopes == nil) + #expect(source.events == ["open verbatimRead", "close verbatimRead"]) + #expect(target.events == ["open verbatimWrite"]) + } + } + + @Test("A source with no scope, or one that fails to open, never touches the target") + func sourceWithoutScopeLeavesTargetAlone() async { + for answer in [ScopeDriver.Answer.missing, .failure] { + let source = ScopeDriver(answer: answer) + let target = ScopeDriver() + + let scopes = await openPair(source, target) + + #expect(scopes == nil) + #expect(source.events == ["open verbatimRead"]) + #expect(target.events.isEmpty) + } + } + + @Test("A close that fails still closes the other side") + func failedCloseStillClosesTheOtherSide() async throws { + let source = ScopeDriver() + let target = ScopeDriver(failsClose: true) + + let scopes = try #require(await openPair(source, target)) + await scopes.close() + + #expect(target.events.last == "close verbatimWrite") + #expect(source.events.last == "close verbatimRead") + } + + @Test("The write-only scope opens one side") + func writeOnlyOpensOneSide() async throws { + let target = ScopeDriver() + + let scopes = try #require(await VerbatimTransferScopes.open(writing: SessionScopeHandle(target))) + await scopes.close() + + #expect(target.events == ["open verbatimWrite", "close verbatimWrite"]) + } + + /// A Stop cancels the task that runs the copy, and a driver that checks for cancellation would + /// otherwise skip the statement that puts the session back. + @Test("Closing from a cancelled task reaches the driver outside the cancellation") + func closeIsShieldedFromCancellation() async throws { + let source = ScopeDriver() + let target = ScopeDriver() + let scopes = try #require(await openPair(source, target)) + + let closing = Task { + withUnsafeCurrentTask { task in + guard let task else { return } + task.cancel() + } + await scopes.close() + } + await closing.value + + #expect(source.closesSawCancellation == [false]) + #expect(target.closesSawCancellation == [false]) + } + + /// An unpooled engine serves both sides from one session, and closing in opening order left it + /// on the read scope's settings. + @Test("Scopes on one session close in reverse order and leave it as it was") + func oneSessionIsRestoredInReverseOrder() async throws { + let session = OneSessionDriver(mode: "STRICT_TRANS_TABLES") + + let scopes = try #require(await VerbatimTransferScopes.open( + reading: SessionScopeHandle(session), writing: SessionScopeHandle(session) + )) + #expect(session.currentMode == "verbatimWrite") + + await scopes.close() + #expect(session.currentMode == "STRICT_TRANS_TABLES") + } + + @Test("Only a row filter keeps the sessions as they are") + func onlyARowFilterCounts() { + #expect(VerbatimTransferScopes.filtersRows(PluginExportRowScope(filter: "id > 5"))) + #expect(VerbatimTransferScopes.filtersRows(PluginExportRowScope(filter: "id > 5;", rowLimit: 3))) + #expect(!VerbatimTransferScopes.filtersRows(PluginExportRowScope(rowLimit: 10))) + #expect(!VerbatimTransferScopes.filtersRows(PluginExportRowScope(columns: ["id"]))) + #expect(!VerbatimTransferScopes.filtersRows(PluginExportRowScope(filter: " "))) + #expect(!VerbatimTransferScopes.filtersRows(nil)) + } + + @Test("Same engine follows Object Copy's type families") + func sameEngineFollowsTypeFamilies() { + #expect(VerbatimTransferScopes.isSameEngine(source: .mysql, target: .mysql)) + #expect(VerbatimTransferScopes.isSameEngine(source: .mariadb, target: .mysql)) + #expect(VerbatimTransferScopes.isSameEngine(source: .postgresql, target: .postgresql)) + #expect(!VerbatimTransferScopes.isSameEngine(source: .postgresql, target: .mysql)) + #expect(!VerbatimTransferScopes.isSameEngine(source: .mysql, target: .sqlite)) + + let unnamed = DatabaseType(rawValue: "PluginOnlyEngine") + #expect(VerbatimTransferScopes.isSameEngine(source: unnamed, target: unnamed)) + #expect(!VerbatimTransferScopes.isSameEngine(source: unnamed, target: DatabaseType(rawValue: "OtherPluginEngine"))) + } +} diff --git a/TableProTests/Core/ObjectCopy/ObjectCopyCreationScopeReadTests.swift b/TableProTests/Core/ObjectCopy/ObjectCopyCreationScopeReadTests.swift new file mode 100644 index 0000000000..903b3e40ef --- /dev/null +++ b/TableProTests/Core/ObjectCopy/ObjectCopyCreationScopeReadTests.swift @@ -0,0 +1,132 @@ +// +// ObjectCopyCreationScopeReadTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct ReadFailure: Error {} + +private final class CreationScopeSourceDriver: PluginDriverStub, @unchecked Sendable { + private let routines: Result<[PluginRoutineInfo], any Error> + private let triggers: [PluginTriggerInfo] + private let scopes: [String: Result] + + init( + routines: Result<[PluginRoutineInfo], any Error> = .success([]), + triggers: [PluginTriggerInfo] = [], + scopes: [String: Result] = [:] + ) { + self.routines = routines + self.triggers = triggers + self.scopes = scopes + } + + var providesBulkTriggerFetch: Bool { true } + + func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { + try routines.get() + } + + func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { + triggers + } + + func fetchRoutineCreationScope(_ routine: PluginRoutineInfo) async throws -> PluginStatementScope? { + try (scopes["\(routine.kind == .procedure ? "procedure" : "function") \(routine.name)"] ?? .success(nil)).get() + } + + func fetchTriggerCreationScope(_ trigger: PluginTriggerInfo) async throws -> PluginStatementScope? { + try (scopes["trigger \(trigger.name) on \(trigger.table ?? "")"] ?? .success(nil)).get() + } +} + +struct ObjectCopyCreationScopeReadTests { + private func scope(_ mode: String) -> PluginStatementScope { + PluginStatementScope(opening: ["SET sql_mode = '\(mode)'"], closing: ["SET sql_mode = @saved_sql_mode"]) + } + + private func makeTrigger(_ name: String, on table: String) -> PluginTriggerInfo { + PluginTriggerInfo(name: name, table: table, timing: "BEFORE", event: "INSERT", statement: "SET NEW.d = NEW.d") + } + + private func readScopes( + _ selections: [ObjectCopySelection], + from driver: CreationScopeSourceDriver + ) async throws -> ObjectCopyCreationScopes { + try await ObjectCopyPlanner.creationScopes( + for: selections, triggerTables: [], schema: nil, using: driver + ) + } + + @Test("Each routine and trigger gets the settings its source recorded, matched like its definition") + func routinesAndTriggersGetTheirOwnSettings() async throws { + let procedure = ObjectCopySelection(kind: .procedure, name: "Tally", schema: nil) + let function = ObjectCopySelection(kind: .function, name: "tally", schema: nil) + let stamp = ObjectCopySelection(kind: .trigger, name: "stamp", schema: nil, owner: "orders") + let driver = CreationScopeSourceDriver( + routines: .success([ + PluginRoutineInfo(name: "tally", kind: .procedure), + PluginRoutineInfo(name: "tally", kind: .function) + ]), + triggers: [makeTrigger("stamp", on: "invoices"), makeTrigger("stamp", on: "orders")], + scopes: [ + "procedure tally": .success(scope("NO_ENGINE_SUBSTITUTION")), + "function tally": .success(scope("ANSI_QUOTES")), + "trigger stamp on invoices": .success(scope("TRADITIONAL")), + "trigger stamp on orders": .success(scope("ONLY_FULL_GROUP_BY")) + ] + ) + + let read = try await readScopes([procedure, function, stamp], from: driver) + + #expect(read.scopes[procedure.id] == scope("NO_ENGINE_SUBSTITUTION")) + #expect(read.scopes[function.id] == scope("ANSI_QUOTES")) + #expect(read.scopes[stamp.id] == scope("ONLY_FULL_GROUP_BY")) + #expect(read.unreadable.isEmpty) + } + + @Test("A failed read marks only its own object, and no settings is not a failure") + func failedReadMarksOnlyItsObject() async throws { + let failing = ObjectCopySelection(kind: .procedure, name: "broken", schema: nil) + let unrecorded = ObjectCopySelection(kind: .procedure, name: "plain", schema: nil) + let driver = CreationScopeSourceDriver( + routines: .success([ + PluginRoutineInfo(name: "broken", kind: .procedure), + PluginRoutineInfo(name: "plain", kind: .procedure) + ]), + scopes: ["procedure broken": .failure(ReadFailure())] + ) + + let read = try await readScopes([failing, unrecorded], from: driver) + + #expect(read.unreadable == [failing.id]) + #expect(read.scopes.isEmpty) + } + + @Test("A routine listing that fails marks every routine unreadable") + func failedListingMarksEveryRoutine() async throws { + let first = ObjectCopySelection(kind: .procedure, name: "one", schema: nil) + let second = ObjectCopySelection(kind: .function, name: "two", schema: nil) + let driver = CreationScopeSourceDriver(routines: .failure(ReadFailure())) + + let read = try await readScopes([first, second], from: driver) + + #expect(read.unreadable == [first.id, second.id]) + #expect(read.scopes.isEmpty) + } + + @Test("Views and tables are not read for creation settings") + func viewsAndTablesAreLeftAlone() async throws { + let view = ObjectCopySelection(kind: .view, name: "recent", schema: nil) + let table = ObjectCopySelection(kind: .table, name: "orders", schema: nil) + + let read = try await readScopes([view, table], from: CreationScopeSourceDriver(routines: .failure(ReadFailure()))) + + #expect(read.scopes.isEmpty) + #expect(read.unreadable.isEmpty) + } +} diff --git a/TableProTests/Core/ObjectCopy/ObjectCopyDDLSessionTests.swift b/TableProTests/Core/ObjectCopy/ObjectCopyDDLSessionTests.swift new file mode 100644 index 0000000000..535157f743 --- /dev/null +++ b/TableProTests/Core/ObjectCopy/ObjectCopyDDLSessionTests.swift @@ -0,0 +1,357 @@ +// +// ObjectCopyDDLSessionTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct StatementFailure: Error {} + +private final class DDLRecordingDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var log: [String] = [] + private let failing: Set + + init(failing: Set = []) { + self.failing = failing + } + + var events: [String] { lock.withLock { log } } + + private func record(_ event: String) { + lock.withLock { log.append(event) } + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + record("open \(purpose)") + return PluginSessionScope(purpose: purpose, saved: [], applied: []) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + record("close \(scope.purpose)") + } + + func execute(query: String) async throws -> PluginQueryResult { + record(query) + if failing.contains(query) { throw StatementFailure() } + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } +} + +private final class SourceTableDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var log: [String] = [] + private let opensScope: Bool + private let failsListing: Bool + + init(opensScope: Bool = true, failsListing: Bool = false) { + self.opensScope = opensScope + self.failsListing = failsListing + } + + var events: [String] { lock.withLock { log } } + + private func record(_ event: String) { + lock.withLock { log.append(event) } + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + record("open \(purpose)") + guard opensScope else { return nil } + return PluginSessionScope(purpose: purpose, saved: [], applied: []) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + record("close \(scope.purpose)") + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + record("fetchTables") + if failsListing { throw StatementFailure() } + return [ + PluginTableInfo(name: "dd", comment: nil, partitionCount: nil), + PluginTableInfo(name: "unrelated", comment: nil, partitionCount: nil) + ] + } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + record("fetchColumns \(table)") + return [PluginColumnInfo(name: "d", dataType: "TIMESTAMP", defaultValue: "'2024-01-01 07:00:00'")] + } +} + +@MainActor +private final class TargetOnlyProvider: ScopedMetadataProviding { + private let driver: DatabaseDriver + + init(driver: DatabaseDriver) { + self.driver = driver + } + + func withMetadataDriver( + scope: DatabaseScope, + workload: MetadataConnectionPool.Workload, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T + ) async throws -> T { + try await body(driver) + } + + func browseScope(for connectionId: UUID) -> DatabaseScope? { nil } +} + +@MainActor +struct ObjectCopyDDLSessionTests { + private let createTable = "CREATE TABLE `dd` (`d` DATE NOT NULL DEFAULT '0000-00-00')" + private let createProcedure = "CREATE PROCEDURE `add_c`(d DATE) INSERT INTO `dd` VALUES (d)" + private let createTrigger = "CREATE TRIGGER `stamp` BEFORE INSERT ON `dd` FOR EACH ROW SET NEW.d = NEW.d" + private let saveMode = "SET @saved_sql_mode = @@sql_mode" + private let setMode = "SET sql_mode = 'NO_ENGINE_SUBSTITUTION'" + private let restoreMode = "SET sql_mode = @saved_sql_mode" + + private var creationScope: PluginStatementScope { + PluginStatementScope(opening: [saveMode, setMode], closing: [restoreMode]) + } + + private func endpoint(_ type: DatabaseType, _ database: String) -> DatabaseEndpoint { + DatabaseEndpoint( + scope: DatabaseScope(connectionId: UUID(), database: database, schema: nil), + connectionName: database, + databaseType: type, + safeModeLevel: .silent, + color: .blue + ) + } + + private func statement(_ sql: String, _ name: String) -> SyncStatement { + SyncStatement(sql: sql, objectName: name, summary: "create") + } + + private func tableStep() -> ObjectCopyTableStep { + ObjectCopyTableStep( + selection: ObjectCopySelection(kind: .table, name: "dd", schema: nil), + dropStatements: [], + sequenceStatements: [], + createStatements: [statement(createTable, "dd")], + truncateStatements: [], + columns: [], + primaryKeyColumns: [], + sourceQuery: "SELECT `d` FROM `dd`", + targetTable: "dd", + targetSchema: nil, + estimatedRows: nil, + copiesData: false, + copiesIdentityColumn: false, + note: nil + ) + } + + private func definitionStep( + _ kind: CompareObjectKind, + _ name: String, + create sql: String, + scope: PluginStatementScope? + ) -> ObjectCopyDefinitionStep { + ObjectCopyDefinitionStep( + selection: ObjectCopySelection(kind: kind, name: name, schema: nil), + dropStatements: [], + createStatements: [statement(sql, name)], + creationScope: scope + ) + } + + private func plan( + from sourceType: DatabaseType = .mysql, + tables: [ObjectCopyTableStep] = [], + definitions: [ObjectCopyDefinitionStep] = [], + isStructureReadPinned: Bool = true + ) -> ObjectCopyPlan { + let selections = tables.map(\.selection) + definitions.map(\.selection) + return ObjectCopyPlan( + request: ObjectCopyRequest( + source: endpoint(sourceType, "shop"), + destination: .existing(endpoint(.mysql, "staging")), + objects: selections, + content: .structure, + existingPolicy: .replace + ), + createsDatabase: false, + tableSteps: tables, + definitionSteps: definitions, + isStructureReadPinned: isStructureReadPinned + ) + } + + private func run(_ plan: ObjectCopyPlan, on driver: DDLRecordingDriver) async throws -> ObjectCopyRunResult { + let adapter = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .mysql), pluginDriver: driver) + let runner = ObjectCopyRunner(manager: TargetOnlyProvider(driver: adapter), gate: AlwaysAllowGate()) + return try await runner.run(plan, progress: ObjectCopyProgress(progress: Progress())) + } + + // MARK: - Tables + + /// A strict MySQL 8 session refuses this `CREATE TABLE` with 1067 before any row is copied. + @Test("A same-engine copy creates its tables under the target's write scope") + func sameEngineTableDDLIsPinned() async throws { + let driver = DDLRecordingDriver() + + let result = try await run(plan(tables: [tableStep()]), on: driver) + + #expect(result.failedCount == 0) + #expect(driver.events == ["open verbatimWrite", createTable, "close verbatimWrite"]) + } + + @Test("A failed CREATE TABLE still closes the write scope") + func failedTableDDLClosesTheWriteScope() async throws { + let driver = DDLRecordingDriver(failing: [createTable]) + + let result = try await run(plan(tables: [tableStep()]), on: driver) + + #expect(result.failedCount == 1) + #expect(driver.events == ["open verbatimWrite", createTable, "close verbatimWrite"]) + } + + /// MariaDB renders a TIMESTAMP default in the session's zone, so DDL read as the session is + /// would move by the offset if it were written under the pinned zone. + @Test("Tables read as the session is are created as the session is") + func unpinnedReadRunsDDLUnscoped() async throws { + let driver = DDLRecordingDriver() + + _ = try await run(plan(tables: [tableStep()], isStructureReadPinned: false), on: driver) + + #expect(driver.events == [createTable]) + } + + @Test("A copy between engines creates its tables under the session as it is") + func crossEngineTableDDLIsNotPinned() async throws { + let driver = DDLRecordingDriver() + + _ = try await run(plan(from: .postgresql, tables: [tableStep()]), on: driver) + + #expect(driver.events == [createTable]) + } + + // MARK: - Reading the source tables + + @Test("The source tables are read inside the read scope and the read reports it pinned") + func sourceTablesAreReadPinned() async throws { + let source = SourceTableDriver() + + let read = try await ObjectCopyPlanner.pinnedTableReads( + names: ["DD"], schema: nil, databaseType: .mariadb, using: source + ) + + #expect(read.isPinned) + #expect(read.reads.map(\.table.name) == ["dd"]) + #expect(read.reads.first?.columns.first?.defaultValue == "'2024-01-01 07:00:00'") + #expect(source.events.first == "open verbatimRead") + #expect(source.events.last == "close verbatimRead") + #expect(source.events.contains("fetchColumns dd")) + #expect(!source.events.contains("fetchColumns unrelated")) + } + + @Test("A source with no read scope is read as the session is and reports it") + func sourceWithoutScopeReadsUnpinned() async throws { + let source = SourceTableDriver(opensScope: false) + + let read = try await ObjectCopyPlanner.pinnedTableReads( + names: ["dd"], schema: nil, databaseType: .mariadb, using: source + ) + + #expect(!read.isPinned) + #expect(read.reads.map(\.table.name) == ["dd"]) + #expect(!source.events.contains { $0.hasPrefix("close") }) + } + + @Test("A failed source read still closes the read scope") + func failedSourceReadClosesTheScope() async { + let source = SourceTableDriver(failsListing: true) + + await #expect(throws: StatementFailure.self) { + _ = try await ObjectCopyPlanner.pinnedTableReads( + names: ["dd"], schema: nil, databaseType: .mariadb, using: source + ) + } + + #expect(source.events == ["open verbatimRead", "fetchTables", "close verbatimRead"]) + } + + // MARK: - Routines and triggers + + @Test("A routine is created under its own settings, outside the write scope") + func routineUsesItsCreationSettings() async throws { + let driver = DDLRecordingDriver() + let procedure = definitionStep(.procedure, "add_c", create: createProcedure, scope: creationScope) + + let result = try await run(plan(tables: [tableStep()], definitions: [procedure]), on: driver) + + #expect(result.failedCount == 0) + #expect(driver.events == [ + "open verbatimWrite", createTable, "close verbatimWrite", + saveMode, setMode, createProcedure, restoreMode + ]) + } + + @Test("A routine that fails still gets the session's settings back") + func failedRoutineRestoresSettings() async throws { + let driver = DDLRecordingDriver(failing: [createProcedure]) + let procedure = definitionStep(.procedure, "add_c", create: createProcedure, scope: creationScope) + + let result = try await run(plan(definitions: [procedure]), on: driver) + + #expect(result.failedCount == 1) + #expect(driver.events == [saveMode, setMode, createProcedure, restoreMode]) + } + + @Test("Settings that fail to apply leave the routine to be created as before") + func unappliedSettingsStillCreateTheRoutine() async throws { + let partly = DDLRecordingDriver(failing: [setMode]) + let procedure = definitionStep(.procedure, "add_c", create: createProcedure, scope: creationScope) + + let partlyApplied = try await run(plan(definitions: [procedure]), on: partly) + + #expect(partlyApplied.failedCount == 0) + #expect(partly.events == [saveMode, setMode, createProcedure, restoreMode]) + + let none = DDLRecordingDriver(failing: [saveMode]) + _ = try await run(plan(definitions: [procedure]), on: none) + + #expect(none.events == [saveMode, createProcedure]) + } + + @Test("A routine with no recorded settings runs its CREATE alone") + func routineWithoutSettingsRunsAlone() async throws { + let driver = DDLRecordingDriver() + let procedure = definitionStep(.procedure, "add_c", create: createProcedure, scope: nil) + + _ = try await run(plan(definitions: [procedure]), on: driver) + + #expect(driver.events == [createProcedure]) + } + + @Test("A trigger installed after the rows gets its own settings") + func triggerUsesItsCreationSettings() async throws { + let driver = DDLRecordingDriver() + let trigger = definitionStep(.trigger, "stamp", create: createTrigger, scope: creationScope) + + let result = try await run(plan(definitions: [trigger]), on: driver) + + #expect(result.failedCount == 0) + #expect(driver.events == [saveMode, setMode, createTrigger, restoreMode]) + } + + // MARK: - The review + + @Test("The script under review shows the settings around the definition") + func scriptShowsCreationSettings() { + let procedure = definitionStep(.procedure, "add_c", create: createProcedure, scope: creationScope) + let trigger = definitionStep(.trigger, "stamp", create: createTrigger, scope: creationScope) + + let reviewed = plan(definitions: [procedure, trigger]) + + #expect(reviewed.creationStatements.map(\.sql) == [saveMode, setMode, createProcedure, restoreMode]) + #expect(reviewed.afterDataStatements.map(\.sql) == [saveMode, setMode, createTrigger, restoreMode]) + } +} diff --git a/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift b/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift index c0fe77d9b5..a9eee6587a 100644 --- a/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift +++ b/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift @@ -94,7 +94,7 @@ struct ObjectCopyRowCopierTests { source.streamed = rows(5) let target = CopyDriver() - let outcome = try await ObjectCopyRowCopier(step: step(), targetDatabaseType: .mysql) + let outcome = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { _ in } #expect(outcome.inserted == 5) @@ -110,7 +110,7 @@ struct ObjectCopyRowCopierTests { source.streamed = rows(2_500) let target = CopyDriver() - _ = try await ObjectCopyRowCopier(step: step(), targetDatabaseType: .mysql) + _ = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { _ in } #expect(target.executedQueries.count == 3, "1,000 rows per statement at the row cap") @@ -126,7 +126,7 @@ struct ObjectCopyRowCopierTests { source.batchSize = 2 let target = CopyDriver() - _ = try await ObjectCopyRowCopier(step: step(), targetDatabaseType: .mysql) + _ = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { _ in } #expect(target.executedParameters.first?.map(\.sortKey) == ["1", "a", "2", "b"]) @@ -138,7 +138,7 @@ struct ObjectCopyRowCopierTests { source.streamed = rows(1) let target = CopyDriver() - _ = try await ObjectCopyRowCopier(step: step(), targetDatabaseType: .mysql) + _ = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { _ in } #expect(target.executedQueries.first == "INSERT INTO `public`.`orders` (`id`, `name`) VALUES (?, ?), (?, ?)".replacingOccurrences( @@ -152,7 +152,7 @@ struct ObjectCopyRowCopierTests { source.streamed = rows(1) let target = CopyDriver() - _ = try await ObjectCopyRowCopier(step: step(schema: nil), targetDatabaseType: .mysql) + _ = try await ObjectCopyRowCopier(step: step(schema: nil), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { _ in } #expect(target.executedQueries.first == "INSERT INTO `orders` (`id`, `name`) VALUES (?, ?)") @@ -163,7 +163,7 @@ struct ObjectCopyRowCopierTests { let source = CopyDriver() let target = CopyDriver() - let outcome = try await ObjectCopyRowCopier(step: step(), targetDatabaseType: .mysql) + let outcome = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { _ in } #expect(outcome.inserted == 0) @@ -179,7 +179,7 @@ struct ObjectCopyRowCopierTests { let target = CopyDriver() do { - _ = try await ObjectCopyRowCopier(step: step(), targetDatabaseType: .mysql) + _ = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { _ in } Issue.record("A row of the wrong width must not be written") } catch { @@ -194,7 +194,7 @@ struct ObjectCopyRowCopierTests { let target = CopyDriver() let reported = Reported() - _ = try await ObjectCopyRowCopier(step: step(), targetDatabaseType: .mysql) + _ = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { reported.append($0) } #expect(reported.values == [1_000, 2_000, 2_500]) @@ -213,7 +213,7 @@ struct ObjectCopyRowCopierTests { source.batchSize = 6 let target = CopyDriver() - _ = try await ObjectCopyRowCopier(step: step(), targetDatabaseType: .mysql) + _ = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { _ in } #expect(target.executedQueries.count > 1, "one mebibyte holds two of these rows") @@ -234,7 +234,7 @@ struct ObjectCopyRowCopierTests { source.batchSize = 1 let target = CopyDriver() - let outcome = try await ObjectCopyRowCopier(step: step(), targetDatabaseType: .mysql) + let outcome = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) .copy(from: source, to: target) { _ in } #expect(outcome.inserted == 1) @@ -259,7 +259,9 @@ struct ObjectCopyRowCopierTests { ) _ = try await ObjectCopyRowCopier( - step: step(columns: ["id", "live"], coercer: coercer), targetDatabaseType: .mysql + step: step(columns: ["id", "live"], coercer: coercer), + sourceDatabaseType: .postgresql, + targetDatabaseType: .mysql ).copy(from: source, to: target) { _ in } #expect(target.executedParameters.first?.map(\.sortKey) == ["1", "1"]) @@ -280,7 +282,9 @@ struct ObjectCopyRowCopierTests { ) let outcome = try await ObjectCopyRowCopier( - step: step(serverSideInsert: statement, estimatedRows: 5), targetDatabaseType: .mysql + step: step(serverSideInsert: statement, estimatedRows: 5), + sourceDatabaseType: .mysql, + targetDatabaseType: .mysql ).copy(from: source, to: target) { _ in } #expect(target.executedQueries == [statement.sql]) @@ -299,6 +303,7 @@ struct ObjectCopyRowCopierTests { serverSideInsert: SyncStatement(sql: "INSERT INTO x SELECT 1;", objectName: "x", summary: ""), estimatedRows: 42 ), + sourceDatabaseType: .mysql, targetDatabaseType: .mysql ).copy(from: CopyDriver(), to: CopyDriver()) { _ in } diff --git a/TableProTests/Core/ObjectCopy/ObjectCopySessionScopeTests.swift b/TableProTests/Core/ObjectCopy/ObjectCopySessionScopeTests.swift new file mode 100644 index 0000000000..c89b4dd285 --- /dev/null +++ b/TableProTests/Core/ObjectCopy/ObjectCopySessionScopeTests.swift @@ -0,0 +1,373 @@ +// +// ObjectCopySessionScopeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct WriteFailure: Error {} + +private final class ScopedCopyDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var log: [String] = [] + private var cancelledAtClose: [Bool] = [] + private let streamed: [[PluginCellValue]] + private let opensScope: Bool + private let failsWrites: Bool + private let cancelsOnWrite: Bool + + init( + streamed: [[PluginCellValue]] = [], + opensScope: Bool = true, + failsWrites: Bool = false, + cancelsOnWrite: Bool = false + ) { + self.streamed = streamed + self.opensScope = opensScope + self.failsWrites = failsWrites + self.cancelsOnWrite = cancelsOnWrite + } + + var events: [String] { lock.withLock { log } } + var closesSawCancellation: [Bool] { lock.withLock { cancelledAtClose } } + + var supportsTransactions: Bool { true } + + private func record(_ event: String) { + lock.withLock { log.append(event) } + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + record("open \(purpose)") + guard opensScope else { return nil } + return PluginSessionScope( + purpose: purpose, saved: [], applied: [PluginSessionSetting(name: "sql_mode", value: "")]) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + let isCancelled = Task.isCancelled + lock.withLock { + log.append("close \(scope.purpose)") + cancelledAtClose.append(isCancelled) + } + } + + func sessionTransactionState() async -> PluginSessionTransactionState { .idle } + + func beginTransaction() async throws { record("begin") } + func commitTransaction() async throws { record("commit") } + func rollbackTransaction() async throws { record("rollback") } + + func quoteIdentifier(_ name: String) -> String { "`\(name)`" } + + func streamRows(query: String) -> AsyncThrowingStream { + record("stream") + let rows = streamed + return AsyncThrowingStream { continuation in + continuation.yield(.header(PluginStreamHeader(columns: ["id", "name"], columnTypeNames: []))) + continuation.yield(.rows(rows)) + continuation.finish() + } + } + + func execute(query: String) async throws -> PluginQueryResult { + record("execute") + if failsWrites { throw WriteFailure() } + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 2, executionTime: 0) + } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + record("insert") + if cancelsOnWrite { + withUnsafeCurrentTask { task in + guard let task else { return } + task.cancel() + } + } + if failsWrites { throw WriteFailure() } + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } +} + +@MainActor +private final class TwoConnectionProvider: ScopedMetadataProviding { + private let drivers: [UUID: DatabaseDriver] + + init(drivers: [UUID: DatabaseDriver]) { + self.drivers = drivers + } + + func withMetadataDriver( + scope: DatabaseScope, + workload: MetadataConnectionPool.Workload, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T + ) async throws -> T { + guard let driver = drivers[scope.connectionId] else { + throw ObjectCopyError.refused("No driver for \(scope.connectionId)") + } + return try await body(driver) + } + + func browseScope(for connectionId: UUID) -> DatabaseScope? { nil } +} + +struct ObjectCopySessionScopeTests { + private func step(serverSideInsert: SyncStatement? = nil, hasRowFilter: Bool = false) -> ObjectCopyTableStep { + ObjectCopyTableStep( + selection: ObjectCopySelection(kind: .table, name: "orders", schema: nil), + dropStatements: [], + sequenceStatements: [], + createStatements: [], + truncateStatements: [], + columns: ["id", "name"], + primaryKeyColumns: ["id"], + sourceQuery: "SELECT `id`, `name` FROM `orders`", + targetTable: "orders", + targetSchema: nil, + estimatedRows: nil, + copiesData: true, + copiesIdentityColumn: false, + serverSideInsert: serverSideInsert, + hasRowFilter: hasRowFilter, + note: nil + ) + } + + private let zeroDates: [[PluginCellValue]] = [ + [.text("0"), .text("0000-00-00")], + [.text("1"), .text("2020-00-15")] + ] + + private let serverSideInsert = SyncStatement( + sql: "INSERT INTO `orders` (`id`, `name`) SELECT `id`, `name` FROM `shop`.`orders`", + objectName: "orders", + summary: "server side" + ) + + // MARK: - Streamed rows + + @Test("A same-engine copy reads under the source's scope and writes under the target's", arguments: [ + DatabaseType.mysql, .mariadb + ]) + func sameEngineCopyPinsBothSessions(sourceType: DatabaseType) async throws { + let source = ScopedCopyDriver(streamed: zeroDates) + let target = ScopedCopyDriver() + + let outcome = try await ObjectCopyRowCopier( + step: step(), sourceDatabaseType: sourceType, targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + #expect(outcome.inserted == 2) + #expect(source.events == ["open verbatimRead", "stream", "close verbatimRead"]) + #expect(target.events == ["open verbatimWrite", "insert", "close verbatimWrite"]) + } + + @Test("A write that fails still closes both scopes") + func failedWriteClosesBothScopes() async { + let source = ScopedCopyDriver(streamed: zeroDates) + let target = ScopedCopyDriver(failsWrites: true) + + await #expect(throws: WriteFailure.self) { + _ = try await ObjectCopyRowCopier( + step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + } + + #expect(source.events == ["open verbatimRead", "stream", "close verbatimRead"]) + #expect(target.events == ["open verbatimWrite", "insert", "close verbatimWrite"]) + } + + @Test("A stopped copy closes both scopes outside the cancellation") + func stoppedCopyClosesBothScopes() async throws { + let source = ScopedCopyDriver(streamed: zeroDates) + let target = ScopedCopyDriver(cancelsOnWrite: true) + let copier = ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) + + let outcome = try await Task { + try await copier.copy(from: source, to: target) { _ in } + }.value + + #expect(outcome.cancelled) + #expect(source.events.last == "close verbatimRead") + #expect(target.events.last == "close verbatimWrite") + #expect(source.closesSawCancellation == [false]) + #expect(target.closesSawCancellation == [false]) + } + + @Test("A copy between engines opens no scope") + func crossEngineCopyOpensNoScope() async throws { + let source = ScopedCopyDriver(streamed: zeroDates) + let target = ScopedCopyDriver() + + let outcome = try await ObjectCopyRowCopier( + step: step(), sourceDatabaseType: .postgresql, targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + #expect(outcome.inserted == 2) + #expect(source.events == ["stream"]) + #expect(target.events == ["insert"]) + } + + @Test("A target with no scope closes the source's and still copies every row") + func targetWithoutScopeStillCopies() async throws { + let source = ScopedCopyDriver(streamed: zeroDates) + let target = ScopedCopyDriver(opensScope: false) + + let outcome = try await ObjectCopyRowCopier( + step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + #expect(outcome.inserted == 2) + #expect(source.events == ["open verbatimRead", "close verbatimRead", "stream"]) + #expect(target.events == ["open verbatimWrite", "insert"]) + } + + // MARK: - Row filters + + /// The `WHERE` is the user's SQL for their own session, so pinned settings could select other rows. + @Test("A copy narrowed by a row filter reads and writes on the sessions as they are") + func filteredCopyOpensNoScope() async throws { + let source = ScopedCopyDriver(streamed: zeroDates) + let target = ScopedCopyDriver() + let filtered = PluginExportRowScope(filter: "created_at >= '2024-06-01 00:00:00'") + + let outcome = try await ObjectCopyRowCopier( + step: step(hasRowFilter: VerbatimTransferScopes.filtersRows(filtered)), + sourceDatabaseType: .mysql, + targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + #expect(outcome.inserted == 2) + #expect(source.events == ["stream"]) + #expect(target.events == ["insert"]) + } + + @Test("A filtered INSERT ... SELECT runs on the session as it is") + func filteredServerSideInsertOpensNoScope() async throws { + let target = ScopedCopyDriver() + + _ = try await ObjectCopyRowCopier( + step: step(serverSideInsert: serverSideInsert, hasRowFilter: true), + sourceDatabaseType: .mysql, + targetDatabaseType: .mysql + ).copy(from: ScopedCopyDriver(), to: target) { _ in } + + #expect(target.events == ["execute"]) + } + + @Test("A row limit or a column list alone keeps the scopes", arguments: [ + PluginExportRowScope(rowLimit: 10), PluginExportRowScope(columns: ["id"]) + ]) + func limitOnlyCopyKeepsScopes(rowScope: PluginExportRowScope) async throws { + let source = ScopedCopyDriver(streamed: zeroDates) + let target = ScopedCopyDriver() + + _ = try await ObjectCopyRowCopier( + step: step(hasRowFilter: VerbatimTransferScopes.filtersRows(rowScope)), + sourceDatabaseType: .mysql, + targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + #expect(source.events == ["open verbatimRead", "stream", "close verbatimRead"]) + #expect(target.events == ["open verbatimWrite", "insert", "close verbatimWrite"]) + } + + // MARK: - Server side + + @Test("INSERT ... SELECT opens only the write scope, on its one connection") + func serverSideInsertOpensOnlyTheWriteScope() async throws { + let source = ScopedCopyDriver() + let target = ScopedCopyDriver() + + let outcome = try await ObjectCopyRowCopier( + step: step(serverSideInsert: serverSideInsert), sourceDatabaseType: .mysql, targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + #expect(outcome.inserted == 2) + #expect(source.events.isEmpty) + #expect(target.events == ["open verbatimWrite", "execute", "close verbatimWrite"]) + } + + @Test("A failed INSERT ... SELECT still closes its write scope") + func failedServerSideInsertClosesTheWriteScope() async { + let target = ScopedCopyDriver(failsWrites: true) + + await #expect(throws: WriteFailure.self) { + _ = try await ObjectCopyRowCopier( + step: step(serverSideInsert: serverSideInsert), sourceDatabaseType: .mysql, targetDatabaseType: .mysql + ).copy(from: ScopedCopyDriver(), to: target) { _ in } + } + + #expect(target.events == ["open verbatimWrite", "execute", "close verbatimWrite"]) + } + + // MARK: - Through the runner + + @MainActor + private func runCopy( + from sourceType: DatabaseType, + source: ScopedCopyDriver, + target: ScopedCopyDriver + ) async throws -> ObjectCopyRunResult { + let sourceScope = DatabaseScope(connectionId: UUID(), database: "shop", schema: nil) + let targetScope = DatabaseScope(connectionId: UUID(), database: "staging", schema: nil) + let sourceEndpoint = DatabaseEndpoint( + scope: sourceScope, connectionName: "source", databaseType: sourceType, safeModeLevel: .silent, color: .blue + ) + let targetEndpoint = DatabaseEndpoint( + scope: targetScope, connectionName: "target", databaseType: .mysql, safeModeLevel: .silent, color: .blue + ) + let selection = ObjectCopySelection(kind: .table, name: "orders", schema: nil) + let plan = ObjectCopyPlan( + request: ObjectCopyRequest( + source: sourceEndpoint, + destination: .existing(targetEndpoint), + objects: [selection], + content: .data, + existingPolicy: .appendData + ), + createsDatabase: false, + tableSteps: [step()], + definitionSteps: [] + ) + let provider = TwoConnectionProvider(drivers: [ + sourceScope.connectionId: PluginDriverAdapter( + connection: TestFixtures.makeConnection(type: sourceType), pluginDriver: source + ), + targetScope.connectionId: PluginDriverAdapter( + connection: TestFixtures.makeConnection(type: .mysql), pluginDriver: target + ) + ]) + let runner = ObjectCopyRunner(manager: provider, gate: AlwaysAllowGate()) + return try await runner.run(plan, progress: ObjectCopyProgress(progress: Progress())) + } + + /// The scopes sit inside the table's transaction: a `SET` commits nothing on MySQL, and the + /// transaction logic stays as it was. + @MainActor @Test("The runner pins a same-engine copy inside the table's transaction") + func runnerPinsSameEngineCopy() async throws { + let source = ScopedCopyDriver(streamed: zeroDates) + let target = ScopedCopyDriver() + + let result = try await runCopy(from: .mysql, source: source, target: target) + + #expect(result.rowsCopied == 2) + #expect(source.events == ["open verbatimRead", "stream", "close verbatimRead"]) + #expect(target.events == ["begin", "open verbatimWrite", "insert", "close verbatimWrite", "commit"]) + } + + @MainActor @Test("The runner opens no scope for a copy between engines") + func runnerLeavesCrossEngineCopyUnscoped() async throws { + let source = ScopedCopyDriver(streamed: zeroDates) + let target = ScopedCopyDriver() + + let result = try await runCopy(from: .postgresql, source: source, target: target) + + #expect(result.rowsCopied == 2) + #expect(source.events == ["stream"]) + #expect(target.events == ["begin", "insert", "commit"]) + } +} diff --git a/TableProTests/Core/Plugins/ExportDataSourceAdapterSessionScopeTests.swift b/TableProTests/Core/Plugins/ExportDataSourceAdapterSessionScopeTests.swift new file mode 100644 index 0000000000..8c0894770c --- /dev/null +++ b/TableProTests/Core/Plugins/ExportDataSourceAdapterSessionScopeTests.swift @@ -0,0 +1,189 @@ +// +// ExportDataSourceAdapterSessionScopeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class CreationScopeDriver: PluginDriverStub, @unchecked Sendable { + var supportsSchemas: Bool { false } + var currentSchema: String? { nil } + + private(set) var openedPurposes: [PluginSessionPurpose] = [] + private(set) var closedScopes: [PluginSessionScope] = [] + private(set) var scopedRoutines: [String] = [] + private(set) var scopedTriggers: [String] = [] + private(set) var scopedEvents: [String] = [] + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + openedPurposes.append(purpose) + return PluginSessionScope( + purpose: purpose, + saved: [PluginSessionSetting(name: "time_zone", value: "+07:00")], + applied: [PluginSessionSetting(name: "time_zone", value: "+00:00")] + ) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + closedScopes.append(scope) + } + + func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { + [ + PluginRoutineInfo(name: "add_course", kind: .procedure, schema: schema, argumentSignature: "IN d DATE"), + PluginRoutineInfo(name: "add_course", kind: .procedure, schema: schema, argumentSignature: "IN d DATETIME"), + ] + } + + func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { + [ + PluginTriggerInfo(name: "stamp", table: "courses", timing: "BEFORE", event: "INSERT", statement: "SET x = 1"), + PluginTriggerInfo(name: "stamp", table: "lessons", timing: "BEFORE", event: "INSERT", statement: "SET y = 1"), + ] + } + + func fetchEvents(schema: String?) async throws -> [PluginEventInfo] { + [PluginEventInfo(name: "purge", schema: schema)] + } + + func fetchRoutineCreationScope(_ routine: PluginRoutineInfo) async throws -> PluginStatementScope? { + scopedRoutines.append(routine.argumentSignature ?? "") + return Self.scope(for: "routine") + } + + func fetchTriggerCreationScope(_ trigger: PluginTriggerInfo) async throws -> PluginStatementScope? { + scopedTriggers.append(trigger.table ?? "") + return Self.scope(for: "trigger") + } + + func fetchEventCreationScope(_ event: PluginEventInfo) async throws -> PluginStatementScope? { + scopedEvents.append(event.name) + return Self.scope(for: "event") + } + + static func scope(for kind: String) -> PluginStatementScope { + PluginStatementScope(opening: ["SET @\(kind) = 1"], closing: ["SET @\(kind) = 0"]) + } +} + +private final class PlainDatabaseDriver: PluginDriverStub, @unchecked Sendable {} + +struct ExportDataSourceAdapterSessionScopeTests { + private func adapter(_ driver: any PluginDatabaseDriver) -> ExportDataSourceAdapter { + let connection = TestFixtures.makeConnection(type: .mysql) + return ExportDataSourceAdapter( + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver), + databaseType: .mysql + ) + } + + private func object( + _ name: String, + kind: PluginExportObjectKind, + identity: String? = nil, + parentTable: String? = nil + ) -> PluginExportTable { + PluginExportTable( + name: name, + databaseName: "school", + tableType: kind.rawValue, + schema: nil, + kind: kind, + identity: identity, + parentTable: parentTable + ) + } + + @Test("Opening and closing a scope reaches the driver the export reads from") + func scopeCallsReachTheDriver() async throws { + let driver = CreationScopeDriver() + let source = adapter(driver) + + let scope = try #require(try await source.openSessionScope(for: .verbatimRead)) + try await source.closeSessionScope(scope) + + #expect(driver.openedPurposes == [.verbatimRead]) + #expect(driver.closedScopes == [scope]) + #expect(scope.appliedValue(of: "TIME_ZONE") == "+00:00") + } + + @Test("A driver with nothing to hold opens no scope") + func driverWithoutScopesOpensNone() async throws { + let source = adapter(PlainDatabaseDriver()) + + #expect(try await source.openSessionScope(for: .verbatimRead) == nil) + } + + @Test("A routine's scope comes from the overload the export item names") + func routineScopeMatchesTheSignature() async throws { + let driver = CreationScopeDriver() + let source = adapter(driver) + + let scope = try await source.fetchCreationScope( + object("add_course", kind: .routine, identity: "IN d DATETIME") + ) + + #expect(scope == CreationScopeDriver.scope(for: "routine")) + #expect(driver.scopedRoutines == ["IN d DATETIME"]) + } + + @Test("A trigger's scope comes from the trigger on the export item's table") + func triggerScopeMatchesTheTable() async throws { + let driver = CreationScopeDriver() + let source = adapter(driver) + + let scope = try await source.fetchCreationScope(object("stamp", kind: .trigger, parentTable: "lessons")) + + #expect(scope == CreationScopeDriver.scope(for: "trigger")) + #expect(driver.scopedTriggers == ["lessons"]) + } + + @Test("An event's scope comes from the event of that name") + func eventScopeMatchesTheName() async throws { + let driver = CreationScopeDriver() + let source = adapter(driver) + + let scope = try await source.fetchCreationScope(object("purge", kind: .event)) + + #expect(scope == CreationScopeDriver.scope(for: "event")) + #expect(driver.scopedEvents == ["purge"]) + } + + @Test("A table, a view or a sequence has no creation scope") + func otherKindsHaveNone() async throws { + let driver = CreationScopeDriver() + let source = adapter(driver) + + for kind in [PluginExportObjectKind.table, .view, .sequence, .userType] { + #expect(try await source.fetchCreationScope(object("courses", kind: kind)) == nil) + } + #expect(driver.scopedRoutines.isEmpty) + #expect(driver.scopedTriggers.isEmpty) + #expect(driver.scopedEvents.isEmpty) + } + + @Test("An object the driver does not list fails the way its definition does") + func unmatchedObjectThrows() async { + let source = adapter(CreationScopeDriver()) + + await #expect(throws: PluginObjectSourceError.self) { + try await source.fetchCreationScope(object("add_course", kind: .routine, identity: "IN d TIME")) + } + await #expect(throws: PluginObjectSourceError.self) { + try await source.fetchCreationScope(object("stamp", kind: .trigger, parentTable: "rooms")) + } + await #expect(throws: PluginObjectSourceError.self) { + try await source.fetchCreationScope(object("archive", kind: .event)) + } + } + + @Test("A source with no plugin driver has no creation scope") + func noPluginDriverHasNone() async throws { + let source = ExportDataSourceAdapter(driver: MockDatabaseDriver(), databaseType: .mysql) + + #expect(try await source.fetchCreationScope(object("purge", kind: .event)) == nil) + } +} diff --git a/TableProTests/Core/Plugins/ImportDataSinkAdapterRollbackTests.swift b/TableProTests/Core/Plugins/ImportDataSinkAdapterRollbackTests.swift new file mode 100644 index 0000000000..a83fde3f4b --- /dev/null +++ b/TableProTests/Core/Plugins/ImportDataSinkAdapterRollbackTests.swift @@ -0,0 +1,334 @@ +// +// ImportDataSinkAdapterRollbackTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +/// Holds a transaction the way MySQL does: open from `BEGIN`, or from any statement under `autocommit = 0`, until a +/// `COMMIT`, a `ROLLBACK` other than to a savepoint or, unless DDL is transactional as on PostgreSQL, a `CREATE`, +/// `ALTER` or `DROP`, which commits even when it then fails. It reports the state from its last success, as an error +/// packet carries no status flags, unless `refreshesAfterError` reads it again after a failure. +private final class StatusFlagDriver: PluginDriverStub, @unchecked Sendable { + static let failingInsert = "INSERT INTO courses (start_date) VALUES ('0000-00-00')" + + let supportsTransactionalDDL: Bool + private let reportsState: Bool + private let autocommits: Bool + private let refreshesAfterError: Bool + private let failureEndsTransaction: Bool + private let failingStatements: Set + private var isInTransaction = false + private var reportsInTransaction = false + private(set) var executed: [String] = [] + private(set) var stateReads = 0 + + init( + reportsState: Bool = true, + transactionalDDL: Bool = false, + autocommits: Bool = true, + refreshesAfterError: Bool = false, + failureEndsTransaction: Bool = false, + failingStatements: Set = [StatusFlagDriver.failingInsert] + ) { + self.reportsState = reportsState + self.supportsTransactionalDDL = transactionalDDL + self.autocommits = autocommits + self.refreshesAfterError = refreshesAfterError + self.failureEndsTransaction = failureEndsTransaction + self.failingStatements = failingStatements + } + + func execute(query: String) async throws -> PluginQueryResult { + executed.append(query) + let head = query.split(separator: " ").first.map { $0.uppercased() } ?? "" + let isDDL = ["CREATE", "ALTER", "DROP"].contains(head) + if isDDL, !supportsTransactionalDDL { isInTransaction = false } + if failingStatements.contains(query) { + if failureEndsTransaction { isInTransaction = false } + if refreshesAfterError { reportsInTransaction = isInTransaction } + throw DatabaseError.queryFailed("statement refused") + } + switch head { + case "BEGIN": + isInTransaction = true + case "COMMIT", "ROLLBACK": + if !query.uppercased().hasPrefix("ROLLBACK TO") { isInTransaction = false } + default: + if !isDDL, !autocommits { isInTransaction = true } + } + reportsInTransaction = isInTransaction + return .empty + } + + func sessionTransactionState() async -> PluginSessionTransactionState { + stateReads += 1 + guard reportsState else { return .unknown } + return reportsInTransaction ? .inTransaction : .idle + } +} + +/// "Stop and Rollback" promised a rollback that MySQL had already made impossible: the dump's first `DROP TABLE` +/// committed the import's transaction, every statement after it autocommitted, and the `ROLLBACK` took nothing back. +struct ImportDataSinkAdapterRollbackTests { + private func makeSink(on driver: StatusFlagDriver) -> (ImportDataSinkAdapter, PluginDriverAdapter) { + let connection = TestFixtures.makeConnection(type: .mysql) + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: driver) + return (ImportDataSinkAdapter(driver: adapter, databaseType: .mysql), adapter) + } + + private func runUntilFailure( + _ statements: [String], + failingAt failing: String = StatusFlagDriver.failingInsert, + on sink: ImportDataSinkAdapter + ) async throws { + try await sink.beginTransaction() + for statement in statements { + try await sink.execute(statement: statement, line: 1) + } + await #expect(throws: DatabaseError.self) { + try await sink.execute(statement: failing, line: 1) + } + } + + @Test("A transaction a DROP already committed is not rolled back, and the sink says so") + func committedTransactionIsNotRolledBack() async throws { + let driver = StatusFlagDriver() + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure( + ["DROP TABLE IF EXISTS courses", "CREATE TABLE courses (start_date DATE)", "INSERT INTO courses VALUES (NULL)"], + on: sink + ) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .committedBeforeRollback) + #expect(!driver.executed.contains("ROLLBACK")) + } + + @Test("The import no longer owns a transaction it found committed") + func committedTransactionIsReleased() async throws { + let driver = StatusFlagDriver() + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure(["DROP TABLE IF EXISTS courses"], on: sink) + try await sink.rollbackTransaction() + try await sink.rollbackTransaction() + try await sink.commitTransaction() + + #expect(driver.executed == ["BEGIN", "DROP TABLE IF EXISTS courses", StatusFlagDriver.failingInsert]) + } + + /// Under `autocommit = 0` the statement after the DROP opens a new transaction, so at the rollback the session + /// reads as in one again, and only the statements after the DROP are rolled back. + @Test("A commit in the middle is still recorded when a new transaction is open by the rollback") + func commitBeforeAReopenedTransactionIsRecorded() async throws { + let driver = StatusFlagDriver(autocommits: false) + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure( + ["INSERT INTO courses VALUES (1)", "DROP TABLE IF EXISTS lessons", "INSERT INTO courses VALUES (2)"], + on: sink + ) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .committedBeforeRollback) + #expect(driver.executed.last == "ROLLBACK") + } + + /// MySQL commits before it runs a DDL statement, so one that then fails has ended the transaction, but the session + /// reads the same as after a deadlock, so the sink names no commit. + @Test("A failing DDL statement that ended the transaction is reported as ended by the server") + func failingDDLEndIsNotCalledACommit() async throws { + let failingCreate = "CREATE TABLE courses (start_date DATE DEFAULT '0000-00-00')" + let driver = StatusFlagDriver(refreshesAfterError: true, failingStatements: [failingCreate]) + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure(["INSERT INTO lessons VALUES (1)"], failingAt: failingCreate, on: sink) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .endedByServer) + #expect(!driver.executed.contains("ROLLBACK")) + } + + /// A deadlock rolls the whole transaction back on the server. Calling that a commit would tell the user rows were + /// kept that are gone. + @Test("A transaction a deadlock ended is not reported as committed") + func deadlockIsNotCalledACommit() async throws { + let driver = StatusFlagDriver(refreshesAfterError: true, failureEndsTransaction: true) + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure(["INSERT INTO courses VALUES (1)", "INSERT INTO courses VALUES (2)"], on: sink) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .endedByServer) + #expect(!driver.executed.contains("ROLLBACK")) + } + + @Test("A transaction still open is rolled back as before") + func openTransactionIsRolledBack() async throws { + let driver = StatusFlagDriver() + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure(["INSERT INTO courses VALUES (NULL)"], on: sink) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .rolledBack) + #expect(driver.executed.last == "ROLLBACK") + } + + @Test("A driver that cannot tell rolls back as before") + func unknownStateRollsBack() async throws { + let driver = StatusFlagDriver(reportsState: false) + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure(["DROP TABLE IF EXISTS courses"], on: sink) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .rolledBack) + #expect(driver.executed.last == "ROLLBACK") + } + + @Test("An engine that rolls DDL back rolls a DROP back with the rest") + func transactionalDDLIsRolledBack() async throws { + let driver = StatusFlagDriver(transactionalDDL: true) + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure(["DROP TABLE IF EXISTS courses"], on: sink) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .rolledBack) + #expect(driver.executed.last == "ROLLBACK") + } + + /// The file's own `COMMIT` ends the import's transaction on PostgreSQL too, and a Stop's `ROLLBACK` then undoes + /// nothing, so calling it a rollback told the user rows were gone that are still there. + @Test("A file's own COMMIT on an engine that rolls DDL back is reported as a commit, not a rollback") + func fileCommitOnTransactionalDDLEngine() async throws { + let driver = StatusFlagDriver(transactionalDDL: true) + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure(["INSERT INTO courses VALUES (1)", "COMMIT"], on: sink) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .committedBeforeRollback) + #expect(!driver.executed.contains("ROLLBACK")) + } + + /// The file's second transaction is open by the Stop, so its `ROLLBACK` undoes that one only. The session reads + /// the same as if the first had never committed. + @Test("A file's own COMMIT and BEGIN on an engine that rolls DDL back still count as a commit") + func fileCommitThenBeginOnTransactionalDDLEngine() async throws { + let driver = StatusFlagDriver(transactionalDDL: true) + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure( + ["INSERT INTO courses VALUES (1)", "COMMIT", "BEGIN", "INSERT INTO courses VALUES (2)"], + on: sink + ) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .committedBeforeRollback) + #expect(driver.executed.last == "ROLLBACK") + } + + /// MySQL reads idle after the file's own `ROLLBACK` exactly as after a DDL's commit, so the text decides. + @Test("A file's own ROLLBACK is not called a commit") + func fileRollbackIsNotCalledACommit() async throws { + let driver = StatusFlagDriver() + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure(["INSERT INTO courses VALUES (1)", "ROLLBACK"], on: sink) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .endedByFile) + #expect(driver.executed.filter { $0 == "ROLLBACK" }.count == 1) + } + + @Test("A ROLLBACK TO SAVEPOINT in the file leaves the import's transaction to roll back") + func rollbackToSavepointChangesNothing() async throws { + let driver = StatusFlagDriver() + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure( + ["SAVEPOINT before_courses", "INSERT INTO courses VALUES (1)", "ROLLBACK TO SAVEPOINT before_courses"], + on: sink + ) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .rolledBack) + #expect(driver.executed.last == "ROLLBACK") + } + + @Test("A transaction the server ended on an engine that rolls DDL back is not called a commit") + func serverEndedTransactionOnTransactionalDDLEngine() async throws { + let driver = StatusFlagDriver(transactionalDDL: true, refreshesAfterError: true, failureEndsTransaction: true) + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure(["DROP TABLE IF EXISTS courses"], on: sink) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .endedByServer) + #expect(!driver.executed.contains("ROLLBACK")) + } + + /// SQL Server reads its transaction state with a query, so reading it after every statement would double an + /// import's round trips. Only an engine whose DDL commits needs the read. + @Test("An engine that rolls DDL back is asked its state at the begin and the rollback only") + func transactionalDDLEngineReadsNoStatePerStatement() async throws { + let driver = StatusFlagDriver(transactionalDDL: true) + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure( + ["INSERT INTO courses VALUES (1)", "INSERT INTO courses VALUES (2)", "INSERT INTO courses VALUES (3)"], + on: sink + ) + try await sink.rollbackTransaction() + + #expect(driver.stateReads == 2) + } + + @Test("An engine whose DDL commits is asked its state after every statement that succeeds") + func implicitCommitEngineReadsStatePerStatement() async throws { + let driver = StatusFlagDriver() + let (sink, _) = makeSink(on: driver) + + try await runUntilFailure( + ["INSERT INTO courses VALUES (1)", "INSERT INTO courses VALUES (2)", "INSERT INTO courses VALUES (3)"], + on: sink + ) + try await sink.rollbackTransaction() + + #expect(driver.stateReads == 5) + } + + @Test("A transaction the user opened is neither rolled back nor reported") + func usersTransactionIsLeftAlone() async throws { + let driver = StatusFlagDriver() + let (sink, adapter) = makeSink(on: driver) + _ = try await adapter.execute(query: "BEGIN") + + try await runUntilFailure(["DROP TABLE IF EXISTS courses"], on: sink) + try await sink.rollbackTransaction() + + #expect(sink.transactionOutcome == .untracked) + #expect(!driver.executed.contains("ROLLBACK")) + #expect(driver.executed.filter { $0 == "BEGIN" }.count == 1) + } + + @Test("An import that commits reports a commit, even after a DDL statement committed part of it") + func commitIsReported() async throws { + let driver = StatusFlagDriver() + let (sink, _) = makeSink(on: driver) + + try await sink.beginTransaction() + try await sink.execute(statement: "DROP TABLE IF EXISTS courses", line: 1) + try await sink.execute(statement: "INSERT INTO lessons VALUES (1)", line: 2) + try await sink.commitTransaction() + + #expect(sink.transactionOutcome == .committed) + } +} diff --git a/TableProTests/Core/Services/ImportServiceSessionScopeTests.swift b/TableProTests/Core/Services/ImportServiceSessionScopeTests.swift new file mode 100644 index 0000000000..bfbd9f6f2b --- /dev/null +++ b/TableProTests/Core/Services/ImportServiceSessionScopeTests.swift @@ -0,0 +1,378 @@ +// +// ImportServiceSessionScopeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct ScopeRefusal: Error {} + +private final class SessionScopeDriver: PluginDriverStub, @unchecked Sendable { + static let failure = "[1292] Incorrect date value: '0000-00-00' for column 'start_date' at row 1" + static let preserved = PluginSessionScope( + purpose: .preserve, + saved: [PluginSessionSetting(name: "sql_mode", value: "STRICT_TRANS_TABLES,NO_ZERO_DATE")], + applied: [] + ) + + var transactionState: PluginSessionTransactionState = .unknown + var statesAfter: [String: PluginSessionTransactionState] = [:] + var failingStatements: Set = [] + var opensScope = true + var openError: (any Error)? + var closeError: (any Error)? + private(set) var events: [String] = [] + private(set) var openedPurposes: [PluginSessionPurpose] = [] + private(set) var closedScopes: [PluginSessionScope] = [] + + func execute(query: String) async throws -> PluginQueryResult { + events.append(query) + if let next = statesAfter[query] { transactionState = next } + if failingStatements.contains(query) { + throw DatabaseError.queryFailed(Self.failure) + } + return .empty + } + + func switchDatabase(to database: String) async throws {} + + func sessionTransactionState() async -> PluginSessionTransactionState { + transactionState + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + events.append("open") + openedPurposes.append(purpose) + if let openError { throw openError } + return opensScope ? Self.preserved : nil + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + events.append("close") + closedScopes.append(scope) + if let closeError { throw closeError } + } +} + +/// Runs its statements the way the SQL import's "Stop and Rollback" does: one transaction, a failure or a cancel +/// rolls it back, then the error goes on. +private final class ScriptedImportPlugin: ImportFormatPlugin { + static let pluginName = "Scripted Import" + static let pluginVersion = "1.0" + static let pluginDescription = "An import format that runs a fixed list of statements" + static let formatId = "tablepro-tests-scripted-import" + static let formatDisplayName = "Scripted" + static let acceptedFileExtensions = ["sql"] + static let iconName = "doc" + static let requiresTargetTable = true + + private let statements: [String] + private let cancelsBeforeLine: Int? + + required init() { + statements = [] + cancelsBeforeLine = nil + } + + init(statements: [String], cancelsBeforeLine: Int? = nil) { + self.statements = statements + self.cancelsBeforeLine = cancelsBeforeLine + } + + func performImport( + source: any PluginImportSource, + sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws -> PluginImportResult { + try await sink.beginTransaction() + do { + for (index, statement) in statements.enumerated() { + let line = index + 1 + if line == cancelsBeforeLine { progress.cancel() } + try progress.checkCancellation() + do { + try await sink.execute(statement: statement, line: line) + } catch { + throw PluginImportError.statementFailed(statement: statement, line: line, underlyingError: error) + } + } + try await sink.commitTransaction() + } catch { + try? await sink.rollbackTransaction() + throw error + } + return PluginImportResult(executedStatements: statements.count, executionTime: 0) + } +} + +private actor DiscardingHistory: QueryHistoryRecording { + func record(_ request: QueryHistoryRecordRequest) async -> Bool { true } +} + +private struct ImportRun { + let service: ImportService + let result: Result + + var error: (any Error)? { + guard case .failure(let error) = result else { return nil } + return error + } +} + +/// The import runs the file on the editor's own session, so what the file sets has to be put back however the +/// import ends. A failed import left a dump's `SET NAMES` and `SQL_MODE` on the session the user kept typing into. +@MainActor +struct ImportServiceSessionScopeTests { + private func runImport(_ plugin: ScriptedImportPlugin, on driver: SessionScopeDriver) async throws -> ImportRun { + let connection = TestFixtures.makeConnection(database: "school", type: .mysql) + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + session.browseDatabase = "school" + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let formatId = "\(ScriptedImportPlugin.formatId)-\(UUID().uuidString)" + PluginManager.shared.importPlugins[formatId] = plugin + defer { PluginManager.shared.importPlugins[formatId] = nil } + + let file = FileManager.default.temporaryDirectory + .appendingPathComponent("session-scope-import-\(UUID().uuidString)") + .appendingPathExtension("sql") + try Data("SELECT 1;\n".utf8).write(to: file) + defer { try? FileManager.default.removeItem(at: file) } + + let service = ImportService(connection: connection, historyRecorder: DiscardingHistory(), gate: AlwaysAllowGate()) + do { + let result = try await service.importFile( + from: file, + formatId: formatId, + encoding: .utf8, + scope: DatabaseScope(connectionId: connection.id, database: "school", schema: nil) + ) + return ImportRun(service: service, result: .success(result)) + } catch { + return ImportRun(service: service, result: .failure(error)) + } + } + + @Test("A successful import opens one preserving scope and closes it after the commit") + func successClosesTheScope() async throws { + let driver = SessionScopeDriver() + + let run = try await runImport(ScriptedImportPlugin(statements: ["INSERT INTO courses VALUES (1)"]), on: driver) + + _ = try run.result.get() + #expect(driver.openedPurposes == [.preserve]) + #expect(driver.events == ["open", "BEGIN", "INSERT INTO courses VALUES (1)", "COMMIT", "close"]) + #expect(driver.closedScopes == [SessionScopeDriver.preserved]) + #expect(run.service.state.transactionOutcome == .committed) + } + + @Test("A failing statement closes the scope after the rollback") + func failureClosesTheScope() async throws { + let driver = SessionScopeDriver() + driver.failingStatements = ["INSERT INTO courses VALUES ('0000-00-00')"] + + let run = try await runImport( + ScriptedImportPlugin(statements: ["SET NAMES utf8mb4", "INSERT INTO courses VALUES ('0000-00-00')"]), + on: driver + ) + + #expect(run.error is PluginImportError) + #expect(driver.events == [ + "open", "BEGIN", "SET NAMES utf8mb4", "INSERT INTO courses VALUES ('0000-00-00')", "ROLLBACK", "close", + ]) + #expect(run.service.state.transactionOutcome == .rolledBack) + } + + @Test("A cancelled import closes the scope after the rollback, and says it rolled back") + func cancelClosesTheScope() async throws { + let driver = SessionScopeDriver() + + let run = try await runImport( + ScriptedImportPlugin(statements: ["SET NAMES utf8mb4", "INSERT INTO courses VALUES (1)"], cancelsBeforeLine: 2), + on: driver + ) + + #expect(run.error is PluginImportCancellationError) + #expect(driver.events == ["open", "BEGIN", "SET NAMES utf8mb4", "ROLLBACK", "close"]) + #expect(run.service.state.transactionOutcome == .rolledBack) + } + + @Test("A cancel after the server committed the transaction says what ran was kept") + func cancelAfterACommitReportsIt() async throws { + let driver = SessionScopeDriver() + driver.transactionState = .idle + driver.statesAfter = ["BEGIN": .inTransaction, "DROP TABLE IF EXISTS courses": .idle] + + let run = try await runImport( + ScriptedImportPlugin( + statements: ["DROP TABLE IF EXISTS courses", "INSERT INTO courses VALUES (1)"], + cancelsBeforeLine: 2 + ), + on: driver + ) + + #expect(run.error is PluginImportCancellationError) + #expect(driver.events == ["open", "BEGIN", "DROP TABLE IF EXISTS courses", "close"]) + #expect(run.service.state.transactionOutcome == .committedBeforeRollback) + } + + @Test("A cancel inside the user's own transaction reports no transaction of the import's") + func cancelInsideTheUsersTransaction() async throws { + let driver = SessionScopeDriver() + driver.transactionState = .inTransaction + + let run = try await runImport( + ScriptedImportPlugin(statements: ["SET NAMES utf8mb4", "INSERT INTO courses VALUES (1)"], cancelsBeforeLine: 2), + on: driver + ) + + #expect(run.error is PluginImportCancellationError) + #expect(driver.events == ["open", "SET NAMES utf8mb4", "close"]) + #expect(run.service.state.transactionOutcome == .untracked) + } + + @Test("A scope that fails to open leaves the import to run as before") + func openFailureDoesNotStopTheImport() async throws { + let driver = SessionScopeDriver() + driver.openError = ScopeRefusal() + + let run = try await runImport(ScriptedImportPlugin(statements: ["INSERT INTO courses VALUES (1)"]), on: driver) + + #expect(try run.result.get().executedStatements == 1) + #expect(driver.events == ["open", "BEGIN", "INSERT INTO courses VALUES (1)", "COMMIT"]) + } + + @Test("A driver with nothing to hold gets no close") + func noScopeGetsNoClose() async throws { + let driver = SessionScopeDriver() + driver.opensScope = false + + let run = try await runImport(ScriptedImportPlugin(statements: ["INSERT INTO courses VALUES (1)"]), on: driver) + + _ = try run.result.get() + #expect(driver.closedScopes.isEmpty) + } + + @Test("A scope that fails to close does not fail the import") + func closeFailureDoesNotFailTheImport() async throws { + let driver = SessionScopeDriver() + driver.closeError = ScopeRefusal() + + let run = try await runImport(ScriptedImportPlugin(statements: ["INSERT INTO courses VALUES (1)"]), on: driver) + + #expect(try run.result.get().executedStatements == 1) + #expect(driver.events.last == "close") + } + + @Test("A failure after the server committed the transaction says nothing was rolled back") + func committedTransactionFailureCarriesTheNote() async throws { + let failing = "INSERT INTO courses VALUES ('0000-00-00')" + let driver = SessionScopeDriver() + driver.transactionState = .idle + driver.statesAfter = ["BEGIN": .inTransaction, "DROP TABLE IF EXISTS courses": .idle] + driver.failingStatements = [failing] + + let run = try await runImport( + ScriptedImportPlugin(statements: ["DROP TABLE IF EXISTS courses", failing]), + on: driver + ) + + guard case .statementFailed(let statement, let line, let underlying)? = run.error as? PluginImportError else { + Issue.record("Expected statementFailed, got \(String(describing: run.error))") + return + } + #expect(statement == failing) + #expect(line == 2) + #expect(underlying.localizedDescription == SessionScopeDriver.failure + "\n" + ImportCommittedTransactionError.note) + #expect(driver.events == ["open", "BEGIN", "DROP TABLE IF EXISTS courses", failing, "close"]) + #expect(run.service.state.transactionOutcome == .committedBeforeRollback) + } + + /// A deadlock rolls the transaction back on the server, and the session then reads exactly as after a commit, so + /// the failure carries no claim either way. + @Test("A failure that ended the transaction on the server carries no note") + func serverEndedTransactionCarriesNoNote() async throws { + let failing = "INSERT INTO courses VALUES (2)" + let driver = SessionScopeDriver() + driver.transactionState = .idle + driver.statesAfter = ["BEGIN": .inTransaction, failing: .idle] + driver.failingStatements = [failing] + + let run = try await runImport( + ScriptedImportPlugin(statements: ["INSERT INTO courses VALUES (1)", failing]), + on: driver + ) + + guard case .statementFailed(_, let line, let underlying)? = run.error as? PluginImportError else { + Issue.record("Expected statementFailed, got \(String(describing: run.error))") + return + } + #expect(line == 2) + #expect(underlying.localizedDescription == SessionScopeDriver.failure) + #expect(driver.events == ["open", "BEGIN", "INSERT INTO courses VALUES (1)", failing, "close"]) + #expect(run.service.state.transactionOutcome == .endedByServer) + } +} + +@MainActor +struct ImportCommittedTransactionErrorTests { + private struct DatabaseMessage: LocalizedError { + let errorDescription: String? + } + + @Test("A statement failure keeps its statement and line, and the alert shows the note under the reason") + func statementFailureKeepsItsShape() throws { + let original = PluginImportError.statementFailed( + statement: "INSERT INTO courses VALUES ('0000-00-00')", + line: 8_264, + underlyingError: DatabaseMessage(errorDescription: "[1292] Incorrect date value") + ) + + let noted = ImportCommittedTransactionError.noting(original) + + guard case .statementFailed(let statement, let line, let underlying)? = noted as? PluginImportError else { + Issue.record("Expected statementFailed, got \(noted)") + return + } + #expect(statement == "INSERT INTO courses VALUES ('0000-00-00')") + #expect(line == 8_264) + #expect(underlying.localizedDescription == "[1292] Incorrect date value\n" + ImportCommittedTransactionError.note) + let text = TransferResultAlert.importFailureText(for: noted) + #expect(text.hasPrefix("Failed at line 8264. [1292] Incorrect date value")) + #expect(text.contains(ImportCommittedTransactionError.note)) + } + + @Test("Any other failure carries the note under its own message") + func otherFailureCarriesTheNote() { + let original = PluginImportError.importFailed("disk full") + + let noted = ImportCommittedTransactionError.noting(original) + + #expect(noted.localizedDescription == original.localizedDescription + "\n" + ImportCommittedTransactionError.note) + } + + /// Under `autocommit = 0` the rollback still undoes what ran after the DDL, so the note never says all of it stayed. + @Test("The note says only some of what ran before the failure was kept") + func noteSaysSomeWasKept() { + #expect( + ImportCommittedTransactionError.note + == "The rollback did not undo everything. A statement in the file, such as COMMIT, CREATE, ALTER or " + + "DROP, committed the import's transaction as it ran, so some of the statements before the failure " + + "were kept." + ) + } + + @Test("A cancel stays a cancel") + func cancelIsUntouched() { + #expect(ImportCommittedTransactionError.noting(PluginImportCancellationError()) is PluginImportCancellationError) + #expect(ImportCommittedTransactionError.noting(CancellationError()) is CancellationError) + } +} diff --git a/TableProTests/Views/ImportCancelledTextTests.swift b/TableProTests/Views/ImportCancelledTextTests.swift new file mode 100644 index 0000000000..d3a5ef1935 --- /dev/null +++ b/TableProTests/Views/ImportCancelledTextTests.swift @@ -0,0 +1,63 @@ +// +// ImportCancelledTextTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +/// A stopped import said what ran "stays committed" even when Wrap in transaction had just rolled all of it back. +@MainActor +struct ImportCancelledTextTests { + @Test("A rolled back import says so") + func rolledBack() { + #expect( + TransferResultAlert.importCancelledText(executedStatements: 12, transaction: .rolledBack) + == "12 statements had already run and were rolled back." + ) + #expect( + TransferResultAlert.importCancelledText(executedStatements: 1, transaction: .rolledBack) + == "1 statement had already run and was rolled back." + ) + } + + /// Under `autocommit = 0` the rollback still undoes what ran after the DDL, so only some of the work was kept. + @Test("An import the database committed part way says some of it was kept, and why") + func committedBeforeRollback() { + #expect( + TransferResultAlert.importCancelledText(executedStatements: 848, transaction: .committedBeforeRollback) + == "848 statements had already run. Some were kept, because a statement in the file, such as " + + "COMMIT, CREATE, ALTER or DROP, committed the import's transaction as it ran." + ) + #expect( + TransferResultAlert.importCancelledText(executedStatements: 1, transaction: .committedBeforeRollback) + == "1 statement had already run and was kept, because a statement in the file, such as COMMIT, " + + "CREATE, ALTER or DROP, committed the import's transaction as it ran." + ) + } + + @Test("An import with no transaction of its own keeps saying what ran stays committed") + func untracked() { + #expect( + TransferResultAlert.importCancelledText(executedStatements: 3, transaction: .untracked) + == "3 statements had already run and stay committed." + ) + #expect( + TransferResultAlert.importCancelledText(executedStatements: 1, transaction: .committed) + == "1 statement had already run and stays committed." + ) + } + + @Test("A transaction the server or the file ended makes no claim about rolled back or kept") + func endedOutsideTheImport() { + #expect( + TransferResultAlert.importCancelledText(executedStatements: 5, transaction: .endedByServer) + == "5 statements had already run and stay committed." + ) + #expect( + TransferResultAlert.importCancelledText(executedStatements: 5, transaction: .endedByFile) + == "5 statements had already run and stay committed." + ) + } +} diff --git a/scripts/probes/check-mysql-dump-session.sh b/scripts/probes/check-mysql-dump-session.sh new file mode 100755 index 0000000000..88f655c544 --- /dev/null +++ b/scripts/probes/check-mysql-dump-session.sh @@ -0,0 +1,516 @@ +#!/usr/bin/env bash +# +# Check a TablePro MySQL dump's session lines, and the session facts the MySQL driver relies on, against a real +# server. +# +# A MySQL dump sets SQL_MODE='NO_AUTO_VALUE_ON_ZERO' in its prologue the way mysqldump does, so zero dates, partial +# dates and an AUTO_INCREMENT id of 0 reload under MySQL 8's strict default mode. The driver's session scopes save +# settings with SHOW SESSION VARIABLES and put them back with SET SESSION, wrap each routine and event in the sql_mode +# and time_zone the server recorded for it, and ask the server again about a transaction after a refused statement. +# Nothing at runtime checks any of that. This compiles the SQL export's declaration and the driver's statement +# builders, then: +# a. loads a dump wrapped in that prologue and epilogue under MySQL 8's default mode and reads the rows back; +# b. loads the same rows without the SQL_MODE line, which must fail with 1292, so a server that accepts zero +# dates anyway cannot pass (a) by accident; +# c. runs the restore forms (SHOW reports ON or OFF, SET SESSION foreign_key_checks = 'OFF' and +# character_set_results = NULL work, '1' is refused), then leaves a session the way a dump file that stopped +# early does and checks the preserve scope's restores, autocommit last, bring every saved setting back, and +# that with the user's own transaction open they leave autocommit alone and commit nothing; +# d. recreates a procedure and an event inside their creation scopes from a mode MariaDB records by default, +# NO_AUTO_CREATE_USER included, and checks the server recorded the scope's mode and zone; +# e. writes under the copy write scope the driver derives from the session's mode: zero and partial dates and +# id 0 are stored, even over TRADITIONAL, while what the session refuses is still refused (1406, 1264, and a +# MyISAM multi-row overflow under STRICT_ALL_TABLES), ALLOW_INVALID_DATES still takes 2024-02-31, and +# NO_BACKSLASH_ESCAPES stays, so DDL spelled for it stores C:\tmp as written; +# f. shows the server committing a transaction when a CREATE TABLE inside it is refused: afterwards the server +# reports no open transaction and ROLLBACK keeps the row. The refusal's ERR packet carries no status flags, so +# a client still holds the last OK's "in transaction". +# +# Usage: +# scripts/probes/check-mysql-dump-session.sh [host] [port] [user] [mysql option...] +# +# An empty host ('') connects the way the mysql client's own option files say, which for a Homebrew MariaDB is the +# local socket as the current user. Options after the user go to every mysql call, such as --skip-ssl for a server +# whose certificate the client cannot verify. Needs the mysql client, xcrun swiftc, and a user that can create and +# drop a database and create events; on a server without @@in_transaction, PROCESS too. The password, if any, comes +# from MYSQL_PWD; the scratch database name from PROBE_DATABASE. Exits 0 when the server agrees, 1 when it does not, +# and 3 when the check could not run. + +set -uo pipefail + +HOST="${1-127.0.0.1}" +PORT="${2:-3306}" +USER_NAME="${3:-}" +EXTRA=("${@:4}") +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +KIT="$ROOT/Plugins/TableProPluginKit" +EXPORT="$ROOT/Plugins/SQLExportPlugin" +PLUGIN="$ROOT/Plugins/MySQLDriverPlugin" +CORE="$ROOT/Packages/TableProCore/Sources/TableProMySQLCore" +PROBE_DATABASE="${PROBE_DATABASE:-tablepro_dump_session_probe}" +STRICT_MODE="ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION" +MARIADB_ROUTINE_MODE="STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" +EVENT_ZONE="+05:00" + +case "$PROBE_DATABASE" in + "" | *[!A-Za-z0-9_]*) + echo "PROBE_DATABASE must be letters, digits and underscores" >&2 + exit 3 + ;; +esac + +command -v mysql > /dev/null || { + echo "mysql client not found" >&2 + exit 3 +} + +if [ -n "$HOST" ]; then + MYSQL=(mysql --no-defaults --comments -h "$HOST" -P "$PORT" -u "${USER_NAME:-root}") + TARGET="$HOST:$PORT" +elif [ -n "$USER_NAME" ]; then + MYSQL=(mysql --comments -u "$USER_NAME") + TARGET="the local socket" +else + MYSQL=(mysql --comments) + TARGET="the local socket" +fi +MYSQL+=(${EXTRA[@]+"${EXTRA[@]}"} -N -B) + +if ! "${MYSQL[@]}" -e "SELECT 1" > /dev/null 2>&1; then + echo "no MySQL at $TARGET" >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +cleanup() { + "${MYSQL[@]}" -e "DROP DATABASE IF EXISTS $PROBE_DATABASE" > /dev/null 2>&1 + rm -rf "$WORK" +} +trap cleanup EXIT + +DECLARATION="$EXPORT/SQLExportSessionDeclaration.swift" +DECLARE='SQLExportSessionDeclaration.forDatabaseType("MySQL", readTimeZone: "+00:00")' +if [ ! -f "$DECLARATION" ]; then + DECLARATION="$EXPORT/SQLExportEncodingDeclaration.swift" + DECLARE='SQLExportEncodingDeclaration.forDatabaseType("MySQL")' +fi +[ -f "$DECLARATION" ] || { + echo "no SQL export declaration under $EXPORT" >&2 + exit 3 +} + +cat > "$WORK/declare.swift" < (prologue: String, epilogue: String) { + let declaration = $DECLARE + return (declaration.prologue, declaration.epilogue) +} +SWIFT + +cat > "$WORK/main.swift" <<'SWIFT' +import Foundation +import TableProMySQLCore +import TableProPluginKit + +let arguments = Array(CommandLine.arguments.dropFirst()) + +func argument(_ index: Int) -> String? { + index < arguments.count ? arguments[index] : nil +} + +func terminated(_ statements: [String]) -> String { + statements.map { $0 + ";" }.joined(separator: "\n") +} + +func settings(inFile path: String, names: [String]) -> [PluginSessionSetting] { + let text = (try? String(contentsOfFile: path, encoding: .utf8)) ?? "" + let rows = text.split(separator: "\n").map { line in + line.split(separator: "\t", omittingEmptySubsequences: false).map { PluginCellValue.text(String($0)) } + } + return MySQLSessionScopePlan.captured(rows: rows, names: names) +} + +switch argument(0) ?? "" { +case "prologue": + print(makeDeclaration().prologue, terminator: "") +case "epilogue": + print(makeDeclaration().epilogue, terminator: "") +case "capture": + print(MySQLSessionStatements.captureQuery(names: Array(arguments.dropFirst())) ?? "") +case "preserve-capture": + print(MySQLSessionStatements.captureQuery(names: MySQLSessionScopePlan.preservedNames) ?? "") +case "assign": + print(MySQLSessionStatements.assignment(name: argument(1) ?? "", value: argument(2)) ?? "") +case "verbatim-capture": + print(MySQLSessionStatements.captureQuery(names: MySQLSessionScopePlan.verbatimNames) ?? "") +case "apply-write": + let saved = settings(inFile: argument(1) ?? "", names: MySQLSessionScopePlan.verbatimNames) + let applied = MySQLSessionScopePlan.applied(for: .verbatimWrite, saved: saved) ?? [] + print(terminated(applied.compactMap { MySQLSessionStatements.assignment(name: $0.name, value: $0.value) })) +case "restore": + let names = MySQLSessionScopePlan.preservedNames + let restore = MySQLSessionScopePlan.restore( + saved: settings(inFile: argument(1) ?? "", names: names), + current: settings(inFile: argument(2) ?? "", names: names), + isTransactionOpen: argument(3) == "in-transaction" + ) + print(terminated(restore.statements)) +case "portable-mode": + print(MySQLCreationSettings(sqlMode: argument(1) ?? "")?.sqlMode ?? "") +case "opening", "closing": + guard let settings = MySQLCreationSettings(sqlMode: argument(1) ?? "", timeZone: argument(2)) else { exit(2) } + print(terminated(argument(0) == "opening" ? settings.opening : settings.closing)) +case "routine-settings": + print(MySQLObjectQueries.routineCreationSettings(schema: argument(1) ?? "", name: argument(2) ?? "", kind: "PROCEDURE")) +case "event-settings": + print(MySQLObjectQueries.eventCreationSettings(schema: argument(1) ?? "", name: argument(2) ?? "")) +default: + exit(2) +} +SWIFT + +mkdir -p "$WORK/modules" +{ + xcrun swiftc -emit-library -emit-module -module-name TableProPluginKit \ + -emit-module-path "$WORK/modules/TableProPluginKit.swiftmodule" -o "$WORK/libTableProPluginKit.dylib" \ + "$KIT/SqlDialect.swift" "$KIT/PluginSessionScope.swift" "$KIT/PluginCellValue.swift" "$KIT/ArrayExtension.swift" && + xcrun swiftc -emit-library -emit-module -module-name TableProMySQLCore \ + -emit-module-path "$WORK/modules/TableProMySQLCore.swiftmodule" -o "$WORK/libTableProMySQLCore.dylib" \ + "$CORE/MySQLLiteralSQL.swift" "$CORE/MySQLSessionStatements.swift" "$CORE/MySQLCreationSettings.swift" \ + "$CORE/MySQLObjectQueries.swift" && + xcrun swiftc -I "$WORK/modules" -L "$WORK" -lTableProPluginKit -lTableProMySQLCore \ + -Xlinker -rpath -Xlinker "$WORK" -o "$WORK/statements" \ + "$WORK/main.swift" "$WORK/declare.swift" "$DECLARATION" "$PLUGIN/MySQLSessionScopePlan.swift" +} > "$WORK/build.log" 2>&1 || { + cat "$WORK/build.log" >&2 + exit 3 +} +STATEMENTS="$WORK/statements" + +"${MYSQL[@]}" -e "DROP DATABASE IF EXISTS $PROBE_DATABASE; CREATE DATABASE $PROBE_DATABASE" || { + echo "could not create $PROBE_DATABASE" >&2 + exit 3 +} +STRICT=("${MYSQL[@]}" "--init-command=SET SESSION sql_mode='$STRICT_MODE'") +STRICT_REPORTED="$("${STRICT[@]}" -e "SELECT @@SESSION.sql_mode")" +echo "$("${MYSQL[@]}" -e "SELECT VERSION()"), declaration from $(basename "$DECLARATION")" +FAILED=0 + +fail() { + echo "FAIL $*" + FAILED=1 +} + +# a. The dump under MySQL 8's default mode. +"$STATEMENTS" prologue > "$WORK/prologue.sql" +"$STATEMENTS" epilogue > "$WORK/epilogue.sql" +ROWS="(0, '0000-00-00', '2020-00-15', '0000-00-00 00:00:00'), (1, '2024-01-31', '2024-01-31', '2024-01-31 10:00:00')" +{ + cat "$WORK/prologue.sql" + cat < "$WORK/dump.sql" + +if MODE_AFTER="$("${STRICT[@]}" "$PROBE_DATABASE" < "$WORK/dump.sql" 2>&1)"; then + READ_BACK="$("${MYSQL[@]}" -e "SELECT id, start_date, partial_date, ends_at FROM $PROBE_DATABASE.courses ORDER BY id")" + EXPECTED="$(printf '0\t0000-00-00\t2020-00-15\t0000-00-00 00:00:00\n1\t2024-01-31\t2024-01-31\t2024-01-31 10:00:00')" + if [ "$READ_BACK" = "$EXPECTED" ]; then + echo "OK (a): zero date, partial date, zero DATETIME and id 0 came back unchanged" + else + fail "(a): rows came back as: $READ_BACK" + fi + if [ "$MODE_AFTER" = "$STRICT_REPORTED" ]; then + echo "OK (a): the epilogue put the session's sql_mode back" + else + fail "(a): sql_mode after the epilogue is '$MODE_AFTER'" + fi +else + fail "(a): the dump did not load: $MODE_AFTER" +fi + +# b. The control: the same rows without the SQL_MODE line. +{ + grep -v SQL_MODE "$WORK/prologue.sql" + cat < "$WORK/control.sql" + +if CONTROL="$("${STRICT[@]}" "$PROBE_DATABASE" < "$WORK/control.sql" 2>&1)"; then + echo "control failed: the rows loaded without the SQL_MODE line, so the strict mode never took effect" >&2 + exit 3 +fi +case "$CONTROL" in + *"ERROR 1292"*) echo "OK (b): without the SQL_MODE line the load fails with 1292" ;; + *) + echo "control failed with something other than 1292: $CONTROL" >&2 + exit 3 + ;; +esac + +# c. The restore forms the driver's session scopes send. +CAPTURE="$("$STATEMENTS" capture foreign_key_checks unique_checks)" +REPORTED="$("${MYSQL[@]}" -e "$CAPTURE" | cut -f2 | sort -u | tr '\n' ' ')" +case "$REPORTED" in + "ON " | "OFF " | "OFF ON ") echo "OK (c): SHOW reports booleans as ${REPORTED% }" ;; + *) fail "(c): SHOW reported '$REPORTED' for foreign_key_checks and unique_checks" ;; +esac + +FK_OFF="$("$STATEMENTS" assign foreign_key_checks OFF)" +RESULTS_NULL="$("$STATEMENTS" assign character_set_results)" +FK_STATE="$("${MYSQL[@]}" -e "$FK_OFF; $("$STATEMENTS" capture foreign_key_checks)" 2>&1)" +RESULTS_STATE="$("${MYSQL[@]}" -e "$RESULTS_NULL; SELECT IFNULL(@@SESSION.character_set_results, 'NULL'); $("$STATEMENTS" capture character_set_results)" 2>&1)" +if [ "$(echo "$FK_STATE" | cut -f2)" = "OFF" ]; then + echo "OK (c): $FK_OFF" +else + fail "(c): $FK_OFF gave: $FK_STATE" +fi +if [ "$RESULTS_STATE" = "$(printf 'NULL\ncharacter_set_results\t')" ]; then + echo "OK (c): $RESULTS_NULL, and SHOW reports the NULL as an empty value" +elif [ "$RESULTS_STATE" = "$(printf 'NULL\ncharacter_set_results\tNULL')" ]; then + echo "OK (c): $RESULTS_NULL, and SHOW reports the NULL as NULL" +else + fail "(c): $RESULTS_NULL gave: $RESULTS_STATE" +fi +if REFUSED="$("${MYSQL[@]}" -e "SET SESSION foreign_key_checks = '1'" 2>&1)"; then + fail "(c): SET SESSION foreign_key_checks = '1' was accepted, so a quoted SELECT @@ value would do" +else + case "$REFUSED" in + *"ERROR 1231"*) echo "OK (c): SET SESSION foreign_key_checks = '1' is refused with 1231" ;; + *) fail "(c): SET SESSION foreign_key_checks = '1' failed with something other than 1231: $REFUSED" ;; + esac +fi + +# A dump file that stopped early: every session starts the same, so one session saves, one reads what the file left, +# and a third runs the file again and then the preserve scope's restores. +PRESERVE_CAPTURE="$("$STATEMENTS" preserve-capture)" +"${MYSQL[@]}" "$PROBE_DATABASE" -e "CREATE TABLE preserve_rows (id int) ENGINE=InnoDB" +cat > "$WORK/stopped.sql" <<'SQL' +/*!40101 SET NAMES latin1 */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET UNIQUE_CHECKS=0, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET SQL_NOTES=0 */; +/*M!100616 SET NOTE_VERBOSITY=0 */; +SET @@AUTOCOMMIT=0; +INSERT INTO preserve_rows VALUES (1); +SQL +"${MYSQL[@]}" "$PROBE_DATABASE" -e "$PRESERVE_CAPTURE" > "$WORK/saved.tsv" +{ + cat "$WORK/stopped.sql" + echo "$PRESERVE_CAPTURE;" +} | "${MYSQL[@]}" "$PROBE_DATABASE" > "$WORK/left.tsv" +"$STATEMENTS" restore "$WORK/saved.tsv" "$WORK/left.tsv" > "$WORK/restore.sql" +if RESTORED="$({ + cat "$WORK/stopped.sql" "$WORK/restore.sql" + echo "$PRESERVE_CAPTURE;" +} | "${MYSQL[@]}" "$PROBE_DATABASE" 2>&1)"; then + if [ "$RESTORED" = "$(cat "$WORK/saved.tsv")" ]; then + echo "OK (c): $(wc -l < "$WORK/restore.sql" | tr -d ' ') restores put back every setting the stopped file changed" + else + fail "(c): after the restores the session reported: $RESTORED" + fi + case "$(tail -n 1 "$WORK/restore.sql")" in + "SET SESSION autocommit = "*) echo "OK (c): autocommit is restored last" ;; + *) fail "(c): the last restore is $(tail -n 1 "$WORK/restore.sql")" ;; + esac + PENDING="$("${MYSQL[@]}" -e "SELECT COUNT(*) FROM $PROBE_DATABASE.preserve_rows")" + echo "note (c): rows the stopped file left pending and the autocommit restore committed: $PENDING" +else + fail "(c): the restores failed: $RESTORED" +fi + +# The same file run inside a transaction the user opened first. The restores the driver sends while the session +# reports it open leave autocommit alone; sent anyway, SET autocommit = 'ON' commits the user's row. +"$STATEMENTS" restore "$WORK/saved.tsv" "$WORK/left.tsv" in-transaction > "$WORK/restore-open.sql" +run_inside_user_transaction() { + { + echo "START TRANSACTION;" + echo "INSERT INTO preserve_rows VALUES ($1);" + cat "$WORK/stopped.sql" "$2" + } | "${MYSQL[@]}" "$PROBE_DATABASE" 2>&1 +} +if OPEN_RUN="$(run_inside_user_transaction 20 "$WORK/restore-open.sql")"; then + KEPT_OPEN="$("${MYSQL[@]}" -e "SELECT COUNT(*) FROM $PROBE_DATABASE.preserve_rows WHERE id = 20")" + if grep -q autocommit "$WORK/restore-open.sql"; then + fail "(c): with a transaction open the restores still set autocommit" + elif [ "$KEPT_OPEN" = "0" ]; then + echo "OK (c): with the user's transaction open the restores leave autocommit alone and commit nothing" + else + fail "(c): with the user's transaction open the restores committed its row" + fi +else + fail "(c): the restores inside a transaction failed: $OPEN_RUN" +fi +run_inside_user_transaction 21 "$WORK/restore.sql" > /dev/null +echo "note (c): rows of the user's transaction an autocommit restore would have committed: $("${MYSQL[@]}" -e "SELECT COUNT(*) FROM $PROBE_DATABASE.preserve_rows WHERE id = 21")" + +# d. A procedure and an event recreated inside their creation scopes, from MariaDB's default recorded mode. +PORTABLE_MODE="$("$STATEMENTS" portable-mode "$MARIADB_ROUTINE_MODE")" +PORTABLE_REPORTED="$("${MYSQL[@]}" -e "SET SESSION sql_mode = '$PORTABLE_MODE'; SELECT @@SESSION.sql_mode")" +if NATIVE="$("${MYSQL[@]}" -e "SET SESSION sql_mode = '$MARIADB_ROUTINE_MODE'" 2>&1)"; then + echo "note (d): this server accepts NO_AUTO_CREATE_USER itself" +else + echo "note (d): this server refuses NO_AUTO_CREATE_USER itself: $(echo "$NATIVE" | grep "^ERROR")" +fi +{ + cat "$WORK/prologue.sql" + "$STATEMENTS" opening "$MARIADB_ROUTINE_MODE" + echo "CREATE PROCEDURE probe_routine() SELECT 1;" + "$STATEMENTS" closing "$MARIADB_ROUTINE_MODE" + "$STATEMENTS" opening "$MARIADB_ROUTINE_MODE" "$EVENT_ZONE" + echo "CREATE EVENT probe_event ON SCHEDULE EVERY 1 DAY DISABLE DO SELECT 1;" + "$STATEMENTS" closing "$MARIADB_ROUTINE_MODE" "$EVENT_ZONE" + echo "SELECT @@SESSION.sql_mode, @@SESSION.time_zone;" + cat "$WORK/epilogue.sql" +} > "$WORK/objects.sql" + +if SCRIPT_STATE="$("${STRICT[@]}" "$PROBE_DATABASE" < "$WORK/objects.sql" 2>&1)"; then + if [ "$SCRIPT_STATE" = "$(printf 'NO_AUTO_VALUE_ON_ZERO\t+00:00')" ]; then + echo "OK (d): after each object the script's own sql_mode and time_zone came back" + else + fail "(d): after the objects the script's session was: $SCRIPT_STATE" + fi + ROUTINE_RECORDED="$("${MYSQL[@]}" -e "$("$STATEMENTS" routine-settings "$PROBE_DATABASE" probe_routine)" | cut -f2)" + EVENT_RECORDED="$("${MYSQL[@]}" -e "$("$STATEMENTS" event-settings "$PROBE_DATABASE" probe_event)" | cut -f2,3)" + if [ "$ROUTINE_RECORDED" = "$PORTABLE_REPORTED" ]; then + echo "OK (d): the procedure recorded $ROUTINE_RECORDED" + else + fail "(d): the procedure recorded '$ROUTINE_RECORDED'" + fi + if [ "$EVENT_RECORDED" = "$(printf '%s\t%s' "$PORTABLE_REPORTED" "$EVENT_ZONE")" ]; then + echo "OK (d): the event recorded the same mode in $EVENT_ZONE" + else + fail "(d): the event recorded '$EVENT_RECORDED'" + fi +else + fail "(d): the creation scopes did not run: $SCRIPT_STATE" +fi + +# e. The copy write scope accepts what the session accepts, plus zero dates and id 0. +write_scope_over() { + "${MYSQL[@]}" -e "SET SESSION sql_mode = '$1'; $("$STATEMENTS" verbatim-capture)" > "$WORK/write-saved.tsv" + "$STATEMENTS" apply-write "$WORK/write-saved.tsv" +} +run_under_write_scope() { + "${MYSQL[@]}" "$PROBE_DATABASE" -e "SET SESSION sql_mode = '$1'; +$(write_scope_over "$1") +$2" 2>&1 +} +expect_refusal() { + if OUTPUT="$(run_under_write_scope "$1" "$2")"; then + fail "(e): under $1 the write scope accepted: $2" + else + case "$OUTPUT" in + *"ERROR $3"*) echo "OK (e): $4 is refused with $3" ;; + *) fail "(e): $4 failed with something other than $3: $OUTPUT" ;; + esac + fi +} + +"${MYSQL[@]}" "$PROBE_DATABASE" -e "CREATE TABLE copy_target ( + id int NOT NULL AUTO_INCREMENT PRIMARY KEY, + d date NULL, + dt datetime NULL, + short_text varchar(5) NULL, + tiny tinyint NULL +) ENGINE=InnoDB; CREATE TABLE copy_myisam (id int PRIMARY KEY, short_text varchar(5)) ENGINE=MyISAM" +if COPIED="$(run_under_write_scope "$STRICT_MODE" "CREATE TABLE copy_zero_default (d date NOT NULL DEFAULT '0000-00-00'); +INSERT INTO copy_target (id, d, dt) VALUES (0, '0000-00-00', '2020-01-00 10:00:00'), (1, '2020-00-15', '0000-00-00 00:00:00'); +SELECT id, d, dt FROM copy_target ORDER BY id")"; then + if [ "$COPIED" = "$(printf '0\t0000-00-00\t2020-01-00 10:00:00\n1\t2020-00-15\t0000-00-00 00:00:00')" ]; then + echo "OK (e): over MySQL 8's default mode, a zero DEFAULT, zero and partial dates and id 0 are stored as written" + else + fail "(e): rows came back as: $COPIED" + fi +else + fail "(e): the write scope refused the rows: $COPIED" +fi +expect_refusal "$STRICT_MODE" "INSERT INTO copy_target (id, short_text) VALUES (2, 'abcdefghij')" 1406 \ + "over MySQL 8's default mode, 'abcdefghij' into VARCHAR(5)" +expect_refusal "$STRICT_MODE" "INSERT INTO copy_target (id, tiny) VALUES (2, 300)" 1264 \ + "over MySQL 8's default mode, 300 into TINYINT" +expect_refusal "STRICT_ALL_TABLES" "INSERT INTO copy_myisam VALUES (1, 'ok'), (2, 'abcdefghij')" 1406 \ + "over STRICT_ALL_TABLES, a MyISAM multi-row INSERT whose second row overflows" +if TRUNCATED="$("${MYSQL[@]}" "$PROBE_DATABASE" -e "SET SESSION sql_mode = 'STRICT_TRANS_TABLES,NO_AUTO_VALUE_ON_ZERO'; +DELETE FROM copy_myisam; INSERT INTO copy_myisam VALUES (1, 'ok'), (2, 'abcdefghij'); SELECT short_text FROM copy_myisam WHERE id = 2" 2>&1)"; then + echo "note (e): a fixed STRICT_TRANS_TABLES write mode stores that MyISAM row as '$TRUNCATED'" +fi +TRADITIONAL_RUN="$(run_under_write_scope "TRADITIONAL" "INSERT INTO copy_target (id, d, dt) VALUES (10, '0000-00-00', '0000-00-00 00:00:00'), (11, '2020-00-15', NULL); +SELECT d FROM copy_target WHERE id IN (10, 11) ORDER BY id; SELECT @@SESSION.sql_mode")" +if [ "$(echo "$TRADITIONAL_RUN" | head -n 2)" = "$(printf '0000-00-00\n2020-00-15')" ]; then + echo "OK (e): over TRADITIONAL, zero and partial dates go in, with sql_mode $(echo "$TRADITIONAL_RUN" | tail -n 1)" +else + fail "(e): over TRADITIONAL the write scope gave: $TRADITIONAL_RUN" +fi +expect_refusal "TRADITIONAL" "INSERT INTO copy_target (id, short_text) VALUES (12, 'abcdefghij')" 1406 \ + "over TRADITIONAL, 'abcdefghij' into VARCHAR(5)" +INVALID_MODE="STRICT_TRANS_TABLES,ALLOW_INVALID_DATES" +if INVALID_RUN="$(run_under_write_scope "$INVALID_MODE" "INSERT INTO copy_target (id, d) VALUES (20, '2024-02-31'); +SELECT d FROM copy_target WHERE id = 20")" && [ "$INVALID_RUN" = "2024-02-31" ]; then + echo "OK (e): over ALLOW_INVALID_DATES, 2024-02-31 still goes in" +else + fail "(e): over ALLOW_INVALID_DATES the write scope gave: $INVALID_RUN" +fi + +# A session that reads backslashes as text keeps doing so, so DDL spelled for it (a single backslash) stores as written. +NBE_MODE="STRICT_TRANS_TABLES,NO_BACKSLASH_ESCAPES" +APPLY_NBE="$(write_scope_over "$NBE_MODE")" +if NBE_COMMENT="$(run_under_write_scope "$NBE_MODE" "CREATE TABLE nbe_target (id int) COMMENT 'C:\tmp'; +SELECT HEX(TABLE_COMMENT) FROM information_schema.TABLES WHERE TABLE_SCHEMA = '$PROBE_DATABASE' AND TABLE_NAME = 'nbe_target'")"; then + case "$APPLY_NBE" in + *"NO_BACKSLASH_ESCAPES"*) ;; + *) fail "(e): the write scope dropped NO_BACKSLASH_ESCAPES: $APPLY_NBE" ;; + esac + if [ "$NBE_COMMENT" = "433A5C746D70" ]; then + echo "OK (e): over NO_BACKSLASH_ESCAPES the write scope keeps it, and C:\\tmp is stored as written" + else + fail "(e): the comment spelled for NO_BACKSLASH_ESCAPES was stored as hex $NBE_COMMENT" + fi +else + fail "(e): the write scope over NO_BACKSLASH_ESCAPES failed: $NBE_COMMENT" +fi + +# f. A refused CREATE TABLE inside a transaction commits it anyway. +if "${MYSQL[@]}" -e "SELECT @@in_transaction" > /dev/null 2>&1; then + IN_TRANSACTION="SELECT @@in_transaction" +else + # The InnoDB transaction tables are a cache refreshed when it is more than 0.1 s old. + IN_TRANSACTION="DO SLEEP(0.2); SELECT COUNT(*) FROM information_schema.INNODB_TRX WHERE trx_mysql_thread_id = CONNECTION_ID() AND trx_rows_modified > 0" +fi +cat > "$WORK/stale.sql" < "$WORK/stale.err")" +if ! grep -q "ERROR 1050" "$WORK/stale.err"; then + fail "(f): the second CREATE TABLE was not refused with 1050: $(cat "$WORK/stale.err")" +elif [ "$STALE" = "$(printf '1\n0\n1')" ]; then + echo "OK (f): after the refused CREATE TABLE the server reports no open transaction, and ROLLBACK keeps the row" +else + fail "(f): open before, open after, rows after ROLLBACK were: $(echo "$STALE" | tr '\n' ' ')" +fi + +exit "$FAILED" diff --git a/scripts/probes/check-postgresql-dump-session.sh b/scripts/probes/check-postgresql-dump-session.sh new file mode 100755 index 0000000000..7bd6c31d27 --- /dev/null +++ b/scripts/probes/check-postgresql-dump-session.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# +# Check the PostgreSQL session facts the driver's session scopes rely on (#3414). +# +# TablePro runs an imported SQL file on the user's own editor session. A pg_dump plain file sets +# search_path to '' and a dozen other settings and never puts them back, so after an import the +# editor session resolves no unqualified name. The preserve scope saves those settings from +# pg_settings and, at close, writes back only the ones that changed. An export reads values as text +# in the session's DateStyle, IntervalStyle and extra_float_digits, and some of those texts read +# back as other values. The read scope sets what pg_dump's own reading session sets. This measures +# both on a live server, plus the transaction facts the scope's close depends on. +# +# Usage: +# scripts/probes/check-postgresql-dump-session.sh [conninfo] +# +# conninfo is a libpq connection string without dbname, default empty (libpq defaults: the local +# socket as the current user). The password, if any, comes from PGPASSWORD. Needs psql and a +# pg_dump no older than the server. Creates and drops two scratch databases named +# zd3414_impl_pg_*, and a NOLOGIN role of the same prefix when the user may create one. +# Exits 0 when every check holds, 1 on a failed check, 3 when psql, pg_dump or the server is +# missing. + +set -uo pipefail + +BASE="${1:-}" +SUFFIX="$$" +SRC_DB="zd3414_impl_pg_src_$SUFFIX" +DST_DB="zd3414_impl_pg_dst_$SUFFIX" +ROLE="zd3414_impl_pg_role_$SUFFIX" + +for tool in psql pg_dump; do + command -v "$tool" > /dev/null || { + echo "$tool not found" >&2 + exit 3 + } +done + +PSQL=(psql -X -A -t -q -v ON_ERROR_STOP=1) +ADMIN="$BASE dbname=postgres" +SRC="$BASE dbname=$SRC_DB" +DST="$BASE dbname=$DST_DB" + +if ! "${PSQL[@]}" -d "$ADMIN" -c "SELECT 1" > /dev/null 2>&1; then + echo "no PostgreSQL answers '$ADMIN'" >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +role_created=0 +cleanup() { + "${PSQL[@]}" -d "$ADMIN" -c "DROP DATABASE IF EXISTS $SRC_DB" > /dev/null 2>&1 + "${PSQL[@]}" -d "$ADMIN" -c "DROP DATABASE IF EXISTS $DST_DB" > /dev/null 2>&1 + if [ "$role_created" -eq 1 ]; then + "${PSQL[@]}" -d "$ADMIN" -c "DROP ROLE IF EXISTS $ROLE" > /dev/null 2>&1 + fi + rm -rf "$WORK" +} +trap cleanup EXIT + +failures=0 +check() { + local label=$1 expected=$2 actual=$3 + if [ "$expected" = "$actual" ]; then + echo "ok $label: $actual" + else + echo "FAIL $label: expected '$expected', got '$actual'" + failures=$((failures + 1)) + fi +} + +q() { + local conn=$1 + shift + "${PSQL[@]}" -d "$conn" "$@" 2>&1 +} + +echo "server: $(q "$ADMIN" -c "SHOW server_version")" +echo "pg_dump: $(pg_dump --version)" + +if ! q "$ADMIN" -c "CREATE DATABASE $SRC_DB" > /dev/null || ! q "$ADMIN" -c "CREATE DATABASE $DST_DB" > /dev/null; then + echo "cannot create the scratch databases" >&2 + exit 3 +fi + +SETTINGS="'statement_timeout', 'lock_timeout', 'idle_in_transaction_session_timeout', 'transaction_timeout', + 'client_encoding', 'standard_conforming_strings', 'search_path', 'check_function_bodies', 'xmloption', + 'client_min_messages', 'row_security', 'default_tablespace', 'default_table_access_method', 'datestyle', + 'intervalstyle', 'extra_float_digits', 'timezone', 'session_replication_role'" + +echo +echo "== pg_dump's settings outlive the import on the session that ran it" + +q "$SRC" -c "CREATE TABLE t (id int PRIMARY KEY, d date, i interval, f float8)" \ + -c "INSERT INTO t VALUES (1, make_date(2024, 6, 1), make_interval(days => -1, hours => -2), 0.1::float8 + 0.2::float8)" \ + > /dev/null +if ! pg_dump -d "$SRC" > "$WORK/dump.sql" 2> "$WORK/pg_dump.err"; then + echo "pg_dump failed: $(cat "$WORK/pg_dump.err")" >&2 + exit 3 +fi +check "the dump empties search_path" "1" \ + "$(grep -cF "SELECT pg_catalog.set_config('search_path', '', false);" "$WORK/dump.sql")" + +session=$(q "$DST" -v dumpfile="$WORK/dump.sql" <