From af50a358d24dff2e0e6a2d251862dcf997089eb9 Mon Sep 17 00:00:00 2001 From: Shawn Jackson Date: Tue, 9 Jun 2026 17:08:56 -0500 Subject: [PATCH 1/4] RE1-T117 Migration fix and utf8 fix --- Core/Resgrid.Config/SystemBehaviorConfig.cs | 43 +++ Core/Resgrid.Framework/Text/Utf8Sanitizer.cs | 323 +++++++++++++++++ .../IUtf8MaintenanceRepository.cs | 81 +++++ .../Migrations/M0068_ChatbotTables.cs | 158 +++++---- .../Migrations/M0069_ChatbotLinkingCodes.cs | 43 ++- .../M0070_ChatbotDepartmentConfigColumns.cs | 32 +- .../Migrations/M0071_AddingFeatureToggles.cs | 246 +++++++------ .../M0072_AddingChatbotTwilioFeatureFlag.cs | 17 +- ...IsPermanentFailureToWeatherAlertSources.cs | 9 +- .../M0075_AddUtf8CleanupProgress.cs | 31 ++ .../Migrations/M0068_ChatbotTablesPg.cs | 158 +++++---- .../Migrations/M0069_ChatbotLinkingCodesPg.cs | 43 ++- .../M0070_ChatbotDepartmentConfigColumnsPg.cs | 32 +- .../M0071_AddingFeatureTogglesPg.cs | 246 +++++++------ .../M0072_AddingChatbotTwilioFeatureFlagPg.cs | 17 +- ...PermanentFailureToWeatherAlertSourcesPg.cs | 9 +- .../M0075_AddUtf8CleanupProgressPg.cs | 31 ++ .../Extensions/DynamicParametersExtension.cs | 10 + .../IdentityRoleRepository.cs | 6 + .../IdentityUserRepository.cs | 2 + .../Modules/ApiDataModule.cs | 3 + .../Modules/DataModule.cs | 3 + .../RepositoryBase.cs | 6 +- .../Utf8MaintenanceRepository.cs | 326 ++++++++++++++++++ .../Utf8WriteGuard.cs | 28 ++ .../Framework/Utf8SanitizerTests.cs | 206 +++++++++++ .../Commands/CleanUtf8Command.cs | 49 +++ Tools/Resgrid.Console/Program.cs | 1 + .../Services/ApplicationHostedService.cs | 5 + .../Commands/Utf8CleanupCommand.cs | 18 + Workers/Resgrid.Workers.Console/Program.cs | 13 + .../Tasks/Utf8CleanupTask.cs | 46 +++ .../Logic/Utf8CleanupLogic.cs | 148 ++++++++ 33 files changed, 1943 insertions(+), 446 deletions(-) create mode 100644 Core/Resgrid.Framework/Text/Utf8Sanitizer.cs create mode 100644 Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs create mode 100644 Providers/Resgrid.Providers.Migrations/Migrations/M0075_AddUtf8CleanupProgress.cs create mode 100644 Providers/Resgrid.Providers.MigrationsPg/Migrations/M0075_AddUtf8CleanupProgressPg.cs create mode 100644 Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs create mode 100644 Repositories/Resgrid.Repositories.DataRepository/Utf8WriteGuard.cs create mode 100644 Tests/Resgrid.Tests/Framework/Utf8SanitizerTests.cs create mode 100644 Tools/Resgrid.Console/Commands/CleanUtf8Command.cs create mode 100644 Workers/Resgrid.Workers.Console/Commands/Utf8CleanupCommand.cs create mode 100644 Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs create mode 100644 Workers/Resgrid.Workers.Framework/Logic/Utf8CleanupLogic.cs diff --git a/Core/Resgrid.Config/SystemBehaviorConfig.cs b/Core/Resgrid.Config/SystemBehaviorConfig.cs index 8d9ab5c76..c03dfca01 100644 --- a/Core/Resgrid.Config/SystemBehaviorConfig.cs +++ b/Core/Resgrid.Config/SystemBehaviorConfig.cs @@ -196,6 +196,49 @@ public static class SystemBehaviorConfig /// public static string LocationName = "US-West"; + /// + /// When true, free-form text is sanitized at the data-repository write chokepoints so it is + /// always safe to store in a PostgreSQL UTF-8 database (strips NUL, repairs Windows-1252 + /// mojibake, replaces unpaired UTF-16 surrogates). Guards the SQL Server -> PostgreSQL migration. + /// + public static bool SanitizeTextForUtf8 = true; + + /// + /// When true (and is enabled), the sanitizer also normalizes + /// text to Unicode NFC. Not required for PostgreSQL UTF-8 validity, so off by default to avoid + /// altering otherwise-valid user text on the write hot-path; mainly useful for the cleanup sweep. + /// + public static bool Utf8NormalizeToNfc = false; + + /// + /// When true (and is enabled), the sanitizer additionally + /// attempts a conservative repair of double-encoded UTF-8 content. Off by default because the + /// heuristic can misfire on legitimately Latin-1-looking text. + /// + public static bool Utf8RepairDoubleEncoding = false; + + /// + /// Enables the nightly UTF-8 data cleanup worker that sweeps every text column and repairs + /// existing content in place so the database stays migration-clean. + /// + public static bool Utf8CleanupEnabled = true; + + /// + /// Number of rows the cleanup worker reads per keyset-paginated batch. + /// + public static int Utf8CleanupBatchSize = 1000; + + /// + /// Upper bound on the number of rows the cleanup worker will scan in a single run before + /// stopping (it resumes from its saved watermark on the next run). 0 means unbounded. + /// + public static int Utf8CleanupMaxRowsPerRun = 250000; + + /// + /// UTC hour (0-23) at which the nightly UTF-8 data cleanup worker is scheduled to run. + /// + public static int Utf8CleanupHourUtc = 4; + public static string GetEnvPrefix() { switch (Environment) diff --git a/Core/Resgrid.Framework/Text/Utf8Sanitizer.cs b/Core/Resgrid.Framework/Text/Utf8Sanitizer.cs new file mode 100644 index 000000000..dc5f36107 --- /dev/null +++ b/Core/Resgrid.Framework/Text/Utf8Sanitizer.cs @@ -0,0 +1,323 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Reflection; +using System.Text; + +namespace Resgrid.Framework +{ + /// + /// Sanitizes free-form text so it is safe to store in a PostgreSQL UTF-8 database. + /// + /// PostgreSQL's UTF-8 text/citext columns reject content that SQL Server + /// tolerates: the NUL character (U+0000) is forbidden outright, and invalid Unicode + /// (unpaired UTF-16 surrogates) cannot be encoded as UTF-8 at all. This class neutralizes + /// that content and, on a best-effort basis, repairs the most common Windows-1252 mojibake + /// so the recovered character (smart quotes, dashes, etc.) is preserved instead of discarded. + /// + /// The transform is pure and allocation-free for already-clean strings (the common case): + /// returns the original reference unchanged when nothing needs fixing. + /// + public static class Utf8Sanitizer + { + private static readonly UTF8Encoding StrictUtf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + + private static readonly ConcurrentDictionary StringPropertyCache = new ConcurrentDictionary(); + + // C1 control range (U+0080 - U+009F). When Windows-1252 bytes are decoded as ISO-8859-1 + // (Latin-1) their punctuation lands here; these are the candidates for mojibake repair. + private const char C1RangeStart = (char)0x80; + private const char C1RangeEnd = (char)0x9F; + + // Highest ISO-8859-1 (Latin-1) code point; above this a char cannot be a single Latin-1 byte. + private const char Latin1Max = (char)0xFF; + + /// The Unicode replacement character (U+FFFD) used for content that cannot be repaired. + public const char ReplacementChar = (char)0xFFFD; + + /// + /// Returns a copy of that is safe for a PostgreSQL UTF-8 column. + /// Returns the original reference (no allocation) when the value is already clean. + /// + /// + /// When true, also attempts a conservative repair of double-encoded UTF-8. Off by default + /// because it can false-positive on legitimately Latin-1-looking text. + /// + /// + /// When non-null, the (possibly repaired) result is normalized to this form. NUL/surrogate + /// safety does not require normalization, so the write hot-path leaves this null. + /// + public static string Clean(string input, bool repairDoubleEncoding = false, NormalizationForm? normalization = null) + { + TryClean(input, out var cleaned, repairDoubleEncoding, normalization); + return cleaned; + } + + /// + /// Cleans and reports whether any change was required. + /// is the original reference when no change was needed. + /// + public static bool TryClean(string input, out string cleaned, bool repairDoubleEncoding = false, NormalizationForm? normalization = null) + { + cleaned = input; + + if (string.IsNullOrEmpty(input)) + return false; + + if (NeedsCleaning(input)) + cleaned = Rebuild(input); + + if (repairDoubleEncoding) + { + var repaired = RepairDoubleEncodedUtf8(cleaned); + if (!ReferenceEquals(repaired, cleaned)) + cleaned = repaired; + } + + if (normalization.HasValue && cleaned.Length > 0 && !cleaned.IsNormalized(normalization.Value)) + cleaned = cleaned.Normalize(normalization.Value); + + // Defensive final guard: the rebuild above removes NUL and lone surrogates, so this + // should never trip, but it guarantees the contract that output is valid UTF-8. + if (!IsValidUtf8(cleaned)) + cleaned = ForceValidUtf8(cleaned); + + return !ReferenceEquals(cleaned, input); + } + + /// Returns true when the value can be stored in a PostgreSQL UTF-8 column as-is. + public static bool IsClean(string input) + { + return string.IsNullOrEmpty(input) || !NeedsCleaning(input); + } + + /// + /// In-place sanitizes every public readable/writable string property of . + /// Used to guard the generic new DynamicParameters(entity) write path where individual + /// values do not flow through DynamicParametersExtension.Add. + /// + public static void CleanEntity(T entity, bool repairDoubleEncoding = false, NormalizationForm? normalization = null) where T : class + { + if (entity == null) + return; + + var properties = StringPropertyCache.GetOrAdd(entity.GetType(), GetStringProperties); + + for (int i = 0; i < properties.Length; i++) + { + var property = properties[i]; + var current = (string)property.GetValue(entity); + + if (current == null) + continue; + + if (TryClean(current, out var cleaned, repairDoubleEncoding, normalization)) + property.SetValue(entity, cleaned); + } + } + + private static PropertyInfo[] GetStringProperties(Type type) + { + var result = new List(); + + foreach (var property in type.GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (property.PropertyType != typeof(string)) + continue; + + if (!property.CanRead || !property.CanWrite) + continue; + + // Skip indexers. + if (property.GetIndexParameters().Length > 0) + continue; + + result.Add(property); + } + + return result.ToArray(); + } + + private static bool NeedsCleaning(string input) + { + for (int i = 0; i < input.Length; i++) + { + char c = input[i]; + + if (c == '\0') + return true; + + if (c >= C1RangeStart && c <= C1RangeEnd) + return true; // C1 range: candidate for Windows-1252 mojibake repair. + + if (char.IsHighSurrogate(c)) + { + if (i + 1 >= input.Length || !char.IsLowSurrogate(input[i + 1])) + return true; // unpaired high surrogate + + i++; // valid pair, skip the low surrogate + } + else if (char.IsLowSurrogate(c)) + { + return true; // lone low surrogate + } + } + + return false; + } + + private static string Rebuild(string input) + { + var sb = new StringBuilder(input.Length); + + for (int i = 0; i < input.Length; i++) + { + char c = input[i]; + + if (c == '\0') + continue; // strip NUL: forbidden in PostgreSQL text + + if (c >= C1RangeStart && c <= C1RangeEnd) + { + char mapped = MapCp1252C1(c); + if (mapped != '\0') + sb.Append(mapped); + // else: undefined Windows-1252 byte -> drop + continue; + } + + if (char.IsHighSurrogate(c)) + { + if (i + 1 < input.Length && char.IsLowSurrogate(input[i + 1])) + { + sb.Append(c); + sb.Append(input[i + 1]); + i++; + } + else + { + sb.Append(ReplacementChar); // unpaired high surrogate + } + + continue; + } + + if (char.IsLowSurrogate(c)) + { + sb.Append(ReplacementChar); // lone low surrogate + continue; + } + + sb.Append(c); + } + + return sb.ToString(); + } + + /// + /// Maps a C1-range char (U+0080-U+009F) to the Windows-1252 character it most likely + /// represents. These show up when Windows-1252 bytes were decoded as ISO-8859-1 (Latin-1). + /// Returns '\0' for the five undefined Windows-1252 bytes (0x81, 0x8D, 0x8F, 0x90, 0x9D). + /// + private static char MapCp1252C1(char c) + { + switch ((int)c) + { + case 0x80: return (char)0x20AC; // EURO SIGN + case 0x82: return (char)0x201A; // SINGLE LOW-9 QUOTATION MARK + case 0x83: return (char)0x0192; // LATIN SMALL LETTER F WITH HOOK + case 0x84: return (char)0x201E; // DOUBLE LOW-9 QUOTATION MARK + case 0x85: return (char)0x2026; // HORIZONTAL ELLIPSIS + case 0x86: return (char)0x2020; // DAGGER + case 0x87: return (char)0x2021; // DOUBLE DAGGER + case 0x88: return (char)0x02C6; // MODIFIER LETTER CIRCUMFLEX ACCENT + case 0x89: return (char)0x2030; // PER MILLE SIGN + case 0x8A: return (char)0x0160; // LATIN CAPITAL LETTER S WITH CARON + case 0x8B: return (char)0x2039; // SINGLE LEFT-POINTING ANGLE QUOTATION MARK + case 0x8C: return (char)0x0152; // LATIN CAPITAL LIGATURE OE + case 0x8E: return (char)0x017D; // LATIN CAPITAL LETTER Z WITH CARON + case 0x91: return (char)0x2018; // LEFT SINGLE QUOTATION MARK + case 0x92: return (char)0x2019; // RIGHT SINGLE QUOTATION MARK + case 0x93: return (char)0x201C; // LEFT DOUBLE QUOTATION MARK + case 0x94: return (char)0x201D; // RIGHT DOUBLE QUOTATION MARK + case 0x95: return (char)0x2022; // BULLET + case 0x96: return (char)0x2013; // EN DASH + case 0x97: return (char)0x2014; // EM DASH + case 0x98: return (char)0x02DC; // SMALL TILDE + case 0x99: return (char)0x2122; // TRADE MARK SIGN + case 0x9A: return (char)0x0161; // LATIN SMALL LETTER S WITH CARON + case 0x9B: return (char)0x203A; // SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + case 0x9C: return (char)0x0153; // LATIN SMALL LIGATURE OE + case 0x9E: return (char)0x017E; // LATIN SMALL LETTER Z WITH CARON + case 0x9F: return (char)0x0178; // LATIN CAPITAL LETTER Y WITH DIAERESIS + default: return '\0'; // 0x81, 0x8D, 0x8F, 0x90, 0x9D are undefined in Windows-1252 + } + } + + /// + /// Conservative double-encoded-UTF-8 repair: if the whole string round-trips as + /// Latin-1 bytes back into valid UTF-8 that differs from the input, return the decoded + /// form. Only triggers when the string actually contains the high-Latin-1 range, so plain + /// ASCII is never touched. Opt-in because it can misfire on genuine Latin-1 text. + /// + private static string RepairDoubleEncodedUtf8(string input) + { + if (string.IsNullOrEmpty(input)) + return input; + + bool hasHighLatin = false; + for (int i = 0; i < input.Length; i++) + { + char c = input[i]; + if (c > Latin1Max) + return input; // contains non-Latin-1: not a single-byte mojibake candidate + if (c >= C1RangeStart) + hasHighLatin = true; + } + + if (!hasHighLatin) + return input; + + try + { + var bytes = new byte[input.Length]; + for (int i = 0; i < input.Length; i++) + bytes[i] = (byte)input[i]; + + var decoded = StrictUtf8.GetString(bytes); // throws if not valid UTF-8 + return string.Equals(decoded, input, StringComparison.Ordinal) ? input : decoded; + } + catch (Exception) + { + return input; // bytes were not valid UTF-8 -> genuine Latin-1, leave alone + } + } + + private static bool IsValidUtf8(string input) + { + if (string.IsNullOrEmpty(input)) + return true; + + if (input.IndexOf('\0') >= 0) + return false; + + try + { + StrictUtf8.GetByteCount(input); + return true; + } + catch (EncoderFallbackException) + { + return false; + } + } + + private static string ForceValidUtf8(string input) + { + var permissive = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false); + var roundTripped = permissive.GetString(permissive.GetBytes(input)); + return roundTripped.Replace("\0", string.Empty); + } + } +} diff --git a/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs b/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs new file mode 100644 index 000000000..5e7497bb0 --- /dev/null +++ b/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Resgrid.Model.Repositories +{ + /// + /// Low-level maintenance access used by the nightly UTF-8 data cleanup worker. It enumerates + /// every free-form text column in the database, reads rows in keyset-paginated batches, and + /// writes back repaired values so the data stays safe for a PostgreSQL UTF-8 migration. Works + /// against whichever backend is currently configured (SQL Server source or PostgreSQL target). + /// + public interface IUtf8MaintenanceRepository + { + /// + /// Returns every base-table text column grouped by table, restricted to tables that have a + /// single-column primary key (required for stable keyset pagination). + /// + Task> GetTextColumnTargetsAsync(CancellationToken cancellationToken); + + /// + /// Reads the next batch of rows for ordered by primary key, starting + /// after (null/empty for the first page). + /// + Task GetRowBatchAsync(Utf8TextColumnTarget target, string lastKey, int batchSize, CancellationToken cancellationToken); + + /// + /// Applies the supplied repaired rows. Each row carries only the columns that changed. + /// Returns the number of rows updated. + /// + Task UpdateRowsAsync(Utf8TextColumnTarget target, IReadOnlyList rows, CancellationToken cancellationToken); + + /// Loads the saved cleanup watermark for a table, or null if none exists yet. + Task GetProgressAsync(string tableKey, CancellationToken cancellationToken); + + /// Inserts or updates the cleanup watermark for a table. + Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationToken cancellationToken); + } + + /// A table plus its single-column primary key and the free-form text columns to scan. + public class Utf8TextColumnTarget + { + public string Schema { get; set; } + public string TableName { get; set; } + public string PrimaryKeyColumn { get; set; } + public List TextColumns { get; set; } = new List(); + + /// Stable identifier used as the watermark key (schema-qualified table name). + public string Key => string.IsNullOrEmpty(Schema) ? TableName : Schema + "." + TableName; + } + + /// A page of rows plus the primary key of the last row (the next page's start cursor). + public class Utf8RowBatch + { + public List Rows { get; set; } = new List(); + public string LastKey { get; set; } + } + + /// + /// A single row's primary key and text-column values. When returned from a read it holds all + /// scanned columns; when passed to it + /// holds only the columns that changed. + /// + public class Utf8TextRow + { + public string Key { get; set; } + public Dictionary Columns { get; set; } = new Dictionary(); + } + + /// Per-table cleanup watermark, persisted so each nightly run resumes where it stopped. + public class Utf8CleanupProgress + { + public string TableName { get; set; } + public string LastProcessedKey { get; set; } + public DateTime? LastCompletedUtc { get; set; } + public long RowsScanned { get; set; } + public long RowsFixed { get; set; } + public DateTime UpdatedOnUtc { get; set; } + } +} diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0068_ChatbotTables.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0068_ChatbotTables.cs index 50240e9ea..66974b778 100644 --- a/Providers/Resgrid.Providers.Migrations/Migrations/M0068_ChatbotTables.cs +++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0068_ChatbotTables.cs @@ -7,91 +7,107 @@ public class M0068_ChatbotTables : Migration { public override void Up() { + // Each table is guarded independently: databases upgraded before the 68->74 migration + // renumber may already have some of these (created under a prior version), so skip any + // table that already exists rather than failing the whole migration. + // ChatbotUserIdentities - Links platform identities to Resgrid users - Create.Table("ChatbotUserIdentities") - .WithColumn("Id").AsString(128).NotNullable().PrimaryKey() - .WithColumn("UserId").AsString(450).NotNullable() - .WithColumn("Platform").AsInt32().NotNullable() - .WithColumn("PlatformUserId").AsString(256).NotNullable() - .WithColumn("PlatformUserName").AsString(256).Nullable() - .WithColumn("IsActive").AsBoolean().NotNullable().WithDefaultValue(true) - .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("LastUsedAt").AsDateTime2().Nullable() - .WithColumn("LinkingMethod").AsString(50).Nullable(); + if (!Schema.Table("ChatbotUserIdentities").Exists()) + { + Create.Table("ChatbotUserIdentities") + .WithColumn("Id").AsString(128).NotNullable().PrimaryKey() + .WithColumn("UserId").AsString(450).NotNullable() + .WithColumn("Platform").AsInt32().NotNullable() + .WithColumn("PlatformUserId").AsString(256).NotNullable() + .WithColumn("PlatformUserName").AsString(256).Nullable() + .WithColumn("IsActive").AsBoolean().NotNullable().WithDefaultValue(true) + .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("LastUsedAt").AsDateTime2().Nullable() + .WithColumn("LinkingMethod").AsString(50).Nullable(); - Create.Index("IX_ChatbotUserIdentities_User_Platform") - .OnTable("ChatbotUserIdentities") - .OnColumn("UserId").Ascending() - .OnColumn("Platform").Ascending(); + Create.Index("IX_ChatbotUserIdentities_User_Platform") + .OnTable("ChatbotUserIdentities") + .OnColumn("UserId").Ascending() + .OnColumn("Platform").Ascending(); - Create.Index("IX_ChatbotUserIdentities_Platform_PlatformUserId") - .OnTable("ChatbotUserIdentities") - .OnColumn("Platform").Ascending() - .OnColumn("PlatformUserId").Ascending(); + Create.Index("IX_ChatbotUserIdentities_Platform_PlatformUserId") + .OnTable("ChatbotUserIdentities") + .OnColumn("Platform").Ascending() + .OnColumn("PlatformUserId").Ascending(); + } // ChatbotSessions - Active conversation session state - Create.Table("ChatbotSessions") - .WithColumn("SessionId").AsString(128).NotNullable().PrimaryKey() - .WithColumn("UserId").AsString(450).NotNullable() - .WithColumn("DepartmentId").AsInt32().NotNullable() - .WithColumn("Platform").AsInt32().NotNullable() - .WithColumn("State").AsInt32().NotNullable().WithDefaultValue(0) - .WithColumn("PendingIntent").AsInt32().Nullable() - .WithColumn("ContextJson").AsString(int.MaxValue).Nullable() - .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("LastActivity").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("TtlMinutes").AsInt32().NotNullable().WithDefaultValue(30); + if (!Schema.Table("ChatbotSessions").Exists()) + { + Create.Table("ChatbotSessions") + .WithColumn("SessionId").AsString(128).NotNullable().PrimaryKey() + .WithColumn("UserId").AsString(450).NotNullable() + .WithColumn("DepartmentId").AsInt32().NotNullable() + .WithColumn("Platform").AsInt32().NotNullable() + .WithColumn("State").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("PendingIntent").AsInt32().Nullable() + .WithColumn("ContextJson").AsString(int.MaxValue).Nullable() + .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("LastActivity").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("TtlMinutes").AsInt32().NotNullable().WithDefaultValue(30); - Create.Index("IX_ChatbotSessions_UserId_Department") - .OnTable("ChatbotSessions") - .OnColumn("UserId").Ascending() - .OnColumn("DepartmentId").Ascending() - .OnColumn("Platform").Ascending(); + Create.Index("IX_ChatbotSessions_UserId_Department") + .OnTable("ChatbotSessions") + .OnColumn("UserId").Ascending() + .OnColumn("DepartmentId").Ascending() + .OnColumn("Platform").Ascending(); - Create.Index("IX_ChatbotSessions_LastActivity") - .OnTable("ChatbotSessions") - .OnColumn("LastActivity").Ascending(); + Create.Index("IX_ChatbotSessions_LastActivity") + .OnTable("ChatbotSessions") + .OnColumn("LastActivity").Ascending(); + } // ChatbotMessageLog - Audit log of all chatbot interactions - Create.Table("ChatbotMessageLog") - .WithColumn("Id").AsString(128).NotNullable().PrimaryKey() - .WithColumn("DepartmentId").AsInt32().NotNullable() - .WithColumn("UserId").AsString(450).Nullable() - .WithColumn("SessionId").AsString(128).Nullable() - .WithColumn("Platform").AsInt32().NotNullable() - .WithColumn("Direction").AsString(10).NotNullable() - .WithColumn("MessageText").AsString(int.MaxValue).Nullable() - .WithColumn("IntentType").AsInt32().Nullable() - .WithColumn("Processed").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("ErrorInfo").AsString(500).Nullable() - .WithColumn("Timestamp").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime); + if (!Schema.Table("ChatbotMessageLog").Exists()) + { + Create.Table("ChatbotMessageLog") + .WithColumn("Id").AsString(128).NotNullable().PrimaryKey() + .WithColumn("DepartmentId").AsInt32().NotNullable() + .WithColumn("UserId").AsString(450).Nullable() + .WithColumn("SessionId").AsString(128).Nullable() + .WithColumn("Platform").AsInt32().NotNullable() + .WithColumn("Direction").AsString(10).NotNullable() + .WithColumn("MessageText").AsString(int.MaxValue).Nullable() + .WithColumn("IntentType").AsInt32().Nullable() + .WithColumn("Processed").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("ErrorInfo").AsString(500).Nullable() + .WithColumn("Timestamp").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime); - Create.Index("IX_ChatbotMessageLog_Department_Timestamp") - .OnTable("ChatbotMessageLog") - .OnColumn("DepartmentId").Ascending() - .OnColumn("Timestamp").Descending(); + Create.Index("IX_ChatbotMessageLog_Department_Timestamp") + .OnTable("ChatbotMessageLog") + .OnColumn("DepartmentId").Ascending() + .OnColumn("Timestamp").Descending(); - Create.Index("IX_ChatbotMessageLog_UserId") - .OnTable("ChatbotMessageLog") - .OnColumn("UserId").Ascending(); + Create.Index("IX_ChatbotMessageLog_UserId") + .OnTable("ChatbotMessageLog") + .OnColumn("UserId").Ascending(); + } // ChatbotDepartmentConfigs - Per-department chatbot configuration - Create.Table("ChatbotDepartmentConfigs") - .WithColumn("Id").AsString(128).NotNullable().PrimaryKey() - .WithColumn("DepartmentId").AsInt32().NotNullable() - .WithColumn("IsEnabled").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("NluProvider").AsString(50).NotNullable().WithDefaultValue("keyword") - .WithColumn("AllowedPlatforms").AsString(500).NotNullable().WithDefaultValue("*") - .WithColumn("MaxSessionsPerUser").AsInt32().NotNullable().WithDefaultValue(3) - .WithColumn("SessionTtlMinutes").AsInt32().NotNullable().WithDefaultValue(30) - .WithColumn("AllowDispatchViaChatbot").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("RequireConfirmationForStatusChange").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("UpdatedAt").AsDateTime2().Nullable(); + if (!Schema.Table("ChatbotDepartmentConfigs").Exists()) + { + Create.Table("ChatbotDepartmentConfigs") + .WithColumn("Id").AsString(128).NotNullable().PrimaryKey() + .WithColumn("DepartmentId").AsInt32().NotNullable() + .WithColumn("IsEnabled").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("NluProvider").AsString(50).NotNullable().WithDefaultValue("keyword") + .WithColumn("AllowedPlatforms").AsString(500).NotNullable().WithDefaultValue("*") + .WithColumn("MaxSessionsPerUser").AsInt32().NotNullable().WithDefaultValue(3) + .WithColumn("SessionTtlMinutes").AsInt32().NotNullable().WithDefaultValue(30) + .WithColumn("AllowDispatchViaChatbot").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("RequireConfirmationForStatusChange").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("UpdatedAt").AsDateTime2().Nullable(); - Create.Index("IX_ChatbotDepartmentConfigs_DepartmentId") - .OnTable("ChatbotDepartmentConfigs") - .OnColumn("DepartmentId").Ascending(); + Create.Index("IX_ChatbotDepartmentConfigs_DepartmentId") + .OnTable("ChatbotDepartmentConfigs") + .OnColumn("DepartmentId").Ascending(); + } } public override void Down() diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0069_ChatbotLinkingCodes.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0069_ChatbotLinkingCodes.cs index eff46947c..13f839199 100644 --- a/Providers/Resgrid.Providers.Migrations/Migrations/M0069_ChatbotLinkingCodes.cs +++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0069_ChatbotLinkingCodes.cs @@ -8,28 +8,33 @@ public class M0069_ChatbotLinkingCodes : Migration public override void Up() { // ChatbotLinkingCodes - Short-lived codes for linking platform accounts - Create.Table("ChatbotLinkingCodes") - .WithColumn("Id").AsString(128).NotNullable().PrimaryKey() - .WithColumn("UserId").AsString(450).NotNullable() - .WithColumn("Code").AsString(10).NotNullable() - .WithColumn("Platform").AsInt32().Nullable() - .WithColumn("PlatformUserId").AsString(256).Nullable() - .WithColumn("IsUsed").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("ExpiresAt").AsDateTime2().NotNullable() - .WithColumn("UsedAt").AsDateTime2().Nullable(); + // Guarded: databases upgraded before the 68->74 migration renumber may already have + // this table (it ran under a prior version), so skip creation if it already exists. + if (!Schema.Table("ChatbotLinkingCodes").Exists()) + { + Create.Table("ChatbotLinkingCodes") + .WithColumn("Id").AsString(128).NotNullable().PrimaryKey() + .WithColumn("UserId").AsString(450).NotNullable() + .WithColumn("Code").AsString(10).NotNullable() + .WithColumn("Platform").AsInt32().Nullable() + .WithColumn("PlatformUserId").AsString(256).Nullable() + .WithColumn("IsUsed").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("ExpiresAt").AsDateTime2().NotNullable() + .WithColumn("UsedAt").AsDateTime2().Nullable(); - Create.Index("IX_ChatbotLinkingCodes_Code") - .OnTable("ChatbotLinkingCodes") - .OnColumn("Code").Ascending(); + Create.Index("IX_ChatbotLinkingCodes_Code") + .OnTable("ChatbotLinkingCodes") + .OnColumn("Code").Ascending(); - Create.Index("IX_ChatbotLinkingCodes_UserId") - .OnTable("ChatbotLinkingCodes") - .OnColumn("UserId").Ascending(); + Create.Index("IX_ChatbotLinkingCodes_UserId") + .OnTable("ChatbotLinkingCodes") + .OnColumn("UserId").Ascending(); - Create.Index("IX_ChatbotLinkingCodes_ExpiresAt") - .OnTable("ChatbotLinkingCodes") - .OnColumn("ExpiresAt").Ascending(); + Create.Index("IX_ChatbotLinkingCodes_ExpiresAt") + .OnTable("ChatbotLinkingCodes") + .OnColumn("ExpiresAt").Ascending(); + } } public override void Down() diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0070_ChatbotDepartmentConfigColumns.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0070_ChatbotDepartmentConfigColumns.cs index 361016648..efdcae37d 100644 --- a/Providers/Resgrid.Providers.Migrations/Migrations/M0070_ChatbotDepartmentConfigColumns.cs +++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0070_ChatbotDepartmentConfigColumns.cs @@ -8,14 +8,30 @@ public class M0070_ChatbotDepartmentConfigColumns : Migration public override void Up() { // Per-department LLM/AI override + rate limits + linking/notification preferences. - Alter.Table("ChatbotDepartmentConfigs") - .AddColumn("LlmApiEndpoint").AsString(500).Nullable() - .AddColumn("LlmApiKey").AsString(1000).Nullable() - .AddColumn("LlmModelName").AsString(200).Nullable() - .AddColumn("MessagesPerUserPerMinute").AsInt32().Nullable() - .AddColumn("MessagesPerDepartmentPerMinute").AsInt32().Nullable() - .AddColumn("RequireLinkingConfirmation").AsBoolean().NotNullable().WithDefaultValue(true) - .AddColumn("ProactiveNotificationsEnabled").AsBoolean().NotNullable().WithDefaultValue(false); + // Each column is guarded so the migration is safe on databases where a prior partial + // apply (or a version renumber) already added some of them. + const string table = "ChatbotDepartmentConfigs"; + + if (!Schema.Table(table).Column("LlmApiEndpoint").Exists()) + Alter.Table(table).AddColumn("LlmApiEndpoint").AsString(500).Nullable(); + + if (!Schema.Table(table).Column("LlmApiKey").Exists()) + Alter.Table(table).AddColumn("LlmApiKey").AsString(1000).Nullable(); + + if (!Schema.Table(table).Column("LlmModelName").Exists()) + Alter.Table(table).AddColumn("LlmModelName").AsString(200).Nullable(); + + if (!Schema.Table(table).Column("MessagesPerUserPerMinute").Exists()) + Alter.Table(table).AddColumn("MessagesPerUserPerMinute").AsInt32().Nullable(); + + if (!Schema.Table(table).Column("MessagesPerDepartmentPerMinute").Exists()) + Alter.Table(table).AddColumn("MessagesPerDepartmentPerMinute").AsInt32().Nullable(); + + if (!Schema.Table(table).Column("RequireLinkingConfirmation").Exists()) + Alter.Table(table).AddColumn("RequireLinkingConfirmation").AsBoolean().NotNullable().WithDefaultValue(true); + + if (!Schema.Table(table).Column("ProactiveNotificationsEnabled").Exists()) + Alter.Table(table).AddColumn("ProactiveNotificationsEnabled").AsBoolean().NotNullable().WithDefaultValue(false); } public override void Down() diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0071_AddingFeatureToggles.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0071_AddingFeatureToggles.cs index 09fc550bf..816700c32 100644 --- a/Providers/Resgrid.Providers.Migrations/Migrations/M0071_AddingFeatureToggles.cs +++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0071_AddingFeatureToggles.cs @@ -7,129 +7,147 @@ public class M0071_AddingFeatureToggles : Migration { public override void Up() { + // Each table (with its foreign keys and indexes) is guarded so the migration is safe to + // re-run / safe on databases where a prior partial apply already created some of them. + // FeatureFlags - system-wide flag definitions with a global default. - Create.Table("FeatureFlags") - .WithColumn("FeatureFlagId").AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FlagKey").AsString(256).NotNullable() - .WithColumn("Name").AsString(256).NotNullable() - .WithColumn("Description").AsString(1000).Nullable() - .WithColumn("Category").AsString(128).Nullable() - .WithColumn("Tags").AsString(512).Nullable() - .WithColumn("FlagType").AsInt32().NotNullable().WithDefaultValue(0) - .WithColumn("IsEnabledGlobally").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("DefaultValue").AsString(int.MaxValue).Nullable() - .WithColumn("OffValue").AsString(int.MaxValue).Nullable() - .WithColumn("RolloutPercentage").AsInt32().Nullable() - .WithColumn("MinimumPlanType").AsInt32().Nullable() - .WithColumn("Environment").AsInt32().Nullable() - .WithColumn("EnableOn").AsDateTime2().Nullable() - .WithColumn("DisableOn").AsDateTime2().Nullable() - .WithColumn("IsArchived").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("IsPermanent").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("LastEvaluatedOn").AsDateTime2().Nullable() - .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("CreatedByUserId").AsString(450).Nullable() - .WithColumn("UpdatedOn").AsDateTime2().Nullable() - .WithColumn("UpdatedByUserId").AsString(450).Nullable(); - - Create.Index("UX_FeatureFlags_FlagKey") - .OnTable("FeatureFlags") - .OnColumn("FlagKey").Ascending() - .WithOptions().Unique(); + if (!Schema.Table("FeatureFlags").Exists()) + { + Create.Table("FeatureFlags") + .WithColumn("FeatureFlagId").AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FlagKey").AsString(256).NotNullable() + .WithColumn("Name").AsString(256).NotNullable() + .WithColumn("Description").AsString(1000).Nullable() + .WithColumn("Category").AsString(128).Nullable() + .WithColumn("Tags").AsString(512).Nullable() + .WithColumn("FlagType").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("IsEnabledGlobally").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("DefaultValue").AsString(int.MaxValue).Nullable() + .WithColumn("OffValue").AsString(int.MaxValue).Nullable() + .WithColumn("RolloutPercentage").AsInt32().Nullable() + .WithColumn("MinimumPlanType").AsInt32().Nullable() + .WithColumn("Environment").AsInt32().Nullable() + .WithColumn("EnableOn").AsDateTime2().Nullable() + .WithColumn("DisableOn").AsDateTime2().Nullable() + .WithColumn("IsArchived").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("IsPermanent").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("LastEvaluatedOn").AsDateTime2().Nullable() + .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("CreatedByUserId").AsString(450).Nullable() + .WithColumn("UpdatedOn").AsDateTime2().Nullable() + .WithColumn("UpdatedByUserId").AsString(450).Nullable(); + + Create.Index("UX_FeatureFlags_FlagKey") + .OnTable("FeatureFlags") + .OnColumn("FlagKey").Ascending() + .WithOptions().Unique(); + } // FeatureFlagOverrides - per-department override of a flag's value. - Create.Table("FeatureFlagOverrides") - .WithColumn("FeatureFlagOverrideId").AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FeatureFlagId").AsInt32().NotNullable() - .WithColumn("DepartmentId").AsInt32().NotNullable() - .WithColumn("IsEnabled").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("FlagValue").AsString(int.MaxValue).Nullable() - .WithColumn("Reason").AsString(512).Nullable() - .WithColumn("ExpiresOn").AsDateTime2().Nullable() - .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("CreatedByUserId").AsString(450).Nullable() - .WithColumn("UpdatedOn").AsDateTime2().Nullable() - .WithColumn("UpdatedByUserId").AsString(450).Nullable(); - - Create.ForeignKey("FK_FeatureFlagOverrides_FeatureFlags") - .FromTable("FeatureFlagOverrides").ForeignColumn("FeatureFlagId") - .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); - - Create.ForeignKey("FK_FeatureFlagOverrides_Departments") - .FromTable("FeatureFlagOverrides").ForeignColumn("DepartmentId") - .ToTable("Departments").PrimaryColumn("DepartmentId"); - - Create.Index("UX_FeatureFlagOverrides_Flag_Department") - .OnTable("FeatureFlagOverrides") - .OnColumn("FeatureFlagId").Ascending() - .OnColumn("DepartmentId").Ascending() - .WithOptions().Unique(); + if (!Schema.Table("FeatureFlagOverrides").Exists()) + { + Create.Table("FeatureFlagOverrides") + .WithColumn("FeatureFlagOverrideId").AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FeatureFlagId").AsInt32().NotNullable() + .WithColumn("DepartmentId").AsInt32().NotNullable() + .WithColumn("IsEnabled").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("FlagValue").AsString(int.MaxValue).Nullable() + .WithColumn("Reason").AsString(512).Nullable() + .WithColumn("ExpiresOn").AsDateTime2().Nullable() + .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("CreatedByUserId").AsString(450).Nullable() + .WithColumn("UpdatedOn").AsDateTime2().Nullable() + .WithColumn("UpdatedByUserId").AsString(450).Nullable(); + + Create.ForeignKey("FK_FeatureFlagOverrides_FeatureFlags") + .FromTable("FeatureFlagOverrides").ForeignColumn("FeatureFlagId") + .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); + + Create.ForeignKey("FK_FeatureFlagOverrides_Departments") + .FromTable("FeatureFlagOverrides").ForeignColumn("DepartmentId") + .ToTable("Departments").PrimaryColumn("DepartmentId"); + + Create.Index("UX_FeatureFlagOverrides_Flag_Department") + .OnTable("FeatureFlagOverrides") + .OnColumn("FeatureFlagId").Ascending() + .OnColumn("DepartmentId").Ascending() + .WithOptions().Unique(); + } // FeatureFlagTargetingRules - attribute/segment targeting rules. - Create.Table("FeatureFlagTargetingRules") - .WithColumn("FeatureFlagTargetingRuleId").AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FeatureFlagId").AsInt32().NotNullable() - .WithColumn("Priority").AsInt32().NotNullable().WithDefaultValue(0) - .WithColumn("AttributeType").AsInt32().NotNullable() - .WithColumn("OperatorType").AsInt32().NotNullable() - .WithColumn("ComparisonValue").AsString(int.MaxValue).Nullable() - .WithColumn("ResultEnabled").AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("ResultValue").AsString(int.MaxValue).Nullable() - .WithColumn("RolloutPercentage").AsInt32().Nullable() - .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("CreatedByUserId").AsString(450).Nullable() - .WithColumn("UpdatedOn").AsDateTime2().Nullable() - .WithColumn("UpdatedByUserId").AsString(450).Nullable(); - - Create.ForeignKey("FK_FeatureFlagTargetingRules_FeatureFlags") - .FromTable("FeatureFlagTargetingRules").ForeignColumn("FeatureFlagId") - .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); - - Create.Index("IX_FeatureFlagTargetingRules_Flag") - .OnTable("FeatureFlagTargetingRules") - .OnColumn("FeatureFlagId").Ascending(); + if (!Schema.Table("FeatureFlagTargetingRules").Exists()) + { + Create.Table("FeatureFlagTargetingRules") + .WithColumn("FeatureFlagTargetingRuleId").AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FeatureFlagId").AsInt32().NotNullable() + .WithColumn("Priority").AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("AttributeType").AsInt32().NotNullable() + .WithColumn("OperatorType").AsInt32().NotNullable() + .WithColumn("ComparisonValue").AsString(int.MaxValue).Nullable() + .WithColumn("ResultEnabled").AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("ResultValue").AsString(int.MaxValue).Nullable() + .WithColumn("RolloutPercentage").AsInt32().Nullable() + .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("CreatedByUserId").AsString(450).Nullable() + .WithColumn("UpdatedOn").AsDateTime2().Nullable() + .WithColumn("UpdatedByUserId").AsString(450).Nullable(); + + Create.ForeignKey("FK_FeatureFlagTargetingRules_FeatureFlags") + .FromTable("FeatureFlagTargetingRules").ForeignColumn("FeatureFlagId") + .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); + + Create.Index("IX_FeatureFlagTargetingRules_Flag") + .OnTable("FeatureFlagTargetingRules") + .OnColumn("FeatureFlagId").Ascending(); + } // FeatureFlagPrerequisites - flag dependency edges. - Create.Table("FeatureFlagPrerequisites") - .WithColumn("FeatureFlagPrerequisiteId").AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FeatureFlagId").AsInt32().NotNullable() - .WithColumn("RequiredFeatureFlagId").AsInt32().NotNullable() - .WithColumn("RequiredValue").AsString(int.MaxValue).Nullable(); - - Create.ForeignKey("FK_FeatureFlagPrerequisites_FeatureFlags") - .FromTable("FeatureFlagPrerequisites").ForeignColumn("FeatureFlagId") - .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); - - Create.ForeignKey("FK_FeatureFlagPrerequisites_RequiredFeatureFlag") - .FromTable("FeatureFlagPrerequisites").ForeignColumn("RequiredFeatureFlagId") - .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); - - Create.Index("IX_FeatureFlagPrerequisites_Flag") - .OnTable("FeatureFlagPrerequisites") - .OnColumn("FeatureFlagId").Ascending(); + if (!Schema.Table("FeatureFlagPrerequisites").Exists()) + { + Create.Table("FeatureFlagPrerequisites") + .WithColumn("FeatureFlagPrerequisiteId").AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FeatureFlagId").AsInt32().NotNullable() + .WithColumn("RequiredFeatureFlagId").AsInt32().NotNullable() + .WithColumn("RequiredValue").AsString(int.MaxValue).Nullable(); + + Create.ForeignKey("FK_FeatureFlagPrerequisites_FeatureFlags") + .FromTable("FeatureFlagPrerequisites").ForeignColumn("FeatureFlagId") + .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); + + Create.ForeignKey("FK_FeatureFlagPrerequisites_RequiredFeatureFlag") + .FromTable("FeatureFlagPrerequisites").ForeignColumn("RequiredFeatureFlagId") + .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); + + Create.Index("IX_FeatureFlagPrerequisites_Flag") + .OnTable("FeatureFlagPrerequisites") + .OnColumn("FeatureFlagId").Ascending(); + } // FeatureFlagUsages - aggregated daily evaluation counts (append-only flushes). - Create.Table("FeatureFlagUsages") - .WithColumn("FeatureFlagUsageId").AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FeatureFlagId").AsInt32().NotNullable() - .WithColumn("DepartmentId").AsInt32().Nullable() - .WithColumn("UsageDate").AsDateTime2().NotNullable() - .WithColumn("EvaluationCount").AsInt64().NotNullable().WithDefaultValue(0) - .WithColumn("EnabledCount").AsInt64().NotNullable().WithDefaultValue(0) - .WithColumn("DisabledCount").AsInt64().NotNullable().WithDefaultValue(0); - - Create.ForeignKey("FK_FeatureFlagUsages_FeatureFlags") - .FromTable("FeatureFlagUsages").ForeignColumn("FeatureFlagId") - .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); - - Create.ForeignKey("FK_FeatureFlagUsages_Departments") - .FromTable("FeatureFlagUsages").ForeignColumn("DepartmentId") - .ToTable("Departments").PrimaryColumn("DepartmentId"); - - Create.Index("IX_FeatureFlagUsages_Flag_Date") - .OnTable("FeatureFlagUsages") - .OnColumn("FeatureFlagId").Ascending() - .OnColumn("UsageDate").Ascending(); + if (!Schema.Table("FeatureFlagUsages").Exists()) + { + Create.Table("FeatureFlagUsages") + .WithColumn("FeatureFlagUsageId").AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FeatureFlagId").AsInt32().NotNullable() + .WithColumn("DepartmentId").AsInt32().Nullable() + .WithColumn("UsageDate").AsDateTime2().NotNullable() + .WithColumn("EvaluationCount").AsInt64().NotNullable().WithDefaultValue(0) + .WithColumn("EnabledCount").AsInt64().NotNullable().WithDefaultValue(0) + .WithColumn("DisabledCount").AsInt64().NotNullable().WithDefaultValue(0); + + Create.ForeignKey("FK_FeatureFlagUsages_FeatureFlags") + .FromTable("FeatureFlagUsages").ForeignColumn("FeatureFlagId") + .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId"); + + Create.ForeignKey("FK_FeatureFlagUsages_Departments") + .FromTable("FeatureFlagUsages").ForeignColumn("DepartmentId") + .ToTable("Departments").PrimaryColumn("DepartmentId"); + + Create.Index("IX_FeatureFlagUsages_Flag_Date") + .OnTable("FeatureFlagUsages") + .OnColumn("FeatureFlagId").Ascending() + .OnColumn("UsageDate").Ascending(); + } } public override void Down() diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0072_AddingChatbotTwilioFeatureFlag.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0072_AddingChatbotTwilioFeatureFlag.cs index dd283ecd9..01574570d 100644 --- a/Providers/Resgrid.Providers.Migrations/Migrations/M0072_AddingChatbotTwilioFeatureFlag.cs +++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0072_AddingChatbotTwilioFeatureFlag.cs @@ -13,14 +13,15 @@ public override void Up() // Seeded OFF (IsEnabledGlobally = false). Inbound Twilio SMS keeps the original text-command // handling until this flag is enabled globally or via a per-department override. FlagType, // IsArchived, IsPermanent and CreatedOn fall back to their table defaults. - Insert.IntoTable("FeatureFlags").Row(new - { - FlagKey = FlagKey, - Name = "Chatbot Twilio Text Integration", - Description = "When enabled, inbound Twilio SMS is processed by the new chatbot ingress pipeline. When off (globally or for a department) the original text-command handling is used.", - Category = "Chatbot", - IsEnabledGlobally = false - }); + // Guarded with IF NOT EXISTS so re-running the migration does not violate the unique + // FlagKey index. + Execute.Sql( + "IF NOT EXISTS (SELECT 1 FROM [FeatureFlags] WHERE [FlagKey] = '" + FlagKey + "') " + + "INSERT INTO [FeatureFlags] ([FlagKey], [Name], [Description], [Category], [IsEnabledGlobally]) " + + "VALUES ('" + FlagKey + "', " + + "'Chatbot Twilio Text Integration', " + + "'When enabled, inbound Twilio SMS is processed by the new chatbot ingress pipeline. When off (globally or for a department) the original text-command handling is used.', " + + "'Chatbot', 0);"); } public override void Down() diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSources.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSources.cs index 39184020e..9ba580979 100644 --- a/Providers/Resgrid.Providers.Migrations/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSources.cs +++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSources.cs @@ -7,8 +7,13 @@ public class M0074_AddIsPermanentFailureToWeatherAlertSources : Migration { public override void Up() { - Alter.Table("WeatherAlertSources") - .AddColumn("IsPermanentFailure").AsBoolean().NotNullable().WithDefaultValue(false); + // Guarded: this migration was renumbered (68 -> 74), so databases upgraded before the + // renumber already added this column under version 68. Skip the add if it exists. + if (!Schema.Table("WeatherAlertSources").Column("IsPermanentFailure").Exists()) + { + Alter.Table("WeatherAlertSources") + .AddColumn("IsPermanentFailure").AsBoolean().NotNullable().WithDefaultValue(false); + } } public override void Down() diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0075_AddUtf8CleanupProgress.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0075_AddUtf8CleanupProgress.cs new file mode 100644 index 000000000..73e1263d9 --- /dev/null +++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0075_AddUtf8CleanupProgress.cs @@ -0,0 +1,31 @@ +using FluentMigrator; + +namespace Resgrid.Providers.Migrations.Migrations +{ + [Migration(75)] + public class M0075_AddUtf8CleanupProgress : Migration + { + public override void Up() + { + // Per-table watermark for the nightly UTF-8 data cleanup worker so each run resumes + // where the previous one stopped while staying idempotent. + // Guarded so the migration is safe to re-run / safe on databases where a prior partial + // apply already created the table. + if (!Schema.Table("Utf8CleanupProgress").Exists()) + { + Create.Table("Utf8CleanupProgress") + .WithColumn("TableName").AsString(256).NotNullable().PrimaryKey() + .WithColumn("LastProcessedKey").AsString(450).Nullable() + .WithColumn("LastCompletedUtc").AsDateTime().Nullable() + .WithColumn("RowsScanned").AsInt64().NotNullable().WithDefaultValue(0) + .WithColumn("RowsFixed").AsInt64().NotNullable().WithDefaultValue(0) + .WithColumn("UpdatedOnUtc").AsDateTime().NotNullable(); + } + } + + public override void Down() + { + Delete.Table("Utf8CleanupProgress"); + } + } +} diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0068_ChatbotTablesPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0068_ChatbotTablesPg.cs index fcb371cdf..163b9b250 100644 --- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0068_ChatbotTablesPg.cs +++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0068_ChatbotTablesPg.cs @@ -7,91 +7,107 @@ public class M0068_ChatbotTablesPg : Migration { public override void Up() { + // Each table is guarded independently: databases upgraded before the 68->74 migration + // renumber may already have some of these (created under a prior version), so skip any + // table that already exists rather than failing the whole migration. + // ChatbotUserIdentities - Links platform identities to Resgrid users - Create.Table("ChatbotUserIdentities".ToLower()) - .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() - .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable() - .WithColumn("Platform".ToLower()).AsInt32().NotNullable() - .WithColumn("PlatformUserId".ToLower()).AsCustom("citext").NotNullable() - .WithColumn("PlatformUserName".ToLower()).AsCustom("citext").Nullable() - .WithColumn("IsActive".ToLower()).AsBoolean().NotNullable().WithDefaultValue(true) - .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("LastUsedAt".ToLower()).AsDateTime2().Nullable() - .WithColumn("LinkingMethod".ToLower()).AsCustom("citext").Nullable(); + if (!Schema.Table("ChatbotUserIdentities".ToLower()).Exists()) + { + Create.Table("ChatbotUserIdentities".ToLower()) + .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() + .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable() + .WithColumn("Platform".ToLower()).AsInt32().NotNullable() + .WithColumn("PlatformUserId".ToLower()).AsCustom("citext").NotNullable() + .WithColumn("PlatformUserName".ToLower()).AsCustom("citext").Nullable() + .WithColumn("IsActive".ToLower()).AsBoolean().NotNullable().WithDefaultValue(true) + .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("LastUsedAt".ToLower()).AsDateTime2().Nullable() + .WithColumn("LinkingMethod".ToLower()).AsCustom("citext").Nullable(); - Create.Index("IX_ChatbotUserIdentities_User_Platform".ToLower()) - .OnTable("ChatbotUserIdentities".ToLower()) - .OnColumn("UserId".ToLower()).Ascending() - .OnColumn("Platform".ToLower()).Ascending(); + Create.Index("IX_ChatbotUserIdentities_User_Platform".ToLower()) + .OnTable("ChatbotUserIdentities".ToLower()) + .OnColumn("UserId".ToLower()).Ascending() + .OnColumn("Platform".ToLower()).Ascending(); - Create.Index("IX_ChatbotUserIdentities_Platform_PlatformUserId".ToLower()) - .OnTable("ChatbotUserIdentities".ToLower()) - .OnColumn("Platform".ToLower()).Ascending() - .OnColumn("PlatformUserId".ToLower()).Ascending(); + Create.Index("IX_ChatbotUserIdentities_Platform_PlatformUserId".ToLower()) + .OnTable("ChatbotUserIdentities".ToLower()) + .OnColumn("Platform".ToLower()).Ascending() + .OnColumn("PlatformUserId".ToLower()).Ascending(); + } // ChatbotSessions - Active conversation session state - Create.Table("ChatbotSessions".ToLower()) - .WithColumn("SessionId".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() - .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable() - .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable() - .WithColumn("Platform".ToLower()).AsInt32().NotNullable() - .WithColumn("State".ToLower()).AsInt32().NotNullable().WithDefaultValue(0) - .WithColumn("PendingIntent".ToLower()).AsInt32().Nullable() - .WithColumn("ContextJson".ToLower()).AsCustom("text").Nullable() - .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("LastActivity".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("TtlMinutes".ToLower()).AsInt32().NotNullable().WithDefaultValue(30); + if (!Schema.Table("ChatbotSessions".ToLower()).Exists()) + { + Create.Table("ChatbotSessions".ToLower()) + .WithColumn("SessionId".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() + .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable() + .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable() + .WithColumn("Platform".ToLower()).AsInt32().NotNullable() + .WithColumn("State".ToLower()).AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("PendingIntent".ToLower()).AsInt32().Nullable() + .WithColumn("ContextJson".ToLower()).AsCustom("text").Nullable() + .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("LastActivity".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("TtlMinutes".ToLower()).AsInt32().NotNullable().WithDefaultValue(30); - Create.Index("IX_ChatbotSessions_UserId_Department".ToLower()) - .OnTable("ChatbotSessions".ToLower()) - .OnColumn("UserId".ToLower()).Ascending() - .OnColumn("DepartmentId".ToLower()).Ascending() - .OnColumn("Platform".ToLower()).Ascending(); + Create.Index("IX_ChatbotSessions_UserId_Department".ToLower()) + .OnTable("ChatbotSessions".ToLower()) + .OnColumn("UserId".ToLower()).Ascending() + .OnColumn("DepartmentId".ToLower()).Ascending() + .OnColumn("Platform".ToLower()).Ascending(); - Create.Index("IX_ChatbotSessions_LastActivity".ToLower()) - .OnTable("ChatbotSessions".ToLower()) - .OnColumn("LastActivity".ToLower()).Ascending(); + Create.Index("IX_ChatbotSessions_LastActivity".ToLower()) + .OnTable("ChatbotSessions".ToLower()) + .OnColumn("LastActivity".ToLower()).Ascending(); + } // ChatbotMessageLog - Audit log of all chatbot interactions - Create.Table("ChatbotMessageLog".ToLower()) - .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() - .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable() - .WithColumn("UserId".ToLower()).AsCustom("citext").Nullable() - .WithColumn("SessionId".ToLower()).AsCustom("citext").Nullable() - .WithColumn("Platform".ToLower()).AsInt32().NotNullable() - .WithColumn("Direction".ToLower()).AsCustom("citext").NotNullable() - .WithColumn("MessageText".ToLower()).AsCustom("text").Nullable() - .WithColumn("IntentType".ToLower()).AsInt32().Nullable() - .WithColumn("Processed".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("ErrorInfo".ToLower()).AsCustom("citext").Nullable() - .WithColumn("Timestamp".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime); + if (!Schema.Table("ChatbotMessageLog".ToLower()).Exists()) + { + Create.Table("ChatbotMessageLog".ToLower()) + .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() + .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable() + .WithColumn("UserId".ToLower()).AsCustom("citext").Nullable() + .WithColumn("SessionId".ToLower()).AsCustom("citext").Nullable() + .WithColumn("Platform".ToLower()).AsInt32().NotNullable() + .WithColumn("Direction".ToLower()).AsCustom("citext").NotNullable() + .WithColumn("MessageText".ToLower()).AsCustom("text").Nullable() + .WithColumn("IntentType".ToLower()).AsInt32().Nullable() + .WithColumn("Processed".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("ErrorInfo".ToLower()).AsCustom("citext").Nullable() + .WithColumn("Timestamp".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime); - Create.Index("IX_ChatbotMessageLog_Department_Timestamp".ToLower()) - .OnTable("ChatbotMessageLog".ToLower()) - .OnColumn("DepartmentId".ToLower()).Ascending() - .OnColumn("Timestamp".ToLower()).Descending(); + Create.Index("IX_ChatbotMessageLog_Department_Timestamp".ToLower()) + .OnTable("ChatbotMessageLog".ToLower()) + .OnColumn("DepartmentId".ToLower()).Ascending() + .OnColumn("Timestamp".ToLower()).Descending(); - Create.Index("IX_ChatbotMessageLog_UserId".ToLower()) - .OnTable("ChatbotMessageLog".ToLower()) - .OnColumn("UserId".ToLower()).Ascending(); + Create.Index("IX_ChatbotMessageLog_UserId".ToLower()) + .OnTable("ChatbotMessageLog".ToLower()) + .OnColumn("UserId".ToLower()).Ascending(); + } // ChatbotDepartmentConfigs - Per-department chatbot configuration - Create.Table("ChatbotDepartmentConfigs".ToLower()) - .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() - .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable() - .WithColumn("IsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("NluProvider".ToLower()).AsCustom("citext").NotNullable().WithDefaultValue("keyword") - .WithColumn("AllowedPlatforms".ToLower()).AsCustom("citext").NotNullable().WithDefaultValue("*") - .WithColumn("MaxSessionsPerUser".ToLower()).AsInt32().NotNullable().WithDefaultValue(3) - .WithColumn("SessionTtlMinutes".ToLower()).AsInt32().NotNullable().WithDefaultValue(30) - .WithColumn("AllowDispatchViaChatbot".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("RequireConfirmationForStatusChange".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("UpdatedAt".ToLower()).AsDateTime2().Nullable(); + if (!Schema.Table("ChatbotDepartmentConfigs".ToLower()).Exists()) + { + Create.Table("ChatbotDepartmentConfigs".ToLower()) + .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() + .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable() + .WithColumn("IsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("NluProvider".ToLower()).AsCustom("citext").NotNullable().WithDefaultValue("keyword") + .WithColumn("AllowedPlatforms".ToLower()).AsCustom("citext").NotNullable().WithDefaultValue("*") + .WithColumn("MaxSessionsPerUser".ToLower()).AsInt32().NotNullable().WithDefaultValue(3) + .WithColumn("SessionTtlMinutes".ToLower()).AsInt32().NotNullable().WithDefaultValue(30) + .WithColumn("AllowDispatchViaChatbot".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("RequireConfirmationForStatusChange".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("UpdatedAt".ToLower()).AsDateTime2().Nullable(); - Create.Index("IX_ChatbotDepartmentConfigs_DepartmentId".ToLower()) - .OnTable("ChatbotDepartmentConfigs".ToLower()) - .OnColumn("DepartmentId".ToLower()).Ascending(); + Create.Index("IX_ChatbotDepartmentConfigs_DepartmentId".ToLower()) + .OnTable("ChatbotDepartmentConfigs".ToLower()) + .OnColumn("DepartmentId".ToLower()).Ascending(); + } } public override void Down() diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0069_ChatbotLinkingCodesPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0069_ChatbotLinkingCodesPg.cs index decc6e5a5..b35fde6b8 100644 --- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0069_ChatbotLinkingCodesPg.cs +++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0069_ChatbotLinkingCodesPg.cs @@ -8,28 +8,33 @@ public class M0069_ChatbotLinkingCodesPg : Migration public override void Up() { // ChatbotLinkingCodes - Short-lived codes for linking platform accounts - Create.Table("ChatbotLinkingCodes".ToLower()) - .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() - .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable() - .WithColumn("Code".ToLower()).AsCustom("citext").NotNullable() - .WithColumn("Platform".ToLower()).AsInt32().Nullable() - .WithColumn("PlatformUserId".ToLower()).AsCustom("citext").Nullable() - .WithColumn("IsUsed".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("ExpiresAt".ToLower()).AsDateTime2().NotNullable() - .WithColumn("UsedAt".ToLower()).AsDateTime2().Nullable(); + // Guarded: databases upgraded before the 68->74 migration renumber may already have + // this table (it ran under a prior version), so skip creation if it already exists. + if (!Schema.Table("ChatbotLinkingCodes".ToLower()).Exists()) + { + Create.Table("ChatbotLinkingCodes".ToLower()) + .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey() + .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable() + .WithColumn("Code".ToLower()).AsCustom("citext").NotNullable() + .WithColumn("Platform".ToLower()).AsInt32().Nullable() + .WithColumn("PlatformUserId".ToLower()).AsCustom("citext").Nullable() + .WithColumn("IsUsed".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("ExpiresAt".ToLower()).AsDateTime2().NotNullable() + .WithColumn("UsedAt".ToLower()).AsDateTime2().Nullable(); - Create.Index("IX_ChatbotLinkingCodes_Code".ToLower()) - .OnTable("ChatbotLinkingCodes".ToLower()) - .OnColumn("Code".ToLower()).Ascending(); + Create.Index("IX_ChatbotLinkingCodes_Code".ToLower()) + .OnTable("ChatbotLinkingCodes".ToLower()) + .OnColumn("Code".ToLower()).Ascending(); - Create.Index("IX_ChatbotLinkingCodes_UserId".ToLower()) - .OnTable("ChatbotLinkingCodes".ToLower()) - .OnColumn("UserId".ToLower()).Ascending(); + Create.Index("IX_ChatbotLinkingCodes_UserId".ToLower()) + .OnTable("ChatbotLinkingCodes".ToLower()) + .OnColumn("UserId".ToLower()).Ascending(); - Create.Index("IX_ChatbotLinkingCodes_ExpiresAt".ToLower()) - .OnTable("ChatbotLinkingCodes".ToLower()) - .OnColumn("ExpiresAt".ToLower()).Ascending(); + Create.Index("IX_ChatbotLinkingCodes_ExpiresAt".ToLower()) + .OnTable("ChatbotLinkingCodes".ToLower()) + .OnColumn("ExpiresAt".ToLower()).Ascending(); + } } public override void Down() diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0070_ChatbotDepartmentConfigColumnsPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0070_ChatbotDepartmentConfigColumnsPg.cs index 167f6aa88..d41f52d9d 100644 --- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0070_ChatbotDepartmentConfigColumnsPg.cs +++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0070_ChatbotDepartmentConfigColumnsPg.cs @@ -8,14 +8,30 @@ public class M0070_ChatbotDepartmentConfigColumnsPg : Migration public override void Up() { // Per-department LLM/AI override + rate limits + linking/notification preferences. - Alter.Table("ChatbotDepartmentConfigs".ToLower()) - .AddColumn("LlmApiEndpoint".ToLower()).AsCustom("citext").Nullable() - .AddColumn("LlmApiKey".ToLower()).AsCustom("citext").Nullable() - .AddColumn("LlmModelName".ToLower()).AsCustom("citext").Nullable() - .AddColumn("MessagesPerUserPerMinute".ToLower()).AsInt32().Nullable() - .AddColumn("MessagesPerDepartmentPerMinute".ToLower()).AsInt32().Nullable() - .AddColumn("RequireLinkingConfirmation".ToLower()).AsBoolean().NotNullable().WithDefaultValue(true) - .AddColumn("ProactiveNotificationsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false); + // Each column is guarded so the migration is safe on databases where a prior partial + // apply (or a version renumber) already added some of them. + var table = "ChatbotDepartmentConfigs".ToLower(); + + if (!Schema.Table(table).Column("LlmApiEndpoint".ToLower()).Exists()) + Alter.Table(table).AddColumn("LlmApiEndpoint".ToLower()).AsCustom("citext").Nullable(); + + if (!Schema.Table(table).Column("LlmApiKey".ToLower()).Exists()) + Alter.Table(table).AddColumn("LlmApiKey".ToLower()).AsCustom("citext").Nullable(); + + if (!Schema.Table(table).Column("LlmModelName".ToLower()).Exists()) + Alter.Table(table).AddColumn("LlmModelName".ToLower()).AsCustom("citext").Nullable(); + + if (!Schema.Table(table).Column("MessagesPerUserPerMinute".ToLower()).Exists()) + Alter.Table(table).AddColumn("MessagesPerUserPerMinute".ToLower()).AsInt32().Nullable(); + + if (!Schema.Table(table).Column("MessagesPerDepartmentPerMinute".ToLower()).Exists()) + Alter.Table(table).AddColumn("MessagesPerDepartmentPerMinute".ToLower()).AsInt32().Nullable(); + + if (!Schema.Table(table).Column("RequireLinkingConfirmation".ToLower()).Exists()) + Alter.Table(table).AddColumn("RequireLinkingConfirmation".ToLower()).AsBoolean().NotNullable().WithDefaultValue(true); + + if (!Schema.Table(table).Column("ProactiveNotificationsEnabled".ToLower()).Exists()) + Alter.Table(table).AddColumn("ProactiveNotificationsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false); } public override void Down() diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0071_AddingFeatureTogglesPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0071_AddingFeatureTogglesPg.cs index d841fdf2a..84a041f14 100644 --- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0071_AddingFeatureTogglesPg.cs +++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0071_AddingFeatureTogglesPg.cs @@ -7,129 +7,147 @@ public class M0071_AddingFeatureTogglesPg : Migration { public override void Up() { + // Each table (with its foreign keys and indexes) is guarded so the migration is safe to + // re-run / safe on databases where a prior partial apply already created some of them. + // FeatureFlags - system-wide flag definitions with a global default. - Create.Table("FeatureFlags".ToLower()) - .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FlagKey".ToLower()).AsCustom("citext").NotNullable() - .WithColumn("Name".ToLower()).AsCustom("citext").NotNullable() - .WithColumn("Description".ToLower()).AsCustom("text").Nullable() - .WithColumn("Category".ToLower()).AsCustom("citext").Nullable() - .WithColumn("Tags".ToLower()).AsCustom("text").Nullable() - .WithColumn("FlagType".ToLower()).AsInt32().NotNullable().WithDefaultValue(0) - .WithColumn("IsEnabledGlobally".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("DefaultValue".ToLower()).AsCustom("text").Nullable() - .WithColumn("OffValue".ToLower()).AsCustom("text").Nullable() - .WithColumn("RolloutPercentage".ToLower()).AsInt32().Nullable() - .WithColumn("MinimumPlanType".ToLower()).AsInt32().Nullable() - .WithColumn("Environment".ToLower()).AsInt32().Nullable() - .WithColumn("EnableOn".ToLower()).AsDateTime2().Nullable() - .WithColumn("DisableOn".ToLower()).AsDateTime2().Nullable() - .WithColumn("IsArchived".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("IsPermanent".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("LastEvaluatedOn".ToLower()).AsDateTime2().Nullable() - .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable() - .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable() - .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable(); - - Create.Index("UX_FeatureFlags_FlagKey".ToLower()) - .OnTable("FeatureFlags".ToLower()) - .OnColumn("FlagKey".ToLower()).Ascending() - .WithOptions().Unique(); + if (!Schema.Table("FeatureFlags".ToLower()).Exists()) + { + Create.Table("FeatureFlags".ToLower()) + .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FlagKey".ToLower()).AsCustom("citext").NotNullable() + .WithColumn("Name".ToLower()).AsCustom("citext").NotNullable() + .WithColumn("Description".ToLower()).AsCustom("text").Nullable() + .WithColumn("Category".ToLower()).AsCustom("citext").Nullable() + .WithColumn("Tags".ToLower()).AsCustom("text").Nullable() + .WithColumn("FlagType".ToLower()).AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("IsEnabledGlobally".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("DefaultValue".ToLower()).AsCustom("text").Nullable() + .WithColumn("OffValue".ToLower()).AsCustom("text").Nullable() + .WithColumn("RolloutPercentage".ToLower()).AsInt32().Nullable() + .WithColumn("MinimumPlanType".ToLower()).AsInt32().Nullable() + .WithColumn("Environment".ToLower()).AsInt32().Nullable() + .WithColumn("EnableOn".ToLower()).AsDateTime2().Nullable() + .WithColumn("DisableOn".ToLower()).AsDateTime2().Nullable() + .WithColumn("IsArchived".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("IsPermanent".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("LastEvaluatedOn".ToLower()).AsDateTime2().Nullable() + .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable() + .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable() + .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable(); + + Create.Index("UX_FeatureFlags_FlagKey".ToLower()) + .OnTable("FeatureFlags".ToLower()) + .OnColumn("FlagKey".ToLower()).Ascending() + .WithOptions().Unique(); + } // FeatureFlagOverrides - per-department override of a flag's value. - Create.Table("FeatureFlagOverrides".ToLower()) - .WithColumn("FeatureFlagOverrideId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable() - .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable() - .WithColumn("IsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("FlagValue".ToLower()).AsCustom("text").Nullable() - .WithColumn("Reason".ToLower()).AsCustom("text").Nullable() - .WithColumn("ExpiresOn".ToLower()).AsDateTime2().Nullable() - .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable() - .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable() - .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable(); - - Create.ForeignKey("FK_FeatureFlagOverrides_FeatureFlags".ToLower()) - .FromTable("FeatureFlagOverrides".ToLower()).ForeignColumn("FeatureFlagId".ToLower()) - .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); - - Create.ForeignKey("FK_FeatureFlagOverrides_Departments".ToLower()) - .FromTable("FeatureFlagOverrides".ToLower()).ForeignColumn("DepartmentId".ToLower()) - .ToTable("Departments".ToLower()).PrimaryColumn("DepartmentId".ToLower()); - - Create.Index("UX_FeatureFlagOverrides_Flag_Department".ToLower()) - .OnTable("FeatureFlagOverrides".ToLower()) - .OnColumn("FeatureFlagId".ToLower()).Ascending() - .OnColumn("DepartmentId".ToLower()).Ascending() - .WithOptions().Unique(); + if (!Schema.Table("FeatureFlagOverrides".ToLower()).Exists()) + { + Create.Table("FeatureFlagOverrides".ToLower()) + .WithColumn("FeatureFlagOverrideId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable() + .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable() + .WithColumn("IsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("FlagValue".ToLower()).AsCustom("text").Nullable() + .WithColumn("Reason".ToLower()).AsCustom("text").Nullable() + .WithColumn("ExpiresOn".ToLower()).AsDateTime2().Nullable() + .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable() + .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable() + .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable(); + + Create.ForeignKey("FK_FeatureFlagOverrides_FeatureFlags".ToLower()) + .FromTable("FeatureFlagOverrides".ToLower()).ForeignColumn("FeatureFlagId".ToLower()) + .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); + + Create.ForeignKey("FK_FeatureFlagOverrides_Departments".ToLower()) + .FromTable("FeatureFlagOverrides".ToLower()).ForeignColumn("DepartmentId".ToLower()) + .ToTable("Departments".ToLower()).PrimaryColumn("DepartmentId".ToLower()); + + Create.Index("UX_FeatureFlagOverrides_Flag_Department".ToLower()) + .OnTable("FeatureFlagOverrides".ToLower()) + .OnColumn("FeatureFlagId".ToLower()).Ascending() + .OnColumn("DepartmentId".ToLower()).Ascending() + .WithOptions().Unique(); + } // FeatureFlagTargetingRules - attribute/segment targeting rules. - Create.Table("FeatureFlagTargetingRules".ToLower()) - .WithColumn("FeatureFlagTargetingRuleId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable() - .WithColumn("Priority".ToLower()).AsInt32().NotNullable().WithDefaultValue(0) - .WithColumn("AttributeType".ToLower()).AsInt32().NotNullable() - .WithColumn("OperatorType".ToLower()).AsInt32().NotNullable() - .WithColumn("ComparisonValue".ToLower()).AsCustom("text").Nullable() - .WithColumn("ResultEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) - .WithColumn("ResultValue".ToLower()).AsCustom("text").Nullable() - .WithColumn("RolloutPercentage".ToLower()).AsInt32().Nullable() - .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) - .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable() - .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable() - .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable(); - - Create.ForeignKey("FK_FeatureFlagTargetingRules_FeatureFlags".ToLower()) - .FromTable("FeatureFlagTargetingRules".ToLower()).ForeignColumn("FeatureFlagId".ToLower()) - .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); - - Create.Index("IX_FeatureFlagTargetingRules_Flag".ToLower()) - .OnTable("FeatureFlagTargetingRules".ToLower()) - .OnColumn("FeatureFlagId".ToLower()).Ascending(); + if (!Schema.Table("FeatureFlagTargetingRules".ToLower()).Exists()) + { + Create.Table("FeatureFlagTargetingRules".ToLower()) + .WithColumn("FeatureFlagTargetingRuleId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable() + .WithColumn("Priority".ToLower()).AsInt32().NotNullable().WithDefaultValue(0) + .WithColumn("AttributeType".ToLower()).AsInt32().NotNullable() + .WithColumn("OperatorType".ToLower()).AsInt32().NotNullable() + .WithColumn("ComparisonValue".ToLower()).AsCustom("text").Nullable() + .WithColumn("ResultEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false) + .WithColumn("ResultValue".ToLower()).AsCustom("text").Nullable() + .WithColumn("RolloutPercentage".ToLower()).AsInt32().Nullable() + .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime) + .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable() + .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable() + .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable(); + + Create.ForeignKey("FK_FeatureFlagTargetingRules_FeatureFlags".ToLower()) + .FromTable("FeatureFlagTargetingRules".ToLower()).ForeignColumn("FeatureFlagId".ToLower()) + .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); + + Create.Index("IX_FeatureFlagTargetingRules_Flag".ToLower()) + .OnTable("FeatureFlagTargetingRules".ToLower()) + .OnColumn("FeatureFlagId".ToLower()).Ascending(); + } // FeatureFlagPrerequisites - flag dependency edges. - Create.Table("FeatureFlagPrerequisites".ToLower()) - .WithColumn("FeatureFlagPrerequisiteId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable() - .WithColumn("RequiredFeatureFlagId".ToLower()).AsInt32().NotNullable() - .WithColumn("RequiredValue".ToLower()).AsCustom("text").Nullable(); - - Create.ForeignKey("FK_FeatureFlagPrerequisites_FeatureFlags".ToLower()) - .FromTable("FeatureFlagPrerequisites".ToLower()).ForeignColumn("FeatureFlagId".ToLower()) - .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); - - Create.ForeignKey("FK_FeatureFlagPrerequisites_RequiredFeatureFlag".ToLower()) - .FromTable("FeatureFlagPrerequisites".ToLower()).ForeignColumn("RequiredFeatureFlagId".ToLower()) - .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); - - Create.Index("IX_FeatureFlagPrerequisites_Flag".ToLower()) - .OnTable("FeatureFlagPrerequisites".ToLower()) - .OnColumn("FeatureFlagId".ToLower()).Ascending(); + if (!Schema.Table("FeatureFlagPrerequisites".ToLower()).Exists()) + { + Create.Table("FeatureFlagPrerequisites".ToLower()) + .WithColumn("FeatureFlagPrerequisiteId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable() + .WithColumn("RequiredFeatureFlagId".ToLower()).AsInt32().NotNullable() + .WithColumn("RequiredValue".ToLower()).AsCustom("text").Nullable(); + + Create.ForeignKey("FK_FeatureFlagPrerequisites_FeatureFlags".ToLower()) + .FromTable("FeatureFlagPrerequisites".ToLower()).ForeignColumn("FeatureFlagId".ToLower()) + .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); + + Create.ForeignKey("FK_FeatureFlagPrerequisites_RequiredFeatureFlag".ToLower()) + .FromTable("FeatureFlagPrerequisites".ToLower()).ForeignColumn("RequiredFeatureFlagId".ToLower()) + .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); + + Create.Index("IX_FeatureFlagPrerequisites_Flag".ToLower()) + .OnTable("FeatureFlagPrerequisites".ToLower()) + .OnColumn("FeatureFlagId".ToLower()).Ascending(); + } // FeatureFlagUsages - aggregated daily evaluation counts (append-only flushes). - Create.Table("FeatureFlagUsages".ToLower()) - .WithColumn("FeatureFlagUsageId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() - .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable() - .WithColumn("DepartmentId".ToLower()).AsInt32().Nullable() - .WithColumn("UsageDate".ToLower()).AsDateTime2().NotNullable() - .WithColumn("EvaluationCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0) - .WithColumn("EnabledCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0) - .WithColumn("DisabledCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0); - - Create.ForeignKey("FK_FeatureFlagUsages_FeatureFlags".ToLower()) - .FromTable("FeatureFlagUsages".ToLower()).ForeignColumn("FeatureFlagId".ToLower()) - .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); - - Create.ForeignKey("FK_FeatureFlagUsages_Departments".ToLower()) - .FromTable("FeatureFlagUsages".ToLower()).ForeignColumn("DepartmentId".ToLower()) - .ToTable("Departments".ToLower()).PrimaryColumn("DepartmentId".ToLower()); - - Create.Index("IX_FeatureFlagUsages_Flag_Date".ToLower()) - .OnTable("FeatureFlagUsages".ToLower()) - .OnColumn("FeatureFlagId".ToLower()).Ascending() - .OnColumn("UsageDate".ToLower()).Ascending(); + if (!Schema.Table("FeatureFlagUsages".ToLower()).Exists()) + { + Create.Table("FeatureFlagUsages".ToLower()) + .WithColumn("FeatureFlagUsageId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity() + .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable() + .WithColumn("DepartmentId".ToLower()).AsInt32().Nullable() + .WithColumn("UsageDate".ToLower()).AsDateTime2().NotNullable() + .WithColumn("EvaluationCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0) + .WithColumn("EnabledCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0) + .WithColumn("DisabledCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0); + + Create.ForeignKey("FK_FeatureFlagUsages_FeatureFlags".ToLower()) + .FromTable("FeatureFlagUsages".ToLower()).ForeignColumn("FeatureFlagId".ToLower()) + .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower()); + + Create.ForeignKey("FK_FeatureFlagUsages_Departments".ToLower()) + .FromTable("FeatureFlagUsages".ToLower()).ForeignColumn("DepartmentId".ToLower()) + .ToTable("Departments".ToLower()).PrimaryColumn("DepartmentId".ToLower()); + + Create.Index("IX_FeatureFlagUsages_Flag_Date".ToLower()) + .OnTable("FeatureFlagUsages".ToLower()) + .OnColumn("FeatureFlagId".ToLower()).Ascending() + .OnColumn("UsageDate".ToLower()).Ascending(); + } } public override void Down() diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0072_AddingChatbotTwilioFeatureFlagPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0072_AddingChatbotTwilioFeatureFlagPg.cs index c65515123..3279e6b01 100644 --- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0072_AddingChatbotTwilioFeatureFlagPg.cs +++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0072_AddingChatbotTwilioFeatureFlagPg.cs @@ -14,14 +14,15 @@ public override void Up() // handling until this flag is enabled globally or via a per-department override. flagtype, // isarchived, ispermanent and createdon fall back to their table defaults; the identity PK is // omitted so Postgres assigns it. - Insert.IntoTable("FeatureFlags".ToLower()).Row(new - { - flagkey = FlagKey, - name = "Chatbot Twilio Text Integration", - description = "When enabled, inbound Twilio SMS is processed by the new chatbot ingress pipeline. When off (globally or for a department) the original text-command handling is used.", - category = "Chatbot", - isenabledglobally = false - }); + // Guarded with WHERE NOT EXISTS so re-running the migration does not violate the unique + // flagkey index. + Execute.Sql( + "INSERT INTO featureflags (flagkey, name, description, category, isenabledglobally) " + + "SELECT '" + FlagKey + "', " + + "'Chatbot Twilio Text Integration', " + + "'When enabled, inbound Twilio SMS is processed by the new chatbot ingress pipeline. When off (globally or for a department) the original text-command handling is used.', " + + "'Chatbot', false " + + "WHERE NOT EXISTS (SELECT 1 FROM featureflags WHERE flagkey = '" + FlagKey + "');"); } public override void Down() diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSourcesPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSourcesPg.cs index 9ae4bf745..0d16a6b25 100644 --- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSourcesPg.cs +++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSourcesPg.cs @@ -7,8 +7,13 @@ public class M0074_AddIsPermanentFailureToWeatherAlertSourcesPg : Migration { public override void Up() { - Alter.Table("weatheralertsources") - .AddColumn("ispermanentfailure").AsBoolean().NotNullable().WithDefaultValue(false); + // Guarded: this migration was renumbered (68 -> 74), so databases upgraded before the + // renumber already added this column under version 68. Skip the add if it exists. + if (!Schema.Table("weatheralertsources").Column("ispermanentfailure").Exists()) + { + Alter.Table("weatheralertsources") + .AddColumn("ispermanentfailure").AsBoolean().NotNullable().WithDefaultValue(false); + } } public override void Down() diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0075_AddUtf8CleanupProgressPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0075_AddUtf8CleanupProgressPg.cs new file mode 100644 index 000000000..4fff46c7f --- /dev/null +++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0075_AddUtf8CleanupProgressPg.cs @@ -0,0 +1,31 @@ +using FluentMigrator; + +namespace Resgrid.Providers.MigrationsPg.Migrations +{ + [Migration(75)] + public class M0075_AddUtf8CleanupProgressPg : Migration + { + public override void Up() + { + // Per-table watermark for the nightly UTF-8 data cleanup worker so each run resumes + // where the previous one stopped while staying idempotent. + // Guarded so the migration is safe to re-run / safe on databases where a prior partial + // apply already created the table. + if (!Schema.Table("utf8cleanupprogress").Exists()) + { + Create.Table("utf8cleanupprogress") + .WithColumn("tablename").AsCustom("citext").NotNullable().PrimaryKey() + .WithColumn("lastprocessedkey").AsCustom("text").Nullable() + .WithColumn("lastcompletedutc").AsDateTime().Nullable() + .WithColumn("rowsscanned").AsInt64().NotNullable().WithDefaultValue(0) + .WithColumn("rowsfixed").AsInt64().NotNullable().WithDefaultValue(0) + .WithColumn("updatedonutc").AsDateTime().NotNullable(); + } + } + + public override void Down() + { + Delete.Table("utf8cleanupprogress"); + } + } +} diff --git a/Repositories/Resgrid.Repositories.DataRepository/Extensions/DynamicParametersExtension.cs b/Repositories/Resgrid.Repositories.DataRepository/Extensions/DynamicParametersExtension.cs index b5caaf299..dd82b286e 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/Extensions/DynamicParametersExtension.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/Extensions/DynamicParametersExtension.cs @@ -1,8 +1,10 @@ using Dapper; using Resgrid.Config; +using Resgrid.Framework; using Resgrid.Repositories.DataRepository.Extensions; using System; using System.Data; +using System.Text; namespace Resgrid.Repositories.DataRepository { @@ -19,6 +21,14 @@ public DynamicParametersExtension(object template) public void Add(string name, object? value = null, DbType? dbType = null, ParameterDirection? direction = null, int? size = null, byte? precision = null, byte? scale = null) { + // Guard the write path: scrub any string value so it is safe for a PostgreSQL UTF-8 + // column (strips NUL, repairs Windows-1252 mojibake, replaces unpaired surrogates). + // This is the central manual-parameter chokepoint used across all repositories. + if (value is string stringValue && SystemBehaviorConfig.SanitizeTextForUtf8) + value = Utf8Sanitizer.Clean(stringValue, + SystemBehaviorConfig.Utf8RepairDoubleEncoding, + SystemBehaviorConfig.Utf8NormalizeToNfc ? NormalizationForm.FormC : (NormalizationForm?)null); + if (DataConfig.DatabaseType == DatabaseTypes.Postgres) { name = name.ToLower(); diff --git a/Repositories/Resgrid.Repositories.DataRepository/IdentityRoleRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/IdentityRoleRepository.cs index 466565af1..f87c18979 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/IdentityRoleRepository.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/IdentityRoleRepository.cs @@ -157,6 +157,8 @@ public async Task InsertAsync(IdentityRole role, CancellationToken cancell { var insertFunction = new Func>(async x => { + Utf8WriteGuard.Sanitize(role); + var dynamicParameters = new DynamicParameters(role); var query = _queryFactory.GetInsertQuery(role); @@ -201,6 +203,8 @@ public async Task InsertClaimAsync(IdentityRole role, Claim claim, Cancell roleClaim.ClaimValue = claim.Value; roleClaim.RoleId = role.Id; + Utf8WriteGuard.Sanitize(roleClaim); + var dynamicParameters = new DynamicParameters(roleClaim); var query = _queryFactory.GetInsertQuery(roleClaim); @@ -322,6 +326,8 @@ public async Task UpdateAsync(IdentityRole role, CancellationToken cancell { var updateFunction = new Func>(async x => { + Utf8WriteGuard.Sanitize(role); + var dynamicParameters = new DynamicParameters(role); var query = _queryFactory.GetUpdateQuery(role); diff --git a/Repositories/Resgrid.Repositories.DataRepository/IdentityUserRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/IdentityUserRepository.cs index 3af54c35a..38169c4a3 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/IdentityUserRepository.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/IdentityUserRepository.cs @@ -234,6 +234,8 @@ public async Task InsertAsync(IdentityUser user, CancellationToken cance { try { + Utf8WriteGuard.Sanitize(user); + var dynamicParameters = new DynamicParameters(user); var query = _queryFactory.GetInsertQuery(user); diff --git a/Repositories/Resgrid.Repositories.DataRepository/Modules/ApiDataModule.cs b/Repositories/Resgrid.Repositories.DataRepository/Modules/ApiDataModule.cs index 361a1411a..4c770908c 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/Modules/ApiDataModule.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/Modules/ApiDataModule.cs @@ -205,6 +205,9 @@ protected override void Load(ContainerBuilder builder) // SSO Repositories builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); + + // UTF-8 / PostgreSQL-migration data maintenance + builder.RegisterType().As().InstancePerLifetimeScope(); } } } diff --git a/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs b/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs index 946e09df8..b3fce716a 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs @@ -230,6 +230,9 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); builder.RegisterType().As().InstancePerLifetimeScope(); + + // UTF-8 / PostgreSQL-migration data maintenance + builder.RegisterType().As().InstancePerLifetimeScope(); } } } diff --git a/Repositories/Resgrid.Repositories.DataRepository/RepositoryBase.cs b/Repositories/Resgrid.Repositories.DataRepository/RepositoryBase.cs index 624179588..ef44781b3 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/RepositoryBase.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/RepositoryBase.cs @@ -164,6 +164,8 @@ public virtual async Task InsertAsync(T entity, CancellationToken cancellatio { try { + Utf8WriteGuard.Sanitize(entity); + var insertFunction = new Func>(async x => { var dynamicParameters = new DynamicParameters(entity); @@ -214,10 +216,10 @@ public virtual async Task UpdateAsync(T entity, CancellationToken cancellatio { try { + Utf8WriteGuard.Sanitize(entity); + var updateFunction = new Func>(async x => { - - var dynamicParameters = new DynamicParameters(entity); var query = _queryFactory.GetUpdateQuery(entity); diff --git a/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs new file mode 100644 index 000000000..e602087fe --- /dev/null +++ b/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs @@ -0,0 +1,326 @@ +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Dapper; +using Resgrid.Config; +using Resgrid.Framework; +using Resgrid.Model.Repositories; +using Resgrid.Model.Repositories.Connection; + +namespace Resgrid.Repositories.DataRepository +{ + /// + /// Backend-aware (SQL Server / PostgreSQL) maintenance access for the nightly UTF-8 data cleanup + /// worker. Builds metadata and row queries dynamically from INFORMATION_SCHEMA so it covers every + /// text column without a hand-maintained list. + /// + public class Utf8MaintenanceRepository : IUtf8MaintenanceRepository + { + private const string ProgressTableName = "utf8cleanupprogress"; + + // Tables the sweep should never touch (its own bookkeeping + the migration history table). + private static readonly HashSet ExcludedTables = new HashSet(StringComparer.OrdinalIgnoreCase) + { + ProgressTableName, + "versioninfo" + }; + + private readonly IConnectionProvider _connectionProvider; + + public Utf8MaintenanceRepository(IConnectionProvider connectionProvider) + { + _connectionProvider = connectionProvider; + } + + private static bool IsPostgres => DataConfig.DatabaseType == DatabaseTypes.Postgres; + + private static string Quote(string identifier) + { + return IsPostgres ? "\"" + identifier + "\"" : "[" + identifier + "]"; + } + + private static string QualifiedTable(Utf8TextColumnTarget target) + { + return string.IsNullOrEmpty(target.Schema) + ? Quote(target.TableName) + : Quote(target.Schema) + "." + Quote(target.TableName); + } + + private static string ProgressTable => (IsPostgres ? "public." : "dbo.") + ProgressTableName; + + public async Task> GetTextColumnTargetsAsync(CancellationToken cancellationToken) + { + try + { + string columnsSql; + string pkSql; + + if (IsPostgres) + { + columnsSql = @" + SELECT c.table_schema AS TableSchema, c.table_name AS TableName, c.column_name AS ColumnName + FROM information_schema.columns c + INNER JOIN information_schema.tables t + ON t.table_schema = c.table_schema AND t.table_name = c.table_name AND t.table_type = 'BASE TABLE' + WHERE (c.data_type IN ('character', 'character varying', 'text') OR c.udt_name = 'citext') + AND c.table_schema NOT IN ('pg_catalog', 'information_schema')"; + + pkSql = @" + SELECT tc.table_schema AS TableSchema, tc.table_name AS TableName, kcu.column_name AS ColumnName + FROM information_schema.table_constraints tc + INNER JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + AND tc.table_name = kcu.table_name + WHERE tc.constraint_type = 'PRIMARY KEY' + AND tc.table_schema NOT IN ('pg_catalog', 'information_schema')"; + } + else + { + columnsSql = @" + SELECT c.TABLE_SCHEMA AS TableSchema, c.TABLE_NAME AS TableName, c.COLUMN_NAME AS ColumnName + FROM INFORMATION_SCHEMA.COLUMNS c + INNER JOIN INFORMATION_SCHEMA.TABLES t + ON t.TABLE_SCHEMA = c.TABLE_SCHEMA AND t.TABLE_NAME = c.TABLE_NAME AND t.TABLE_TYPE = 'BASE TABLE' + WHERE c.DATA_TYPE IN ('char', 'varchar', 'nchar', 'nvarchar', 'text', 'ntext')"; + + pkSql = @" + SELECT tc.TABLE_SCHEMA AS TableSchema, tc.TABLE_NAME AS TableName, kcu.COLUMN_NAME AS ColumnName + FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc + INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu + ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME + AND tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA + AND tc.TABLE_NAME = kcu.TABLE_NAME + WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY'"; + } + + using (DbConnection conn = _connectionProvider.Create()) + { + await conn.OpenAsync(cancellationToken); + + var columns = (await conn.QueryAsync(new CommandDefinition(columnsSql, cancellationToken: cancellationToken))).ToList(); + var pkRows = (await conn.QueryAsync(new CommandDefinition(pkSql, cancellationToken: cancellationToken))).ToList(); + + // Keep only tables with EXACTLY one primary-key column (needed for keyset paging). + var singlePk = pkRows + .GroupBy(p => p.TableSchema + "." + p.TableName) + .Where(g => g.Count() == 1) + .ToDictionary(g => g.Key, g => g.First().ColumnName, StringComparer.OrdinalIgnoreCase); + + var targets = new List(); + + foreach (var group in columns.GroupBy(c => c.TableSchema + "." + c.TableName)) + { + var first = group.First(); + + if (ExcludedTables.Contains(first.TableName)) + continue; + + if (!singlePk.TryGetValue(group.Key, out var pkColumn)) + continue; // no single-column PK -> cannot page safely + + // Never clean the primary-key column itself. + var textColumns = group + .Select(c => c.ColumnName) + .Where(name => !string.Equals(name, pkColumn, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (textColumns.Count == 0) + continue; + + targets.Add(new Utf8TextColumnTarget + { + Schema = first.TableSchema, + TableName = first.TableName, + PrimaryKeyColumn = pkColumn, + TextColumns = textColumns + }); + } + + return targets; + } + } + catch (Exception ex) + { + Logging.LogException(ex); + throw; + } + } + + public async Task GetRowBatchAsync(Utf8TextColumnTarget target, string lastKey, int batchSize, CancellationToken cancellationToken) + { + try + { + var table = QualifiedTable(target); + var pk = Quote(target.PrimaryKeyColumn); + var columnList = string.Join(", ", target.TextColumns.Select(Quote)); + var hasCursor = !string.IsNullOrEmpty(lastKey); + var whereClause = hasCursor ? "WHERE " + pk + " > @lastKey " : string.Empty; + + string sql = IsPostgres + ? $"SELECT {pk} AS rg_pk, {columnList} FROM {table} {whereClause}ORDER BY {pk} LIMIT @batchSize" + : $"SELECT TOP (@batchSize) {pk} AS rg_pk, {columnList} FROM {table} {whereClause}ORDER BY {pk}"; + + var dynamicParameters = new DynamicParameters(); + dynamicParameters.Add("batchSize", batchSize); + if (hasCursor) + dynamicParameters.Add("lastKey", lastKey); + + using (DbConnection conn = _connectionProvider.Create()) + { + await conn.OpenAsync(cancellationToken); + + var rows = await conn.QueryAsync(new CommandDefinition(sql, dynamicParameters, cancellationToken: cancellationToken)); + + var batch = new Utf8RowBatch(); + + foreach (var item in rows) + { + var dict = (IDictionary)item; + var key = Convert.ToString(dict["rg_pk"], CultureInfo.InvariantCulture); + + var row = new Utf8TextRow { Key = key }; + + foreach (var column in target.TextColumns) + { + dict.TryGetValue(column, out var value); + row.Columns[column] = value as string; + } + + batch.Rows.Add(row); + batch.LastKey = key; + } + + return batch; + } + } + catch (Exception ex) + { + Logging.LogException(ex, "Utf8 cleanup failed reading batch for " + target.Key); + throw; + } + } + + public async Task UpdateRowsAsync(Utf8TextColumnTarget target, IReadOnlyList rows, CancellationToken cancellationToken) + { + if (rows == null || rows.Count == 0) + return 0; + + try + { + var table = QualifiedTable(target); + var pk = Quote(target.PrimaryKeyColumn); + var updated = 0; + + using (DbConnection conn = _connectionProvider.Create()) + { + await conn.OpenAsync(cancellationToken); + + using (var transaction = await conn.BeginTransactionAsync(cancellationToken)) + { + foreach (var row in rows) + { + if (row.Columns.Count == 0) + continue; + + var setClauses = new List(row.Columns.Count); + var dynamicParameters = new DynamicParameters(); + var index = 0; + + foreach (var column in row.Columns) + { + var paramName = "c" + index++; + setClauses.Add(Quote(column.Key) + " = @" + paramName); + dynamicParameters.Add(paramName, (object)column.Value ?? DBNull.Value); + } + + dynamicParameters.Add("rg_key", row.Key); + + var sql = $"UPDATE {table} SET {string.Join(", ", setClauses)} WHERE {pk} = @rg_key"; + + updated += await conn.ExecuteAsync(new CommandDefinition(sql, dynamicParameters, transaction, cancellationToken: cancellationToken)); + } + + await transaction.CommitAsync(cancellationToken); + } + } + + return updated; + } + catch (Exception ex) + { + Logging.LogException(ex, "Utf8 cleanup failed updating rows for " + target.Key); + throw; + } + } + + public async Task GetProgressAsync(string tableKey, CancellationToken cancellationToken) + { + try + { + var sql = $@" + SELECT tablename AS TableName, lastprocessedkey AS LastProcessedKey, lastcompletedutc AS LastCompletedUtc, + rowsscanned AS RowsScanned, rowsfixed AS RowsFixed, updatedonutc AS UpdatedOnUtc + FROM {ProgressTable} + WHERE tablename = @tablename"; + + var dynamicParameters = new DynamicParameters(); + dynamicParameters.Add("tablename", tableKey); + + using (DbConnection conn = _connectionProvider.Create()) + { + await conn.OpenAsync(cancellationToken); + + return await conn.QueryFirstOrDefaultAsync(new CommandDefinition(sql, dynamicParameters, cancellationToken: cancellationToken)); + } + } + catch (Exception ex) + { + Logging.LogException(ex); + throw; + } + } + + public async Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationToken cancellationToken) + { + try + { + var updateSql = $@" + UPDATE {ProgressTable} + SET lastprocessedkey = @LastProcessedKey, lastcompletedutc = @LastCompletedUtc, + rowsscanned = @RowsScanned, rowsfixed = @RowsFixed, updatedonutc = @UpdatedOnUtc + WHERE tablename = @TableName"; + + var insertSql = $@" + INSERT INTO {ProgressTable} (tablename, lastprocessedkey, lastcompletedutc, rowsscanned, rowsfixed, updatedonutc) + VALUES (@TableName, @LastProcessedKey, @LastCompletedUtc, @RowsScanned, @RowsFixed, @UpdatedOnUtc)"; + + using (DbConnection conn = _connectionProvider.Create()) + { + await conn.OpenAsync(cancellationToken); + + var affected = await conn.ExecuteAsync(new CommandDefinition(updateSql, progress, cancellationToken: cancellationToken)); + + if (affected == 0) + await conn.ExecuteAsync(new CommandDefinition(insertSql, progress, cancellationToken: cancellationToken)); + } + } + catch (Exception ex) + { + Logging.LogException(ex); + throw; + } + } + + private class MetaColumn + { + public string TableSchema { get; set; } + public string TableName { get; set; } + public string ColumnName { get; set; } + } + } +} diff --git a/Repositories/Resgrid.Repositories.DataRepository/Utf8WriteGuard.cs b/Repositories/Resgrid.Repositories.DataRepository/Utf8WriteGuard.cs new file mode 100644 index 000000000..7371480ff --- /dev/null +++ b/Repositories/Resgrid.Repositories.DataRepository/Utf8WriteGuard.cs @@ -0,0 +1,28 @@ +using System.Text; +using Resgrid.Config; +using Resgrid.Framework; + +namespace Resgrid.Repositories.DataRepository +{ + /// + /// Central, config-gated entry point for scrubbing entity string values before they are bound + /// to a SQL command via Dapper's DynamicParameters(entity) (which bypasses + /// ). Keeps the UTF-8 guard logic in one place. + /// + internal static class Utf8WriteGuard + { + /// + /// In-place sanitizes the string properties of when UTF-8 guarding + /// is enabled. No-op when disabled or when the entity is null. + /// + public static void Sanitize(object entity) + { + if (entity == null || !SystemBehaviorConfig.SanitizeTextForUtf8) + return; + + Utf8Sanitizer.CleanEntity(entity, + SystemBehaviorConfig.Utf8RepairDoubleEncoding, + SystemBehaviorConfig.Utf8NormalizeToNfc ? NormalizationForm.FormC : (NormalizationForm?)null); + } + } +} diff --git a/Tests/Resgrid.Tests/Framework/Utf8SanitizerTests.cs b/Tests/Resgrid.Tests/Framework/Utf8SanitizerTests.cs new file mode 100644 index 000000000..1acc3c199 --- /dev/null +++ b/Tests/Resgrid.Tests/Framework/Utf8SanitizerTests.cs @@ -0,0 +1,206 @@ +using System.Text; +using FluentAssertions; +using NUnit.Framework; +using Resgrid.Framework; + +namespace Resgrid.Tests.Framework +{ + /// + /// Verifies produces content that is always safe for a PostgreSQL + /// UTF-8 column: no NUL, no unpaired surrogates, with best-effort Windows-1252 mojibake repair, + /// while leaving already-clean text untouched (and allocation-free). + /// + /// Non-ASCII characters are built from explicit (char)0xNN code points so the byte-level intent + /// is exact and unambiguous. + /// + [TestFixture] + public class Utf8SanitizerTests + { + private const char Win1252RightQuote = (char)0x92; // repairs to U+2019 + private const char Win1252EnDash = (char)0x96; // repairs to U+2013 + private const char Undefined1252 = (char)0x81; // dropped (no Windows-1252 mapping) + private const char RightSingleQuote = (char)0x2019; + private const char EnDash = (char)0x2013; + private const char HighSurrogate = (char)0xD83D; // lead of U+1F600 + private const char LowSurrogate = (char)0xDE00; // trail of U+1F600 + private const char CombiningAcute = (char)0x0301; + private const char EAcute = (char)0x00E9; // precomposed "é" + private const char UDiaeresis = (char)0x00FC; // "ü" + private const char CapitalATilde = (char)0x00C3; // "Ã" + private const char CopyrightSign = (char)0x00A9; // "©" (second byte of double-encoded é) + + [Test] + public void Clean_Null_ReturnsNull() + { + Utf8Sanitizer.Clean(null).Should().BeNull(); + } + + [Test] + public void Clean_Empty_ReturnsEmpty() + { + Utf8Sanitizer.Clean(string.Empty).Should().BeEmpty(); + } + + [Test] + public void Clean_AlreadyCleanString_ReturnsSameReference() + { + // ASCII + Latin-1 accented chars (above the C1 range) + a valid emoji surrogate pair. + var input = "Engine 51 responding " + EAcute + UDiaeresis + " " + HighSurrogate + LowSurrogate; + + var result = Utf8Sanitizer.Clean(input); + + result.Should().BeSameAs(input); + Utf8Sanitizer.IsClean(input).Should().BeTrue(); + } + + [Test] + public void Clean_StripsNulCharacters() + { + var input = "a\0b\0c"; + + var result = Utf8Sanitizer.Clean(input); + + result.Should().Be("abc"); + result.Should().NotContain("\0"); + } + + [Test] + public void Clean_ReplacesUnpairedHighSurrogate() + { + var input = "before" + HighSurrogate + "after"; // high surrogate with no trailing low surrogate + + var result = Utf8Sanitizer.Clean(input); + + result.Should().Be("before" + Utf8Sanitizer.ReplacementChar + "after"); + } + + [Test] + public void Clean_ReplacesLoneLowSurrogate() + { + var input = "x" + LowSurrogate + "y"; // low surrogate with no preceding high surrogate + + var result = Utf8Sanitizer.Clean(input); + + result.Should().Be("x" + Utf8Sanitizer.ReplacementChar + "y"); + } + + [Test] + public void Clean_PreservesValidSurrogatePair() + { + var input = "grin " + HighSurrogate + LowSurrogate + " done"; // U+1F600 + + var result = Utf8Sanitizer.Clean(input); + + result.Should().BeSameAs(input); + } + + [Test] + public void Clean_RepairsWindows1252SmartQuote() + { + var input = "It" + Win1252RightQuote + "s here"; + + var result = Utf8Sanitizer.Clean(input); + + result.Should().Be("It" + RightSingleQuote + "s here"); + } + + [Test] + public void Clean_RepairsWindows1252EnDash() + { + var input = "9" + Win1252EnDash + "10"; + + Utf8Sanitizer.Clean(input).Should().Be("9" + EnDash + "10"); + } + + [Test] + public void Clean_DropsUndefinedWindows1252Bytes() + { + var input = "a" + Undefined1252 + "b"; + + Utf8Sanitizer.Clean(input).Should().Be("ab"); + } + + [Test] + public void TryClean_ReturnsFalseForCleanInput() + { + var changed = Utf8Sanitizer.TryClean("plain text", out var cleaned); + + changed.Should().BeFalse(); + cleaned.Should().Be("plain text"); + } + + [Test] + public void TryClean_ReturnsTrueWhenChanged() + { + var changed = Utf8Sanitizer.TryClean("bad\0value", out var cleaned); + + changed.Should().BeTrue(); + cleaned.Should().Be("badvalue"); + } + + [Test] + public void Clean_Output_IsAlwaysValidUtf8WithoutNul() + { + // NUL + lone high surrogate + C1 mojibake + lone low surrogate. + var input = "mix\0" + HighSurrogate + " C1" + Win1252EnDash + " " + LowSurrogate + "tail"; + + var result = Utf8Sanitizer.Clean(input); + + result.Should().NotContain("\0"); + + var strict = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + var act = () => strict.GetByteCount(result); + act.Should().NotThrow(); + } + + [Test] + public void Clean_AppliesNfcNormalizationWhenRequested() + { + // "e" + combining acute accent; NFC composes to precomposed "é" (U+00E9). + var input = "e" + CombiningAcute; + + var withoutNorm = Utf8Sanitizer.Clean(input); + var withNorm = Utf8Sanitizer.Clean(input, repairDoubleEncoding: false, normalization: NormalizationForm.FormC); + + withoutNorm.Should().Be(input); + withNorm.Should().Be(string.Empty + EAcute); + } + + [Test] + public void Clean_DoubleEncodingRepair_OptIn() + { + // CapitalATilde + CopyrightSign is the double-encoded (Latin-1-read) form of "é" (U+00E9). + var input = "caf" + CapitalATilde + CopyrightSign; + + Utf8Sanitizer.Clean(input).Should().Be(input); // off by default + Utf8Sanitizer.Clean(input, repairDoubleEncoding: true).Should().Be("caf" + EAcute); + } + + [Test] + public void CleanEntity_ScrubsStringProperties() + { + var entity = new SampleEntity + { + Name = "Unit\0 1", + Notes = "Quote" + Win1252RightQuote + "s", + Ignored = null, + Number = 42 + }; + + Utf8Sanitizer.CleanEntity(entity); + + entity.Name.Should().Be("Unit 1"); + entity.Notes.Should().Be("Quote" + RightSingleQuote + "s"); + entity.Ignored.Should().BeNull(); + entity.Number.Should().Be(42); + } + + private class SampleEntity + { + public string Name { get; set; } + public string Notes { get; set; } + public string Ignored { get; set; } + public int Number { get; set; } + } + } +} diff --git a/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs b/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs new file mode 100644 index 000000000..dc571db6a --- /dev/null +++ b/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs @@ -0,0 +1,49 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Consolas2.Core; +using Microsoft.Extensions.Logging; +using Resgrid.Console.Models; +using Resgrid.Model.Repositories; +using Resgrid.Workers.Framework.Logic; + +namespace Resgrid.Console.Commands +{ + /// + /// On-demand UTF-8 data cleanup. Runs the same sweep as the nightly worker so the database can + /// be made migration-clean ad hoc (e.g. immediately before a SQL Server -> PostgreSQL cutover). + /// + public sealed class CleanUtf8Command( + ILogger logger, + IUtf8MaintenanceRepository utf8MaintenanceRepository) : ICommandService + { + public async Task ExecuteMainAsync(string[] args, CancellationToken cancellationToken) + { + logger.LogInformation("Starting the Resgrid UTF-8 Data Cleanup (pre-migration sweep)"); + logger.LogInformation("Please Wait..."); + + try + { + var logic = new Utf8CleanupLogic(utf8MaintenanceRepository); + var result = await logic.Process(cancellationToken); + + if (!result.Item1) + { + logger.LogError("UTF-8 data cleanup did not complete successfully: " + result.Item2); + return ExitCode.Failed; + } + + logger.LogInformation(result.Item2); + logger.LogInformation("Completed the Resgrid UTF-8 Data Cleanup!"); + } + catch (Exception ex) + { + logger.LogError("There was an error running the UTF-8 data cleanup, see the error output below:"); + logger.LogError(ex.ToString()); + return ExitCode.Failed; + } + + return ExitCode.Success; + } + } +} diff --git a/Tools/Resgrid.Console/Program.cs b/Tools/Resgrid.Console/Program.cs index 071ec32cc..f75fe44c9 100644 --- a/Tools/Resgrid.Console/Program.cs +++ b/Tools/Resgrid.Console/Program.cs @@ -107,6 +107,7 @@ static async Task Main(string[] args) services.AddKeyedTransient("DbUpdateCommand"); services.AddKeyedTransient("GenOidcCertsCommand"); services.AddKeyedTransient("MigrateDocsDbCommand"); + services.AddKeyedTransient("CleanUtf8Command"); services.AddKeyedTransient("OidcUpdateCommand"); services.AddKeyedTransient("SecurityRefreshCommand"); services.AddKeyedTransient("HelpCommand"); diff --git a/Tools/Resgrid.Console/Services/ApplicationHostedService.cs b/Tools/Resgrid.Console/Services/ApplicationHostedService.cs index a0b003684..7239c2fc3 100644 --- a/Tools/Resgrid.Console/Services/ApplicationHostedService.cs +++ b/Tools/Resgrid.Console/Services/ApplicationHostedService.cs @@ -30,6 +30,7 @@ public sealed class ApplicationHostedService : IHostedService, IDisposable private ICommandService _dbUpdateCommand; private ICommandService _genOidcCertsCommand; private ICommandService _migrateDocsDbCommand; + private ICommandService _cleanUtf8Command; private ICommandService _oidcUpdateCommand; private ICommandService _securityRefreshCommand; private ICommandService _helpCommand; @@ -55,6 +56,7 @@ public ApplicationHostedService( [FromKeyedServices("DbUpdateCommand")] ICommandService dbUpdateCommand, [FromKeyedServices("GenOidcCertsCommand")] ICommandService genOidcCertsCommand, [FromKeyedServices("MigrateDocsDbCommand")] ICommandService migrateDocsDbCommand, + [FromKeyedServices("CleanUtf8Command")] ICommandService cleanUtf8Command, [FromKeyedServices("OidcUpdateCommand")] ICommandService oidcUpdateCommand, [FromKeyedServices("SecurityRefreshCommand")] ICommandService securityRefreshCommand, [FromKeyedServices("HelpCommand")] ICommandService helpCommand) @@ -67,6 +69,7 @@ public ApplicationHostedService( _dbUpdateCommand = dbUpdateCommand; _genOidcCertsCommand = genOidcCertsCommand; _migrateDocsDbCommand = migrateDocsDbCommand; + _cleanUtf8Command = cleanUtf8Command; _oidcUpdateCommand = oidcUpdateCommand; _securityRefreshCommand = securityRefreshCommand; _helpCommand = helpCommand; @@ -192,6 +195,8 @@ private async Task ExecuteMainAsync(string[] args, CancellationToken c return await _genOidcCertsCommand.ExecuteMainAsync(args, cancellationToken).ConfigureAwait(false); else if (args.Contains("--MigrateDocsDb")) return await _migrateDocsDbCommand.ExecuteMainAsync(args, cancellationToken).ConfigureAwait(false); + else if (args.Contains("--CleanUtf8")) + return await _cleanUtf8Command.ExecuteMainAsync(args, cancellationToken).ConfigureAwait(false); else if (args.Contains("--OidcUpdate")) return await _oidcUpdateCommand.ExecuteMainAsync(args, cancellationToken).ConfigureAwait(false); else if (args.Contains("--SecurityRefresh")) diff --git a/Workers/Resgrid.Workers.Console/Commands/Utf8CleanupCommand.cs b/Workers/Resgrid.Workers.Console/Commands/Utf8CleanupCommand.cs new file mode 100644 index 000000000..3a43a4d90 --- /dev/null +++ b/Workers/Resgrid.Workers.Console/Commands/Utf8CleanupCommand.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using Quidjibo.Commands; + +namespace Resgrid.Workers.Console.Commands +{ + public class Utf8CleanupCommand : IQuidjiboCommand + { + public int Id { get; } + public Guid? CorrelationId { get; set; } + public Dictionary Metadata { get; set; } + + public Utf8CleanupCommand(int id) + { + Id = id; + } + } +} diff --git a/Workers/Resgrid.Workers.Console/Program.cs b/Workers/Resgrid.Workers.Console/Program.cs index e5088e834..f319a208e 100644 --- a/Workers/Resgrid.Workers.Console/Program.cs +++ b/Workers/Resgrid.Workers.Console/Program.cs @@ -420,6 +420,19 @@ await Client.ScheduleAsync("Reporting Rollup", new Commands.ReportingRollupCommand(21), Cron.Daily(3, 30), stoppingToken); + + if (SystemBehaviorConfig.Utf8CleanupEnabled) + { + var utf8CleanupHour = SystemBehaviorConfig.Utf8CleanupHourUtc >= 0 && SystemBehaviorConfig.Utf8CleanupHourUtc <= 23 + ? SystemBehaviorConfig.Utf8CleanupHourUtc + : 4; + + _logger.Log(LogLevel.Information, "Scheduling UTF-8 Data Cleanup"); + await Client.ScheduleAsync("UTF-8 Data Cleanup", + new Commands.Utf8CleanupCommand(22), + Cron.Daily(utf8CleanupHour, 0), + stoppingToken); + } } else { diff --git a/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs b/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs new file mode 100644 index 000000000..9868cba08 --- /dev/null +++ b/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs @@ -0,0 +1,46 @@ +using Microsoft.Extensions.Logging; +using Quidjibo.Handlers; +using Quidjibo.Misc; +using Resgrid.Workers.Console.Commands; +using Resgrid.Workers.Framework.Logic; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Resgrid.Workers.Console.Tasks +{ + public class Utf8CleanupTask : IQuidjiboHandler + { + public string Name => "UTF-8 Data Cleanup"; + public int Priority => 1; + public ILogger _logger; + + public Utf8CleanupTask(ILogger logger) + { + _logger = logger; + } + + public async Task ProcessAsync(Utf8CleanupCommand command, IQuidjiboProgress progress, CancellationToken cancellationToken) + { + try + { + progress.Report(1, $"Starting the {Name} Task"); + + var logic = new Utf8CleanupLogic(); + var result = await logic.Process(cancellationToken); + + if (result.Item1) + _logger.LogInformation("Utf8Cleanup::" + result.Item2); + else + _logger.LogError("Utf8Cleanup::Failed: " + result.Item2); + + progress.Report(100, $"Finishing the {Name} Task"); + } + catch (Exception ex) + { + Resgrid.Framework.Logging.LogException(ex); + _logger.LogError(ex.ToString()); + } + } + } +} diff --git a/Workers/Resgrid.Workers.Framework/Logic/Utf8CleanupLogic.cs b/Workers/Resgrid.Workers.Framework/Logic/Utf8CleanupLogic.cs new file mode 100644 index 000000000..4b2f144a7 --- /dev/null +++ b/Workers/Resgrid.Workers.Framework/Logic/Utf8CleanupLogic.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using Resgrid.Config; +using Resgrid.Framework; +using Resgrid.Model.Repositories; + +namespace Resgrid.Workers.Framework.Logic +{ + /// + /// Sweeps every free-form text column in the database and repairs content that would block a + /// SQL Server -> PostgreSQL (UTF-8) migration (NUL, unpaired surrogates, Windows-1252 mojibake). + /// Resumable via a per-table watermark, idempotent (clean values are no-ops), and bounded by + /// configuration so a single run never scans an unbounded number of rows. + /// + public class Utf8CleanupLogic + { + private readonly IUtf8MaintenanceRepository _repository; + + public Utf8CleanupLogic() + : this(Bootstrapper.GetKernel().Resolve()) + { + } + + public Utf8CleanupLogic(IUtf8MaintenanceRepository repository) + { + _repository = repository; + } + + public async Task> Process(CancellationToken cancellationToken) + { + if (!SystemBehaviorConfig.Utf8CleanupEnabled) + return new Tuple(true, "UTF-8 cleanup is disabled by configuration."); + + try + { + var batchSize = SystemBehaviorConfig.Utf8CleanupBatchSize > 0 ? SystemBehaviorConfig.Utf8CleanupBatchSize : 1000; + var maxRows = SystemBehaviorConfig.Utf8CleanupMaxRowsPerRun; // 0 == unbounded + var repairDoubleEncoding = SystemBehaviorConfig.Utf8RepairDoubleEncoding; + NormalizationForm? normalization = SystemBehaviorConfig.Utf8NormalizeToNfc ? NormalizationForm.FormC : (NormalizationForm?)null; + + var targets = await _repository.GetTextColumnTargetsAsync(cancellationToken); + + long totalScanned = 0; + long totalFixed = 0; + + foreach (var target in targets) + { + if (cancellationToken.IsCancellationRequested) + break; + + if (maxRows > 0 && totalScanned >= maxRows) + break; + + var progress = await _repository.GetProgressAsync(target.Key, cancellationToken) + ?? new Utf8CleanupProgress { TableName = target.Key }; + + var lastKey = progress.LastProcessedKey; + + while (true) + { + if (cancellationToken.IsCancellationRequested) + break; + + if (maxRows > 0 && totalScanned >= maxRows) + break; + + var batch = await _repository.GetRowBatchAsync(target, lastKey, batchSize, cancellationToken); + + if (batch.Rows.Count == 0) + { + MarkTableComplete(progress); + await _repository.SaveProgressAsync(progress, cancellationToken); + break; + } + + var changedRows = new List(); + + foreach (var row in batch.Rows) + { + Utf8TextRow changed = null; + + foreach (var column in row.Columns) + { + if (column.Value == null) + continue; + + if (Utf8Sanitizer.TryClean(column.Value, out var cleaned, repairDoubleEncoding, normalization)) + { + if (changed == null) + changed = new Utf8TextRow { Key = row.Key }; + + changed.Columns[column.Key] = cleaned; + } + } + + if (changed != null) + changedRows.Add(changed); + } + + if (changedRows.Count > 0) + { + var updated = await _repository.UpdateRowsAsync(target, changedRows, cancellationToken); + totalFixed += updated; + progress.RowsFixed += updated; + } + + totalScanned += batch.Rows.Count; + progress.RowsScanned += batch.Rows.Count; + lastKey = batch.LastKey; + progress.LastProcessedKey = lastKey; + progress.UpdatedOnUtc = DateTime.UtcNow; + await _repository.SaveProgressAsync(progress, cancellationToken); + + if (batch.Rows.Count < batchSize) + { + MarkTableComplete(progress); + await _repository.SaveProgressAsync(progress, cancellationToken); + break; + } + } + } + + var summary = $"UTF-8 cleanup scanned {totalScanned} row(s) and repaired {totalFixed} across {targets.Count} table(s)."; + Logging.LogInfo(summary); + + return new Tuple(true, summary); + } + catch (Exception ex) + { + Logging.LogException(ex); + return new Tuple(false, ex.ToString()); + } + } + + private static void MarkTableComplete(Utf8CleanupProgress progress) + { + // Reset the cursor so the next scheduled run re-sweeps the table from the start + // (catching anything written since this pass) while staying idempotent. + progress.LastProcessedKey = null; + progress.LastCompletedUtc = DateTime.UtcNow; + progress.UpdatedOnUtc = DateTime.UtcNow; + } + } +} From a52f9f2189a4cd725b9feca7ff26808acc19bfb5 Mon Sep 17 00:00:00 2001 From: Shawn Jackson Date: Tue, 9 Jun 2026 17:54:07 -0500 Subject: [PATCH 2/4] RE1-T117 PR#403 fixes --- .../IUtf8MaintenanceRepository.cs | 30 +++- Core/Resgrid.Services/UsersService.cs | 6 + .../IdentityRepository.cs | 16 +- .../Utf8MaintenanceRepository.cs | 140 +++++++++++++++--- .../Commands/CleanUtf8Command.cs | 3 +- .../Tasks/Utf8CleanupTask.cs | 2 +- 6 files changed, 168 insertions(+), 29 deletions(-) diff --git a/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs b/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs index 5e7497bb0..dd9f656da 100644 --- a/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs +++ b/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs @@ -15,13 +15,17 @@ public interface IUtf8MaintenanceRepository { /// /// Returns every base-table text column grouped by table, restricted to tables that have a - /// single-column primary key (required for stable keyset pagination). + /// single-column primary key of a pageable type — text/citext, an integer type, or uuid + /// (required for stable keyset pagination). Tables whose PK is some other type are skipped. /// Task> GetTextColumnTargetsAsync(CancellationToken cancellationToken); /// /// Reads the next batch of rows for ordered by primary key, starting - /// after (null/empty for the first page). + /// after (null/empty for the first page). The cursor is the previous + /// page's last key rendered as an invariant string; the implementation re-binds it to the PK's + /// native type (see ) so the keyset comparison + /// works for text, integer and uuid keys alike. /// Task GetRowBatchAsync(Utf8TextColumnTarget target, string lastKey, int batchSize, CancellationToken cancellationToken); @@ -38,12 +42,33 @@ public interface IUtf8MaintenanceRepository Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationToken cancellationToken); } + /// + /// The pageable category of a table's primary key. Keyset pagination renders the cursor as an + /// invariant string and re-binds it as this type for the > / = predicates, so only + /// these PK types are swept; tables with any other PK type are skipped. + /// + public enum Utf8PrimaryKeyType + { + /// char/varchar/nchar/nvarchar/text/ntext and PostgreSQL citext — bound as a string. + Text = 0, + + /// tinyint/smallint/int/bigint (and PostgreSQL integer types) — bound as Int64. + Integer = 1, + + /// SQL Server uniqueidentifier / PostgreSQL uuid — bound as a Guid. + Guid = 2 + } + /// A table plus its single-column primary key and the free-form text columns to scan. public class Utf8TextColumnTarget { public string Schema { get; set; } public string TableName { get; set; } public string PrimaryKeyColumn { get; set; } + + /// The PK's value category, so keyset cursors are bound as the PK's native type. + public Utf8PrimaryKeyType PrimaryKeyType { get; set; } + public List TextColumns { get; set; } = new List(); /// Stable identifier used as the watermark key (schema-qualified table name). @@ -64,6 +89,7 @@ public class Utf8RowBatch /// public class Utf8TextRow { + /// The row's primary key rendered as an invariant string (re-bound to its native type for queries). public string Key { get; set; } public Dictionary Columns { get; set; } = new Dictionary(); } diff --git a/Core/Resgrid.Services/UsersService.cs b/Core/Resgrid.Services/UsersService.cs index 10d051266..5b7fac5c3 100644 --- a/Core/Resgrid.Services/UsersService.cs +++ b/Core/Resgrid.Services/UsersService.cs @@ -98,6 +98,12 @@ public IdentityUser GetUserByEmail(string emailAddress) public async Task DoesUserHaveAnyActiveDepartments(string userName) { var user = await GetUserByNameAsync(userName); + + // Defensive: the user has already authenticated by this point, but if the lookup returns + // null (e.g. a stale/edge data state) treat it as no active departments rather than NRE. + if (user == null) + return false; + var memberships = await _departmentMembersRepository.GetAllByUserIdAsync(user.UserId); return memberships.Any(x => x.IsDeleted == false && (x.IsDisabled == null || x.IsDisabled == false)); diff --git a/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs index 37a250918..9de3f893f 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs @@ -63,7 +63,10 @@ public async Task GetUserByUserNameAsync(string userName) { using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString)) { - var result = await db.QueryAsync($"SELECT * FROM aspnetusers WHERE username = @userName", new { userName = userName }); + // PostgreSQL text comparison is case-sensitive (unlike SQL Server's default collation), + // so match on normalizedusername the same way ASP.NET Identity sign-in does. Otherwise a + // user who authenticated with different casing than the stored username is not found here. + var result = await db.QueryAsync($"SELECT * FROM aspnetusers WHERE normalizedusername = @normalizedUserName", new { normalizedUserName = userName?.ToUpperInvariant() }); return result.FirstOrDefault(); } @@ -87,7 +90,9 @@ public IdentityUser GetUserByEmail(string email) { using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString)) { - return db.Query($"SELECT * FROM aspnetusers WHERE email = @email", new { email = email }).FirstOrDefault(); + // PostgreSQL text comparison is case-sensitive; LOWER() both sides so a different-cased email + // still matches (normalizedemail is not maintained on the PG update path, so it can't be relied on). + return db.Query($"SELECT * FROM aspnetusers WHERE LOWER(email) = LOWER(@email)", new { email = email }).FirstOrDefault(); } } else @@ -107,7 +112,8 @@ public void UpdateUsername(string oldUsername, string newUsername) { using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString)) { - db.Execute($"UPDATE public.aspnetusers SET username = @newUsername, normalizedusername = @newUsernameUpper WHERE username = @oldUsername", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpper(), oldUsername = oldUsername }); + // Case-insensitive match on the old username (PostgreSQL text comparison is case-sensitive). + db.Execute($"UPDATE public.aspnetusers SET username = @newUsername, normalizedusername = @newUsernameUpper WHERE LOWER(username) = LOWER(@oldUsername)", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpper(), oldUsername = oldUsername }); } } else @@ -125,7 +131,9 @@ public void UpdateEmail(string userId, string newEmail) { using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString)) { - db.Execute($"UPDATE public.aspnetusers SET email = @newEmail WHERE id = @userId", new { userId = userId, newEmail = newEmail }); + // Keep normalizedemail in sync (ASP.NET Identity's FindByEmailAsync looks up by it); the + // SQL Server branch already does this. Without it, email lookups go stale after a change. + db.Execute($"UPDATE public.aspnetusers SET email = @newEmail, normalizedemail = @newEmailUpper WHERE id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail?.ToUpper() }); } } else diff --git a/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs index e602087fe..0ce0aeb4b 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs @@ -70,12 +70,17 @@ INNER JOIN information_schema.tables t AND c.table_schema NOT IN ('pg_catalog', 'information_schema')"; pkSql = @" - SELECT tc.table_schema AS TableSchema, tc.table_name AS TableName, kcu.column_name AS ColumnName + SELECT tc.table_schema AS TableSchema, tc.table_name AS TableName, kcu.column_name AS ColumnName, + col.data_type AS DataType, col.udt_name AS UdtName FROM information_schema.table_constraints tc INNER JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema AND tc.table_name = kcu.table_name + INNER JOIN information_schema.columns col + ON col.table_schema = kcu.table_schema + AND col.table_name = kcu.table_name + AND col.column_name = kcu.column_name WHERE tc.constraint_type = 'PRIMARY KEY' AND tc.table_schema NOT IN ('pg_catalog', 'information_schema')"; } @@ -89,12 +94,17 @@ INNER JOIN INFORMATION_SCHEMA.TABLES t WHERE c.DATA_TYPE IN ('char', 'varchar', 'nchar', 'nvarchar', 'text', 'ntext')"; pkSql = @" - SELECT tc.TABLE_SCHEMA AS TableSchema, tc.TABLE_NAME AS TableName, kcu.COLUMN_NAME AS ColumnName + SELECT tc.TABLE_SCHEMA AS TableSchema, tc.TABLE_NAME AS TableName, kcu.COLUMN_NAME AS ColumnName, + col.DATA_TYPE AS DataType FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME AND tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA AND tc.TABLE_NAME = kcu.TABLE_NAME + INNER JOIN INFORMATION_SCHEMA.COLUMNS col + ON col.TABLE_SCHEMA = kcu.TABLE_SCHEMA + AND col.TABLE_NAME = kcu.TABLE_NAME + AND col.COLUMN_NAME = kcu.COLUMN_NAME WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY'"; } @@ -109,7 +119,7 @@ INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu var singlePk = pkRows .GroupBy(p => p.TableSchema + "." + p.TableName) .Where(g => g.Count() == 1) - .ToDictionary(g => g.Key, g => g.First().ColumnName, StringComparer.OrdinalIgnoreCase); + .ToDictionary(g => g.Key, g => g.First(), StringComparer.OrdinalIgnoreCase); var targets = new List(); @@ -120,13 +130,20 @@ INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu if (ExcludedTables.Contains(first.TableName)) continue; - if (!singlePk.TryGetValue(group.Key, out var pkColumn)) + if (!singlePk.TryGetValue(group.Key, out var pk)) continue; // no single-column PK -> cannot page safely + // The keyset cursor is a string re-bound to the PK's native type; only text, + // integer and uuid PKs are supported. Skip anything else (e.g. date/decimal PKs) + // so we never emit a query that compares an incompatible type against the cursor. + var keyType = ClassifyKeyType(pk); + if (keyType == null) + continue; + // Never clean the primary-key column itself. var textColumns = group .Select(c => c.ColumnName) - .Where(name => !string.Equals(name, pkColumn, StringComparison.OrdinalIgnoreCase)) + .Where(name => !string.Equals(name, pk.ColumnName, StringComparison.OrdinalIgnoreCase)) .ToList(); if (textColumns.Count == 0) @@ -136,7 +153,8 @@ INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu { Schema = first.TableSchema, TableName = first.TableName, - PrimaryKeyColumn = pkColumn, + PrimaryKeyColumn = pk.ColumnName, + PrimaryKeyType = keyType.Value, TextColumns = textColumns }); } @@ -168,7 +186,7 @@ public async Task GetRowBatchAsync(Utf8TextColumnTarget target, st var dynamicParameters = new DynamicParameters(); dynamicParameters.Add("batchSize", batchSize); if (hasCursor) - dynamicParameters.Add("lastKey", lastKey); + dynamicParameters.Add("lastKey", ConvertKey(lastKey, target.PrimaryKeyType)); using (DbConnection conn = _connectionProvider.Create()) { @@ -238,7 +256,7 @@ public async Task UpdateRowsAsync(Utf8TextColumnTarget target, IReadOnlyLis dynamicParameters.Add(paramName, (object)column.Value ?? DBNull.Value); } - dynamicParameters.Add("rg_key", row.Key); + dynamicParameters.Add("rg_key", ConvertKey(row.Key, target.PrimaryKeyType)); var sql = $"UPDATE {table} SET {string.Join(", ", setClauses)} WHERE {pk} = @rg_key"; @@ -289,24 +307,45 @@ public async Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationTo { try { - var updateSql = $@" - UPDATE {ProgressTable} - SET lastprocessedkey = @LastProcessedKey, lastcompletedutc = @LastCompletedUtc, - rowsscanned = @RowsScanned, rowsfixed = @RowsFixed, updatedonutc = @UpdatedOnUtc - WHERE tablename = @TableName"; + // Atomic upsert: overlapping cleanup runs (long sweeps or multiple workers) can save the + // first watermark for the same table concurrently, which the old UPDATE-then-INSERT raced + // on (duplicate-PK on the second INSERT). The tablename PK makes both forms below race-safe. + string upsertSql; - var insertSql = $@" - INSERT INTO {ProgressTable} (tablename, lastprocessedkey, lastcompletedutc, rowsscanned, rowsfixed, updatedonutc) - VALUES (@TableName, @LastProcessedKey, @LastCompletedUtc, @RowsScanned, @RowsFixed, @UpdatedOnUtc)"; + if (IsPostgres) + { + upsertSql = $@" + INSERT INTO {ProgressTable} (tablename, lastprocessedkey, lastcompletedutc, rowsscanned, rowsfixed, updatedonutc) + VALUES (@TableName, @LastProcessedKey, @LastCompletedUtc, @RowsScanned, @RowsFixed, @UpdatedOnUtc) + ON CONFLICT (tablename) DO UPDATE SET + lastprocessedkey = EXCLUDED.lastprocessedkey, + lastcompletedutc = EXCLUDED.lastcompletedutc, + rowsscanned = EXCLUDED.rowsscanned, + rowsfixed = EXCLUDED.rowsfixed, + updatedonutc = EXCLUDED.updatedonutc"; + } + else + { + // UPDLOCK + SERIALIZABLE takes a key-range lock so the row-absent case serializes: + // one run inserts, the other blocks then updates. No MERGE (its upsert races are well known). + upsertSql = $@" + SET XACT_ABORT ON; + BEGIN TRANSACTION; + UPDATE {ProgressTable} WITH (UPDLOCK, SERIALIZABLE) + SET lastprocessedkey = @LastProcessedKey, lastcompletedutc = @LastCompletedUtc, + rowsscanned = @RowsScanned, rowsfixed = @RowsFixed, updatedonutc = @UpdatedOnUtc + WHERE tablename = @TableName; + IF @@ROWCOUNT = 0 + INSERT INTO {ProgressTable} (tablename, lastprocessedkey, lastcompletedutc, rowsscanned, rowsfixed, updatedonutc) + VALUES (@TableName, @LastProcessedKey, @LastCompletedUtc, @RowsScanned, @RowsFixed, @UpdatedOnUtc); + COMMIT TRANSACTION;"; + } using (DbConnection conn = _connectionProvider.Create()) { await conn.OpenAsync(cancellationToken); - var affected = await conn.ExecuteAsync(new CommandDefinition(updateSql, progress, cancellationToken: cancellationToken)); - - if (affected == 0) - await conn.ExecuteAsync(new CommandDefinition(insertSql, progress, cancellationToken: cancellationToken)); + await conn.ExecuteAsync(new CommandDefinition(upsertSql, progress, cancellationToken: cancellationToken)); } } catch (Exception ex) @@ -316,11 +355,72 @@ public async Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationTo } } + /// + /// Maps a primary key column's declared type to the pageable category used for keyset cursors, + /// or null when the type cannot be paged with a string cursor (e.g. date/decimal PKs). + /// + private static Utf8PrimaryKeyType? ClassifyKeyType(MetaColumn pk) + { + var udtName = (pk.UdtName ?? string.Empty).ToLowerInvariant(); + if (udtName == "citext") + return Utf8PrimaryKeyType.Text; + + var dataType = (pk.DataType ?? string.Empty).ToLowerInvariant(); + switch (dataType) + { + case "char": + case "varchar": + case "nchar": + case "nvarchar": + case "text": + case "ntext": + case "character": + case "character varying": + return Utf8PrimaryKeyType.Text; + + case "tinyint": + case "smallint": + case "int": + case "integer": + case "bigint": + return Utf8PrimaryKeyType.Integer; + + case "uniqueidentifier": + case "uuid": + return Utf8PrimaryKeyType.Guid; + + default: + return null; // unsupported PK type -> table is skipped + } + } + + /// + /// Re-binds a string cursor to the primary key's native CLR type so the keyset > / = + /// predicates compare like types (PostgreSQL has no implicit text-to-integer/uuid coercion). + /// + private static object ConvertKey(string key, Utf8PrimaryKeyType keyType) + { + if (string.IsNullOrEmpty(key)) + return key; + + switch (keyType) + { + case Utf8PrimaryKeyType.Integer: + return long.Parse(key, CultureInfo.InvariantCulture); + case Utf8PrimaryKeyType.Guid: + return Guid.Parse(key); + default: + return key; + } + } + private class MetaColumn { public string TableSchema { get; set; } public string TableName { get; set; } public string ColumnName { get; set; } + public string DataType { get; set; } + public string UdtName { get; set; } } } } diff --git a/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs b/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs index dc571db6a..205a960a1 100644 --- a/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs +++ b/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs @@ -38,8 +38,7 @@ public async Task ExecuteMainAsync(string[] args, CancellationToken ca } catch (Exception ex) { - logger.LogError("There was an error running the UTF-8 data cleanup, see the error output below:"); - logger.LogError(ex.ToString()); + Resgrid.Framework.Logging.LogException(ex, "There was an error running the UTF-8 data cleanup"); return ExitCode.Failed; } diff --git a/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs b/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs index 9868cba08..aae287a39 100644 --- a/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs +++ b/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs @@ -13,7 +13,7 @@ public class Utf8CleanupTask : IQuidjiboHandler { public string Name => "UTF-8 Data Cleanup"; public int Priority => 1; - public ILogger _logger; + private readonly ILogger _logger; public Utf8CleanupTask(ILogger logger) { From bc3515642f2cb18c01fa1517883eabb7dde8120f Mon Sep 17 00:00:00 2001 From: Shawn Jackson Date: Tue, 9 Jun 2026 18:20:38 -0500 Subject: [PATCH 3/4] RE1-T117 PR#403 fixes --- .../IdentityRepository.cs | 51 +++++++++++++------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs index 9de3f893f..d0ee6dbde 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs @@ -63,9 +63,9 @@ public async Task GetUserByUserNameAsync(string userName) { using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString)) { - // PostgreSQL text comparison is case-sensitive (unlike SQL Server's default collation), - // so match on normalizedusername the same way ASP.NET Identity sign-in does. Otherwise a - // user who authenticated with different casing than the stored username is not found here. + // aspnetusers columns are citext (case-insensitive), so match on the same key ASP.NET + // Identity authenticated against -- normalizedusername -- so a row whose username and + // normalizedusername have drifted apart is still found here. var result = await db.QueryAsync($"SELECT * FROM aspnetusers WHERE normalizedusername = @normalizedUserName", new { normalizedUserName = userName?.ToUpperInvariant() }); return result.FirstOrDefault(); @@ -90,9 +90,9 @@ public IdentityUser GetUserByEmail(string email) { using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString)) { - // PostgreSQL text comparison is case-sensitive; LOWER() both sides so a different-cased email - // still matches (normalizedemail is not maintained on the PG update path, so it can't be relied on). - return db.Query($"SELECT * FROM aspnetusers WHERE LOWER(email) = LOWER(@email)", new { email = email }).FirstOrDefault(); + // email is citext (case-insensitive), so plain equality is already case-insensitive and can + // use the index -- no LOWER() needed. + return db.Query($"SELECT * FROM aspnetusers WHERE email = @email", new { email = email }).FirstOrDefault(); } } else @@ -112,15 +112,15 @@ public void UpdateUsername(string oldUsername, string newUsername) { using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString)) { - // Case-insensitive match on the old username (PostgreSQL text comparison is case-sensitive). - db.Execute($"UPDATE public.aspnetusers SET username = @newUsername, normalizedusername = @newUsernameUpper WHERE LOWER(username) = LOWER(@oldUsername)", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpper(), oldUsername = oldUsername }); + // username is citext, so equality is already case-insensitive and index-friendly. + db.Execute($"UPDATE public.aspnetusers SET username = @newUsername, normalizedusername = @newUsernameUpper WHERE username = @oldUsername", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpperInvariant(), oldUsername = oldUsername }); } } else { using (IDbConnection db = new SqlConnection(DataConfig.CoreConnectionString)) { - db.Execute($"UPDATE [AspNetUsers] SET [UserName] = @newUsername, [NormalizedUserName] = @newUsernameUpper WHERE UserName = @oldUsername", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpper(), oldUsername = oldUsername }); + db.Execute($"UPDATE [AspNetUsers] SET [UserName] = @newUsername, [NormalizedUserName] = @newUsernameUpper WHERE UserName = @oldUsername", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpperInvariant(), oldUsername = oldUsername }); } } } @@ -133,14 +133,14 @@ public void UpdateEmail(string userId, string newEmail) { // Keep normalizedemail in sync (ASP.NET Identity's FindByEmailAsync looks up by it); the // SQL Server branch already does this. Without it, email lookups go stale after a change. - db.Execute($"UPDATE public.aspnetusers SET email = @newEmail, normalizedemail = @newEmailUpper WHERE id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail?.ToUpper() }); + db.Execute($"UPDATE public.aspnetusers SET email = @newEmail, normalizedemail = @newEmailUpper WHERE id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail?.ToUpperInvariant() }); } } else { using (IDbConnection db = new SqlConnection(DataConfig.CoreConnectionString)) { - db.Execute($"UPDATE [AspNetUsers] SET [Email] = @newEmail, [NormalizedEmail] = @newEmailUpper WHERE Id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail.ToUpper() }); + db.Execute($"UPDATE [AspNetUsers] SET [Email] = @newEmail, [NormalizedEmail] = @newEmailUpper WHERE Id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail.ToUpperInvariant() }); } } } @@ -469,22 +469,41 @@ public async Task ClearOutUserLoginAsync(string userId) { var deleteId = Guid.NewGuid().ToString(); var maskedEmail = deleteId + "@resgrid.del"; + // Full de-provisioning: mask the normalized columns too (so ASP.NET Identity's normalized + // lookups can't find the row), null the password hash, rotate the security stamp, and lock + // the account so a deleted user can no longer authenticate. var result = await db.ExecuteAsync(@"UPDATE public.aspnetusers SET username = @deleteId, - email = @maskedEmail + normalizedusername = @normalizedDeleteId, + email = @maskedEmail, + normalizedemail = @normalizedMaskedEmail, + passwordhash = NULL, + securitystamp = @securityStamp, + emailconfirmed = false, + lockoutenabled = true, + lockoutend = @lockoutEnd WHERE id = @userId", - new { userId = userId, deleteId = deleteId, maskedEmail = maskedEmail }); + new { userId = userId, deleteId = deleteId, normalizedDeleteId = deleteId.ToUpperInvariant(), maskedEmail = maskedEmail, normalizedMaskedEmail = maskedEmail.ToUpperInvariant(), securityStamp = Guid.NewGuid().ToString(), lockoutEnd = new DateTimeOffset(9999, 12, 31, 23, 59, 59, TimeSpan.Zero) }); } } else { using (IDbConnection db = new SqlConnection(DataConfig.CoreConnectionString)) { + var deleteId = Guid.NewGuid().ToString(); + var maskedEmail = deleteId + "@resgrid.del"; var result = await db.ExecuteAsync(@"UPDATE AspNetUsers - SET UserName = @deleteid, - Email = @deleteid + '@resgrid.del' + SET UserName = @deleteId, + NormalizedUserName = @normalizedDeleteId, + Email = @maskedEmail, + NormalizedEmail = @normalizedMaskedEmail, + PasswordHash = NULL, + SecurityStamp = @securityStamp, + EmailConfirmed = 0, + LockoutEnabled = 1, + LockoutEnd = @lockoutEnd WHERE Id = @userId", - new { userId = userId, deleteid = Guid.NewGuid().ToString() }); + new { userId = userId, deleteId = deleteId, normalizedDeleteId = deleteId.ToUpperInvariant(), maskedEmail = maskedEmail, normalizedMaskedEmail = maskedEmail.ToUpperInvariant(), securityStamp = Guid.NewGuid().ToString(), lockoutEnd = new DateTimeOffset(9999, 12, 31, 23, 59, 59, TimeSpan.Zero) }); } } From 3b497fcff33c6db8bd5f867d8f555117f7902393 Mon Sep 17 00:00:00 2001 From: Shawn Jackson Date: Tue, 9 Jun 2026 18:27:35 -0500 Subject: [PATCH 4/4] RE1-T117 PR#403 fixes --- .../Resgrid.Repositories.DataRepository/IdentityRepository.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs index d0ee6dbde..a85a86206 100644 --- a/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs +++ b/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs @@ -140,7 +140,7 @@ public void UpdateEmail(string userId, string newEmail) { using (IDbConnection db = new SqlConnection(DataConfig.CoreConnectionString)) { - db.Execute($"UPDATE [AspNetUsers] SET [Email] = @newEmail, [NormalizedEmail] = @newEmailUpper WHERE Id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail.ToUpperInvariant() }); + db.Execute($"UPDATE [AspNetUsers] SET [Email] = @newEmail, [NormalizedEmail] = @newEmailUpper WHERE Id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail?.ToUpperInvariant() }); } } }