diff --git a/Core/Resgrid.Config/SystemBehaviorConfig.cs b/Core/Resgrid.Config/SystemBehaviorConfig.cs
index 8d9ab5c76..c03dfca01 100644
--- a/Core/Resgrid.Config/SystemBehaviorConfig.cs
+++ b/Core/Resgrid.Config/SystemBehaviorConfig.cs
@@ -196,6 +196,49 @@ public static class SystemBehaviorConfig
///
public static string LocationName = "US-West";
+ ///
+ /// When true, free-form text is sanitized at the data-repository write chokepoints so it is
+ /// always safe to store in a PostgreSQL UTF-8 database (strips NUL, repairs Windows-1252
+ /// mojibake, replaces unpaired UTF-16 surrogates). Guards the SQL Server -> PostgreSQL migration.
+ ///
+ public static bool SanitizeTextForUtf8 = true;
+
+ ///
+ /// When true (and is enabled), the sanitizer also normalizes
+ /// text to Unicode NFC. Not required for PostgreSQL UTF-8 validity, so off by default to avoid
+ /// altering otherwise-valid user text on the write hot-path; mainly useful for the cleanup sweep.
+ ///
+ public static bool Utf8NormalizeToNfc = false;
+
+ ///
+ /// When true (and is enabled), the sanitizer additionally
+ /// attempts a conservative repair of double-encoded UTF-8 content. Off by default because the
+ /// heuristic can misfire on legitimately Latin-1-looking text.
+ ///
+ public static bool Utf8RepairDoubleEncoding = false;
+
+ ///
+ /// Enables the nightly UTF-8 data cleanup worker that sweeps every text column and repairs
+ /// existing content in place so the database stays migration-clean.
+ ///
+ public static bool Utf8CleanupEnabled = true;
+
+ ///
+ /// Number of rows the cleanup worker reads per keyset-paginated batch.
+ ///
+ public static int Utf8CleanupBatchSize = 1000;
+
+ ///
+ /// Upper bound on the number of rows the cleanup worker will scan in a single run before
+ /// stopping (it resumes from its saved watermark on the next run). 0 means unbounded.
+ ///
+ public static int Utf8CleanupMaxRowsPerRun = 250000;
+
+ ///
+ /// UTC hour (0-23) at which the nightly UTF-8 data cleanup worker is scheduled to run.
+ ///
+ public static int Utf8CleanupHourUtc = 4;
+
public static string GetEnvPrefix()
{
switch (Environment)
diff --git a/Core/Resgrid.Framework/Text/Utf8Sanitizer.cs b/Core/Resgrid.Framework/Text/Utf8Sanitizer.cs
new file mode 100644
index 000000000..dc5f36107
--- /dev/null
+++ b/Core/Resgrid.Framework/Text/Utf8Sanitizer.cs
@@ -0,0 +1,323 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Reflection;
+using System.Text;
+
+namespace Resgrid.Framework
+{
+ ///
+ /// Sanitizes free-form text so it is safe to store in a PostgreSQL UTF-8 database.
+ ///
+ /// PostgreSQL's UTF-8 text/citext columns reject content that SQL Server
+ /// tolerates: the NUL character (U+0000) is forbidden outright, and invalid Unicode
+ /// (unpaired UTF-16 surrogates) cannot be encoded as UTF-8 at all. This class neutralizes
+ /// that content and, on a best-effort basis, repairs the most common Windows-1252 mojibake
+ /// so the recovered character (smart quotes, dashes, etc.) is preserved instead of discarded.
+ ///
+ /// The transform is pure and allocation-free for already-clean strings (the common case):
+ /// returns the original reference unchanged when nothing needs fixing.
+ ///
+ public static class Utf8Sanitizer
+ {
+ private static readonly UTF8Encoding StrictUtf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
+
+ private static readonly ConcurrentDictionary StringPropertyCache = new ConcurrentDictionary();
+
+ // C1 control range (U+0080 - U+009F). When Windows-1252 bytes are decoded as ISO-8859-1
+ // (Latin-1) their punctuation lands here; these are the candidates for mojibake repair.
+ private const char C1RangeStart = (char)0x80;
+ private const char C1RangeEnd = (char)0x9F;
+
+ // Highest ISO-8859-1 (Latin-1) code point; above this a char cannot be a single Latin-1 byte.
+ private const char Latin1Max = (char)0xFF;
+
+ /// The Unicode replacement character (U+FFFD) used for content that cannot be repaired.
+ public const char ReplacementChar = (char)0xFFFD;
+
+ ///
+ /// Returns a copy of that is safe for a PostgreSQL UTF-8 column.
+ /// Returns the original reference (no allocation) when the value is already clean.
+ ///
+ ///
+ /// When true, also attempts a conservative repair of double-encoded UTF-8. Off by default
+ /// because it can false-positive on legitimately Latin-1-looking text.
+ ///
+ ///
+ /// When non-null, the (possibly repaired) result is normalized to this form. NUL/surrogate
+ /// safety does not require normalization, so the write hot-path leaves this null.
+ ///
+ public static string Clean(string input, bool repairDoubleEncoding = false, NormalizationForm? normalization = null)
+ {
+ TryClean(input, out var cleaned, repairDoubleEncoding, normalization);
+ return cleaned;
+ }
+
+ ///
+ /// Cleans and reports whether any change was required.
+ /// is the original reference when no change was needed.
+ ///
+ public static bool TryClean(string input, out string cleaned, bool repairDoubleEncoding = false, NormalizationForm? normalization = null)
+ {
+ cleaned = input;
+
+ if (string.IsNullOrEmpty(input))
+ return false;
+
+ if (NeedsCleaning(input))
+ cleaned = Rebuild(input);
+
+ if (repairDoubleEncoding)
+ {
+ var repaired = RepairDoubleEncodedUtf8(cleaned);
+ if (!ReferenceEquals(repaired, cleaned))
+ cleaned = repaired;
+ }
+
+ if (normalization.HasValue && cleaned.Length > 0 && !cleaned.IsNormalized(normalization.Value))
+ cleaned = cleaned.Normalize(normalization.Value);
+
+ // Defensive final guard: the rebuild above removes NUL and lone surrogates, so this
+ // should never trip, but it guarantees the contract that output is valid UTF-8.
+ if (!IsValidUtf8(cleaned))
+ cleaned = ForceValidUtf8(cleaned);
+
+ return !ReferenceEquals(cleaned, input);
+ }
+
+ /// Returns true when the value can be stored in a PostgreSQL UTF-8 column as-is.
+ public static bool IsClean(string input)
+ {
+ return string.IsNullOrEmpty(input) || !NeedsCleaning(input);
+ }
+
+ ///
+ /// In-place sanitizes every public readable/writable string property of .
+ /// Used to guard the generic new DynamicParameters(entity) write path where individual
+ /// values do not flow through DynamicParametersExtension.Add.
+ ///
+ public static void CleanEntity(T entity, bool repairDoubleEncoding = false, NormalizationForm? normalization = null) where T : class
+ {
+ if (entity == null)
+ return;
+
+ var properties = StringPropertyCache.GetOrAdd(entity.GetType(), GetStringProperties);
+
+ for (int i = 0; i < properties.Length; i++)
+ {
+ var property = properties[i];
+ var current = (string)property.GetValue(entity);
+
+ if (current == null)
+ continue;
+
+ if (TryClean(current, out var cleaned, repairDoubleEncoding, normalization))
+ property.SetValue(entity, cleaned);
+ }
+ }
+
+ private static PropertyInfo[] GetStringProperties(Type type)
+ {
+ var result = new List();
+
+ foreach (var property in type.GetProperties(BindingFlags.Public | BindingFlags.Instance))
+ {
+ if (property.PropertyType != typeof(string))
+ continue;
+
+ if (!property.CanRead || !property.CanWrite)
+ continue;
+
+ // Skip indexers.
+ if (property.GetIndexParameters().Length > 0)
+ continue;
+
+ result.Add(property);
+ }
+
+ return result.ToArray();
+ }
+
+ private static bool NeedsCleaning(string input)
+ {
+ for (int i = 0; i < input.Length; i++)
+ {
+ char c = input[i];
+
+ if (c == '\0')
+ return true;
+
+ if (c >= C1RangeStart && c <= C1RangeEnd)
+ return true; // C1 range: candidate for Windows-1252 mojibake repair.
+
+ if (char.IsHighSurrogate(c))
+ {
+ if (i + 1 >= input.Length || !char.IsLowSurrogate(input[i + 1]))
+ return true; // unpaired high surrogate
+
+ i++; // valid pair, skip the low surrogate
+ }
+ else if (char.IsLowSurrogate(c))
+ {
+ return true; // lone low surrogate
+ }
+ }
+
+ return false;
+ }
+
+ private static string Rebuild(string input)
+ {
+ var sb = new StringBuilder(input.Length);
+
+ for (int i = 0; i < input.Length; i++)
+ {
+ char c = input[i];
+
+ if (c == '\0')
+ continue; // strip NUL: forbidden in PostgreSQL text
+
+ if (c >= C1RangeStart && c <= C1RangeEnd)
+ {
+ char mapped = MapCp1252C1(c);
+ if (mapped != '\0')
+ sb.Append(mapped);
+ // else: undefined Windows-1252 byte -> drop
+ continue;
+ }
+
+ if (char.IsHighSurrogate(c))
+ {
+ if (i + 1 < input.Length && char.IsLowSurrogate(input[i + 1]))
+ {
+ sb.Append(c);
+ sb.Append(input[i + 1]);
+ i++;
+ }
+ else
+ {
+ sb.Append(ReplacementChar); // unpaired high surrogate
+ }
+
+ continue;
+ }
+
+ if (char.IsLowSurrogate(c))
+ {
+ sb.Append(ReplacementChar); // lone low surrogate
+ continue;
+ }
+
+ sb.Append(c);
+ }
+
+ return sb.ToString();
+ }
+
+ ///
+ /// Maps a C1-range char (U+0080-U+009F) to the Windows-1252 character it most likely
+ /// represents. These show up when Windows-1252 bytes were decoded as ISO-8859-1 (Latin-1).
+ /// Returns '\0' for the five undefined Windows-1252 bytes (0x81, 0x8D, 0x8F, 0x90, 0x9D).
+ ///
+ private static char MapCp1252C1(char c)
+ {
+ switch ((int)c)
+ {
+ case 0x80: return (char)0x20AC; // EURO SIGN
+ case 0x82: return (char)0x201A; // SINGLE LOW-9 QUOTATION MARK
+ case 0x83: return (char)0x0192; // LATIN SMALL LETTER F WITH HOOK
+ case 0x84: return (char)0x201E; // DOUBLE LOW-9 QUOTATION MARK
+ case 0x85: return (char)0x2026; // HORIZONTAL ELLIPSIS
+ case 0x86: return (char)0x2020; // DAGGER
+ case 0x87: return (char)0x2021; // DOUBLE DAGGER
+ case 0x88: return (char)0x02C6; // MODIFIER LETTER CIRCUMFLEX ACCENT
+ case 0x89: return (char)0x2030; // PER MILLE SIGN
+ case 0x8A: return (char)0x0160; // LATIN CAPITAL LETTER S WITH CARON
+ case 0x8B: return (char)0x2039; // SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+ case 0x8C: return (char)0x0152; // LATIN CAPITAL LIGATURE OE
+ case 0x8E: return (char)0x017D; // LATIN CAPITAL LETTER Z WITH CARON
+ case 0x91: return (char)0x2018; // LEFT SINGLE QUOTATION MARK
+ case 0x92: return (char)0x2019; // RIGHT SINGLE QUOTATION MARK
+ case 0x93: return (char)0x201C; // LEFT DOUBLE QUOTATION MARK
+ case 0x94: return (char)0x201D; // RIGHT DOUBLE QUOTATION MARK
+ case 0x95: return (char)0x2022; // BULLET
+ case 0x96: return (char)0x2013; // EN DASH
+ case 0x97: return (char)0x2014; // EM DASH
+ case 0x98: return (char)0x02DC; // SMALL TILDE
+ case 0x99: return (char)0x2122; // TRADE MARK SIGN
+ case 0x9A: return (char)0x0161; // LATIN SMALL LETTER S WITH CARON
+ case 0x9B: return (char)0x203A; // SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
+ case 0x9C: return (char)0x0153; // LATIN SMALL LIGATURE OE
+ case 0x9E: return (char)0x017E; // LATIN SMALL LETTER Z WITH CARON
+ case 0x9F: return (char)0x0178; // LATIN CAPITAL LETTER Y WITH DIAERESIS
+ default: return '\0'; // 0x81, 0x8D, 0x8F, 0x90, 0x9D are undefined in Windows-1252
+ }
+ }
+
+ ///
+ /// Conservative double-encoded-UTF-8 repair: if the whole string round-trips as
+ /// Latin-1 bytes back into valid UTF-8 that differs from the input, return the decoded
+ /// form. Only triggers when the string actually contains the high-Latin-1 range, so plain
+ /// ASCII is never touched. Opt-in because it can misfire on genuine Latin-1 text.
+ ///
+ private static string RepairDoubleEncodedUtf8(string input)
+ {
+ if (string.IsNullOrEmpty(input))
+ return input;
+
+ bool hasHighLatin = false;
+ for (int i = 0; i < input.Length; i++)
+ {
+ char c = input[i];
+ if (c > Latin1Max)
+ return input; // contains non-Latin-1: not a single-byte mojibake candidate
+ if (c >= C1RangeStart)
+ hasHighLatin = true;
+ }
+
+ if (!hasHighLatin)
+ return input;
+
+ try
+ {
+ var bytes = new byte[input.Length];
+ for (int i = 0; i < input.Length; i++)
+ bytes[i] = (byte)input[i];
+
+ var decoded = StrictUtf8.GetString(bytes); // throws if not valid UTF-8
+ return string.Equals(decoded, input, StringComparison.Ordinal) ? input : decoded;
+ }
+ catch (Exception)
+ {
+ return input; // bytes were not valid UTF-8 -> genuine Latin-1, leave alone
+ }
+ }
+
+ private static bool IsValidUtf8(string input)
+ {
+ if (string.IsNullOrEmpty(input))
+ return true;
+
+ if (input.IndexOf('\0') >= 0)
+ return false;
+
+ try
+ {
+ StrictUtf8.GetByteCount(input);
+ return true;
+ }
+ catch (EncoderFallbackException)
+ {
+ return false;
+ }
+ }
+
+ private static string ForceValidUtf8(string input)
+ {
+ var permissive = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false);
+ var roundTripped = permissive.GetString(permissive.GetBytes(input));
+ return roundTripped.Replace("\0", string.Empty);
+ }
+ }
+}
diff --git a/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs b/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs
new file mode 100644
index 000000000..dd9f656da
--- /dev/null
+++ b/Core/Resgrid.Model/Repositories/IUtf8MaintenanceRepository.cs
@@ -0,0 +1,107 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Resgrid.Model.Repositories
+{
+ ///
+ /// Low-level maintenance access used by the nightly UTF-8 data cleanup worker. It enumerates
+ /// every free-form text column in the database, reads rows in keyset-paginated batches, and
+ /// writes back repaired values so the data stays safe for a PostgreSQL UTF-8 migration. Works
+ /// against whichever backend is currently configured (SQL Server source or PostgreSQL target).
+ ///
+ public interface IUtf8MaintenanceRepository
+ {
+ ///
+ /// Returns every base-table text column grouped by table, restricted to tables that have a
+ /// single-column primary key of a pageable type — text/citext, an integer type, or uuid
+ /// (required for stable keyset pagination). Tables whose PK is some other type are skipped.
+ ///
+ Task> GetTextColumnTargetsAsync(CancellationToken cancellationToken);
+
+ ///
+ /// Reads the next batch of rows for ordered by primary key, starting
+ /// after (null/empty for the first page). The cursor is the previous
+ /// page's last key rendered as an invariant string; the implementation re-binds it to the PK's
+ /// native type (see ) so the keyset comparison
+ /// works for text, integer and uuid keys alike.
+ ///
+ Task GetRowBatchAsync(Utf8TextColumnTarget target, string lastKey, int batchSize, CancellationToken cancellationToken);
+
+ ///
+ /// Applies the supplied repaired rows. Each row carries only the columns that changed.
+ /// Returns the number of rows updated.
+ ///
+ Task UpdateRowsAsync(Utf8TextColumnTarget target, IReadOnlyList rows, CancellationToken cancellationToken);
+
+ /// Loads the saved cleanup watermark for a table, or null if none exists yet.
+ Task GetProgressAsync(string tableKey, CancellationToken cancellationToken);
+
+ /// Inserts or updates the cleanup watermark for a table.
+ Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationToken cancellationToken);
+ }
+
+ ///
+ /// The pageable category of a table's primary key. Keyset pagination renders the cursor as an
+ /// invariant string and re-binds it as this type for the > / = predicates, so only
+ /// these PK types are swept; tables with any other PK type are skipped.
+ ///
+ public enum Utf8PrimaryKeyType
+ {
+ /// char/varchar/nchar/nvarchar/text/ntext and PostgreSQL citext — bound as a string.
+ Text = 0,
+
+ /// tinyint/smallint/int/bigint (and PostgreSQL integer types) — bound as Int64.
+ Integer = 1,
+
+ /// SQL Server uniqueidentifier / PostgreSQL uuid — bound as a Guid.
+ Guid = 2
+ }
+
+ /// A table plus its single-column primary key and the free-form text columns to scan.
+ public class Utf8TextColumnTarget
+ {
+ public string Schema { get; set; }
+ public string TableName { get; set; }
+ public string PrimaryKeyColumn { get; set; }
+
+ /// The PK's value category, so keyset cursors are bound as the PK's native type.
+ public Utf8PrimaryKeyType PrimaryKeyType { get; set; }
+
+ public List TextColumns { get; set; } = new List();
+
+ /// Stable identifier used as the watermark key (schema-qualified table name).
+ public string Key => string.IsNullOrEmpty(Schema) ? TableName : Schema + "." + TableName;
+ }
+
+ /// A page of rows plus the primary key of the last row (the next page's start cursor).
+ public class Utf8RowBatch
+ {
+ public List Rows { get; set; } = new List();
+ public string LastKey { get; set; }
+ }
+
+ ///
+ /// A single row's primary key and text-column values. When returned from a read it holds all
+ /// scanned columns; when passed to it
+ /// holds only the columns that changed.
+ ///
+ public class Utf8TextRow
+ {
+ /// The row's primary key rendered as an invariant string (re-bound to its native type for queries).
+ public string Key { get; set; }
+ public Dictionary Columns { get; set; } = new Dictionary();
+ }
+
+ /// Per-table cleanup watermark, persisted so each nightly run resumes where it stopped.
+ public class Utf8CleanupProgress
+ {
+ public string TableName { get; set; }
+ public string LastProcessedKey { get; set; }
+ public DateTime? LastCompletedUtc { get; set; }
+ public long RowsScanned { get; set; }
+ public long RowsFixed { get; set; }
+ public DateTime UpdatedOnUtc { get; set; }
+ }
+}
diff --git a/Core/Resgrid.Services/UsersService.cs b/Core/Resgrid.Services/UsersService.cs
index 10d051266..5b7fac5c3 100644
--- a/Core/Resgrid.Services/UsersService.cs
+++ b/Core/Resgrid.Services/UsersService.cs
@@ -98,6 +98,12 @@ public IdentityUser GetUserByEmail(string emailAddress)
public async Task DoesUserHaveAnyActiveDepartments(string userName)
{
var user = await GetUserByNameAsync(userName);
+
+ // Defensive: the user has already authenticated by this point, but if the lookup returns
+ // null (e.g. a stale/edge data state) treat it as no active departments rather than NRE.
+ if (user == null)
+ return false;
+
var memberships = await _departmentMembersRepository.GetAllByUserIdAsync(user.UserId);
return memberships.Any(x => x.IsDeleted == false && (x.IsDisabled == null || x.IsDisabled == false));
diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0068_ChatbotTables.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0068_ChatbotTables.cs
index 50240e9ea..66974b778 100644
--- a/Providers/Resgrid.Providers.Migrations/Migrations/M0068_ChatbotTables.cs
+++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0068_ChatbotTables.cs
@@ -7,91 +7,107 @@ public class M0068_ChatbotTables : Migration
{
public override void Up()
{
+ // Each table is guarded independently: databases upgraded before the 68->74 migration
+ // renumber may already have some of these (created under a prior version), so skip any
+ // table that already exists rather than failing the whole migration.
+
// ChatbotUserIdentities - Links platform identities to Resgrid users
- Create.Table("ChatbotUserIdentities")
- .WithColumn("Id").AsString(128).NotNullable().PrimaryKey()
- .WithColumn("UserId").AsString(450).NotNullable()
- .WithColumn("Platform").AsInt32().NotNullable()
- .WithColumn("PlatformUserId").AsString(256).NotNullable()
- .WithColumn("PlatformUserName").AsString(256).Nullable()
- .WithColumn("IsActive").AsBoolean().NotNullable().WithDefaultValue(true)
- .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("LastUsedAt").AsDateTime2().Nullable()
- .WithColumn("LinkingMethod").AsString(50).Nullable();
+ if (!Schema.Table("ChatbotUserIdentities").Exists())
+ {
+ Create.Table("ChatbotUserIdentities")
+ .WithColumn("Id").AsString(128).NotNullable().PrimaryKey()
+ .WithColumn("UserId").AsString(450).NotNullable()
+ .WithColumn("Platform").AsInt32().NotNullable()
+ .WithColumn("PlatformUserId").AsString(256).NotNullable()
+ .WithColumn("PlatformUserName").AsString(256).Nullable()
+ .WithColumn("IsActive").AsBoolean().NotNullable().WithDefaultValue(true)
+ .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("LastUsedAt").AsDateTime2().Nullable()
+ .WithColumn("LinkingMethod").AsString(50).Nullable();
- Create.Index("IX_ChatbotUserIdentities_User_Platform")
- .OnTable("ChatbotUserIdentities")
- .OnColumn("UserId").Ascending()
- .OnColumn("Platform").Ascending();
+ Create.Index("IX_ChatbotUserIdentities_User_Platform")
+ .OnTable("ChatbotUserIdentities")
+ .OnColumn("UserId").Ascending()
+ .OnColumn("Platform").Ascending();
- Create.Index("IX_ChatbotUserIdentities_Platform_PlatformUserId")
- .OnTable("ChatbotUserIdentities")
- .OnColumn("Platform").Ascending()
- .OnColumn("PlatformUserId").Ascending();
+ Create.Index("IX_ChatbotUserIdentities_Platform_PlatformUserId")
+ .OnTable("ChatbotUserIdentities")
+ .OnColumn("Platform").Ascending()
+ .OnColumn("PlatformUserId").Ascending();
+ }
// ChatbotSessions - Active conversation session state
- Create.Table("ChatbotSessions")
- .WithColumn("SessionId").AsString(128).NotNullable().PrimaryKey()
- .WithColumn("UserId").AsString(450).NotNullable()
- .WithColumn("DepartmentId").AsInt32().NotNullable()
- .WithColumn("Platform").AsInt32().NotNullable()
- .WithColumn("State").AsInt32().NotNullable().WithDefaultValue(0)
- .WithColumn("PendingIntent").AsInt32().Nullable()
- .WithColumn("ContextJson").AsString(int.MaxValue).Nullable()
- .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("LastActivity").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("TtlMinutes").AsInt32().NotNullable().WithDefaultValue(30);
+ if (!Schema.Table("ChatbotSessions").Exists())
+ {
+ Create.Table("ChatbotSessions")
+ .WithColumn("SessionId").AsString(128).NotNullable().PrimaryKey()
+ .WithColumn("UserId").AsString(450).NotNullable()
+ .WithColumn("DepartmentId").AsInt32().NotNullable()
+ .WithColumn("Platform").AsInt32().NotNullable()
+ .WithColumn("State").AsInt32().NotNullable().WithDefaultValue(0)
+ .WithColumn("PendingIntent").AsInt32().Nullable()
+ .WithColumn("ContextJson").AsString(int.MaxValue).Nullable()
+ .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("LastActivity").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("TtlMinutes").AsInt32().NotNullable().WithDefaultValue(30);
- Create.Index("IX_ChatbotSessions_UserId_Department")
- .OnTable("ChatbotSessions")
- .OnColumn("UserId").Ascending()
- .OnColumn("DepartmentId").Ascending()
- .OnColumn("Platform").Ascending();
+ Create.Index("IX_ChatbotSessions_UserId_Department")
+ .OnTable("ChatbotSessions")
+ .OnColumn("UserId").Ascending()
+ .OnColumn("DepartmentId").Ascending()
+ .OnColumn("Platform").Ascending();
- Create.Index("IX_ChatbotSessions_LastActivity")
- .OnTable("ChatbotSessions")
- .OnColumn("LastActivity").Ascending();
+ Create.Index("IX_ChatbotSessions_LastActivity")
+ .OnTable("ChatbotSessions")
+ .OnColumn("LastActivity").Ascending();
+ }
// ChatbotMessageLog - Audit log of all chatbot interactions
- Create.Table("ChatbotMessageLog")
- .WithColumn("Id").AsString(128).NotNullable().PrimaryKey()
- .WithColumn("DepartmentId").AsInt32().NotNullable()
- .WithColumn("UserId").AsString(450).Nullable()
- .WithColumn("SessionId").AsString(128).Nullable()
- .WithColumn("Platform").AsInt32().NotNullable()
- .WithColumn("Direction").AsString(10).NotNullable()
- .WithColumn("MessageText").AsString(int.MaxValue).Nullable()
- .WithColumn("IntentType").AsInt32().Nullable()
- .WithColumn("Processed").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("ErrorInfo").AsString(500).Nullable()
- .WithColumn("Timestamp").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime);
+ if (!Schema.Table("ChatbotMessageLog").Exists())
+ {
+ Create.Table("ChatbotMessageLog")
+ .WithColumn("Id").AsString(128).NotNullable().PrimaryKey()
+ .WithColumn("DepartmentId").AsInt32().NotNullable()
+ .WithColumn("UserId").AsString(450).Nullable()
+ .WithColumn("SessionId").AsString(128).Nullable()
+ .WithColumn("Platform").AsInt32().NotNullable()
+ .WithColumn("Direction").AsString(10).NotNullable()
+ .WithColumn("MessageText").AsString(int.MaxValue).Nullable()
+ .WithColumn("IntentType").AsInt32().Nullable()
+ .WithColumn("Processed").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("ErrorInfo").AsString(500).Nullable()
+ .WithColumn("Timestamp").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime);
- Create.Index("IX_ChatbotMessageLog_Department_Timestamp")
- .OnTable("ChatbotMessageLog")
- .OnColumn("DepartmentId").Ascending()
- .OnColumn("Timestamp").Descending();
+ Create.Index("IX_ChatbotMessageLog_Department_Timestamp")
+ .OnTable("ChatbotMessageLog")
+ .OnColumn("DepartmentId").Ascending()
+ .OnColumn("Timestamp").Descending();
- Create.Index("IX_ChatbotMessageLog_UserId")
- .OnTable("ChatbotMessageLog")
- .OnColumn("UserId").Ascending();
+ Create.Index("IX_ChatbotMessageLog_UserId")
+ .OnTable("ChatbotMessageLog")
+ .OnColumn("UserId").Ascending();
+ }
// ChatbotDepartmentConfigs - Per-department chatbot configuration
- Create.Table("ChatbotDepartmentConfigs")
- .WithColumn("Id").AsString(128).NotNullable().PrimaryKey()
- .WithColumn("DepartmentId").AsInt32().NotNullable()
- .WithColumn("IsEnabled").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("NluProvider").AsString(50).NotNullable().WithDefaultValue("keyword")
- .WithColumn("AllowedPlatforms").AsString(500).NotNullable().WithDefaultValue("*")
- .WithColumn("MaxSessionsPerUser").AsInt32().NotNullable().WithDefaultValue(3)
- .WithColumn("SessionTtlMinutes").AsInt32().NotNullable().WithDefaultValue(30)
- .WithColumn("AllowDispatchViaChatbot").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("RequireConfirmationForStatusChange").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("UpdatedAt").AsDateTime2().Nullable();
+ if (!Schema.Table("ChatbotDepartmentConfigs").Exists())
+ {
+ Create.Table("ChatbotDepartmentConfigs")
+ .WithColumn("Id").AsString(128).NotNullable().PrimaryKey()
+ .WithColumn("DepartmentId").AsInt32().NotNullable()
+ .WithColumn("IsEnabled").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("NluProvider").AsString(50).NotNullable().WithDefaultValue("keyword")
+ .WithColumn("AllowedPlatforms").AsString(500).NotNullable().WithDefaultValue("*")
+ .WithColumn("MaxSessionsPerUser").AsInt32().NotNullable().WithDefaultValue(3)
+ .WithColumn("SessionTtlMinutes").AsInt32().NotNullable().WithDefaultValue(30)
+ .WithColumn("AllowDispatchViaChatbot").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("RequireConfirmationForStatusChange").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("UpdatedAt").AsDateTime2().Nullable();
- Create.Index("IX_ChatbotDepartmentConfigs_DepartmentId")
- .OnTable("ChatbotDepartmentConfigs")
- .OnColumn("DepartmentId").Ascending();
+ Create.Index("IX_ChatbotDepartmentConfigs_DepartmentId")
+ .OnTable("ChatbotDepartmentConfigs")
+ .OnColumn("DepartmentId").Ascending();
+ }
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0069_ChatbotLinkingCodes.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0069_ChatbotLinkingCodes.cs
index eff46947c..13f839199 100644
--- a/Providers/Resgrid.Providers.Migrations/Migrations/M0069_ChatbotLinkingCodes.cs
+++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0069_ChatbotLinkingCodes.cs
@@ -8,28 +8,33 @@ public class M0069_ChatbotLinkingCodes : Migration
public override void Up()
{
// ChatbotLinkingCodes - Short-lived codes for linking platform accounts
- Create.Table("ChatbotLinkingCodes")
- .WithColumn("Id").AsString(128).NotNullable().PrimaryKey()
- .WithColumn("UserId").AsString(450).NotNullable()
- .WithColumn("Code").AsString(10).NotNullable()
- .WithColumn("Platform").AsInt32().Nullable()
- .WithColumn("PlatformUserId").AsString(256).Nullable()
- .WithColumn("IsUsed").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("ExpiresAt").AsDateTime2().NotNullable()
- .WithColumn("UsedAt").AsDateTime2().Nullable();
+ // Guarded: databases upgraded before the 68->74 migration renumber may already have
+ // this table (it ran under a prior version), so skip creation if it already exists.
+ if (!Schema.Table("ChatbotLinkingCodes").Exists())
+ {
+ Create.Table("ChatbotLinkingCodes")
+ .WithColumn("Id").AsString(128).NotNullable().PrimaryKey()
+ .WithColumn("UserId").AsString(450).NotNullable()
+ .WithColumn("Code").AsString(10).NotNullable()
+ .WithColumn("Platform").AsInt32().Nullable()
+ .WithColumn("PlatformUserId").AsString(256).Nullable()
+ .WithColumn("IsUsed").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("CreatedAt").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("ExpiresAt").AsDateTime2().NotNullable()
+ .WithColumn("UsedAt").AsDateTime2().Nullable();
- Create.Index("IX_ChatbotLinkingCodes_Code")
- .OnTable("ChatbotLinkingCodes")
- .OnColumn("Code").Ascending();
+ Create.Index("IX_ChatbotLinkingCodes_Code")
+ .OnTable("ChatbotLinkingCodes")
+ .OnColumn("Code").Ascending();
- Create.Index("IX_ChatbotLinkingCodes_UserId")
- .OnTable("ChatbotLinkingCodes")
- .OnColumn("UserId").Ascending();
+ Create.Index("IX_ChatbotLinkingCodes_UserId")
+ .OnTable("ChatbotLinkingCodes")
+ .OnColumn("UserId").Ascending();
- Create.Index("IX_ChatbotLinkingCodes_ExpiresAt")
- .OnTable("ChatbotLinkingCodes")
- .OnColumn("ExpiresAt").Ascending();
+ Create.Index("IX_ChatbotLinkingCodes_ExpiresAt")
+ .OnTable("ChatbotLinkingCodes")
+ .OnColumn("ExpiresAt").Ascending();
+ }
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0070_ChatbotDepartmentConfigColumns.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0070_ChatbotDepartmentConfigColumns.cs
index 361016648..efdcae37d 100644
--- a/Providers/Resgrid.Providers.Migrations/Migrations/M0070_ChatbotDepartmentConfigColumns.cs
+++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0070_ChatbotDepartmentConfigColumns.cs
@@ -8,14 +8,30 @@ public class M0070_ChatbotDepartmentConfigColumns : Migration
public override void Up()
{
// Per-department LLM/AI override + rate limits + linking/notification preferences.
- Alter.Table("ChatbotDepartmentConfigs")
- .AddColumn("LlmApiEndpoint").AsString(500).Nullable()
- .AddColumn("LlmApiKey").AsString(1000).Nullable()
- .AddColumn("LlmModelName").AsString(200).Nullable()
- .AddColumn("MessagesPerUserPerMinute").AsInt32().Nullable()
- .AddColumn("MessagesPerDepartmentPerMinute").AsInt32().Nullable()
- .AddColumn("RequireLinkingConfirmation").AsBoolean().NotNullable().WithDefaultValue(true)
- .AddColumn("ProactiveNotificationsEnabled").AsBoolean().NotNullable().WithDefaultValue(false);
+ // Each column is guarded so the migration is safe on databases where a prior partial
+ // apply (or a version renumber) already added some of them.
+ const string table = "ChatbotDepartmentConfigs";
+
+ if (!Schema.Table(table).Column("LlmApiEndpoint").Exists())
+ Alter.Table(table).AddColumn("LlmApiEndpoint").AsString(500).Nullable();
+
+ if (!Schema.Table(table).Column("LlmApiKey").Exists())
+ Alter.Table(table).AddColumn("LlmApiKey").AsString(1000).Nullable();
+
+ if (!Schema.Table(table).Column("LlmModelName").Exists())
+ Alter.Table(table).AddColumn("LlmModelName").AsString(200).Nullable();
+
+ if (!Schema.Table(table).Column("MessagesPerUserPerMinute").Exists())
+ Alter.Table(table).AddColumn("MessagesPerUserPerMinute").AsInt32().Nullable();
+
+ if (!Schema.Table(table).Column("MessagesPerDepartmentPerMinute").Exists())
+ Alter.Table(table).AddColumn("MessagesPerDepartmentPerMinute").AsInt32().Nullable();
+
+ if (!Schema.Table(table).Column("RequireLinkingConfirmation").Exists())
+ Alter.Table(table).AddColumn("RequireLinkingConfirmation").AsBoolean().NotNullable().WithDefaultValue(true);
+
+ if (!Schema.Table(table).Column("ProactiveNotificationsEnabled").Exists())
+ Alter.Table(table).AddColumn("ProactiveNotificationsEnabled").AsBoolean().NotNullable().WithDefaultValue(false);
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0071_AddingFeatureToggles.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0071_AddingFeatureToggles.cs
index 09fc550bf..816700c32 100644
--- a/Providers/Resgrid.Providers.Migrations/Migrations/M0071_AddingFeatureToggles.cs
+++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0071_AddingFeatureToggles.cs
@@ -7,129 +7,147 @@ public class M0071_AddingFeatureToggles : Migration
{
public override void Up()
{
+ // Each table (with its foreign keys and indexes) is guarded so the migration is safe to
+ // re-run / safe on databases where a prior partial apply already created some of them.
+
// FeatureFlags - system-wide flag definitions with a global default.
- Create.Table("FeatureFlags")
- .WithColumn("FeatureFlagId").AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FlagKey").AsString(256).NotNullable()
- .WithColumn("Name").AsString(256).NotNullable()
- .WithColumn("Description").AsString(1000).Nullable()
- .WithColumn("Category").AsString(128).Nullable()
- .WithColumn("Tags").AsString(512).Nullable()
- .WithColumn("FlagType").AsInt32().NotNullable().WithDefaultValue(0)
- .WithColumn("IsEnabledGlobally").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("DefaultValue").AsString(int.MaxValue).Nullable()
- .WithColumn("OffValue").AsString(int.MaxValue).Nullable()
- .WithColumn("RolloutPercentage").AsInt32().Nullable()
- .WithColumn("MinimumPlanType").AsInt32().Nullable()
- .WithColumn("Environment").AsInt32().Nullable()
- .WithColumn("EnableOn").AsDateTime2().Nullable()
- .WithColumn("DisableOn").AsDateTime2().Nullable()
- .WithColumn("IsArchived").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("IsPermanent").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("LastEvaluatedOn").AsDateTime2().Nullable()
- .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("CreatedByUserId").AsString(450).Nullable()
- .WithColumn("UpdatedOn").AsDateTime2().Nullable()
- .WithColumn("UpdatedByUserId").AsString(450).Nullable();
-
- Create.Index("UX_FeatureFlags_FlagKey")
- .OnTable("FeatureFlags")
- .OnColumn("FlagKey").Ascending()
- .WithOptions().Unique();
+ if (!Schema.Table("FeatureFlags").Exists())
+ {
+ Create.Table("FeatureFlags")
+ .WithColumn("FeatureFlagId").AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FlagKey").AsString(256).NotNullable()
+ .WithColumn("Name").AsString(256).NotNullable()
+ .WithColumn("Description").AsString(1000).Nullable()
+ .WithColumn("Category").AsString(128).Nullable()
+ .WithColumn("Tags").AsString(512).Nullable()
+ .WithColumn("FlagType").AsInt32().NotNullable().WithDefaultValue(0)
+ .WithColumn("IsEnabledGlobally").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("DefaultValue").AsString(int.MaxValue).Nullable()
+ .WithColumn("OffValue").AsString(int.MaxValue).Nullable()
+ .WithColumn("RolloutPercentage").AsInt32().Nullable()
+ .WithColumn("MinimumPlanType").AsInt32().Nullable()
+ .WithColumn("Environment").AsInt32().Nullable()
+ .WithColumn("EnableOn").AsDateTime2().Nullable()
+ .WithColumn("DisableOn").AsDateTime2().Nullable()
+ .WithColumn("IsArchived").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("IsPermanent").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("LastEvaluatedOn").AsDateTime2().Nullable()
+ .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("CreatedByUserId").AsString(450).Nullable()
+ .WithColumn("UpdatedOn").AsDateTime2().Nullable()
+ .WithColumn("UpdatedByUserId").AsString(450).Nullable();
+
+ Create.Index("UX_FeatureFlags_FlagKey")
+ .OnTable("FeatureFlags")
+ .OnColumn("FlagKey").Ascending()
+ .WithOptions().Unique();
+ }
// FeatureFlagOverrides - per-department override of a flag's value.
- Create.Table("FeatureFlagOverrides")
- .WithColumn("FeatureFlagOverrideId").AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FeatureFlagId").AsInt32().NotNullable()
- .WithColumn("DepartmentId").AsInt32().NotNullable()
- .WithColumn("IsEnabled").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("FlagValue").AsString(int.MaxValue).Nullable()
- .WithColumn("Reason").AsString(512).Nullable()
- .WithColumn("ExpiresOn").AsDateTime2().Nullable()
- .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("CreatedByUserId").AsString(450).Nullable()
- .WithColumn("UpdatedOn").AsDateTime2().Nullable()
- .WithColumn("UpdatedByUserId").AsString(450).Nullable();
-
- Create.ForeignKey("FK_FeatureFlagOverrides_FeatureFlags")
- .FromTable("FeatureFlagOverrides").ForeignColumn("FeatureFlagId")
- .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
-
- Create.ForeignKey("FK_FeatureFlagOverrides_Departments")
- .FromTable("FeatureFlagOverrides").ForeignColumn("DepartmentId")
- .ToTable("Departments").PrimaryColumn("DepartmentId");
-
- Create.Index("UX_FeatureFlagOverrides_Flag_Department")
- .OnTable("FeatureFlagOverrides")
- .OnColumn("FeatureFlagId").Ascending()
- .OnColumn("DepartmentId").Ascending()
- .WithOptions().Unique();
+ if (!Schema.Table("FeatureFlagOverrides").Exists())
+ {
+ Create.Table("FeatureFlagOverrides")
+ .WithColumn("FeatureFlagOverrideId").AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FeatureFlagId").AsInt32().NotNullable()
+ .WithColumn("DepartmentId").AsInt32().NotNullable()
+ .WithColumn("IsEnabled").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("FlagValue").AsString(int.MaxValue).Nullable()
+ .WithColumn("Reason").AsString(512).Nullable()
+ .WithColumn("ExpiresOn").AsDateTime2().Nullable()
+ .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("CreatedByUserId").AsString(450).Nullable()
+ .WithColumn("UpdatedOn").AsDateTime2().Nullable()
+ .WithColumn("UpdatedByUserId").AsString(450).Nullable();
+
+ Create.ForeignKey("FK_FeatureFlagOverrides_FeatureFlags")
+ .FromTable("FeatureFlagOverrides").ForeignColumn("FeatureFlagId")
+ .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
+
+ Create.ForeignKey("FK_FeatureFlagOverrides_Departments")
+ .FromTable("FeatureFlagOverrides").ForeignColumn("DepartmentId")
+ .ToTable("Departments").PrimaryColumn("DepartmentId");
+
+ Create.Index("UX_FeatureFlagOverrides_Flag_Department")
+ .OnTable("FeatureFlagOverrides")
+ .OnColumn("FeatureFlagId").Ascending()
+ .OnColumn("DepartmentId").Ascending()
+ .WithOptions().Unique();
+ }
// FeatureFlagTargetingRules - attribute/segment targeting rules.
- Create.Table("FeatureFlagTargetingRules")
- .WithColumn("FeatureFlagTargetingRuleId").AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FeatureFlagId").AsInt32().NotNullable()
- .WithColumn("Priority").AsInt32().NotNullable().WithDefaultValue(0)
- .WithColumn("AttributeType").AsInt32().NotNullable()
- .WithColumn("OperatorType").AsInt32().NotNullable()
- .WithColumn("ComparisonValue").AsString(int.MaxValue).Nullable()
- .WithColumn("ResultEnabled").AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("ResultValue").AsString(int.MaxValue).Nullable()
- .WithColumn("RolloutPercentage").AsInt32().Nullable()
- .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("CreatedByUserId").AsString(450).Nullable()
- .WithColumn("UpdatedOn").AsDateTime2().Nullable()
- .WithColumn("UpdatedByUserId").AsString(450).Nullable();
-
- Create.ForeignKey("FK_FeatureFlagTargetingRules_FeatureFlags")
- .FromTable("FeatureFlagTargetingRules").ForeignColumn("FeatureFlagId")
- .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
-
- Create.Index("IX_FeatureFlagTargetingRules_Flag")
- .OnTable("FeatureFlagTargetingRules")
- .OnColumn("FeatureFlagId").Ascending();
+ if (!Schema.Table("FeatureFlagTargetingRules").Exists())
+ {
+ Create.Table("FeatureFlagTargetingRules")
+ .WithColumn("FeatureFlagTargetingRuleId").AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FeatureFlagId").AsInt32().NotNullable()
+ .WithColumn("Priority").AsInt32().NotNullable().WithDefaultValue(0)
+ .WithColumn("AttributeType").AsInt32().NotNullable()
+ .WithColumn("OperatorType").AsInt32().NotNullable()
+ .WithColumn("ComparisonValue").AsString(int.MaxValue).Nullable()
+ .WithColumn("ResultEnabled").AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("ResultValue").AsString(int.MaxValue).Nullable()
+ .WithColumn("RolloutPercentage").AsInt32().Nullable()
+ .WithColumn("CreatedOn").AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("CreatedByUserId").AsString(450).Nullable()
+ .WithColumn("UpdatedOn").AsDateTime2().Nullable()
+ .WithColumn("UpdatedByUserId").AsString(450).Nullable();
+
+ Create.ForeignKey("FK_FeatureFlagTargetingRules_FeatureFlags")
+ .FromTable("FeatureFlagTargetingRules").ForeignColumn("FeatureFlagId")
+ .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
+
+ Create.Index("IX_FeatureFlagTargetingRules_Flag")
+ .OnTable("FeatureFlagTargetingRules")
+ .OnColumn("FeatureFlagId").Ascending();
+ }
// FeatureFlagPrerequisites - flag dependency edges.
- Create.Table("FeatureFlagPrerequisites")
- .WithColumn("FeatureFlagPrerequisiteId").AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FeatureFlagId").AsInt32().NotNullable()
- .WithColumn("RequiredFeatureFlagId").AsInt32().NotNullable()
- .WithColumn("RequiredValue").AsString(int.MaxValue).Nullable();
-
- Create.ForeignKey("FK_FeatureFlagPrerequisites_FeatureFlags")
- .FromTable("FeatureFlagPrerequisites").ForeignColumn("FeatureFlagId")
- .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
-
- Create.ForeignKey("FK_FeatureFlagPrerequisites_RequiredFeatureFlag")
- .FromTable("FeatureFlagPrerequisites").ForeignColumn("RequiredFeatureFlagId")
- .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
-
- Create.Index("IX_FeatureFlagPrerequisites_Flag")
- .OnTable("FeatureFlagPrerequisites")
- .OnColumn("FeatureFlagId").Ascending();
+ if (!Schema.Table("FeatureFlagPrerequisites").Exists())
+ {
+ Create.Table("FeatureFlagPrerequisites")
+ .WithColumn("FeatureFlagPrerequisiteId").AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FeatureFlagId").AsInt32().NotNullable()
+ .WithColumn("RequiredFeatureFlagId").AsInt32().NotNullable()
+ .WithColumn("RequiredValue").AsString(int.MaxValue).Nullable();
+
+ Create.ForeignKey("FK_FeatureFlagPrerequisites_FeatureFlags")
+ .FromTable("FeatureFlagPrerequisites").ForeignColumn("FeatureFlagId")
+ .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
+
+ Create.ForeignKey("FK_FeatureFlagPrerequisites_RequiredFeatureFlag")
+ .FromTable("FeatureFlagPrerequisites").ForeignColumn("RequiredFeatureFlagId")
+ .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
+
+ Create.Index("IX_FeatureFlagPrerequisites_Flag")
+ .OnTable("FeatureFlagPrerequisites")
+ .OnColumn("FeatureFlagId").Ascending();
+ }
// FeatureFlagUsages - aggregated daily evaluation counts (append-only flushes).
- Create.Table("FeatureFlagUsages")
- .WithColumn("FeatureFlagUsageId").AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FeatureFlagId").AsInt32().NotNullable()
- .WithColumn("DepartmentId").AsInt32().Nullable()
- .WithColumn("UsageDate").AsDateTime2().NotNullable()
- .WithColumn("EvaluationCount").AsInt64().NotNullable().WithDefaultValue(0)
- .WithColumn("EnabledCount").AsInt64().NotNullable().WithDefaultValue(0)
- .WithColumn("DisabledCount").AsInt64().NotNullable().WithDefaultValue(0);
-
- Create.ForeignKey("FK_FeatureFlagUsages_FeatureFlags")
- .FromTable("FeatureFlagUsages").ForeignColumn("FeatureFlagId")
- .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
-
- Create.ForeignKey("FK_FeatureFlagUsages_Departments")
- .FromTable("FeatureFlagUsages").ForeignColumn("DepartmentId")
- .ToTable("Departments").PrimaryColumn("DepartmentId");
-
- Create.Index("IX_FeatureFlagUsages_Flag_Date")
- .OnTable("FeatureFlagUsages")
- .OnColumn("FeatureFlagId").Ascending()
- .OnColumn("UsageDate").Ascending();
+ if (!Schema.Table("FeatureFlagUsages").Exists())
+ {
+ Create.Table("FeatureFlagUsages")
+ .WithColumn("FeatureFlagUsageId").AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FeatureFlagId").AsInt32().NotNullable()
+ .WithColumn("DepartmentId").AsInt32().Nullable()
+ .WithColumn("UsageDate").AsDateTime2().NotNullable()
+ .WithColumn("EvaluationCount").AsInt64().NotNullable().WithDefaultValue(0)
+ .WithColumn("EnabledCount").AsInt64().NotNullable().WithDefaultValue(0)
+ .WithColumn("DisabledCount").AsInt64().NotNullable().WithDefaultValue(0);
+
+ Create.ForeignKey("FK_FeatureFlagUsages_FeatureFlags")
+ .FromTable("FeatureFlagUsages").ForeignColumn("FeatureFlagId")
+ .ToTable("FeatureFlags").PrimaryColumn("FeatureFlagId");
+
+ Create.ForeignKey("FK_FeatureFlagUsages_Departments")
+ .FromTable("FeatureFlagUsages").ForeignColumn("DepartmentId")
+ .ToTable("Departments").PrimaryColumn("DepartmentId");
+
+ Create.Index("IX_FeatureFlagUsages_Flag_Date")
+ .OnTable("FeatureFlagUsages")
+ .OnColumn("FeatureFlagId").Ascending()
+ .OnColumn("UsageDate").Ascending();
+ }
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0072_AddingChatbotTwilioFeatureFlag.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0072_AddingChatbotTwilioFeatureFlag.cs
index dd283ecd9..01574570d 100644
--- a/Providers/Resgrid.Providers.Migrations/Migrations/M0072_AddingChatbotTwilioFeatureFlag.cs
+++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0072_AddingChatbotTwilioFeatureFlag.cs
@@ -13,14 +13,15 @@ public override void Up()
// Seeded OFF (IsEnabledGlobally = false). Inbound Twilio SMS keeps the original text-command
// handling until this flag is enabled globally or via a per-department override. FlagType,
// IsArchived, IsPermanent and CreatedOn fall back to their table defaults.
- Insert.IntoTable("FeatureFlags").Row(new
- {
- FlagKey = FlagKey,
- Name = "Chatbot Twilio Text Integration",
- Description = "When enabled, inbound Twilio SMS is processed by the new chatbot ingress pipeline. When off (globally or for a department) the original text-command handling is used.",
- Category = "Chatbot",
- IsEnabledGlobally = false
- });
+ // Guarded with IF NOT EXISTS so re-running the migration does not violate the unique
+ // FlagKey index.
+ Execute.Sql(
+ "IF NOT EXISTS (SELECT 1 FROM [FeatureFlags] WHERE [FlagKey] = '" + FlagKey + "') " +
+ "INSERT INTO [FeatureFlags] ([FlagKey], [Name], [Description], [Category], [IsEnabledGlobally]) " +
+ "VALUES ('" + FlagKey + "', " +
+ "'Chatbot Twilio Text Integration', " +
+ "'When enabled, inbound Twilio SMS is processed by the new chatbot ingress pipeline. When off (globally or for a department) the original text-command handling is used.', " +
+ "'Chatbot', 0);");
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSources.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSources.cs
index 39184020e..9ba580979 100644
--- a/Providers/Resgrid.Providers.Migrations/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSources.cs
+++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSources.cs
@@ -7,8 +7,13 @@ public class M0074_AddIsPermanentFailureToWeatherAlertSources : Migration
{
public override void Up()
{
- Alter.Table("WeatherAlertSources")
- .AddColumn("IsPermanentFailure").AsBoolean().NotNullable().WithDefaultValue(false);
+ // Guarded: this migration was renumbered (68 -> 74), so databases upgraded before the
+ // renumber already added this column under version 68. Skip the add if it exists.
+ if (!Schema.Table("WeatherAlertSources").Column("IsPermanentFailure").Exists())
+ {
+ Alter.Table("WeatherAlertSources")
+ .AddColumn("IsPermanentFailure").AsBoolean().NotNullable().WithDefaultValue(false);
+ }
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.Migrations/Migrations/M0075_AddUtf8CleanupProgress.cs b/Providers/Resgrid.Providers.Migrations/Migrations/M0075_AddUtf8CleanupProgress.cs
new file mode 100644
index 000000000..73e1263d9
--- /dev/null
+++ b/Providers/Resgrid.Providers.Migrations/Migrations/M0075_AddUtf8CleanupProgress.cs
@@ -0,0 +1,31 @@
+using FluentMigrator;
+
+namespace Resgrid.Providers.Migrations.Migrations
+{
+ [Migration(75)]
+ public class M0075_AddUtf8CleanupProgress : Migration
+ {
+ public override void Up()
+ {
+ // Per-table watermark for the nightly UTF-8 data cleanup worker so each run resumes
+ // where the previous one stopped while staying idempotent.
+ // Guarded so the migration is safe to re-run / safe on databases where a prior partial
+ // apply already created the table.
+ if (!Schema.Table("Utf8CleanupProgress").Exists())
+ {
+ Create.Table("Utf8CleanupProgress")
+ .WithColumn("TableName").AsString(256).NotNullable().PrimaryKey()
+ .WithColumn("LastProcessedKey").AsString(450).Nullable()
+ .WithColumn("LastCompletedUtc").AsDateTime().Nullable()
+ .WithColumn("RowsScanned").AsInt64().NotNullable().WithDefaultValue(0)
+ .WithColumn("RowsFixed").AsInt64().NotNullable().WithDefaultValue(0)
+ .WithColumn("UpdatedOnUtc").AsDateTime().NotNullable();
+ }
+ }
+
+ public override void Down()
+ {
+ Delete.Table("Utf8CleanupProgress");
+ }
+ }
+}
diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0068_ChatbotTablesPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0068_ChatbotTablesPg.cs
index fcb371cdf..163b9b250 100644
--- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0068_ChatbotTablesPg.cs
+++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0068_ChatbotTablesPg.cs
@@ -7,91 +7,107 @@ public class M0068_ChatbotTablesPg : Migration
{
public override void Up()
{
+ // Each table is guarded independently: databases upgraded before the 68->74 migration
+ // renumber may already have some of these (created under a prior version), so skip any
+ // table that already exists rather than failing the whole migration.
+
// ChatbotUserIdentities - Links platform identities to Resgrid users
- Create.Table("ChatbotUserIdentities".ToLower())
- .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
- .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable()
- .WithColumn("Platform".ToLower()).AsInt32().NotNullable()
- .WithColumn("PlatformUserId".ToLower()).AsCustom("citext").NotNullable()
- .WithColumn("PlatformUserName".ToLower()).AsCustom("citext").Nullable()
- .WithColumn("IsActive".ToLower()).AsBoolean().NotNullable().WithDefaultValue(true)
- .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("LastUsedAt".ToLower()).AsDateTime2().Nullable()
- .WithColumn("LinkingMethod".ToLower()).AsCustom("citext").Nullable();
+ if (!Schema.Table("ChatbotUserIdentities".ToLower()).Exists())
+ {
+ Create.Table("ChatbotUserIdentities".ToLower())
+ .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
+ .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable()
+ .WithColumn("Platform".ToLower()).AsInt32().NotNullable()
+ .WithColumn("PlatformUserId".ToLower()).AsCustom("citext").NotNullable()
+ .WithColumn("PlatformUserName".ToLower()).AsCustom("citext").Nullable()
+ .WithColumn("IsActive".ToLower()).AsBoolean().NotNullable().WithDefaultValue(true)
+ .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("LastUsedAt".ToLower()).AsDateTime2().Nullable()
+ .WithColumn("LinkingMethod".ToLower()).AsCustom("citext").Nullable();
- Create.Index("IX_ChatbotUserIdentities_User_Platform".ToLower())
- .OnTable("ChatbotUserIdentities".ToLower())
- .OnColumn("UserId".ToLower()).Ascending()
- .OnColumn("Platform".ToLower()).Ascending();
+ Create.Index("IX_ChatbotUserIdentities_User_Platform".ToLower())
+ .OnTable("ChatbotUserIdentities".ToLower())
+ .OnColumn("UserId".ToLower()).Ascending()
+ .OnColumn("Platform".ToLower()).Ascending();
- Create.Index("IX_ChatbotUserIdentities_Platform_PlatformUserId".ToLower())
- .OnTable("ChatbotUserIdentities".ToLower())
- .OnColumn("Platform".ToLower()).Ascending()
- .OnColumn("PlatformUserId".ToLower()).Ascending();
+ Create.Index("IX_ChatbotUserIdentities_Platform_PlatformUserId".ToLower())
+ .OnTable("ChatbotUserIdentities".ToLower())
+ .OnColumn("Platform".ToLower()).Ascending()
+ .OnColumn("PlatformUserId".ToLower()).Ascending();
+ }
// ChatbotSessions - Active conversation session state
- Create.Table("ChatbotSessions".ToLower())
- .WithColumn("SessionId".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
- .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable()
- .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable()
- .WithColumn("Platform".ToLower()).AsInt32().NotNullable()
- .WithColumn("State".ToLower()).AsInt32().NotNullable().WithDefaultValue(0)
- .WithColumn("PendingIntent".ToLower()).AsInt32().Nullable()
- .WithColumn("ContextJson".ToLower()).AsCustom("text").Nullable()
- .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("LastActivity".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("TtlMinutes".ToLower()).AsInt32().NotNullable().WithDefaultValue(30);
+ if (!Schema.Table("ChatbotSessions".ToLower()).Exists())
+ {
+ Create.Table("ChatbotSessions".ToLower())
+ .WithColumn("SessionId".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
+ .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable()
+ .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable()
+ .WithColumn("Platform".ToLower()).AsInt32().NotNullable()
+ .WithColumn("State".ToLower()).AsInt32().NotNullable().WithDefaultValue(0)
+ .WithColumn("PendingIntent".ToLower()).AsInt32().Nullable()
+ .WithColumn("ContextJson".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("LastActivity".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("TtlMinutes".ToLower()).AsInt32().NotNullable().WithDefaultValue(30);
- Create.Index("IX_ChatbotSessions_UserId_Department".ToLower())
- .OnTable("ChatbotSessions".ToLower())
- .OnColumn("UserId".ToLower()).Ascending()
- .OnColumn("DepartmentId".ToLower()).Ascending()
- .OnColumn("Platform".ToLower()).Ascending();
+ Create.Index("IX_ChatbotSessions_UserId_Department".ToLower())
+ .OnTable("ChatbotSessions".ToLower())
+ .OnColumn("UserId".ToLower()).Ascending()
+ .OnColumn("DepartmentId".ToLower()).Ascending()
+ .OnColumn("Platform".ToLower()).Ascending();
- Create.Index("IX_ChatbotSessions_LastActivity".ToLower())
- .OnTable("ChatbotSessions".ToLower())
- .OnColumn("LastActivity".ToLower()).Ascending();
+ Create.Index("IX_ChatbotSessions_LastActivity".ToLower())
+ .OnTable("ChatbotSessions".ToLower())
+ .OnColumn("LastActivity".ToLower()).Ascending();
+ }
// ChatbotMessageLog - Audit log of all chatbot interactions
- Create.Table("ChatbotMessageLog".ToLower())
- .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
- .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable()
- .WithColumn("UserId".ToLower()).AsCustom("citext").Nullable()
- .WithColumn("SessionId".ToLower()).AsCustom("citext").Nullable()
- .WithColumn("Platform".ToLower()).AsInt32().NotNullable()
- .WithColumn("Direction".ToLower()).AsCustom("citext").NotNullable()
- .WithColumn("MessageText".ToLower()).AsCustom("text").Nullable()
- .WithColumn("IntentType".ToLower()).AsInt32().Nullable()
- .WithColumn("Processed".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("ErrorInfo".ToLower()).AsCustom("citext").Nullable()
- .WithColumn("Timestamp".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime);
+ if (!Schema.Table("ChatbotMessageLog".ToLower()).Exists())
+ {
+ Create.Table("ChatbotMessageLog".ToLower())
+ .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
+ .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable()
+ .WithColumn("UserId".ToLower()).AsCustom("citext").Nullable()
+ .WithColumn("SessionId".ToLower()).AsCustom("citext").Nullable()
+ .WithColumn("Platform".ToLower()).AsInt32().NotNullable()
+ .WithColumn("Direction".ToLower()).AsCustom("citext").NotNullable()
+ .WithColumn("MessageText".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("IntentType".ToLower()).AsInt32().Nullable()
+ .WithColumn("Processed".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("ErrorInfo".ToLower()).AsCustom("citext").Nullable()
+ .WithColumn("Timestamp".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime);
- Create.Index("IX_ChatbotMessageLog_Department_Timestamp".ToLower())
- .OnTable("ChatbotMessageLog".ToLower())
- .OnColumn("DepartmentId".ToLower()).Ascending()
- .OnColumn("Timestamp".ToLower()).Descending();
+ Create.Index("IX_ChatbotMessageLog_Department_Timestamp".ToLower())
+ .OnTable("ChatbotMessageLog".ToLower())
+ .OnColumn("DepartmentId".ToLower()).Ascending()
+ .OnColumn("Timestamp".ToLower()).Descending();
- Create.Index("IX_ChatbotMessageLog_UserId".ToLower())
- .OnTable("ChatbotMessageLog".ToLower())
- .OnColumn("UserId".ToLower()).Ascending();
+ Create.Index("IX_ChatbotMessageLog_UserId".ToLower())
+ .OnTable("ChatbotMessageLog".ToLower())
+ .OnColumn("UserId".ToLower()).Ascending();
+ }
// ChatbotDepartmentConfigs - Per-department chatbot configuration
- Create.Table("ChatbotDepartmentConfigs".ToLower())
- .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
- .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable()
- .WithColumn("IsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("NluProvider".ToLower()).AsCustom("citext").NotNullable().WithDefaultValue("keyword")
- .WithColumn("AllowedPlatforms".ToLower()).AsCustom("citext").NotNullable().WithDefaultValue("*")
- .WithColumn("MaxSessionsPerUser".ToLower()).AsInt32().NotNullable().WithDefaultValue(3)
- .WithColumn("SessionTtlMinutes".ToLower()).AsInt32().NotNullable().WithDefaultValue(30)
- .WithColumn("AllowDispatchViaChatbot".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("RequireConfirmationForStatusChange".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("UpdatedAt".ToLower()).AsDateTime2().Nullable();
+ if (!Schema.Table("ChatbotDepartmentConfigs".ToLower()).Exists())
+ {
+ Create.Table("ChatbotDepartmentConfigs".ToLower())
+ .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
+ .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable()
+ .WithColumn("IsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("NluProvider".ToLower()).AsCustom("citext").NotNullable().WithDefaultValue("keyword")
+ .WithColumn("AllowedPlatforms".ToLower()).AsCustom("citext").NotNullable().WithDefaultValue("*")
+ .WithColumn("MaxSessionsPerUser".ToLower()).AsInt32().NotNullable().WithDefaultValue(3)
+ .WithColumn("SessionTtlMinutes".ToLower()).AsInt32().NotNullable().WithDefaultValue(30)
+ .WithColumn("AllowDispatchViaChatbot".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("RequireConfirmationForStatusChange".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("UpdatedAt".ToLower()).AsDateTime2().Nullable();
- Create.Index("IX_ChatbotDepartmentConfigs_DepartmentId".ToLower())
- .OnTable("ChatbotDepartmentConfigs".ToLower())
- .OnColumn("DepartmentId".ToLower()).Ascending();
+ Create.Index("IX_ChatbotDepartmentConfigs_DepartmentId".ToLower())
+ .OnTable("ChatbotDepartmentConfigs".ToLower())
+ .OnColumn("DepartmentId".ToLower()).Ascending();
+ }
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0069_ChatbotLinkingCodesPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0069_ChatbotLinkingCodesPg.cs
index decc6e5a5..b35fde6b8 100644
--- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0069_ChatbotLinkingCodesPg.cs
+++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0069_ChatbotLinkingCodesPg.cs
@@ -8,28 +8,33 @@ public class M0069_ChatbotLinkingCodesPg : Migration
public override void Up()
{
// ChatbotLinkingCodes - Short-lived codes for linking platform accounts
- Create.Table("ChatbotLinkingCodes".ToLower())
- .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
- .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable()
- .WithColumn("Code".ToLower()).AsCustom("citext").NotNullable()
- .WithColumn("Platform".ToLower()).AsInt32().Nullable()
- .WithColumn("PlatformUserId".ToLower()).AsCustom("citext").Nullable()
- .WithColumn("IsUsed".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("ExpiresAt".ToLower()).AsDateTime2().NotNullable()
- .WithColumn("UsedAt".ToLower()).AsDateTime2().Nullable();
+ // Guarded: databases upgraded before the 68->74 migration renumber may already have
+ // this table (it ran under a prior version), so skip creation if it already exists.
+ if (!Schema.Table("ChatbotLinkingCodes".ToLower()).Exists())
+ {
+ Create.Table("ChatbotLinkingCodes".ToLower())
+ .WithColumn("Id".ToLower()).AsCustom("citext").NotNullable().PrimaryKey()
+ .WithColumn("UserId".ToLower()).AsCustom("citext").NotNullable()
+ .WithColumn("Code".ToLower()).AsCustom("citext").NotNullable()
+ .WithColumn("Platform".ToLower()).AsInt32().Nullable()
+ .WithColumn("PlatformUserId".ToLower()).AsCustom("citext").Nullable()
+ .WithColumn("IsUsed".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("CreatedAt".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("ExpiresAt".ToLower()).AsDateTime2().NotNullable()
+ .WithColumn("UsedAt".ToLower()).AsDateTime2().Nullable();
- Create.Index("IX_ChatbotLinkingCodes_Code".ToLower())
- .OnTable("ChatbotLinkingCodes".ToLower())
- .OnColumn("Code".ToLower()).Ascending();
+ Create.Index("IX_ChatbotLinkingCodes_Code".ToLower())
+ .OnTable("ChatbotLinkingCodes".ToLower())
+ .OnColumn("Code".ToLower()).Ascending();
- Create.Index("IX_ChatbotLinkingCodes_UserId".ToLower())
- .OnTable("ChatbotLinkingCodes".ToLower())
- .OnColumn("UserId".ToLower()).Ascending();
+ Create.Index("IX_ChatbotLinkingCodes_UserId".ToLower())
+ .OnTable("ChatbotLinkingCodes".ToLower())
+ .OnColumn("UserId".ToLower()).Ascending();
- Create.Index("IX_ChatbotLinkingCodes_ExpiresAt".ToLower())
- .OnTable("ChatbotLinkingCodes".ToLower())
- .OnColumn("ExpiresAt".ToLower()).Ascending();
+ Create.Index("IX_ChatbotLinkingCodes_ExpiresAt".ToLower())
+ .OnTable("ChatbotLinkingCodes".ToLower())
+ .OnColumn("ExpiresAt".ToLower()).Ascending();
+ }
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0070_ChatbotDepartmentConfigColumnsPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0070_ChatbotDepartmentConfigColumnsPg.cs
index 167f6aa88..d41f52d9d 100644
--- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0070_ChatbotDepartmentConfigColumnsPg.cs
+++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0070_ChatbotDepartmentConfigColumnsPg.cs
@@ -8,14 +8,30 @@ public class M0070_ChatbotDepartmentConfigColumnsPg : Migration
public override void Up()
{
// Per-department LLM/AI override + rate limits + linking/notification preferences.
- Alter.Table("ChatbotDepartmentConfigs".ToLower())
- .AddColumn("LlmApiEndpoint".ToLower()).AsCustom("citext").Nullable()
- .AddColumn("LlmApiKey".ToLower()).AsCustom("citext").Nullable()
- .AddColumn("LlmModelName".ToLower()).AsCustom("citext").Nullable()
- .AddColumn("MessagesPerUserPerMinute".ToLower()).AsInt32().Nullable()
- .AddColumn("MessagesPerDepartmentPerMinute".ToLower()).AsInt32().Nullable()
- .AddColumn("RequireLinkingConfirmation".ToLower()).AsBoolean().NotNullable().WithDefaultValue(true)
- .AddColumn("ProactiveNotificationsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false);
+ // Each column is guarded so the migration is safe on databases where a prior partial
+ // apply (or a version renumber) already added some of them.
+ var table = "ChatbotDepartmentConfigs".ToLower();
+
+ if (!Schema.Table(table).Column("LlmApiEndpoint".ToLower()).Exists())
+ Alter.Table(table).AddColumn("LlmApiEndpoint".ToLower()).AsCustom("citext").Nullable();
+
+ if (!Schema.Table(table).Column("LlmApiKey".ToLower()).Exists())
+ Alter.Table(table).AddColumn("LlmApiKey".ToLower()).AsCustom("citext").Nullable();
+
+ if (!Schema.Table(table).Column("LlmModelName".ToLower()).Exists())
+ Alter.Table(table).AddColumn("LlmModelName".ToLower()).AsCustom("citext").Nullable();
+
+ if (!Schema.Table(table).Column("MessagesPerUserPerMinute".ToLower()).Exists())
+ Alter.Table(table).AddColumn("MessagesPerUserPerMinute".ToLower()).AsInt32().Nullable();
+
+ if (!Schema.Table(table).Column("MessagesPerDepartmentPerMinute".ToLower()).Exists())
+ Alter.Table(table).AddColumn("MessagesPerDepartmentPerMinute".ToLower()).AsInt32().Nullable();
+
+ if (!Schema.Table(table).Column("RequireLinkingConfirmation".ToLower()).Exists())
+ Alter.Table(table).AddColumn("RequireLinkingConfirmation".ToLower()).AsBoolean().NotNullable().WithDefaultValue(true);
+
+ if (!Schema.Table(table).Column("ProactiveNotificationsEnabled".ToLower()).Exists())
+ Alter.Table(table).AddColumn("ProactiveNotificationsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false);
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0071_AddingFeatureTogglesPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0071_AddingFeatureTogglesPg.cs
index d841fdf2a..84a041f14 100644
--- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0071_AddingFeatureTogglesPg.cs
+++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0071_AddingFeatureTogglesPg.cs
@@ -7,129 +7,147 @@ public class M0071_AddingFeatureTogglesPg : Migration
{
public override void Up()
{
+ // Each table (with its foreign keys and indexes) is guarded so the migration is safe to
+ // re-run / safe on databases where a prior partial apply already created some of them.
+
// FeatureFlags - system-wide flag definitions with a global default.
- Create.Table("FeatureFlags".ToLower())
- .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FlagKey".ToLower()).AsCustom("citext").NotNullable()
- .WithColumn("Name".ToLower()).AsCustom("citext").NotNullable()
- .WithColumn("Description".ToLower()).AsCustom("text").Nullable()
- .WithColumn("Category".ToLower()).AsCustom("citext").Nullable()
- .WithColumn("Tags".ToLower()).AsCustom("text").Nullable()
- .WithColumn("FlagType".ToLower()).AsInt32().NotNullable().WithDefaultValue(0)
- .WithColumn("IsEnabledGlobally".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("DefaultValue".ToLower()).AsCustom("text").Nullable()
- .WithColumn("OffValue".ToLower()).AsCustom("text").Nullable()
- .WithColumn("RolloutPercentage".ToLower()).AsInt32().Nullable()
- .WithColumn("MinimumPlanType".ToLower()).AsInt32().Nullable()
- .WithColumn("Environment".ToLower()).AsInt32().Nullable()
- .WithColumn("EnableOn".ToLower()).AsDateTime2().Nullable()
- .WithColumn("DisableOn".ToLower()).AsDateTime2().Nullable()
- .WithColumn("IsArchived".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("IsPermanent".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("LastEvaluatedOn".ToLower()).AsDateTime2().Nullable()
- .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable()
- .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable()
- .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable();
-
- Create.Index("UX_FeatureFlags_FlagKey".ToLower())
- .OnTable("FeatureFlags".ToLower())
- .OnColumn("FlagKey".ToLower()).Ascending()
- .WithOptions().Unique();
+ if (!Schema.Table("FeatureFlags".ToLower()).Exists())
+ {
+ Create.Table("FeatureFlags".ToLower())
+ .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FlagKey".ToLower()).AsCustom("citext").NotNullable()
+ .WithColumn("Name".ToLower()).AsCustom("citext").NotNullable()
+ .WithColumn("Description".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("Category".ToLower()).AsCustom("citext").Nullable()
+ .WithColumn("Tags".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("FlagType".ToLower()).AsInt32().NotNullable().WithDefaultValue(0)
+ .WithColumn("IsEnabledGlobally".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("DefaultValue".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("OffValue".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("RolloutPercentage".ToLower()).AsInt32().Nullable()
+ .WithColumn("MinimumPlanType".ToLower()).AsInt32().Nullable()
+ .WithColumn("Environment".ToLower()).AsInt32().Nullable()
+ .WithColumn("EnableOn".ToLower()).AsDateTime2().Nullable()
+ .WithColumn("DisableOn".ToLower()).AsDateTime2().Nullable()
+ .WithColumn("IsArchived".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("IsPermanent".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("LastEvaluatedOn".ToLower()).AsDateTime2().Nullable()
+ .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable()
+ .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable()
+ .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable();
+
+ Create.Index("UX_FeatureFlags_FlagKey".ToLower())
+ .OnTable("FeatureFlags".ToLower())
+ .OnColumn("FlagKey".ToLower()).Ascending()
+ .WithOptions().Unique();
+ }
// FeatureFlagOverrides - per-department override of a flag's value.
- Create.Table("FeatureFlagOverrides".ToLower())
- .WithColumn("FeatureFlagOverrideId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable()
- .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable()
- .WithColumn("IsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("FlagValue".ToLower()).AsCustom("text").Nullable()
- .WithColumn("Reason".ToLower()).AsCustom("text").Nullable()
- .WithColumn("ExpiresOn".ToLower()).AsDateTime2().Nullable()
- .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable()
- .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable()
- .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable();
-
- Create.ForeignKey("FK_FeatureFlagOverrides_FeatureFlags".ToLower())
- .FromTable("FeatureFlagOverrides".ToLower()).ForeignColumn("FeatureFlagId".ToLower())
- .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
-
- Create.ForeignKey("FK_FeatureFlagOverrides_Departments".ToLower())
- .FromTable("FeatureFlagOverrides".ToLower()).ForeignColumn("DepartmentId".ToLower())
- .ToTable("Departments".ToLower()).PrimaryColumn("DepartmentId".ToLower());
-
- Create.Index("UX_FeatureFlagOverrides_Flag_Department".ToLower())
- .OnTable("FeatureFlagOverrides".ToLower())
- .OnColumn("FeatureFlagId".ToLower()).Ascending()
- .OnColumn("DepartmentId".ToLower()).Ascending()
- .WithOptions().Unique();
+ if (!Schema.Table("FeatureFlagOverrides".ToLower()).Exists())
+ {
+ Create.Table("FeatureFlagOverrides".ToLower())
+ .WithColumn("FeatureFlagOverrideId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable()
+ .WithColumn("DepartmentId".ToLower()).AsInt32().NotNullable()
+ .WithColumn("IsEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("FlagValue".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("Reason".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("ExpiresOn".ToLower()).AsDateTime2().Nullable()
+ .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable()
+ .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable()
+ .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable();
+
+ Create.ForeignKey("FK_FeatureFlagOverrides_FeatureFlags".ToLower())
+ .FromTable("FeatureFlagOverrides".ToLower()).ForeignColumn("FeatureFlagId".ToLower())
+ .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
+
+ Create.ForeignKey("FK_FeatureFlagOverrides_Departments".ToLower())
+ .FromTable("FeatureFlagOverrides".ToLower()).ForeignColumn("DepartmentId".ToLower())
+ .ToTable("Departments".ToLower()).PrimaryColumn("DepartmentId".ToLower());
+
+ Create.Index("UX_FeatureFlagOverrides_Flag_Department".ToLower())
+ .OnTable("FeatureFlagOverrides".ToLower())
+ .OnColumn("FeatureFlagId".ToLower()).Ascending()
+ .OnColumn("DepartmentId".ToLower()).Ascending()
+ .WithOptions().Unique();
+ }
// FeatureFlagTargetingRules - attribute/segment targeting rules.
- Create.Table("FeatureFlagTargetingRules".ToLower())
- .WithColumn("FeatureFlagTargetingRuleId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable()
- .WithColumn("Priority".ToLower()).AsInt32().NotNullable().WithDefaultValue(0)
- .WithColumn("AttributeType".ToLower()).AsInt32().NotNullable()
- .WithColumn("OperatorType".ToLower()).AsInt32().NotNullable()
- .WithColumn("ComparisonValue".ToLower()).AsCustom("text").Nullable()
- .WithColumn("ResultEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
- .WithColumn("ResultValue".ToLower()).AsCustom("text").Nullable()
- .WithColumn("RolloutPercentage".ToLower()).AsInt32().Nullable()
- .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
- .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable()
- .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable()
- .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable();
-
- Create.ForeignKey("FK_FeatureFlagTargetingRules_FeatureFlags".ToLower())
- .FromTable("FeatureFlagTargetingRules".ToLower()).ForeignColumn("FeatureFlagId".ToLower())
- .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
-
- Create.Index("IX_FeatureFlagTargetingRules_Flag".ToLower())
- .OnTable("FeatureFlagTargetingRules".ToLower())
- .OnColumn("FeatureFlagId".ToLower()).Ascending();
+ if (!Schema.Table("FeatureFlagTargetingRules".ToLower()).Exists())
+ {
+ Create.Table("FeatureFlagTargetingRules".ToLower())
+ .WithColumn("FeatureFlagTargetingRuleId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable()
+ .WithColumn("Priority".ToLower()).AsInt32().NotNullable().WithDefaultValue(0)
+ .WithColumn("AttributeType".ToLower()).AsInt32().NotNullable()
+ .WithColumn("OperatorType".ToLower()).AsInt32().NotNullable()
+ .WithColumn("ComparisonValue".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("ResultEnabled".ToLower()).AsBoolean().NotNullable().WithDefaultValue(false)
+ .WithColumn("ResultValue".ToLower()).AsCustom("text").Nullable()
+ .WithColumn("RolloutPercentage".ToLower()).AsInt32().Nullable()
+ .WithColumn("CreatedOn".ToLower()).AsDateTime2().NotNullable().WithDefault(SystemMethods.CurrentUTCDateTime)
+ .WithColumn("CreatedByUserId".ToLower()).AsCustom("citext").Nullable()
+ .WithColumn("UpdatedOn".ToLower()).AsDateTime2().Nullable()
+ .WithColumn("UpdatedByUserId".ToLower()).AsCustom("citext").Nullable();
+
+ Create.ForeignKey("FK_FeatureFlagTargetingRules_FeatureFlags".ToLower())
+ .FromTable("FeatureFlagTargetingRules".ToLower()).ForeignColumn("FeatureFlagId".ToLower())
+ .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
+
+ Create.Index("IX_FeatureFlagTargetingRules_Flag".ToLower())
+ .OnTable("FeatureFlagTargetingRules".ToLower())
+ .OnColumn("FeatureFlagId".ToLower()).Ascending();
+ }
// FeatureFlagPrerequisites - flag dependency edges.
- Create.Table("FeatureFlagPrerequisites".ToLower())
- .WithColumn("FeatureFlagPrerequisiteId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable()
- .WithColumn("RequiredFeatureFlagId".ToLower()).AsInt32().NotNullable()
- .WithColumn("RequiredValue".ToLower()).AsCustom("text").Nullable();
-
- Create.ForeignKey("FK_FeatureFlagPrerequisites_FeatureFlags".ToLower())
- .FromTable("FeatureFlagPrerequisites".ToLower()).ForeignColumn("FeatureFlagId".ToLower())
- .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
-
- Create.ForeignKey("FK_FeatureFlagPrerequisites_RequiredFeatureFlag".ToLower())
- .FromTable("FeatureFlagPrerequisites".ToLower()).ForeignColumn("RequiredFeatureFlagId".ToLower())
- .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
-
- Create.Index("IX_FeatureFlagPrerequisites_Flag".ToLower())
- .OnTable("FeatureFlagPrerequisites".ToLower())
- .OnColumn("FeatureFlagId".ToLower()).Ascending();
+ if (!Schema.Table("FeatureFlagPrerequisites".ToLower()).Exists())
+ {
+ Create.Table("FeatureFlagPrerequisites".ToLower())
+ .WithColumn("FeatureFlagPrerequisiteId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable()
+ .WithColumn("RequiredFeatureFlagId".ToLower()).AsInt32().NotNullable()
+ .WithColumn("RequiredValue".ToLower()).AsCustom("text").Nullable();
+
+ Create.ForeignKey("FK_FeatureFlagPrerequisites_FeatureFlags".ToLower())
+ .FromTable("FeatureFlagPrerequisites".ToLower()).ForeignColumn("FeatureFlagId".ToLower())
+ .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
+
+ Create.ForeignKey("FK_FeatureFlagPrerequisites_RequiredFeatureFlag".ToLower())
+ .FromTable("FeatureFlagPrerequisites".ToLower()).ForeignColumn("RequiredFeatureFlagId".ToLower())
+ .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
+
+ Create.Index("IX_FeatureFlagPrerequisites_Flag".ToLower())
+ .OnTable("FeatureFlagPrerequisites".ToLower())
+ .OnColumn("FeatureFlagId".ToLower()).Ascending();
+ }
// FeatureFlagUsages - aggregated daily evaluation counts (append-only flushes).
- Create.Table("FeatureFlagUsages".ToLower())
- .WithColumn("FeatureFlagUsageId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
- .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable()
- .WithColumn("DepartmentId".ToLower()).AsInt32().Nullable()
- .WithColumn("UsageDate".ToLower()).AsDateTime2().NotNullable()
- .WithColumn("EvaluationCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0)
- .WithColumn("EnabledCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0)
- .WithColumn("DisabledCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0);
-
- Create.ForeignKey("FK_FeatureFlagUsages_FeatureFlags".ToLower())
- .FromTable("FeatureFlagUsages".ToLower()).ForeignColumn("FeatureFlagId".ToLower())
- .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
-
- Create.ForeignKey("FK_FeatureFlagUsages_Departments".ToLower())
- .FromTable("FeatureFlagUsages".ToLower()).ForeignColumn("DepartmentId".ToLower())
- .ToTable("Departments".ToLower()).PrimaryColumn("DepartmentId".ToLower());
-
- Create.Index("IX_FeatureFlagUsages_Flag_Date".ToLower())
- .OnTable("FeatureFlagUsages".ToLower())
- .OnColumn("FeatureFlagId".ToLower()).Ascending()
- .OnColumn("UsageDate".ToLower()).Ascending();
+ if (!Schema.Table("FeatureFlagUsages".ToLower()).Exists())
+ {
+ Create.Table("FeatureFlagUsages".ToLower())
+ .WithColumn("FeatureFlagUsageId".ToLower()).AsInt32().NotNullable().PrimaryKey().Identity()
+ .WithColumn("FeatureFlagId".ToLower()).AsInt32().NotNullable()
+ .WithColumn("DepartmentId".ToLower()).AsInt32().Nullable()
+ .WithColumn("UsageDate".ToLower()).AsDateTime2().NotNullable()
+ .WithColumn("EvaluationCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0)
+ .WithColumn("EnabledCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0)
+ .WithColumn("DisabledCount".ToLower()).AsInt64().NotNullable().WithDefaultValue(0);
+
+ Create.ForeignKey("FK_FeatureFlagUsages_FeatureFlags".ToLower())
+ .FromTable("FeatureFlagUsages".ToLower()).ForeignColumn("FeatureFlagId".ToLower())
+ .ToTable("FeatureFlags".ToLower()).PrimaryColumn("FeatureFlagId".ToLower());
+
+ Create.ForeignKey("FK_FeatureFlagUsages_Departments".ToLower())
+ .FromTable("FeatureFlagUsages".ToLower()).ForeignColumn("DepartmentId".ToLower())
+ .ToTable("Departments".ToLower()).PrimaryColumn("DepartmentId".ToLower());
+
+ Create.Index("IX_FeatureFlagUsages_Flag_Date".ToLower())
+ .OnTable("FeatureFlagUsages".ToLower())
+ .OnColumn("FeatureFlagId".ToLower()).Ascending()
+ .OnColumn("UsageDate".ToLower()).Ascending();
+ }
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0072_AddingChatbotTwilioFeatureFlagPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0072_AddingChatbotTwilioFeatureFlagPg.cs
index c65515123..3279e6b01 100644
--- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0072_AddingChatbotTwilioFeatureFlagPg.cs
+++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0072_AddingChatbotTwilioFeatureFlagPg.cs
@@ -14,14 +14,15 @@ public override void Up()
// handling until this flag is enabled globally or via a per-department override. flagtype,
// isarchived, ispermanent and createdon fall back to their table defaults; the identity PK is
// omitted so Postgres assigns it.
- Insert.IntoTable("FeatureFlags".ToLower()).Row(new
- {
- flagkey = FlagKey,
- name = "Chatbot Twilio Text Integration",
- description = "When enabled, inbound Twilio SMS is processed by the new chatbot ingress pipeline. When off (globally or for a department) the original text-command handling is used.",
- category = "Chatbot",
- isenabledglobally = false
- });
+ // Guarded with WHERE NOT EXISTS so re-running the migration does not violate the unique
+ // flagkey index.
+ Execute.Sql(
+ "INSERT INTO featureflags (flagkey, name, description, category, isenabledglobally) " +
+ "SELECT '" + FlagKey + "', " +
+ "'Chatbot Twilio Text Integration', " +
+ "'When enabled, inbound Twilio SMS is processed by the new chatbot ingress pipeline. When off (globally or for a department) the original text-command handling is used.', " +
+ "'Chatbot', false " +
+ "WHERE NOT EXISTS (SELECT 1 FROM featureflags WHERE flagkey = '" + FlagKey + "');");
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSourcesPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSourcesPg.cs
index 9ae4bf745..0d16a6b25 100644
--- a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSourcesPg.cs
+++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0074_AddIsPermanentFailureToWeatherAlertSourcesPg.cs
@@ -7,8 +7,13 @@ public class M0074_AddIsPermanentFailureToWeatherAlertSourcesPg : Migration
{
public override void Up()
{
- Alter.Table("weatheralertsources")
- .AddColumn("ispermanentfailure").AsBoolean().NotNullable().WithDefaultValue(false);
+ // Guarded: this migration was renumbered (68 -> 74), so databases upgraded before the
+ // renumber already added this column under version 68. Skip the add if it exists.
+ if (!Schema.Table("weatheralertsources").Column("ispermanentfailure").Exists())
+ {
+ Alter.Table("weatheralertsources")
+ .AddColumn("ispermanentfailure").AsBoolean().NotNullable().WithDefaultValue(false);
+ }
}
public override void Down()
diff --git a/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0075_AddUtf8CleanupProgressPg.cs b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0075_AddUtf8CleanupProgressPg.cs
new file mode 100644
index 000000000..4fff46c7f
--- /dev/null
+++ b/Providers/Resgrid.Providers.MigrationsPg/Migrations/M0075_AddUtf8CleanupProgressPg.cs
@@ -0,0 +1,31 @@
+using FluentMigrator;
+
+namespace Resgrid.Providers.MigrationsPg.Migrations
+{
+ [Migration(75)]
+ public class M0075_AddUtf8CleanupProgressPg : Migration
+ {
+ public override void Up()
+ {
+ // Per-table watermark for the nightly UTF-8 data cleanup worker so each run resumes
+ // where the previous one stopped while staying idempotent.
+ // Guarded so the migration is safe to re-run / safe on databases where a prior partial
+ // apply already created the table.
+ if (!Schema.Table("utf8cleanupprogress").Exists())
+ {
+ Create.Table("utf8cleanupprogress")
+ .WithColumn("tablename").AsCustom("citext").NotNullable().PrimaryKey()
+ .WithColumn("lastprocessedkey").AsCustom("text").Nullable()
+ .WithColumn("lastcompletedutc").AsDateTime().Nullable()
+ .WithColumn("rowsscanned").AsInt64().NotNullable().WithDefaultValue(0)
+ .WithColumn("rowsfixed").AsInt64().NotNullable().WithDefaultValue(0)
+ .WithColumn("updatedonutc").AsDateTime().NotNullable();
+ }
+ }
+
+ public override void Down()
+ {
+ Delete.Table("utf8cleanupprogress");
+ }
+ }
+}
diff --git a/Repositories/Resgrid.Repositories.DataRepository/Extensions/DynamicParametersExtension.cs b/Repositories/Resgrid.Repositories.DataRepository/Extensions/DynamicParametersExtension.cs
index b5caaf299..dd82b286e 100644
--- a/Repositories/Resgrid.Repositories.DataRepository/Extensions/DynamicParametersExtension.cs
+++ b/Repositories/Resgrid.Repositories.DataRepository/Extensions/DynamicParametersExtension.cs
@@ -1,8 +1,10 @@
using Dapper;
using Resgrid.Config;
+using Resgrid.Framework;
using Resgrid.Repositories.DataRepository.Extensions;
using System;
using System.Data;
+using System.Text;
namespace Resgrid.Repositories.DataRepository
{
@@ -19,6 +21,14 @@ public DynamicParametersExtension(object template)
public void Add(string name, object? value = null, DbType? dbType = null, ParameterDirection? direction = null, int? size = null, byte? precision = null, byte? scale = null)
{
+ // Guard the write path: scrub any string value so it is safe for a PostgreSQL UTF-8
+ // column (strips NUL, repairs Windows-1252 mojibake, replaces unpaired surrogates).
+ // This is the central manual-parameter chokepoint used across all repositories.
+ if (value is string stringValue && SystemBehaviorConfig.SanitizeTextForUtf8)
+ value = Utf8Sanitizer.Clean(stringValue,
+ SystemBehaviorConfig.Utf8RepairDoubleEncoding,
+ SystemBehaviorConfig.Utf8NormalizeToNfc ? NormalizationForm.FormC : (NormalizationForm?)null);
+
if (DataConfig.DatabaseType == DatabaseTypes.Postgres)
{
name = name.ToLower();
diff --git a/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs
index 37a250918..a85a86206 100644
--- a/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs
+++ b/Repositories/Resgrid.Repositories.DataRepository/IdentityRepository.cs
@@ -63,7 +63,10 @@ public async Task GetUserByUserNameAsync(string userName)
{
using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString))
{
- var result = await db.QueryAsync($"SELECT * FROM aspnetusers WHERE username = @userName", new { userName = userName });
+ // aspnetusers columns are citext (case-insensitive), so match on the same key ASP.NET
+ // Identity authenticated against -- normalizedusername -- so a row whose username and
+ // normalizedusername have drifted apart is still found here.
+ var result = await db.QueryAsync($"SELECT * FROM aspnetusers WHERE normalizedusername = @normalizedUserName", new { normalizedUserName = userName?.ToUpperInvariant() });
return result.FirstOrDefault();
}
@@ -87,6 +90,8 @@ public IdentityUser GetUserByEmail(string email)
{
using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString))
{
+ // email is citext (case-insensitive), so plain equality is already case-insensitive and can
+ // use the index -- no LOWER() needed.
return db.Query($"SELECT * FROM aspnetusers WHERE email = @email", new { email = email }).FirstOrDefault();
}
}
@@ -107,14 +112,15 @@ public void UpdateUsername(string oldUsername, string newUsername)
{
using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString))
{
- db.Execute($"UPDATE public.aspnetusers SET username = @newUsername, normalizedusername = @newUsernameUpper WHERE username = @oldUsername", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpper(), oldUsername = oldUsername });
+ // username is citext, so equality is already case-insensitive and index-friendly.
+ db.Execute($"UPDATE public.aspnetusers SET username = @newUsername, normalizedusername = @newUsernameUpper WHERE username = @oldUsername", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpperInvariant(), oldUsername = oldUsername });
}
}
else
{
using (IDbConnection db = new SqlConnection(DataConfig.CoreConnectionString))
{
- db.Execute($"UPDATE [AspNetUsers] SET [UserName] = @newUsername, [NormalizedUserName] = @newUsernameUpper WHERE UserName = @oldUsername", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpper(), oldUsername = oldUsername });
+ db.Execute($"UPDATE [AspNetUsers] SET [UserName] = @newUsername, [NormalizedUserName] = @newUsernameUpper WHERE UserName = @oldUsername", new { newUsername = newUsername, newUsernameUpper = newUsername.ToUpperInvariant(), oldUsername = oldUsername });
}
}
}
@@ -125,14 +131,16 @@ public void UpdateEmail(string userId, string newEmail)
{
using (IDbConnection db = new NpgsqlConnection(DataConfig.CoreConnectionString))
{
- db.Execute($"UPDATE public.aspnetusers SET email = @newEmail WHERE id = @userId", new { userId = userId, newEmail = newEmail });
+ // Keep normalizedemail in sync (ASP.NET Identity's FindByEmailAsync looks up by it); the
+ // SQL Server branch already does this. Without it, email lookups go stale after a change.
+ db.Execute($"UPDATE public.aspnetusers SET email = @newEmail, normalizedemail = @newEmailUpper WHERE id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail?.ToUpperInvariant() });
}
}
else
{
using (IDbConnection db = new SqlConnection(DataConfig.CoreConnectionString))
{
- db.Execute($"UPDATE [AspNetUsers] SET [Email] = @newEmail, [NormalizedEmail] = @newEmailUpper WHERE Id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail.ToUpper() });
+ db.Execute($"UPDATE [AspNetUsers] SET [Email] = @newEmail, [NormalizedEmail] = @newEmailUpper WHERE Id = @userId", new { userId = userId, newEmail = newEmail, newEmailUpper = newEmail?.ToUpperInvariant() });
}
}
}
@@ -461,22 +469,41 @@ public async Task ClearOutUserLoginAsync(string userId)
{
var deleteId = Guid.NewGuid().ToString();
var maskedEmail = deleteId + "@resgrid.del";
+ // Full de-provisioning: mask the normalized columns too (so ASP.NET Identity's normalized
+ // lookups can't find the row), null the password hash, rotate the security stamp, and lock
+ // the account so a deleted user can no longer authenticate.
var result = await db.ExecuteAsync(@"UPDATE public.aspnetusers
SET username = @deleteId,
- email = @maskedEmail
+ normalizedusername = @normalizedDeleteId,
+ email = @maskedEmail,
+ normalizedemail = @normalizedMaskedEmail,
+ passwordhash = NULL,
+ securitystamp = @securityStamp,
+ emailconfirmed = false,
+ lockoutenabled = true,
+ lockoutend = @lockoutEnd
WHERE id = @userId",
- new { userId = userId, deleteId = deleteId, maskedEmail = maskedEmail });
+ new { userId = userId, deleteId = deleteId, normalizedDeleteId = deleteId.ToUpperInvariant(), maskedEmail = maskedEmail, normalizedMaskedEmail = maskedEmail.ToUpperInvariant(), securityStamp = Guid.NewGuid().ToString(), lockoutEnd = new DateTimeOffset(9999, 12, 31, 23, 59, 59, TimeSpan.Zero) });
}
}
else
{
using (IDbConnection db = new SqlConnection(DataConfig.CoreConnectionString))
{
+ var deleteId = Guid.NewGuid().ToString();
+ var maskedEmail = deleteId + "@resgrid.del";
var result = await db.ExecuteAsync(@"UPDATE AspNetUsers
- SET UserName = @deleteid,
- Email = @deleteid + '@resgrid.del'
+ SET UserName = @deleteId,
+ NormalizedUserName = @normalizedDeleteId,
+ Email = @maskedEmail,
+ NormalizedEmail = @normalizedMaskedEmail,
+ PasswordHash = NULL,
+ SecurityStamp = @securityStamp,
+ EmailConfirmed = 0,
+ LockoutEnabled = 1,
+ LockoutEnd = @lockoutEnd
WHERE Id = @userId",
- new { userId = userId, deleteid = Guid.NewGuid().ToString() });
+ new { userId = userId, deleteId = deleteId, normalizedDeleteId = deleteId.ToUpperInvariant(), maskedEmail = maskedEmail, normalizedMaskedEmail = maskedEmail.ToUpperInvariant(), securityStamp = Guid.NewGuid().ToString(), lockoutEnd = new DateTimeOffset(9999, 12, 31, 23, 59, 59, TimeSpan.Zero) });
}
}
diff --git a/Repositories/Resgrid.Repositories.DataRepository/IdentityRoleRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/IdentityRoleRepository.cs
index 466565af1..f87c18979 100644
--- a/Repositories/Resgrid.Repositories.DataRepository/IdentityRoleRepository.cs
+++ b/Repositories/Resgrid.Repositories.DataRepository/IdentityRoleRepository.cs
@@ -157,6 +157,8 @@ public async Task InsertAsync(IdentityRole role, CancellationToken cancell
{
var insertFunction = new Func>(async x =>
{
+ Utf8WriteGuard.Sanitize(role);
+
var dynamicParameters = new DynamicParameters(role);
var query = _queryFactory.GetInsertQuery(role);
@@ -201,6 +203,8 @@ public async Task InsertClaimAsync(IdentityRole role, Claim claim, Cancell
roleClaim.ClaimValue = claim.Value;
roleClaim.RoleId = role.Id;
+ Utf8WriteGuard.Sanitize(roleClaim);
+
var dynamicParameters = new DynamicParameters(roleClaim);
var query = _queryFactory.GetInsertQuery(roleClaim);
@@ -322,6 +326,8 @@ public async Task UpdateAsync(IdentityRole role, CancellationToken cancell
{
var updateFunction = new Func>(async x =>
{
+ Utf8WriteGuard.Sanitize(role);
+
var dynamicParameters = new DynamicParameters(role);
var query = _queryFactory.GetUpdateQuery(role);
diff --git a/Repositories/Resgrid.Repositories.DataRepository/IdentityUserRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/IdentityUserRepository.cs
index 3af54c35a..38169c4a3 100644
--- a/Repositories/Resgrid.Repositories.DataRepository/IdentityUserRepository.cs
+++ b/Repositories/Resgrid.Repositories.DataRepository/IdentityUserRepository.cs
@@ -234,6 +234,8 @@ public async Task InsertAsync(IdentityUser user, CancellationToken cance
{
try
{
+ Utf8WriteGuard.Sanitize(user);
+
var dynamicParameters = new DynamicParameters(user);
var query = _queryFactory.GetInsertQuery(user);
diff --git a/Repositories/Resgrid.Repositories.DataRepository/Modules/ApiDataModule.cs b/Repositories/Resgrid.Repositories.DataRepository/Modules/ApiDataModule.cs
index 361a1411a..4c770908c 100644
--- a/Repositories/Resgrid.Repositories.DataRepository/Modules/ApiDataModule.cs
+++ b/Repositories/Resgrid.Repositories.DataRepository/Modules/ApiDataModule.cs
@@ -205,6 +205,9 @@ protected override void Load(ContainerBuilder builder)
// SSO Repositories
builder.RegisterType().As().InstancePerLifetimeScope();
builder.RegisterType().As().InstancePerLifetimeScope();
+
+ // UTF-8 / PostgreSQL-migration data maintenance
+ builder.RegisterType().As().InstancePerLifetimeScope();
}
}
}
diff --git a/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs b/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs
index 946e09df8..b3fce716a 100644
--- a/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs
+++ b/Repositories/Resgrid.Repositories.DataRepository/Modules/DataModule.cs
@@ -230,6 +230,9 @@ protected override void Load(ContainerBuilder builder)
builder.RegisterType().As().InstancePerLifetimeScope();
builder.RegisterType().As().InstancePerLifetimeScope();
builder.RegisterType().As().InstancePerLifetimeScope();
+
+ // UTF-8 / PostgreSQL-migration data maintenance
+ builder.RegisterType().As().InstancePerLifetimeScope();
}
}
}
diff --git a/Repositories/Resgrid.Repositories.DataRepository/RepositoryBase.cs b/Repositories/Resgrid.Repositories.DataRepository/RepositoryBase.cs
index 624179588..ef44781b3 100644
--- a/Repositories/Resgrid.Repositories.DataRepository/RepositoryBase.cs
+++ b/Repositories/Resgrid.Repositories.DataRepository/RepositoryBase.cs
@@ -164,6 +164,8 @@ public virtual async Task InsertAsync(T entity, CancellationToken cancellatio
{
try
{
+ Utf8WriteGuard.Sanitize(entity);
+
var insertFunction = new Func>(async x =>
{
var dynamicParameters = new DynamicParameters(entity);
@@ -214,10 +216,10 @@ public virtual async Task UpdateAsync(T entity, CancellationToken cancellatio
{
try
{
+ Utf8WriteGuard.Sanitize(entity);
+
var updateFunction = new Func>(async x =>
{
-
-
var dynamicParameters = new DynamicParameters(entity);
var query = _queryFactory.GetUpdateQuery(entity);
diff --git a/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs b/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs
new file mode 100644
index 000000000..0ce0aeb4b
--- /dev/null
+++ b/Repositories/Resgrid.Repositories.DataRepository/Utf8MaintenanceRepository.cs
@@ -0,0 +1,426 @@
+using System;
+using System.Collections.Generic;
+using System.Data.Common;
+using System.Globalization;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Dapper;
+using Resgrid.Config;
+using Resgrid.Framework;
+using Resgrid.Model.Repositories;
+using Resgrid.Model.Repositories.Connection;
+
+namespace Resgrid.Repositories.DataRepository
+{
+ ///
+ /// Backend-aware (SQL Server / PostgreSQL) maintenance access for the nightly UTF-8 data cleanup
+ /// worker. Builds metadata and row queries dynamically from INFORMATION_SCHEMA so it covers every
+ /// text column without a hand-maintained list.
+ ///
+ public class Utf8MaintenanceRepository : IUtf8MaintenanceRepository
+ {
+ private const string ProgressTableName = "utf8cleanupprogress";
+
+ // Tables the sweep should never touch (its own bookkeeping + the migration history table).
+ private static readonly HashSet ExcludedTables = new HashSet(StringComparer.OrdinalIgnoreCase)
+ {
+ ProgressTableName,
+ "versioninfo"
+ };
+
+ private readonly IConnectionProvider _connectionProvider;
+
+ public Utf8MaintenanceRepository(IConnectionProvider connectionProvider)
+ {
+ _connectionProvider = connectionProvider;
+ }
+
+ private static bool IsPostgres => DataConfig.DatabaseType == DatabaseTypes.Postgres;
+
+ private static string Quote(string identifier)
+ {
+ return IsPostgres ? "\"" + identifier + "\"" : "[" + identifier + "]";
+ }
+
+ private static string QualifiedTable(Utf8TextColumnTarget target)
+ {
+ return string.IsNullOrEmpty(target.Schema)
+ ? Quote(target.TableName)
+ : Quote(target.Schema) + "." + Quote(target.TableName);
+ }
+
+ private static string ProgressTable => (IsPostgres ? "public." : "dbo.") + ProgressTableName;
+
+ public async Task> GetTextColumnTargetsAsync(CancellationToken cancellationToken)
+ {
+ try
+ {
+ string columnsSql;
+ string pkSql;
+
+ if (IsPostgres)
+ {
+ columnsSql = @"
+ SELECT c.table_schema AS TableSchema, c.table_name AS TableName, c.column_name AS ColumnName
+ FROM information_schema.columns c
+ INNER JOIN information_schema.tables t
+ ON t.table_schema = c.table_schema AND t.table_name = c.table_name AND t.table_type = 'BASE TABLE'
+ WHERE (c.data_type IN ('character', 'character varying', 'text') OR c.udt_name = 'citext')
+ AND c.table_schema NOT IN ('pg_catalog', 'information_schema')";
+
+ pkSql = @"
+ SELECT tc.table_schema AS TableSchema, tc.table_name AS TableName, kcu.column_name AS ColumnName,
+ col.data_type AS DataType, col.udt_name AS UdtName
+ FROM information_schema.table_constraints tc
+ INNER JOIN information_schema.key_column_usage kcu
+ ON tc.constraint_name = kcu.constraint_name
+ AND tc.table_schema = kcu.table_schema
+ AND tc.table_name = kcu.table_name
+ INNER JOIN information_schema.columns col
+ ON col.table_schema = kcu.table_schema
+ AND col.table_name = kcu.table_name
+ AND col.column_name = kcu.column_name
+ WHERE tc.constraint_type = 'PRIMARY KEY'
+ AND tc.table_schema NOT IN ('pg_catalog', 'information_schema')";
+ }
+ else
+ {
+ columnsSql = @"
+ SELECT c.TABLE_SCHEMA AS TableSchema, c.TABLE_NAME AS TableName, c.COLUMN_NAME AS ColumnName
+ FROM INFORMATION_SCHEMA.COLUMNS c
+ INNER JOIN INFORMATION_SCHEMA.TABLES t
+ ON t.TABLE_SCHEMA = c.TABLE_SCHEMA AND t.TABLE_NAME = c.TABLE_NAME AND t.TABLE_TYPE = 'BASE TABLE'
+ WHERE c.DATA_TYPE IN ('char', 'varchar', 'nchar', 'nvarchar', 'text', 'ntext')";
+
+ pkSql = @"
+ SELECT tc.TABLE_SCHEMA AS TableSchema, tc.TABLE_NAME AS TableName, kcu.COLUMN_NAME AS ColumnName,
+ col.DATA_TYPE AS DataType
+ FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc
+ INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
+ ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME
+ AND tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA
+ AND tc.TABLE_NAME = kcu.TABLE_NAME
+ INNER JOIN INFORMATION_SCHEMA.COLUMNS col
+ ON col.TABLE_SCHEMA = kcu.TABLE_SCHEMA
+ AND col.TABLE_NAME = kcu.TABLE_NAME
+ AND col.COLUMN_NAME = kcu.COLUMN_NAME
+ WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY'";
+ }
+
+ using (DbConnection conn = _connectionProvider.Create())
+ {
+ await conn.OpenAsync(cancellationToken);
+
+ var columns = (await conn.QueryAsync(new CommandDefinition(columnsSql, cancellationToken: cancellationToken))).ToList();
+ var pkRows = (await conn.QueryAsync(new CommandDefinition(pkSql, cancellationToken: cancellationToken))).ToList();
+
+ // Keep only tables with EXACTLY one primary-key column (needed for keyset paging).
+ var singlePk = pkRows
+ .GroupBy(p => p.TableSchema + "." + p.TableName)
+ .Where(g => g.Count() == 1)
+ .ToDictionary(g => g.Key, g => g.First(), StringComparer.OrdinalIgnoreCase);
+
+ var targets = new List();
+
+ foreach (var group in columns.GroupBy(c => c.TableSchema + "." + c.TableName))
+ {
+ var first = group.First();
+
+ if (ExcludedTables.Contains(first.TableName))
+ continue;
+
+ if (!singlePk.TryGetValue(group.Key, out var pk))
+ continue; // no single-column PK -> cannot page safely
+
+ // The keyset cursor is a string re-bound to the PK's native type; only text,
+ // integer and uuid PKs are supported. Skip anything else (e.g. date/decimal PKs)
+ // so we never emit a query that compares an incompatible type against the cursor.
+ var keyType = ClassifyKeyType(pk);
+ if (keyType == null)
+ continue;
+
+ // Never clean the primary-key column itself.
+ var textColumns = group
+ .Select(c => c.ColumnName)
+ .Where(name => !string.Equals(name, pk.ColumnName, StringComparison.OrdinalIgnoreCase))
+ .ToList();
+
+ if (textColumns.Count == 0)
+ continue;
+
+ targets.Add(new Utf8TextColumnTarget
+ {
+ Schema = first.TableSchema,
+ TableName = first.TableName,
+ PrimaryKeyColumn = pk.ColumnName,
+ PrimaryKeyType = keyType.Value,
+ TextColumns = textColumns
+ });
+ }
+
+ return targets;
+ }
+ }
+ catch (Exception ex)
+ {
+ Logging.LogException(ex);
+ throw;
+ }
+ }
+
+ public async Task GetRowBatchAsync(Utf8TextColumnTarget target, string lastKey, int batchSize, CancellationToken cancellationToken)
+ {
+ try
+ {
+ var table = QualifiedTable(target);
+ var pk = Quote(target.PrimaryKeyColumn);
+ var columnList = string.Join(", ", target.TextColumns.Select(Quote));
+ var hasCursor = !string.IsNullOrEmpty(lastKey);
+ var whereClause = hasCursor ? "WHERE " + pk + " > @lastKey " : string.Empty;
+
+ string sql = IsPostgres
+ ? $"SELECT {pk} AS rg_pk, {columnList} FROM {table} {whereClause}ORDER BY {pk} LIMIT @batchSize"
+ : $"SELECT TOP (@batchSize) {pk} AS rg_pk, {columnList} FROM {table} {whereClause}ORDER BY {pk}";
+
+ var dynamicParameters = new DynamicParameters();
+ dynamicParameters.Add("batchSize", batchSize);
+ if (hasCursor)
+ dynamicParameters.Add("lastKey", ConvertKey(lastKey, target.PrimaryKeyType));
+
+ using (DbConnection conn = _connectionProvider.Create())
+ {
+ await conn.OpenAsync(cancellationToken);
+
+ var rows = await conn.QueryAsync(new CommandDefinition(sql, dynamicParameters, cancellationToken: cancellationToken));
+
+ var batch = new Utf8RowBatch();
+
+ foreach (var item in rows)
+ {
+ var dict = (IDictionary)item;
+ var key = Convert.ToString(dict["rg_pk"], CultureInfo.InvariantCulture);
+
+ var row = new Utf8TextRow { Key = key };
+
+ foreach (var column in target.TextColumns)
+ {
+ dict.TryGetValue(column, out var value);
+ row.Columns[column] = value as string;
+ }
+
+ batch.Rows.Add(row);
+ batch.LastKey = key;
+ }
+
+ return batch;
+ }
+ }
+ catch (Exception ex)
+ {
+ Logging.LogException(ex, "Utf8 cleanup failed reading batch for " + target.Key);
+ throw;
+ }
+ }
+
+ public async Task UpdateRowsAsync(Utf8TextColumnTarget target, IReadOnlyList rows, CancellationToken cancellationToken)
+ {
+ if (rows == null || rows.Count == 0)
+ return 0;
+
+ try
+ {
+ var table = QualifiedTable(target);
+ var pk = Quote(target.PrimaryKeyColumn);
+ var updated = 0;
+
+ using (DbConnection conn = _connectionProvider.Create())
+ {
+ await conn.OpenAsync(cancellationToken);
+
+ using (var transaction = await conn.BeginTransactionAsync(cancellationToken))
+ {
+ foreach (var row in rows)
+ {
+ if (row.Columns.Count == 0)
+ continue;
+
+ var setClauses = new List(row.Columns.Count);
+ var dynamicParameters = new DynamicParameters();
+ var index = 0;
+
+ foreach (var column in row.Columns)
+ {
+ var paramName = "c" + index++;
+ setClauses.Add(Quote(column.Key) + " = @" + paramName);
+ dynamicParameters.Add(paramName, (object)column.Value ?? DBNull.Value);
+ }
+
+ dynamicParameters.Add("rg_key", ConvertKey(row.Key, target.PrimaryKeyType));
+
+ var sql = $"UPDATE {table} SET {string.Join(", ", setClauses)} WHERE {pk} = @rg_key";
+
+ updated += await conn.ExecuteAsync(new CommandDefinition(sql, dynamicParameters, transaction, cancellationToken: cancellationToken));
+ }
+
+ await transaction.CommitAsync(cancellationToken);
+ }
+ }
+
+ return updated;
+ }
+ catch (Exception ex)
+ {
+ Logging.LogException(ex, "Utf8 cleanup failed updating rows for " + target.Key);
+ throw;
+ }
+ }
+
+ public async Task GetProgressAsync(string tableKey, CancellationToken cancellationToken)
+ {
+ try
+ {
+ var sql = $@"
+ SELECT tablename AS TableName, lastprocessedkey AS LastProcessedKey, lastcompletedutc AS LastCompletedUtc,
+ rowsscanned AS RowsScanned, rowsfixed AS RowsFixed, updatedonutc AS UpdatedOnUtc
+ FROM {ProgressTable}
+ WHERE tablename = @tablename";
+
+ var dynamicParameters = new DynamicParameters();
+ dynamicParameters.Add("tablename", tableKey);
+
+ using (DbConnection conn = _connectionProvider.Create())
+ {
+ await conn.OpenAsync(cancellationToken);
+
+ return await conn.QueryFirstOrDefaultAsync(new CommandDefinition(sql, dynamicParameters, cancellationToken: cancellationToken));
+ }
+ }
+ catch (Exception ex)
+ {
+ Logging.LogException(ex);
+ throw;
+ }
+ }
+
+ public async Task SaveProgressAsync(Utf8CleanupProgress progress, CancellationToken cancellationToken)
+ {
+ try
+ {
+ // Atomic upsert: overlapping cleanup runs (long sweeps or multiple workers) can save the
+ // first watermark for the same table concurrently, which the old UPDATE-then-INSERT raced
+ // on (duplicate-PK on the second INSERT). The tablename PK makes both forms below race-safe.
+ string upsertSql;
+
+ if (IsPostgres)
+ {
+ upsertSql = $@"
+ INSERT INTO {ProgressTable} (tablename, lastprocessedkey, lastcompletedutc, rowsscanned, rowsfixed, updatedonutc)
+ VALUES (@TableName, @LastProcessedKey, @LastCompletedUtc, @RowsScanned, @RowsFixed, @UpdatedOnUtc)
+ ON CONFLICT (tablename) DO UPDATE SET
+ lastprocessedkey = EXCLUDED.lastprocessedkey,
+ lastcompletedutc = EXCLUDED.lastcompletedutc,
+ rowsscanned = EXCLUDED.rowsscanned,
+ rowsfixed = EXCLUDED.rowsfixed,
+ updatedonutc = EXCLUDED.updatedonutc";
+ }
+ else
+ {
+ // UPDLOCK + SERIALIZABLE takes a key-range lock so the row-absent case serializes:
+ // one run inserts, the other blocks then updates. No MERGE (its upsert races are well known).
+ upsertSql = $@"
+ SET XACT_ABORT ON;
+ BEGIN TRANSACTION;
+ UPDATE {ProgressTable} WITH (UPDLOCK, SERIALIZABLE)
+ SET lastprocessedkey = @LastProcessedKey, lastcompletedutc = @LastCompletedUtc,
+ rowsscanned = @RowsScanned, rowsfixed = @RowsFixed, updatedonutc = @UpdatedOnUtc
+ WHERE tablename = @TableName;
+ IF @@ROWCOUNT = 0
+ INSERT INTO {ProgressTable} (tablename, lastprocessedkey, lastcompletedutc, rowsscanned, rowsfixed, updatedonutc)
+ VALUES (@TableName, @LastProcessedKey, @LastCompletedUtc, @RowsScanned, @RowsFixed, @UpdatedOnUtc);
+ COMMIT TRANSACTION;";
+ }
+
+ using (DbConnection conn = _connectionProvider.Create())
+ {
+ await conn.OpenAsync(cancellationToken);
+
+ await conn.ExecuteAsync(new CommandDefinition(upsertSql, progress, cancellationToken: cancellationToken));
+ }
+ }
+ catch (Exception ex)
+ {
+ Logging.LogException(ex);
+ throw;
+ }
+ }
+
+ ///
+ /// Maps a primary key column's declared type to the pageable category used for keyset cursors,
+ /// or null when the type cannot be paged with a string cursor (e.g. date/decimal PKs).
+ ///
+ private static Utf8PrimaryKeyType? ClassifyKeyType(MetaColumn pk)
+ {
+ var udtName = (pk.UdtName ?? string.Empty).ToLowerInvariant();
+ if (udtName == "citext")
+ return Utf8PrimaryKeyType.Text;
+
+ var dataType = (pk.DataType ?? string.Empty).ToLowerInvariant();
+ switch (dataType)
+ {
+ case "char":
+ case "varchar":
+ case "nchar":
+ case "nvarchar":
+ case "text":
+ case "ntext":
+ case "character":
+ case "character varying":
+ return Utf8PrimaryKeyType.Text;
+
+ case "tinyint":
+ case "smallint":
+ case "int":
+ case "integer":
+ case "bigint":
+ return Utf8PrimaryKeyType.Integer;
+
+ case "uniqueidentifier":
+ case "uuid":
+ return Utf8PrimaryKeyType.Guid;
+
+ default:
+ return null; // unsupported PK type -> table is skipped
+ }
+ }
+
+ ///
+ /// Re-binds a string cursor to the primary key's native CLR type so the keyset > / =
+ /// predicates compare like types (PostgreSQL has no implicit text-to-integer/uuid coercion).
+ ///
+ private static object ConvertKey(string key, Utf8PrimaryKeyType keyType)
+ {
+ if (string.IsNullOrEmpty(key))
+ return key;
+
+ switch (keyType)
+ {
+ case Utf8PrimaryKeyType.Integer:
+ return long.Parse(key, CultureInfo.InvariantCulture);
+ case Utf8PrimaryKeyType.Guid:
+ return Guid.Parse(key);
+ default:
+ return key;
+ }
+ }
+
+ private class MetaColumn
+ {
+ public string TableSchema { get; set; }
+ public string TableName { get; set; }
+ public string ColumnName { get; set; }
+ public string DataType { get; set; }
+ public string UdtName { get; set; }
+ }
+ }
+}
diff --git a/Repositories/Resgrid.Repositories.DataRepository/Utf8WriteGuard.cs b/Repositories/Resgrid.Repositories.DataRepository/Utf8WriteGuard.cs
new file mode 100644
index 000000000..7371480ff
--- /dev/null
+++ b/Repositories/Resgrid.Repositories.DataRepository/Utf8WriteGuard.cs
@@ -0,0 +1,28 @@
+using System.Text;
+using Resgrid.Config;
+using Resgrid.Framework;
+
+namespace Resgrid.Repositories.DataRepository
+{
+ ///
+ /// Central, config-gated entry point for scrubbing entity string values before they are bound
+ /// to a SQL command via Dapper's DynamicParameters(entity) (which bypasses
+ /// ). Keeps the UTF-8 guard logic in one place.
+ ///
+ internal static class Utf8WriteGuard
+ {
+ ///
+ /// In-place sanitizes the string properties of when UTF-8 guarding
+ /// is enabled. No-op when disabled or when the entity is null.
+ ///
+ public static void Sanitize(object entity)
+ {
+ if (entity == null || !SystemBehaviorConfig.SanitizeTextForUtf8)
+ return;
+
+ Utf8Sanitizer.CleanEntity(entity,
+ SystemBehaviorConfig.Utf8RepairDoubleEncoding,
+ SystemBehaviorConfig.Utf8NormalizeToNfc ? NormalizationForm.FormC : (NormalizationForm?)null);
+ }
+ }
+}
diff --git a/Tests/Resgrid.Tests/Framework/Utf8SanitizerTests.cs b/Tests/Resgrid.Tests/Framework/Utf8SanitizerTests.cs
new file mode 100644
index 000000000..1acc3c199
--- /dev/null
+++ b/Tests/Resgrid.Tests/Framework/Utf8SanitizerTests.cs
@@ -0,0 +1,206 @@
+using System.Text;
+using FluentAssertions;
+using NUnit.Framework;
+using Resgrid.Framework;
+
+namespace Resgrid.Tests.Framework
+{
+ ///
+ /// Verifies produces content that is always safe for a PostgreSQL
+ /// UTF-8 column: no NUL, no unpaired surrogates, with best-effort Windows-1252 mojibake repair,
+ /// while leaving already-clean text untouched (and allocation-free).
+ ///
+ /// Non-ASCII characters are built from explicit (char)0xNN code points so the byte-level intent
+ /// is exact and unambiguous.
+ ///
+ [TestFixture]
+ public class Utf8SanitizerTests
+ {
+ private const char Win1252RightQuote = (char)0x92; // repairs to U+2019
+ private const char Win1252EnDash = (char)0x96; // repairs to U+2013
+ private const char Undefined1252 = (char)0x81; // dropped (no Windows-1252 mapping)
+ private const char RightSingleQuote = (char)0x2019;
+ private const char EnDash = (char)0x2013;
+ private const char HighSurrogate = (char)0xD83D; // lead of U+1F600
+ private const char LowSurrogate = (char)0xDE00; // trail of U+1F600
+ private const char CombiningAcute = (char)0x0301;
+ private const char EAcute = (char)0x00E9; // precomposed "é"
+ private const char UDiaeresis = (char)0x00FC; // "ü"
+ private const char CapitalATilde = (char)0x00C3; // "Ã"
+ private const char CopyrightSign = (char)0x00A9; // "©" (second byte of double-encoded é)
+
+ [Test]
+ public void Clean_Null_ReturnsNull()
+ {
+ Utf8Sanitizer.Clean(null).Should().BeNull();
+ }
+
+ [Test]
+ public void Clean_Empty_ReturnsEmpty()
+ {
+ Utf8Sanitizer.Clean(string.Empty).Should().BeEmpty();
+ }
+
+ [Test]
+ public void Clean_AlreadyCleanString_ReturnsSameReference()
+ {
+ // ASCII + Latin-1 accented chars (above the C1 range) + a valid emoji surrogate pair.
+ var input = "Engine 51 responding " + EAcute + UDiaeresis + " " + HighSurrogate + LowSurrogate;
+
+ var result = Utf8Sanitizer.Clean(input);
+
+ result.Should().BeSameAs(input);
+ Utf8Sanitizer.IsClean(input).Should().BeTrue();
+ }
+
+ [Test]
+ public void Clean_StripsNulCharacters()
+ {
+ var input = "a\0b\0c";
+
+ var result = Utf8Sanitizer.Clean(input);
+
+ result.Should().Be("abc");
+ result.Should().NotContain("\0");
+ }
+
+ [Test]
+ public void Clean_ReplacesUnpairedHighSurrogate()
+ {
+ var input = "before" + HighSurrogate + "after"; // high surrogate with no trailing low surrogate
+
+ var result = Utf8Sanitizer.Clean(input);
+
+ result.Should().Be("before" + Utf8Sanitizer.ReplacementChar + "after");
+ }
+
+ [Test]
+ public void Clean_ReplacesLoneLowSurrogate()
+ {
+ var input = "x" + LowSurrogate + "y"; // low surrogate with no preceding high surrogate
+
+ var result = Utf8Sanitizer.Clean(input);
+
+ result.Should().Be("x" + Utf8Sanitizer.ReplacementChar + "y");
+ }
+
+ [Test]
+ public void Clean_PreservesValidSurrogatePair()
+ {
+ var input = "grin " + HighSurrogate + LowSurrogate + " done"; // U+1F600
+
+ var result = Utf8Sanitizer.Clean(input);
+
+ result.Should().BeSameAs(input);
+ }
+
+ [Test]
+ public void Clean_RepairsWindows1252SmartQuote()
+ {
+ var input = "It" + Win1252RightQuote + "s here";
+
+ var result = Utf8Sanitizer.Clean(input);
+
+ result.Should().Be("It" + RightSingleQuote + "s here");
+ }
+
+ [Test]
+ public void Clean_RepairsWindows1252EnDash()
+ {
+ var input = "9" + Win1252EnDash + "10";
+
+ Utf8Sanitizer.Clean(input).Should().Be("9" + EnDash + "10");
+ }
+
+ [Test]
+ public void Clean_DropsUndefinedWindows1252Bytes()
+ {
+ var input = "a" + Undefined1252 + "b";
+
+ Utf8Sanitizer.Clean(input).Should().Be("ab");
+ }
+
+ [Test]
+ public void TryClean_ReturnsFalseForCleanInput()
+ {
+ var changed = Utf8Sanitizer.TryClean("plain text", out var cleaned);
+
+ changed.Should().BeFalse();
+ cleaned.Should().Be("plain text");
+ }
+
+ [Test]
+ public void TryClean_ReturnsTrueWhenChanged()
+ {
+ var changed = Utf8Sanitizer.TryClean("bad\0value", out var cleaned);
+
+ changed.Should().BeTrue();
+ cleaned.Should().Be("badvalue");
+ }
+
+ [Test]
+ public void Clean_Output_IsAlwaysValidUtf8WithoutNul()
+ {
+ // NUL + lone high surrogate + C1 mojibake + lone low surrogate.
+ var input = "mix\0" + HighSurrogate + " C1" + Win1252EnDash + " " + LowSurrogate + "tail";
+
+ var result = Utf8Sanitizer.Clean(input);
+
+ result.Should().NotContain("\0");
+
+ var strict = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
+ var act = () => strict.GetByteCount(result);
+ act.Should().NotThrow();
+ }
+
+ [Test]
+ public void Clean_AppliesNfcNormalizationWhenRequested()
+ {
+ // "e" + combining acute accent; NFC composes to precomposed "é" (U+00E9).
+ var input = "e" + CombiningAcute;
+
+ var withoutNorm = Utf8Sanitizer.Clean(input);
+ var withNorm = Utf8Sanitizer.Clean(input, repairDoubleEncoding: false, normalization: NormalizationForm.FormC);
+
+ withoutNorm.Should().Be(input);
+ withNorm.Should().Be(string.Empty + EAcute);
+ }
+
+ [Test]
+ public void Clean_DoubleEncodingRepair_OptIn()
+ {
+ // CapitalATilde + CopyrightSign is the double-encoded (Latin-1-read) form of "é" (U+00E9).
+ var input = "caf" + CapitalATilde + CopyrightSign;
+
+ Utf8Sanitizer.Clean(input).Should().Be(input); // off by default
+ Utf8Sanitizer.Clean(input, repairDoubleEncoding: true).Should().Be("caf" + EAcute);
+ }
+
+ [Test]
+ public void CleanEntity_ScrubsStringProperties()
+ {
+ var entity = new SampleEntity
+ {
+ Name = "Unit\0 1",
+ Notes = "Quote" + Win1252RightQuote + "s",
+ Ignored = null,
+ Number = 42
+ };
+
+ Utf8Sanitizer.CleanEntity(entity);
+
+ entity.Name.Should().Be("Unit 1");
+ entity.Notes.Should().Be("Quote" + RightSingleQuote + "s");
+ entity.Ignored.Should().BeNull();
+ entity.Number.Should().Be(42);
+ }
+
+ private class SampleEntity
+ {
+ public string Name { get; set; }
+ public string Notes { get; set; }
+ public string Ignored { get; set; }
+ public int Number { get; set; }
+ }
+ }
+}
diff --git a/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs b/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs
new file mode 100644
index 000000000..205a960a1
--- /dev/null
+++ b/Tools/Resgrid.Console/Commands/CleanUtf8Command.cs
@@ -0,0 +1,48 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Consolas2.Core;
+using Microsoft.Extensions.Logging;
+using Resgrid.Console.Models;
+using Resgrid.Model.Repositories;
+using Resgrid.Workers.Framework.Logic;
+
+namespace Resgrid.Console.Commands
+{
+ ///
+ /// On-demand UTF-8 data cleanup. Runs the same sweep as the nightly worker so the database can
+ /// be made migration-clean ad hoc (e.g. immediately before a SQL Server -> PostgreSQL cutover).
+ ///
+ public sealed class CleanUtf8Command(
+ ILogger logger,
+ IUtf8MaintenanceRepository utf8MaintenanceRepository) : ICommandService
+ {
+ public async Task ExecuteMainAsync(string[] args, CancellationToken cancellationToken)
+ {
+ logger.LogInformation("Starting the Resgrid UTF-8 Data Cleanup (pre-migration sweep)");
+ logger.LogInformation("Please Wait...");
+
+ try
+ {
+ var logic = new Utf8CleanupLogic(utf8MaintenanceRepository);
+ var result = await logic.Process(cancellationToken);
+
+ if (!result.Item1)
+ {
+ logger.LogError("UTF-8 data cleanup did not complete successfully: " + result.Item2);
+ return ExitCode.Failed;
+ }
+
+ logger.LogInformation(result.Item2);
+ logger.LogInformation("Completed the Resgrid UTF-8 Data Cleanup!");
+ }
+ catch (Exception ex)
+ {
+ Resgrid.Framework.Logging.LogException(ex, "There was an error running the UTF-8 data cleanup");
+ return ExitCode.Failed;
+ }
+
+ return ExitCode.Success;
+ }
+ }
+}
diff --git a/Tools/Resgrid.Console/Program.cs b/Tools/Resgrid.Console/Program.cs
index 071ec32cc..f75fe44c9 100644
--- a/Tools/Resgrid.Console/Program.cs
+++ b/Tools/Resgrid.Console/Program.cs
@@ -107,6 +107,7 @@ static async Task Main(string[] args)
services.AddKeyedTransient("DbUpdateCommand");
services.AddKeyedTransient("GenOidcCertsCommand");
services.AddKeyedTransient("MigrateDocsDbCommand");
+ services.AddKeyedTransient("CleanUtf8Command");
services.AddKeyedTransient("OidcUpdateCommand");
services.AddKeyedTransient("SecurityRefreshCommand");
services.AddKeyedTransient("HelpCommand");
diff --git a/Tools/Resgrid.Console/Services/ApplicationHostedService.cs b/Tools/Resgrid.Console/Services/ApplicationHostedService.cs
index a0b003684..7239c2fc3 100644
--- a/Tools/Resgrid.Console/Services/ApplicationHostedService.cs
+++ b/Tools/Resgrid.Console/Services/ApplicationHostedService.cs
@@ -30,6 +30,7 @@ public sealed class ApplicationHostedService : IHostedService, IDisposable
private ICommandService _dbUpdateCommand;
private ICommandService _genOidcCertsCommand;
private ICommandService _migrateDocsDbCommand;
+ private ICommandService _cleanUtf8Command;
private ICommandService _oidcUpdateCommand;
private ICommandService _securityRefreshCommand;
private ICommandService _helpCommand;
@@ -55,6 +56,7 @@ public ApplicationHostedService(
[FromKeyedServices("DbUpdateCommand")] ICommandService dbUpdateCommand,
[FromKeyedServices("GenOidcCertsCommand")] ICommandService genOidcCertsCommand,
[FromKeyedServices("MigrateDocsDbCommand")] ICommandService migrateDocsDbCommand,
+ [FromKeyedServices("CleanUtf8Command")] ICommandService cleanUtf8Command,
[FromKeyedServices("OidcUpdateCommand")] ICommandService oidcUpdateCommand,
[FromKeyedServices("SecurityRefreshCommand")] ICommandService securityRefreshCommand,
[FromKeyedServices("HelpCommand")] ICommandService helpCommand)
@@ -67,6 +69,7 @@ public ApplicationHostedService(
_dbUpdateCommand = dbUpdateCommand;
_genOidcCertsCommand = genOidcCertsCommand;
_migrateDocsDbCommand = migrateDocsDbCommand;
+ _cleanUtf8Command = cleanUtf8Command;
_oidcUpdateCommand = oidcUpdateCommand;
_securityRefreshCommand = securityRefreshCommand;
_helpCommand = helpCommand;
@@ -192,6 +195,8 @@ private async Task ExecuteMainAsync(string[] args, CancellationToken c
return await _genOidcCertsCommand.ExecuteMainAsync(args, cancellationToken).ConfigureAwait(false);
else if (args.Contains("--MigrateDocsDb"))
return await _migrateDocsDbCommand.ExecuteMainAsync(args, cancellationToken).ConfigureAwait(false);
+ else if (args.Contains("--CleanUtf8"))
+ return await _cleanUtf8Command.ExecuteMainAsync(args, cancellationToken).ConfigureAwait(false);
else if (args.Contains("--OidcUpdate"))
return await _oidcUpdateCommand.ExecuteMainAsync(args, cancellationToken).ConfigureAwait(false);
else if (args.Contains("--SecurityRefresh"))
diff --git a/Workers/Resgrid.Workers.Console/Commands/Utf8CleanupCommand.cs b/Workers/Resgrid.Workers.Console/Commands/Utf8CleanupCommand.cs
new file mode 100644
index 000000000..3a43a4d90
--- /dev/null
+++ b/Workers/Resgrid.Workers.Console/Commands/Utf8CleanupCommand.cs
@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using Quidjibo.Commands;
+
+namespace Resgrid.Workers.Console.Commands
+{
+ public class Utf8CleanupCommand : IQuidjiboCommand
+ {
+ public int Id { get; }
+ public Guid? CorrelationId { get; set; }
+ public Dictionary Metadata { get; set; }
+
+ public Utf8CleanupCommand(int id)
+ {
+ Id = id;
+ }
+ }
+}
diff --git a/Workers/Resgrid.Workers.Console/Program.cs b/Workers/Resgrid.Workers.Console/Program.cs
index e5088e834..f319a208e 100644
--- a/Workers/Resgrid.Workers.Console/Program.cs
+++ b/Workers/Resgrid.Workers.Console/Program.cs
@@ -420,6 +420,19 @@ await Client.ScheduleAsync("Reporting Rollup",
new Commands.ReportingRollupCommand(21),
Cron.Daily(3, 30),
stoppingToken);
+
+ if (SystemBehaviorConfig.Utf8CleanupEnabled)
+ {
+ var utf8CleanupHour = SystemBehaviorConfig.Utf8CleanupHourUtc >= 0 && SystemBehaviorConfig.Utf8CleanupHourUtc <= 23
+ ? SystemBehaviorConfig.Utf8CleanupHourUtc
+ : 4;
+
+ _logger.Log(LogLevel.Information, "Scheduling UTF-8 Data Cleanup");
+ await Client.ScheduleAsync("UTF-8 Data Cleanup",
+ new Commands.Utf8CleanupCommand(22),
+ Cron.Daily(utf8CleanupHour, 0),
+ stoppingToken);
+ }
}
else
{
diff --git a/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs b/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs
new file mode 100644
index 000000000..aae287a39
--- /dev/null
+++ b/Workers/Resgrid.Workers.Console/Tasks/Utf8CleanupTask.cs
@@ -0,0 +1,46 @@
+using Microsoft.Extensions.Logging;
+using Quidjibo.Handlers;
+using Quidjibo.Misc;
+using Resgrid.Workers.Console.Commands;
+using Resgrid.Workers.Framework.Logic;
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Resgrid.Workers.Console.Tasks
+{
+ public class Utf8CleanupTask : IQuidjiboHandler
+ {
+ public string Name => "UTF-8 Data Cleanup";
+ public int Priority => 1;
+ private readonly ILogger _logger;
+
+ public Utf8CleanupTask(ILogger logger)
+ {
+ _logger = logger;
+ }
+
+ public async Task ProcessAsync(Utf8CleanupCommand command, IQuidjiboProgress progress, CancellationToken cancellationToken)
+ {
+ try
+ {
+ progress.Report(1, $"Starting the {Name} Task");
+
+ var logic = new Utf8CleanupLogic();
+ var result = await logic.Process(cancellationToken);
+
+ if (result.Item1)
+ _logger.LogInformation("Utf8Cleanup::" + result.Item2);
+ else
+ _logger.LogError("Utf8Cleanup::Failed: " + result.Item2);
+
+ progress.Report(100, $"Finishing the {Name} Task");
+ }
+ catch (Exception ex)
+ {
+ Resgrid.Framework.Logging.LogException(ex);
+ _logger.LogError(ex.ToString());
+ }
+ }
+ }
+}
diff --git a/Workers/Resgrid.Workers.Framework/Logic/Utf8CleanupLogic.cs b/Workers/Resgrid.Workers.Framework/Logic/Utf8CleanupLogic.cs
new file mode 100644
index 000000000..4b2f144a7
--- /dev/null
+++ b/Workers/Resgrid.Workers.Framework/Logic/Utf8CleanupLogic.cs
@@ -0,0 +1,148 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Autofac;
+using Resgrid.Config;
+using Resgrid.Framework;
+using Resgrid.Model.Repositories;
+
+namespace Resgrid.Workers.Framework.Logic
+{
+ ///
+ /// Sweeps every free-form text column in the database and repairs content that would block a
+ /// SQL Server -> PostgreSQL (UTF-8) migration (NUL, unpaired surrogates, Windows-1252 mojibake).
+ /// Resumable via a per-table watermark, idempotent (clean values are no-ops), and bounded by
+ /// configuration so a single run never scans an unbounded number of rows.
+ ///
+ public class Utf8CleanupLogic
+ {
+ private readonly IUtf8MaintenanceRepository _repository;
+
+ public Utf8CleanupLogic()
+ : this(Bootstrapper.GetKernel().Resolve())
+ {
+ }
+
+ public Utf8CleanupLogic(IUtf8MaintenanceRepository repository)
+ {
+ _repository = repository;
+ }
+
+ public async Task> Process(CancellationToken cancellationToken)
+ {
+ if (!SystemBehaviorConfig.Utf8CleanupEnabled)
+ return new Tuple(true, "UTF-8 cleanup is disabled by configuration.");
+
+ try
+ {
+ var batchSize = SystemBehaviorConfig.Utf8CleanupBatchSize > 0 ? SystemBehaviorConfig.Utf8CleanupBatchSize : 1000;
+ var maxRows = SystemBehaviorConfig.Utf8CleanupMaxRowsPerRun; // 0 == unbounded
+ var repairDoubleEncoding = SystemBehaviorConfig.Utf8RepairDoubleEncoding;
+ NormalizationForm? normalization = SystemBehaviorConfig.Utf8NormalizeToNfc ? NormalizationForm.FormC : (NormalizationForm?)null;
+
+ var targets = await _repository.GetTextColumnTargetsAsync(cancellationToken);
+
+ long totalScanned = 0;
+ long totalFixed = 0;
+
+ foreach (var target in targets)
+ {
+ if (cancellationToken.IsCancellationRequested)
+ break;
+
+ if (maxRows > 0 && totalScanned >= maxRows)
+ break;
+
+ var progress = await _repository.GetProgressAsync(target.Key, cancellationToken)
+ ?? new Utf8CleanupProgress { TableName = target.Key };
+
+ var lastKey = progress.LastProcessedKey;
+
+ while (true)
+ {
+ if (cancellationToken.IsCancellationRequested)
+ break;
+
+ if (maxRows > 0 && totalScanned >= maxRows)
+ break;
+
+ var batch = await _repository.GetRowBatchAsync(target, lastKey, batchSize, cancellationToken);
+
+ if (batch.Rows.Count == 0)
+ {
+ MarkTableComplete(progress);
+ await _repository.SaveProgressAsync(progress, cancellationToken);
+ break;
+ }
+
+ var changedRows = new List();
+
+ foreach (var row in batch.Rows)
+ {
+ Utf8TextRow changed = null;
+
+ foreach (var column in row.Columns)
+ {
+ if (column.Value == null)
+ continue;
+
+ if (Utf8Sanitizer.TryClean(column.Value, out var cleaned, repairDoubleEncoding, normalization))
+ {
+ if (changed == null)
+ changed = new Utf8TextRow { Key = row.Key };
+
+ changed.Columns[column.Key] = cleaned;
+ }
+ }
+
+ if (changed != null)
+ changedRows.Add(changed);
+ }
+
+ if (changedRows.Count > 0)
+ {
+ var updated = await _repository.UpdateRowsAsync(target, changedRows, cancellationToken);
+ totalFixed += updated;
+ progress.RowsFixed += updated;
+ }
+
+ totalScanned += batch.Rows.Count;
+ progress.RowsScanned += batch.Rows.Count;
+ lastKey = batch.LastKey;
+ progress.LastProcessedKey = lastKey;
+ progress.UpdatedOnUtc = DateTime.UtcNow;
+ await _repository.SaveProgressAsync(progress, cancellationToken);
+
+ if (batch.Rows.Count < batchSize)
+ {
+ MarkTableComplete(progress);
+ await _repository.SaveProgressAsync(progress, cancellationToken);
+ break;
+ }
+ }
+ }
+
+ var summary = $"UTF-8 cleanup scanned {totalScanned} row(s) and repaired {totalFixed} across {targets.Count} table(s).";
+ Logging.LogInfo(summary);
+
+ return new Tuple(true, summary);
+ }
+ catch (Exception ex)
+ {
+ Logging.LogException(ex);
+ return new Tuple(false, ex.ToString());
+ }
+ }
+
+ private static void MarkTableComplete(Utf8CleanupProgress progress)
+ {
+ // Reset the cursor so the next scheduled run re-sweeps the table from the start
+ // (catching anything written since this pass) while staying idempotent.
+ progress.LastProcessedKey = null;
+ progress.LastCompletedUtc = DateTime.UtcNow;
+ progress.UpdatedOnUtc = DateTime.UtcNow;
+ }
+ }
+}