fix: export backups from a consistent database snapshot

docs/adr/0001-system-configuration-backup-restore.md

@@ -228,6 +228,15 @@ operator to check the operation ID before repeating the restore. Request cancell
 skip this commit reconciliation.
 
 ### K9 — Export only the runtime overrides, as raw file content
+
+Export captures all database sections and their counts in one Serializable transaction,
+inside the EF execution strategy. A retry recaptures every section. PostgreSQL uses its
+transaction snapshot; SQL Server holds serializable read/range locks through capture.
+Runtime overrides are captured once from the atomically replaced file; their count is
+derived from that captured document. The file and database are separate snapshots, not
+a distributed transaction. Encryption and archive serialization finish after the database
+transaction is released. Any capture or transaction failure prevents returning an archive.
+
 What is exported is **only** the **raw JSON content** of `appsettings.runtime.json` (the
 database/file overrides), **not** `IConfigurationRoot` — that one is already decrypted and merged
 with env, CLI and appsettings.json, which would drag host and environment secrets into the file.

src/NodePilot.Api/Services/Backup/BackupService.cs

@@ -1,8 +1,11 @@
 using System.Reflection;
+using System.Data;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using NodePilot.Data.Security;
+using NodePilot.Data;
+using Microsoft.EntityFrameworkCore;
 
 namespace NodePilot.Api.Services.Backup;
 
@@ -29,7 +32,7 @@ public sealed class BackupExportException(string message) : Exception(message);
 /// <see cref="IBackupPart"/>, assembles the current versioned system-backup envelope, and seals it
 /// and authenticates that complete payload with passphrase-derived AES-GCM.
 /// </summary>
-public sealed class BackupService(IEnumerable<IBackupPart> parts)
+public sealed class BackupService(IEnumerable<IBackupPart> parts, NodePilotDbContext db)
 {
     /// <summary>
     /// Minimum backup passphrase length. A backup is a portable secret-bearing artifact.
@@ -87,21 +90,32 @@ public async Task<BackupExportResult> ExportAsync(
 
         var salt = PassphraseSecretProtector.GenerateSalt();
         var protector = PassphraseSecretProtector.Derive(passphrase, salt);
-        var ctx = new BackupExportContext { Protector = protector };
-
-        var sections = new JsonObject();
         var includedOrdered = SectionOrder.Where(effective.Contains).ToList();
-        var counts = new List<BackupSectionCount>(includedOrdered.Count);
-        foreach (var key in includedOrdered)
+        var strategy = db.Database.CreateExecutionStrategy();
+        var (sections, counts) = await strategy.ExecuteAsync(async () =>
         {
-            counts.Add(new BackupSectionCount(key, await byKey[key].CountAsync(ct)));
-            sections[key] = await byKey[key].ExportAsync(ctx, ct);
-        }
-
-        if (ctx.Warnings.Count > 0)
-            throw new BackupExportException(
-                "Backup aborted because one or more selected values could not be exported completely: "
-                + string.Join(" | ", ctx.Warnings));
+            var ctx = new BackupExportContext { Protector = protector };
+            var captured = new JsonObject();
+            var capturedCounts = new List<BackupSectionCount>(includedOrdered.Count);
+            // Serializable prevents cross-section phantoms on both PostgreSQL and SQL Server.
+            await using var snapshot = await db.Database.BeginTransactionAsync(IsolationLevel.Serializable, ct);
+            foreach (var key in includedOrdered)
+            {
+                captured[key] = await byKey[key].ExportAsync(ctx, ct);
+                // Runtime settings are an independent atomic file snapshot, outside DB isolation.
+                var count = key == BackupSections.Settings
+                    ? (captured[key]?["runtimeJson"] as JsonObject)?.Count ?? 0
+                    : await byKey[key].CountAsync(ct);
+                capturedCounts.Add(new BackupSectionCount(key, count));
+            }
+
+            if (ctx.Warnings.Count > 0)
+                throw new BackupExportException(
+                    "Backup aborted because one or more selected values could not be exported completely: "
+                    + string.Join(" | ", ctx.Warnings));
+            await snapshot.CommitAsync(ct);
+            return (captured, capturedCounts);
+        });
 
         var payload = new JsonObject
         {

tests/NodePilot.Api.Tests/Controllers/BackupControllerTests.cs

@@ -54,7 +54,7 @@ public BackupControllerTests()
             new CustomActivityBackupPart(new CustomActivityDefinitionStore(_db)),
             new WorkflowBackupPart(_db),
             new SettingsBackupPart(overrides, _atRest),
-        ]);
+        ], _db);
         _restore = new BackupRestoreService(
             _db, _atRest, overrides, NullLogger<BackupRestoreService>.Instance,
             new NodePilot.Api.Services.WorkflowVersionDefinitionProtector(

tests/NodePilot.Api.Tests/Hosting/DatabaseRecoveryAuditServiceTests.cs

@@ -69,8 +69,7 @@ private void MaybeThrow(DbContextEventData eventData)
     [Fact]
     public async Task RealOutageRecovery_PersistsExactlyOneEpisodeAudit_AndNoTripAudit()
     {
-        using var connection = new SqliteConnection("DataSource=:memory:");
-        connection.Open();
+        using var connection = OpenDatabase();
         await using (var schema = NewDb(connection))
             await schema.Database.EnsureCreatedAsync();
 
@@ -122,8 +121,7 @@ public async Task RealOutageRecovery_PersistsExactlyOneEpisodeAudit_AndNoTripAud
     [Fact]
     public async Task TwoRealOutageRecoveries_PersistOneDistinctAuditPerEpisode()
     {
-        using var connection = new SqliteConnection("DataSource=:memory:");
-        connection.Open();
+        using var connection = OpenDatabase();
         await using (var schema = NewDb(connection))
             await schema.Database.EnsureCreatedAsync();
 
@@ -171,8 +169,7 @@ public async Task TwoRealOutageRecoveries_PersistOneDistinctAuditPerEpisode()
     public async Task RecoveryAuditWriteFailure_RetriesOnFreshContextWithoutDuplicatingUnknownCommit(
         bool commitBeforeAcknowledgementIsLost)
     {
-        using var connection = new SqliteConnection("DataSource=:memory:");
-        connection.Open();
+        using var connection = OpenDatabase();
         await using (var schema = NewDb(connection))
             await schema.Database.EnsureCreatedAsync();
 
@@ -189,7 +186,7 @@ public async Task RecoveryAuditWriteFailure_RetriesOnFreshContextWithoutDuplicat
             Interlocked.Increment(ref contextsCreated);
             return new NodePilotDbContext(
                 new DbContextOptionsBuilder<NodePilotDbContext>()
-                    .UseSqlite(connection)
+                    .UseSqlite(connection.ConnectionString)
                     .AddInterceptors(failure)
                     .Options);
         });
@@ -231,8 +228,16 @@ public async Task RecoveryAuditWriteFailure_RetriesOnFreshContextWithoutDuplicat
         }
     }
 
+    private static SqliteConnection OpenDatabase()
+    {
+        // Keep the shared in-memory database alive; each concurrent context owns its connection.
+        var connection = new SqliteConnection($"Data Source=recovery-audit-{Guid.NewGuid():N};Mode=Memory;Cache=Shared;Pooling=False");
+        connection.Open();
+        return connection;
+    }
+
     private static NodePilotDbContext NewDb(SqliteConnection connection)
-        => new(new DbContextOptionsBuilder<NodePilotDbContext>().UseSqlite(connection).Options);
+        => new(new DbContextOptionsBuilder<NodePilotDbContext>().UseSqlite(connection.ConnectionString).Options);
 
     private static async Task<List<AuditLogEntry>> WaitForAuditRowsAsync(
         SqliteConnection connection,

tests/NodePilot.Api.Tests/Security/ProvisioningSeederTests.cs

@@ -60,7 +60,7 @@ private async Task<byte[]> BuildSeedAsync(string username = "seeded-admin")
         });
         await source.SaveChangesAsync();
 
-        var export = await new BackupService([new UserBackupPart(source)])
+        var export = await new BackupService([new UserBackupPart(source)], source)
             .ExportAsync([BackupSections.Users], Passphrase, "admin", CancellationToken.None);
         return export.Content;
     }

tests/NodePilot.Api.Tests/Services/Backup/BackupAlertingTests.cs

@@ -54,7 +54,7 @@ private IBackupPart[] Parts(NodePilotDbContext db) =>
     ];
 
     private async Task<byte[]> ExportAsync(NodePilotDbContext db, List<string> sections)
-        => (await new BackupService(Parts(db)).ExportAsync(sections, Passphrase, "admin", CancellationToken.None)).Content;
+        => (await new BackupService(Parts(db), db).ExportAsync(sections, Passphrase, "admin", CancellationToken.None)).Content;
 
     private BackupRestoreService Restore(NodePilotDbContext db) =>
         new(db, _atRest, new RuntimeOverridesWriter(TempPath(), NullLogger<RuntimeOverridesWriter>.Instance),

tests/NodePilot.Api.Tests/Services/Backup/BackupRestoreServiceTests.cs

@@ -74,7 +74,7 @@ private NodePilot.Api.Services.WorkflowVersionDefinitionProtector VersionProtect
         new(_atRest, NullLogger<NodePilot.Api.Services.WorkflowVersionDefinitionProtector>.Instance);
 
     private async Task<byte[]> ExportAsync(NodePilotDbContext db, List<string> sections)
-        => (await new BackupService(Parts(db)).ExportAsync(sections, Passphrase, "admin", CancellationToken.None)).Content;
+        => (await new BackupService(Parts(db), db).ExportAsync(sections, Passphrase, "admin", CancellationToken.None)).Content;
 
     private static byte[] MutatePayload(byte[] backup, Action<JsonObject> mutate)
     {
@@ -208,7 +208,7 @@ public async Task Restore_CommitFailure_PreservesSettingsAndLiveScopesAccordingT
         sourceSettings.MutateAndWrite(root => root["Smtp"] = new JsonObject { ["Port"] = 2525 });
         var backup = (await new BackupService(Parts(src)
             .Where(part => part is not SettingsBackupPart)
-            .Append(new SettingsBackupPart(sourceSettings, _atRest)))
+            .Append(new SettingsBackupPart(sourceSettings, _atRest)), src)
             .ExportAsync([BackupSections.Workflows, BackupSections.Settings], Passphrase, "admin", ct)).Content;
 
         using var dst = TestDbFactory.Create();
@@ -1324,7 +1324,8 @@ public async Task Restore_Settings_ReplacesOverrides_RemovingKeysNotInBackup()
         // Backup carries only a Smtp override.
         var srcWriter = new RuntimeOverridesWriter(TempPath(), NullLogger<RuntimeOverridesWriter>.Instance);
         srcWriter.MutateAndWrite(root => root["Smtp"] = new JsonObject { ["Port"] = 2525 });
-        var backup = (await new BackupService([new SettingsBackupPart(srcWriter, _atRest)])
+        using var src = TestDbFactory.Create();
+        var backup = (await new BackupService([new SettingsBackupPart(srcWriter, _atRest)], src)
             .ExportAsync([BackupSections.Settings], Passphrase, "admin", CancellationToken.None)).Content;
 
         // Target has an extra override ("Foo") that is NOT in the backup.

tests/NodePilot.Api.Tests/Services/Backup/BackupServiceExportTests.cs

@@ -52,7 +52,7 @@ public BackupServiceExportTests()
             new WorkflowBackupPart(_db),
             new SettingsBackupPart(_overrides, _atRest),
         };
-        _service = new BackupService(parts);
+        _service = new BackupService(parts, _db);
     }
 
     private static byte[] DeterministicKey()
@@ -371,13 +371,35 @@ public async Task Export_ShortPassphrase_Throws()
     [Fact]
     public async Task Export_AnyPartWarning_AbortsInsteadOfProducingAnIncompleteArchive()
     {
-        var service = new BackupService([new WarningPart()]);
+        var service = new BackupService([new WarningPart()], _db);
 
         var act = () => service.ExportAsync(
             [BackupSections.Users], Passphrase, "admin", CancellationToken.None);
 
         await act.Should().ThrowAsync<BackupExportException>()
             .WithMessage("*could not be exported completely*");
+        _db.Database.CurrentTransaction.Should().BeNull();
+    }
+
+    [Fact]
+    public async Task Export_Settings_CountMatchesTheCapturedFileWithoutASecondRead()
+    {
+        _overrides.MutateAndWrite(root => root["Smtp"] = new JsonObject { ["Port"] = 2525 });
+        var part = new SettingsSnapshotPart(new SettingsBackupPart(_overrides, _atRest));
+        var service = new BackupService([part], _db);
+
+        var result = await service.ExportAsync([BackupSections.Settings], Passphrase, "admin", TestContext.Current.CancellationToken);
+
+        result.Counts.Single().Count.Should().Be(1);
+        Parse(result.Content)["sections"]!["settings"]!["runtimeJson"]!["Smtp"]!["Port"]!.GetValue<int>().Should().Be(2525);
+    }
+
+    private sealed class SettingsSnapshotPart(IBackupPart inner) : IBackupPart
+    {
+        public string Key => inner.Key;
+        public IReadOnlyList<string> DependsOn => inner.DependsOn;
+        public Task<int> CountAsync(CancellationToken ct) => throw new InvalidOperationException("Export must count the captured settings document.");
+        public Task<JsonNode> ExportAsync(BackupExportContext ctx, CancellationToken ct) => inner.ExportAsync(ctx, ct);
     }
 
     private sealed class WarningPart : IBackupPart

tests/NodePilot.Api.Tests/Services/Backup/BackupSnapshotTests.cs

@@ -0,0 +1,100 @@
+using System.Text.Json.Nodes;
+using FluentAssertions;
+using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging.Abstractions;
+using NodePilot.Api.Configuration;
+using NodePilot.Api.Services;
+using NodePilot.Api.Services.Backup;
+using NodePilot.Api.Services.Backup.Parts;
+using NodePilot.Core.Models;
+using NodePilot.Data;
+using NodePilot.Data.Security;
+using NodePilot.TestCommons;
+using Xunit;
+
+namespace NodePilot.Api.Tests.Services.Backup;
+
+public class BackupSnapshotTests
+{
+    [Fact]
+    public async Task Export_ConcurrentCredentialAndMachineCreation_RestoresIntoEmptyDestination()
+    {
+        const string passphrase = "snapshot-backup-passphrase";
+        var databasePath = Path.Combine(Path.GetTempPath(), $"np-backup-snapshot-{Guid.NewGuid():N}.db");
+        var runtimePath = Path.Combine(Path.GetTempPath(), $"np-backup-snapshot-{Guid.NewGuid():N}.json");
+        var options = new DbContextOptionsBuilder<NodePilotDbContext>().UseSqlite(
+            new SqliteConnectionStringBuilder { DataSource = databasePath, Pooling = false, DefaultTimeout = 1 }.ToString()).Options;
+        var atRest = new AesGcmSecretProtector(new byte[32]);
+        var ct = TestContext.Current.CancellationToken;
+        try
+        {
+            await using var source = new NodePilotDbContext(options);
+            await source.Database.EnsureCreatedAsync(ct);
+            var existing = new Credential { Name = "existing", Username = "existing", EncryptedPassword = atRest.Protect("existing-secret") };
+            source.Credentials.Add(existing);
+            source.ManagedMachines.Add(new ManagedMachine { Name = "existing-host", Hostname = "existing.local", DefaultCredentialId = existing.Id });
+            await source.SaveChangesAsync(ct);
+
+            await using var writer = new NodePilotDbContext(options);
+            var concurrent = new Credential { Name = "concurrent", Username = "concurrent", EncryptedPassword = atRest.Protect("concurrent-secret") };
+            writer.Credentials.Add(concurrent);
+            writer.ManagedMachines.Add(new ManagedMachine { Name = "concurrent-host", Hostname = "concurrent.local", DefaultCredentialId = concurrent.Id });
+            var deferred = false;
+            var attempted = false;
+            var credentials = new AfterExportPart(new CredentialBackupPart(source, atRest), async () =>
+            {
+                attempted = true;
+                try { await writer.SaveChangesAsync(ct); }
+                catch (SqliteException ex) when (ex.SqliteErrorCode is 5 or 6)
+                {
+                    deferred = true;
+                }
+                catch (DbUpdateException ex) when (ex.InnerException is SqliteException { SqliteErrorCode: 5 or 6 })
+                {
+                    deferred = true;
+                }
+            });
+            var export = new BackupService([credentials, new MachineBackupPart(source)], source);
+
+            var backup = await export.ExportAsync([BackupSections.Machines], passphrase, "admin", ct);
+            attempted.Should().BeTrue();
+            if (deferred) await writer.SaveChangesAsync(ct);
+            (await writer.ManagedMachines.CountAsync(ct)).Should().Be(2);
+            source.Database.CurrentTransaction.Should().BeNull();
+
+            using var destination = TestDbFactory.Create();
+            var restore = new BackupRestoreService(destination, atRest,
+                new RuntimeOverridesWriter(runtimePath, NullLogger<RuntimeOverridesWriter>.Instance),
+                NullLogger<BackupRestoreService>.Instance,
+                new WorkflowVersionDefinitionProtector(atRest, NullLogger<WorkflowVersionDefinitionProtector>.Instance));
+            await restore.RestoreAsync(backup.Content, passphrase,
+                new Dictionary<string, RestoreConflictPolicy>(), null, ct);
+
+            var machines = await destination.ManagedMachines.ToListAsync(ct);
+            var restoredCredentials = await destination.Credentials.ToListAsync(ct);
+            machines.Should().NotBeEmpty();
+            machines.Should().OnlyContain(machine => restoredCredentials.Any(credential => credential.Id == machine.DefaultCredentialId));
+            machines.Count.Should().Be(backup.Counts.Single(count => count.Section == BackupSections.Machines).Count);
+            restoredCredentials.Count.Should().Be(backup.Counts.Single(count => count.Section == BackupSections.Credentials).Count);
+        }
+        finally
+        {
+            File.Delete(databasePath);
+            File.Delete(runtimePath);
+        }
+    }
+
+    private sealed class AfterExportPart(IBackupPart inner, Func<Task> afterExport) : IBackupPart
+    {
+        public string Key => inner.Key;
+        public IReadOnlyList<string> DependsOn => inner.DependsOn;
+        public Task<int> CountAsync(CancellationToken ct) => inner.CountAsync(ct);
+        public async Task<JsonNode> ExportAsync(BackupExportContext ctx, CancellationToken ct)
+        {
+            var content = await inner.ExportAsync(ctx, ct);
+            await afterExport();
+            return content;
+        }
+    }
+}

tests/NodePilot.Api.Tests/Services/Backup/CustomActivityBackupTests.cs

@@ -91,7 +91,7 @@ private BackupRestoreService Restore(NodePilotDbContext db) =>
                 _atRest, NullLogger<NodePilot.Api.Services.WorkflowVersionDefinitionProtector>.Instance));
 
     private async Task<byte[]> ExportAsync(NodePilotDbContext db) =>
-        (await new BackupService(Parts(db)).ExportAsync(Sections, Passphrase, "admin", CancellationToken.None)).Content;
+        (await new BackupService(Parts(db), db).ExportAsync(Sections, Passphrase, "admin", CancellationToken.None)).Content;
 
     private static async Task<CustomActivityDefinition> SeedAsync(NodePilotDbContext db, bool enabled)
     {
@@ -196,7 +196,7 @@ public async Task WorkflowOnlyExport_AutoIncludesReferencedCustomActivityDefinit
         var definition = await SeedAsync(src, enabled: true);
         SeedWorkflow(src, definition.Id);
 
-        var result = await new BackupService(Parts(src)).ExportAsync(
+        var result = await new BackupService(Parts(src), src).ExportAsync(
             [BackupSections.Workflows], Passphrase, "admin", CancellationToken.None);
 
         result.AutoIncludedSections.Should().Contain(BackupSections.CustomActivities);
@@ -228,7 +228,7 @@ public async Task Restore_LegacyV2Archive_IsRejected()
     {
         using var src = TestDbFactory.Create();
         await SeedAsync(src, enabled: true);
-        var current = await new BackupService(Parts(src)).ExportAsync(
+        var current = await new BackupService(Parts(src), src).ExportAsync(
             [BackupSections.CustomActivities], Passphrase, "admin", CancellationToken.None);
         var envelope = (JsonObject)JsonNode.Parse(Encoding.UTF8.GetString(current.Content))!;
         envelope["schema"] = BackupSections.SchemaV2;