Keep repair-sweep tests from outgrowing the lane hang timeout

backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.Doors.cs

@@ -1,3 +1,4 @@
+using System.Diagnostics;
 using Autofac;
 using CodeSpace.Core.Persistence.Db;
 using CodeSpace.Core.Persistence.Entities;
@@ -361,13 +362,19 @@ public async Task Credentials_whose_repositories_their_successors_connection_can
         var movable = await _kit.SeedRepositoryAsync(elsewhere, movableDead, RepositoryStatus.Error);
         var passes = 0;
 
+        await _kit.RetireOtherSiblingConnectionsAsync();
+
+        var clock = Stopwatch.StartNew();
+
         while ((await _kit.LoadRepositoryAsync(movable)).CredentialId != movableSuccessor && passes < MaxSweeps)
         {
             passes++;
 
             using var scope = _kit.Fixture.BeginScope();
 
             await scope.Resolve<IMediator>().Send(new RepairDisconnectedCredentialBindingsCommand { BatchSize = 1 });
+
+            EnsureWithinDeadline(clock, passes);
         }
 
         (await _kit.LoadRepositoryAsync(movable)).CredentialId.ShouldBe(movableSuccessor, "the repairable credential was reached however many refused ones sit ahead of it");
@@ -498,12 +505,17 @@ private async Task<int> StrandedAsync()
     /// </summary>
     private async Task<int> DrainedStrandedAsync()
     {
+        await _kit.RetireOtherSiblingConnectionsAsync();
+
         var count = await StrandedAsync();
+        var clock = Stopwatch.StartNew();
 
         for (var pass = 1; pass <= MaxSweeps; pass++)
         {
             using (var scope = _kit.Fixture.BeginScope()) await scope.Resolve<IMediator>().Send(new RepairDisconnectedCredentialBindingsCommand { BatchSize = 200 });
 
+            EnsureWithinDeadline(clock, pass);
+
             var next = await StrandedAsync();
 
             if (next == count) return count;

backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.cs

@@ -1,3 +1,4 @@
+using System.Diagnostics;
 using Autofac;
 using CodeSpace.Core.Persistence.Db;
 using CodeSpace.Core.Persistence.Entities;
@@ -155,17 +156,34 @@ private async Task<IReadOnlyList<RelinkSuggestion>> SuggestAsync(BindingMoveKit.
 
     /// <summary>
     /// Sweeps until the row the test owns has moved. The repair is deployment-wide, so what one pass repairs says nothing about this
-    /// test's row: a test waits on the row it owns, never on a tally of the pass.
+    /// test's row: a test waits on the row it owns, never on a tally of the pass. Bounded by <see cref="MaxSweeps"/> passes and by
+    /// <see cref="SweepDeadline"/>, and the deadline names what it was waiting for.
     /// </summary>
     private async Task SweepUntilAsync(Func<Task<bool>> ownedRowHasMoved)
     {
+        await _kit.RetireOtherSiblingConnectionsAsync();
+
+        var clock = Stopwatch.StartNew();
+
         for (var pass = 1; pass <= MaxSweeps; pass++)
         {
             using (var scope = _kit.Fixture.BeginScope()) await scope.Resolve<IMediator>().Send(new RepairDisconnectedCredentialBindingsCommand { BatchSize = 50 });
 
             if (await ownedRowHasMoved()) return;
+
+            EnsureWithinDeadline(clock, pass);
         }
     }
 
+    /// <summary>Fails a wait that has used up <see cref="SweepDeadline"/>, naming the signal it watched and how to look at it by hand.</summary>
+    private static void EnsureWithinDeadline(Stopwatch clock, int passes)
+    {
+        if (clock.Elapsed < SweepDeadline) return;
+
+        throw new TimeoutException($"The repair sweep's pass {passes} ended {clock.Elapsed.TotalSeconds:F0}s into waiting for the row this test owns to move (limit {SweepDeadline.TotalSeconds:F0}s). Each pass costs a query per group of sibling connections in the database — check `SELECT count(*) FROM provider_instance WHERE deleted_date IS NULL` and the 'ok in' time of RepairDisconnectedCredentialBindingsCommand in the log.");
+    }
+
     private const int MaxSweeps = 20;
+
+    private static readonly TimeSpan SweepDeadline = TimeSpan.FromMinutes(2);
 }

backend/tests/CodeSpace.IntegrationTests/OAuth/BindingMoveKit.cs

@@ -30,7 +30,11 @@ namespace CodeSpace.IntegrationTests.OAuth;
 /// </summary>
 internal sealed class BindingMoveKit
 {
+    /// <summary>The display-name prefix of every connection <see cref="SeedSiblingInstanceAsync"/> makes — what <see cref="RetireOtherSiblingConnectionsAsync"/> recognises as its own.</summary>
+    private const string SiblingNamePrefix = "GitLab sibling ";
+
     private readonly PostgresFixture _fixture;
+    private readonly List<Guid> _siblings = new();
 
     public BindingMoveKit(PostgresFixture fixture) { _fixture = fixture; }
 
@@ -82,14 +86,33 @@ public async Task<Guid> SeedSiblingInstanceAsync(World world, Guid? ofInstanceId
         var of = await db.ProviderInstance.AsNoTracking().SingleAsync(p => p.Id == (ofInstanceId ?? world.InstanceId));
         var suffix = Guid.NewGuid().ToString("N")[..8];
 
-        var sibling = new ProviderInstance { Id = Guid.NewGuid(), TeamId = of.TeamId, Provider = of.Provider, DisplayName = $"GitLab sibling {suffix}", BaseUrl = of.BaseUrl, ApiUrl = apiUrl ?? of.ApiUrl, WebhookScope = webhookScope, OauthClientId = $"client-{suffix}", OauthClientSecretEnc = scope.Resolve<IPayloadEncryptor>().Encrypt("client-secret") };
+        var sibling = new ProviderInstance { Id = Guid.NewGuid(), TeamId = of.TeamId, Provider = of.Provider, DisplayName = $"{SiblingNamePrefix}{suffix}", BaseUrl = of.BaseUrl, ApiUrl = apiUrl ?? of.ApiUrl, WebhookScope = webhookScope, OauthClientId = $"client-{suffix}", OauthClientSecretEnc = scope.Resolve<IPayloadEncryptor>().Encrypt("client-secret") };
 
         db.ProviderInstance.Add(sibling);
         await db.SaveChangesAsync();
 
+        _siblings.Add(sibling.Id);
+
         return sibling.Id;
     }
 
+    /// <summary>
+    /// Soft-deletes the sibling connections earlier tests of this database left behind, keeping the ones this kit seeded. The repair sweep reads every
+    /// group of interchangeable connections in the database and builds one query across all of them, on every candidate it attempts, so what it costs grows
+    /// with the number of groups: late in a full lane a pass that took milliseconds takes tens of seconds, and a test that needs a dozen attempts outlasts the
+    /// lane's hang timeout. A connection that is deleted is in no group. The tests of one collection run one after another, so nothing is still using them.
+    /// </summary>
+    public async Task RetireOtherSiblingConnectionsAsync()
+    {
+        var mine = _siblings.ToArray();
+
+        using var scope = _fixture.BeginScope();
+
+        await scope.Resolve<CodeSpaceDbContext>().ProviderInstance
+            .Where(p => p.DeletedDate == null && p.DisplayName.StartsWith(SiblingNamePrefix) && !mine.Contains(p.Id))
+            .ExecuteUpdateAsync(s => s.SetProperty(p => p.DeletedDate, DateTimeOffset.UtcNow));
+    }
+
     /// <summary>The connection moved to <paramref name="webhookScope"/> as an operator's edit leaves it, with nothing registered yet to retire — for a test that needs a connection-wide target.</summary>
     public async Task SetWebhookScopeAsync(Guid instanceId, ProviderWebhookScope webhookScope)
     {