From dc55204251b30db2fadc646983f55b55cf1e5c62 Mon Sep 17 00:00:00 2001 From: "Mars.P" Date: Sun, 8 Nov 2026 03:09:10 +0800 Subject: [PATCH] Keep repair-sweep integration tests from outgrowing the lane hang timeout The stranded-binding repair builds one query across every group of interchangeable provider connections in the database, on each candidate it attempts. Every sibling-connection test leaves a group behind, so late in the Integration lane a pass took ~70s and the refused-candidates test, which needs four passes of four attempts, ran past --blame-hang-timeout. Retire the sibling connections earlier tests left before the tests that sweep repeatedly, and fail their waits with a named timeout. --- .../SameHostConnectionsFlowTests.Doors.cs | 12 +++++++++ .../SameHostConnectionsFlowTests.cs | 20 ++++++++++++++- .../OAuth/BindingMoveKit.cs | 25 ++++++++++++++++++- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.Doors.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.Doors.cs index 31b930d79..c0f62b9fd 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.Doors.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.Doors.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using Autofac; using CodeSpace.Core.Persistence.Db; using CodeSpace.Core.Persistence.Entities; @@ -361,6 +362,10 @@ public async Task Credentials_whose_repositories_their_successors_connection_can var movable = await _kit.SeedRepositoryAsync(elsewhere, movableDead, RepositoryStatus.Error); var passes = 0; + await _kit.RetireOtherSiblingConnectionsAsync(); + + var clock = Stopwatch.StartNew(); + while ((await _kit.LoadRepositoryAsync(movable)).CredentialId != movableSuccessor && passes < MaxSweeps) { passes++; @@ -368,6 +373,8 @@ public async Task Credentials_whose_repositories_their_successors_connection_can using var scope = _kit.Fixture.BeginScope(); await scope.Resolve().Send(new RepairDisconnectedCredentialBindingsCommand { BatchSize = 1 }); + + EnsureWithinDeadline(clock, passes); } (await _kit.LoadRepositoryAsync(movable)).CredentialId.ShouldBe(movableSuccessor, "the repairable credential was reached however many refused ones sit ahead of it"); @@ -498,12 +505,17 @@ private async Task StrandedAsync() /// private async Task DrainedStrandedAsync() { + await _kit.RetireOtherSiblingConnectionsAsync(); + var count = await StrandedAsync(); + var clock = Stopwatch.StartNew(); for (var pass = 1; pass <= MaxSweeps; pass++) { using (var scope = _kit.Fixture.BeginScope()) await scope.Resolve().Send(new RepairDisconnectedCredentialBindingsCommand { BatchSize = 200 }); + EnsureWithinDeadline(clock, pass); + var next = await StrandedAsync(); if (next == count) return count; diff --git a/backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.cs b/backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.cs index ff087f185..638674361 100644 --- a/backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.cs +++ b/backend/tests/CodeSpace.IntegrationTests/Credentials/SameHostConnectionsFlowTests.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using Autofac; using CodeSpace.Core.Persistence.Db; using CodeSpace.Core.Persistence.Entities; @@ -155,17 +156,34 @@ private async Task> SuggestAsync(BindingMoveKit. /// /// Sweeps until the row the test owns has moved. The repair is deployment-wide, so what one pass repairs says nothing about this - /// test's row: a test waits on the row it owns, never on a tally of the pass. + /// test's row: a test waits on the row it owns, never on a tally of the pass. Bounded by passes and by + /// , and the deadline names what it was waiting for. /// private async Task SweepUntilAsync(Func> ownedRowHasMoved) { + await _kit.RetireOtherSiblingConnectionsAsync(); + + var clock = Stopwatch.StartNew(); + for (var pass = 1; pass <= MaxSweeps; pass++) { using (var scope = _kit.Fixture.BeginScope()) await scope.Resolve().Send(new RepairDisconnectedCredentialBindingsCommand { BatchSize = 50 }); if (await ownedRowHasMoved()) return; + + EnsureWithinDeadline(clock, pass); } } + /// Fails a wait that has used up , naming the signal it watched and how to look at it by hand. + private static void EnsureWithinDeadline(Stopwatch clock, int passes) + { + if (clock.Elapsed < SweepDeadline) return; + + throw new TimeoutException($"The repair sweep's pass {passes} ended {clock.Elapsed.TotalSeconds:F0}s into waiting for the row this test owns to move (limit {SweepDeadline.TotalSeconds:F0}s). Each pass costs a query per group of sibling connections in the database — check `SELECT count(*) FROM provider_instance WHERE deleted_date IS NULL` and the 'ok in' time of RepairDisconnectedCredentialBindingsCommand in the log."); + } + private const int MaxSweeps = 20; + + private static readonly TimeSpan SweepDeadline = TimeSpan.FromMinutes(2); } diff --git a/backend/tests/CodeSpace.IntegrationTests/OAuth/BindingMoveKit.cs b/backend/tests/CodeSpace.IntegrationTests/OAuth/BindingMoveKit.cs index 06b141446..22fbfff1f 100644 --- a/backend/tests/CodeSpace.IntegrationTests/OAuth/BindingMoveKit.cs +++ b/backend/tests/CodeSpace.IntegrationTests/OAuth/BindingMoveKit.cs @@ -30,7 +30,11 @@ namespace CodeSpace.IntegrationTests.OAuth; /// internal sealed class BindingMoveKit { + /// The display-name prefix of every connection makes — what recognises as its own. + private const string SiblingNamePrefix = "GitLab sibling "; + private readonly PostgresFixture _fixture; + private readonly List _siblings = new(); public BindingMoveKit(PostgresFixture fixture) { _fixture = fixture; } @@ -82,14 +86,33 @@ public async Task SeedSiblingInstanceAsync(World world, Guid? ofInstanceId var of = await db.ProviderInstance.AsNoTracking().SingleAsync(p => p.Id == (ofInstanceId ?? world.InstanceId)); var suffix = Guid.NewGuid().ToString("N")[..8]; - var sibling = new ProviderInstance { Id = Guid.NewGuid(), TeamId = of.TeamId, Provider = of.Provider, DisplayName = $"GitLab sibling {suffix}", BaseUrl = of.BaseUrl, ApiUrl = apiUrl ?? of.ApiUrl, WebhookScope = webhookScope, OauthClientId = $"client-{suffix}", OauthClientSecretEnc = scope.Resolve().Encrypt("client-secret") }; + var sibling = new ProviderInstance { Id = Guid.NewGuid(), TeamId = of.TeamId, Provider = of.Provider, DisplayName = $"{SiblingNamePrefix}{suffix}", BaseUrl = of.BaseUrl, ApiUrl = apiUrl ?? of.ApiUrl, WebhookScope = webhookScope, OauthClientId = $"client-{suffix}", OauthClientSecretEnc = scope.Resolve().Encrypt("client-secret") }; db.ProviderInstance.Add(sibling); await db.SaveChangesAsync(); + _siblings.Add(sibling.Id); + return sibling.Id; } + /// + /// Soft-deletes the sibling connections earlier tests of this database left behind, keeping the ones this kit seeded. The repair sweep reads every + /// group of interchangeable connections in the database and builds one query across all of them, on every candidate it attempts, so what it costs grows + /// with the number of groups: late in a full lane a pass that took milliseconds takes tens of seconds, and a test that needs a dozen attempts outlasts the + /// lane's hang timeout. A connection that is deleted is in no group. The tests of one collection run one after another, so nothing is still using them. + /// + public async Task RetireOtherSiblingConnectionsAsync() + { + var mine = _siblings.ToArray(); + + using var scope = _fixture.BeginScope(); + + await scope.Resolve().ProviderInstance + .Where(p => p.DeletedDate == null && p.DisplayName.StartsWith(SiblingNamePrefix) && !mine.Contains(p.Id)) + .ExecuteUpdateAsync(s => s.SetProperty(p => p.DeletedDate, DateTimeOffset.UtcNow)); + } + /// The connection moved to as an operator's edit leaves it, with nothing registered yet to retire — for a test that needs a connection-wide target. public async Task SetWebhookScopeAsync(Guid instanceId, ProviderWebhookScope webhookScope) {