Tighten up SetupFixtures and move TestBase to Tsavorite; handle cancellation race condition in maintenance loops

.github/copilot-instructions.md

@@ -389,7 +389,8 @@ To update the version and make a new release, increment the `VersionPrefix` in `
 
 ### PR Protocol
 
-1. Create a GitHub Issue (Enhancement / Bug / Task)
-2. Branch naming: `<username>/branch-name`
-3. Include unit tests (see Test Structure above)
-4. Link PR to the issue in the development section
+1. Do not create an issue unless asked to do so. If such a request is made, create the issue with a clear and complete but concise description, assign it to the current user, and link the PR to the issue in the development section. Once an Issue has been created, do not update it until the PR is opened unless asked; when the PR is opened, update the Issue (again, completely but concisely) with any necessary changes.
+   a. Similarly, do not create a PR until asked. A PR may be created without an underlying Issue; the PR must completely and concisely state the problem. Once a PR has been created, update its description (again, completely but concisely) on subsequent commits with any necessary changes.
+2. It is acceptable and encouraged to consolidate smaller changes into a single larger PR for reviewer convenience and reducing bookkeeping and process, as long as the PR description clearly outlines the different fixes or improvements being made in the work.
+3. Branch naming: `<username>/branch-name`
+4. Include unit tests (see Test Structure above)

libs/cluster/Server/Gossip/Gossip.cs

@@ -355,12 +355,17 @@ async Task GossipMainAsync()
                         else
                             await GossipSampleSendAsync().ConfigureAwait(false);
                     }
-                    catch (Exception ex) when (!ctsGossip.Token.IsCancellationRequested)
+                    catch (Exception ex)
                     {
                         // Nothing restarts this task, so letting a single failed round leave the loop would stop
                         // this node gossiping for the rest of the process. Its view then stops being offered to
                         // the rest of the cluster and no further configuration change can ever converge.
-                        logger?.LogWarning("Gossip round failed {msg}", ex.Message);
+                        // Every round failure is classified here rather than in an exception filter, so one that
+                        // races shutdown is still handled as a round failure: the outer handlers recognize
+                        // cancellation by exception type and treat anything else as terminal. Cancellation makes
+                        // the failure shutdown noise, so the loop ends without logging it.
+                        if (!ctsGossip.Token.IsCancellationRequested)
+                            logger?.LogWarning("Gossip round failed {msg}", ex.Message);
                     }
 
                     await Task.Delay(gossipDelay, ctsGossip.Token).ConfigureAwait(false);

libs/server/StoreWrapper.cs

@@ -736,9 +736,14 @@ internal static async Task RunMaintenanceLoopAsync(Func<CancellationToken, Value
                     {
                         await runCycle(token).ConfigureAwait(false);
                     }
-                    catch (Exception ex) when (!token.IsCancellationRequested)
+                    catch (Exception ex)
                     {
-                        logger?.LogError(ex, "Unknown exception received for {taskDescription}. Retrying on the next cycle.", taskDescription);
+                        // Every cycle failure is classified here rather than in an exception filter, so one that
+                        // races shutdown is still handled as a cycle failure: the outer handlers recognize
+                        // cancellation by exception type and treat anything else as terminal. Cancellation makes
+                        // the failure shutdown noise, so the loop ends without logging it.
+                        if (!token.IsCancellationRequested)
+                            logger?.LogError(ex, "Unknown exception received for {taskDescription}. Retrying on the next cycle.", taskDescription);
                     }
 
                     await Task.Delay(interval, token).ConfigureAwait(false);

libs/storage/Tsavorite/cs/test/AllocateBlockPartialTests.cs

@@ -2,7 +2,6 @@
 // Licensed under the MIT license.
 
 using System.IO;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/BasicLockTests.cs

@@ -5,7 +5,6 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/BasicTests.cs

@@ -5,7 +5,6 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/CancellationTests.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.IO;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/CompletePendingTests.cs

@@ -5,7 +5,6 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/ConcurrentCounterTests.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/EnqueueAndWaitForCommit.cs

@@ -3,7 +3,6 @@
 using System;
 using System.IO;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/EnqueueTests.cs

@@ -5,7 +5,6 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/ExpirationTests.cs

@@ -4,7 +4,6 @@
 using System;
 using System.IO;
 using System.Runtime.InteropServices;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/InitialIORecordSizeTests.cs

@@ -2,7 +2,6 @@
 // Licensed under the MIT license.
 
 using System.IO;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/InputOutputParameterTests.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.IO;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/LowMemoryTests.cs

@@ -2,7 +2,6 @@
 // Licensed under the MIT license.
 
 using System.IO;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/MallocFixedPageSizeTests.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/MiscTests.cs

@@ -4,7 +4,6 @@
 using System;
 using System.IO;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/NamespaceTests.cs

@@ -4,7 +4,6 @@
 using System;
 using System.IO;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/NeedCopyUpdateTests.cs

@@ -2,7 +2,6 @@
 // Licensed under the MIT license.
 
 using System.IO;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/OverflowBucketLockTableTests.cs

@@ -6,7 +6,6 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/PostOperationsTests.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.IO;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/RecordTriggersExtTests.cs

@@ -4,7 +4,6 @@
 using System.Collections.Concurrent;
 using System.IO;
 using System.Threading;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/SharedDirectoryTests.cs

@@ -7,7 +7,6 @@
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
-using Garnet.test;
 using Microsoft.Win32.SafeHandles;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;

libs/storage/Tsavorite/cs/test/SimpleVersionSchemeTest.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.Threading;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 

libs/storage/Tsavorite/cs/test/StateMachineDriverTests.cs

@@ -5,7 +5,6 @@
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/TestBase.cs

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+using System;
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Text;
+using System.Threading.Tasks;
+using NUnit.Framework;
+
+namespace Tsavorite.test
+{
+    /// <summary>
+    /// Base class for tests — tracks currently running tests for diagnostics.
+    /// </summary>
+    public abstract class TestBase
+    {
+        // Thread-safe collection to store currently running tests
+        public static readonly ConcurrentDictionary<string, bool> RunningTests = new();
+
+        [SetUp]
+        public void TrackRunningTest()
+        {
+            // Add test to the running list
+            RunningTests[TestContext.CurrentContext.Test.Name] = true;
+
+            if (TestContext.CurrentContext.CurrentRepeatCount > 0)
+                Debug.WriteLine($"*** Current test iteration {TestContext.CurrentContext.CurrentRepeatCount + 1}: {TestContext.CurrentContext.Test.Name} ***");
+        }
+
+        [TearDown]
+        public void RemoveRunningTest()
+        {
+            Assert.That(RunningTests.TryRemove(TestContext.CurrentContext.Test.Name, out _), Is.True, $"Could not find running test {TestContext.CurrentContext.Test.Name}");
+        }
+
+        /// <summary>
+        /// Installs process-wide handlers that dump the currently running tests when an exception escapes a
+        /// background thread or an unobserved task, which is otherwise reported with no indication of which
+        /// test was in flight. NUnit discovers a <c>[SetUpFixture]</c> only in the assembly under test, never
+        /// in a referenced one, so each test assembly calls this from its own fixture.
+        /// </summary>
+        public static void InstallUnhandledExceptionHandlers()
+        {
+            AppDomain.CurrentDomain.UnhandledException += (s, e) =>
+            {
+                DumpTests();
+            };
+
+            TaskScheduler.UnobservedTaskException += (s, e) =>
+            {
+                DumpTests();
+                e.SetObserved(); // Optionally mark observed so it doesn't escalate later
+            };
+
+            static void DumpTests()
+            {
+                if (RunningTests.Count == 0)
+                    return;
+                var sb = new StringBuilder();
+                _ = sb.AppendLine("*** CURRENTLY RUNNING TESTS ***:");
+                foreach (var key in RunningTests.Keys)
+                    _ = sb.AppendLine(key);
+                Console.WriteLine(sb.ToString());
+            }
+        }
+    }
+}

libs/storage/Tsavorite/cs/test/TestProjectSetup.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using NUnit.Framework;
+
+namespace Tsavorite.test
+{
+    /// <summary>
+    /// Per-assembly setup. Installs the handlers that report which tests were in flight when an exception
+    /// escapes a background thread, which NUnit runs only for the assembly under test.
+    /// </summary>
+    [SetUpFixture]
+    public class TestProjectSetup
+    {
+        [OneTimeSetUp]
+        public void SetUpProject() => TestBase.InstallUnhandledExceptionHandlers();
+    }
+}

libs/storage/Tsavorite/cs/test/TryEnqueueBasicTests.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.IO;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj

@@ -11,7 +11,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <Compile Include="..\..\..\..\..\test\standalone\Garnet.test\TestBase.cs" Link="TestBase.cs" />
     <Compile Remove="test.*/**" />
     <Compile Remove="stress/**" />
   </ItemGroup>

libs/storage/Tsavorite/cs/test/WaitForCommit.cs

@@ -3,7 +3,6 @@
 
 using System.IO;
 using System.Threading;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/test.epoch/helpers/EpochTestBase.cs

@@ -9,10 +9,10 @@
 namespace Tsavorite.test.epoch
 {
     /// <summary>
-    /// Base for every fixture here: the running-test tracking from <see cref="Garnet.test.TestBase"/>,
+    /// Base for every fixture here: the running-test tracking from <see cref="TestBase"/>,
     /// a fresh <see cref="LightEpoch"/> per test, and the join helper they share.
     /// </summary>
-    public abstract class EpochTestBase : Garnet.test.TestBase
+    public abstract class EpochTestBase : TestBase
     {
         protected LightEpoch epoch;
 

libs/storage/Tsavorite/cs/test/test.hlog/BasicStorageTests.cs

@@ -5,7 +5,6 @@
 using System.IO;
 using System.Runtime.InteropServices;
 using System.Threading;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/test.hlog/DeviceLogTests.cs

@@ -9,7 +9,6 @@
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/test.hlog/DeviceTests.cs

@@ -6,7 +6,6 @@
 using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Threading;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/test.hlog/InsertAtTailSpanByteStressTests.cs

@@ -5,7 +5,6 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/test.hlog/LogAndDeviceConfigTests.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 using System.IO;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;

libs/storage/Tsavorite/cs/test/test.hlog/LogReadAsyncTests.cs

@@ -4,7 +4,6 @@
 using System.Buffers;
 using System.IO;
 using System.Threading;
-using Garnet.test;
 using NUnit.Framework;
 using NUnit.Framework.Legacy;
 using Tsavorite.core;