Fixing leadership

Author: Drawaes

src/CondenserDotNet.Client/Leadership/LeaderRegistry.cs

@@ -7,7 +7,7 @@ namespace CondenserDotNet.Client.Leadership
     public class LeaderRegistry : ILeaderRegistry
     {
         private readonly IServiceManager _serviceManager;
-        private readonly Dictionary<string, LeaderWatcherNew> _leaderWatchers = new Dictionary<string, LeaderWatcherNew>(StringComparer.OrdinalIgnoreCase);
+        private readonly Dictionary<string, LeaderWatcher> _leaderWatchers = new Dictionary<string, LeaderWatcher>(StringComparer.OrdinalIgnoreCase);
 
         public LeaderRegistry(IServiceManager serviceManager) => _serviceManager = serviceManager;
 
@@ -23,7 +23,7 @@ public async Task<ILeaderWatcher> GetLeaderWatcherAsync(string keyForLeadership)
             {
                 if (!_leaderWatchers.TryGetValue(keyForLeadership, out var returnValue))
                 {
-                    returnValue = new LeaderWatcherNew(_serviceManager, keyForLeadership);
+                    returnValue = new LeaderWatcher(_serviceManager, keyForLeadership);
                     _leaderWatchers[keyForLeadership] = returnValue;
                 }
                 return returnValue;

src/CondenserDotNet.Client/Leadership/LeaderWatcherNew.cs

@@ -12,7 +12,7 @@
 
 namespace CondenserDotNet.Client.Leadership
 {
-    public class LeaderWatcherNew : ILeaderWatcher
+    public class LeaderWatcher : ILeaderWatcher
     {
         private readonly AsyncManualResetEvent<bool> _electedLeaderEvent = new AsyncManualResetEvent<bool>();
         private Action<InformationService> _callback;
@@ -23,8 +23,10 @@ public class LeaderWatcherNew : ILeaderWatcher
         private string _keyToWatch;
         private AsyncManualResetEvent<InformationService> _currentInfoService = new AsyncManualResetEvent<InformationService>();
 
-        public LeaderWatcherNew(IServiceManager serviceManager, string keyToWatch)
+        public LeaderWatcher(IServiceManager serviceManager, string keyToWatch)
         {
+            _currentInfoService.Reset();
+            _electedLeaderEvent.Reset();
             _keyToWatch = keyToWatch;
             _serviceManager = serviceManager;
             _sessionIdTask = GetSession();
@@ -92,7 +94,7 @@ private async Task KeepLeadershipLoop()
                 }
                 else
                 {
-                    _electedLeaderEvent.Set(false);
+                    _electedLeaderEvent.Reset();
                 }
                 await WaitForLeadershipChange();
             }
@@ -105,6 +107,13 @@ private async Task WaitForLeadershipChange()
                 var leaderResult = await _serviceManager.Client.GetAsync($"{KeyPath}{_keyToWatch}?index={_consulIndex}");
                 if(!leaderResult.IsSuccessStatusCode)
                 {
+                    //Lock deleted
+                    if(leaderResult.StatusCode == System.Net.HttpStatusCode.NotFound)
+                    {
+                        _electedLeaderEvent.Reset();
+                        _currentInfoService.Reset();
+                        return;
+                    }
                     await Task.Delay(500);
                     continue;
                 }

test/Condenser.Tests.Integration/LeadershipFacts.cs

@@ -83,6 +83,21 @@ public async Task TestGetLeadership()
             }
         }
 
+        //[Fact]
+        //public async Task TestHoldingLeadershipForAWhile()
+        //{
+        //    using (var manager = GetConfig("Service1"))
+        //    {
+        //        manager.AddTtlHealthCheck(100);
+        //        var registerResult = await manager.RegisterServiceAsync();
+        //        var ttlResult = await manager.TtlCheck.ReportPassingAsync();
+        //        var leaderRegistry = new LeaderRegistry(manager);
+        //        var watcher = await leaderRegistry.GetLeaderWatcherAsync(leadershipKey);
+        //        await watcher.GetLeadershipAsync();
+        //        await Task.Delay(50000);
+        //    }
+        //}
+
         [Fact]
         public async Task TestLeadershipFailOver()
         {
@@ -116,7 +131,8 @@ public async Task TestLeadershipFailOver()
                 await manager.TtlCheck.ReportFailAsync();
 
                 //Now we wait, the leadership should fall over
-                Assert.True(resetEvent.WaitOne(5000));
+                var waitResult = resetEvent.WaitOne(5000);
+                Assert.True(waitResult);
 
                 //Now check that service 2 is the leader
                 var leader = await watcher2.GetCurrentLeaderAsync();