CCBC-1612: do not throttle CCCP provider in faster failover mode

In faster failover more, we expect server notifications, and we should
not throttle refresh signals because it is done on provider level, and
it ensures that not more than one request is in flight at a time.

Change-Id: I2a95a1780272a74ab57ee070521a46b99302a06c
Reviewed-on: https://review.couchbase.org/c/libcouchbase/+/197336
Reviewed-by: Brett Lawson <brett19@gmail.com>
Tested-by: Build Bot <build@couchbase.com>

Author: avsej

src/bootstrap.cc

@@ -260,13 +260,20 @@ lcb_STATUS Bootstrap::bootstrap(unsigned options)
         next_ts = last_refresh;
         next_ts += LCB_US2NS(LCBT_SETTING(parent, weird_things_delay));
         if (now < next_ts && errcounter < errthresh) {
-            lcb_log(
-                LOGARGS(parent, INFO),
-                "Not requesting a config refresh because of throttling parameters. Next refresh possible in %" PRIu64
-                "ms or %u errors. "
-                "See LCB_CNTL_CONFDELAY_THRESH and LCB_CNTL_CONFERRTHRESH to modify the throttling settings",
-                LCB_NS2US(next_ts - now) / 1000, errthresh - errcounter);
-            return LCB_SUCCESS;
+            const auto *provider = parent->confmon->cur_provider;
+            if (provider == nullptr || !(provider->type == clconfig::CLCONFIG_CCCP &&
+                                         parent->confmon->mode == clconfig::CONFMON_M_PUSH)) {
+                lcb_log(LOGARGS(parent, INFO),
+                        "Not requesting a config refresh because of throttling parameters. Next refresh possible in "
+                        "%" PRIu64 "ms or %u errors. "
+                        "See LCB_CNTL_CONFDELAY_THRESH and LCB_CNTL_CONFERRTHRESH to modify the throttling settings",
+                        LCB_NS2US(next_ts - now) / 1000, errthresh - errcounter);
+                return LCB_SUCCESS;
+            }
+            lcb_log(LOGARGS(parent, INFO),
+                    "A config refresh requested, trigger CCCP provider. (next_ts=%" PRIu64
+                    "ms, errcounter=%u, errthresh=%u)",
+                    LCB_NS2US(next_ts - now) / 1000, errcounter, errthresh);
         }
     }
 

src/bucketconfig/bc_cccp.cc

@@ -220,6 +220,7 @@ lcb_STATUS CccpProvider::schedule_next_request(lcb_STATUS err, bool can_rollover
     if (skip_if_push_supported && nodes->all_hosts_support_config_push()) {
         /* all nodes support configuration push, and this function invoked from periodic poller, so nothing has to be
          * done here */
+        parent->mode = CONFMON_M_PUSH;
         parent->stop();
         return LCB_SUCCESS;
     }
@@ -254,6 +255,7 @@ lcb_STATUS CccpProvider::schedule_next_request(lcb_STATUS err, bool can_rollover
             /* we found the server, but at the same time all sockets support push, so we can stop polling and just
              * expect notifications from KV engine */
             lcb_log(LOGARGS(this, DEBUG), "Stop background polling, as all nodes support configuration push");
+            parent->mode = CONFMON_M_PUSH;
             parent->stop();
             return LCB_SUCCESS;
         }

src/bucketconfig/clconfig.h

@@ -145,6 +145,13 @@ enum State {
     CONFMON_S_ITERGRACE = 1 << 1
 };
 
+enum Mode {
+    /** Periodically poll for configuration */
+    CONFMON_M_POLL,
+    /** Subscribe for notifications from the server */
+    CONFMON_M_PUSH,
+};
+
 struct Provider;
 struct Listener;
 class ConfigInfo;
@@ -399,6 +406,8 @@ struct Confmon {
     /* CONFMON_S_* values. Used internally */
     int state;
 
+    Mode mode{CONFMON_M_POLL};
+
     /** Last time the provider was stopped. As a microsecond timestamp */
     lcb_uint64_t last_stop_us;