frontend: introduce request rate limter to frontend

Motivation:
protect the system from DoS attacks.

Modification:
Enable rate-limiter aware jetty handler list.
Update rate-limiter handler list to log with warn only on too many auth
errors.

New properties are introduced:

frontend.limits.max-blocked-clients

frontend.limits.rate.overall
frontend.limits.rate.per-client.fractions

frontend.limits.error.max-allowed
frontend.limits.error.block.window.time
frontend.limits.error.block.window.time.units

frontend.limits.rate.per-client.block.window.time
frontend.limits.rate.per-client.block.window.time.units

frontend.limits.blocked-clients.idle-time
frontend.limits.blocked-clients.idle-time.units

Result:
frontend now can be protected against DoS attacks.

Ticket: #10371
Acked-by: Dmitry Litvintsev
Target: master
Require-book: no
Require-notes: yes

Author: kofemann

modules/dcache-frontend/src/main/resources/org/dcache/frontend/frontend.xml

@@ -378,8 +378,23 @@
       <description>Loads the factories for producers of well-known endpoint content.</description>
     </bean>
 
-    <bean id="handlers" class="org.eclipse.jetty.server.handler.HandlerList">
+    <bean id="rate-limiter-config" class="org.dcache.util.jetty.RateLimitedHandlerList.Configuration">
+        <description>Configuration of requests rate limiter</description>
+        <property name="maxClientsToTrack" value="${frontend.limits.max-blocked-clients}"/>
+        <property name="globalRequestsPerSecond" value="${frontend.limits.rate.overall}"/>
+        <property name="limitPercentagePerClient" value="${frontend.limits.rate.per-client.fractions}"/>
+        <property name="clientBlockingTime" value="${frontend.limits.rate.per-client.block.window.time}"/>
+        <property name="clientBlockingTimeUnit" value="${frontend.limits.rate.per-client.block.window.time.units}"/>
+        <property name="clientIdleTime" value="${frontend.limits.blocked-clients.idle-time}"/>
+        <property name="clientIdleTimeUnit" value="${frontend.limits.blocked-clients.idle-time.units}"/>
+        <property name="numErrorsBeforeBlocking" value="${frontend.limits.error.max-allowed}"/>
+        <property name="errorCountingWindow" value="${frontend.limits.error.block.window.time}"/>
+        <property name="errorCountingWindowUnit" value="${frontend.limits.error.block.window.time.units}"/>
+    </bean>
+
+    <bean id="handlers" class="org.dcache.util.jetty.RateLimitedHandlerList">
         <description>List of handlers for HTTP requests</description>
+        <constructor-arg ref="rate-limiter-config"/>
         <property name="handlers">
             <list>
                 <bean class="org.eclipse.jetty.server.handler.ContextHandler">

modules/dcache-webdav/src/main/resources/org/dcache/webdav/webdav.xml

@@ -271,6 +271,7 @@
 
 
     <bean id="rate-limiter-config" class="org.dcache.util.jetty.RateLimitedHandlerList.Configuration">
+        <description>Configuration of requests rate limiter</description>
         <property name="maxClientsToTrack" value="${webdav.limits.max-blocked-clients}"/>
         <property name="globalRequestsPerSecond" value="${webdav.limits.rate.overall}"/>
         <property name="limitPercentagePerClient" value="${webdav.limits.rate.per-client.fractions}"/>

modules/dcache/src/main/java/org/dcache/util/jetty/RateLimitedHandlerList.java

@@ -217,23 +217,23 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques
 
         boolean blocked = blockedClients.getIfPresent(client) != null;
         if (blocked) {
-            LOGGER.warn("Blocking client with too many auth errors {}", client);
+            LOGGER.debug("Blocking client with too many auth errors {}", client);
             response.setStatus(HttpStatus.TOO_MANY_REQUESTS_429);
             response.getWriter().write("Server is busy. Please try again later.");
             baseRequest.setHandled(true);
             return;
         }
 
         if (!getClientRateLimiter(client).tryAcquire()) {
-            LOGGER.warn("Blocking client with too many requests {}", client);
+            LOGGER.debug("Blocking client with too many requests {}", client);
             response.setStatus(HttpStatus.TOO_MANY_REQUESTS_429);
             response.getWriter().write("Server is busy. Please try again later.");
             baseRequest.setHandled(true);
             return;
         }
 
         if (!globalRateLimiter.tryAcquire()) {
-            LOGGER.warn("Blocking client due to globally too many requests {}", client);
+            LOGGER.debug("Blocking client due to globally too many requests {}", client);
             response.setStatus(HttpStatus.TOO_MANY_REQUESTS_429);
             response.getWriter().write("Server is busy. Please try again later.");
             baseRequest.setHandled(true);
@@ -249,6 +249,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques
                     if (response.getStatus() >=  400 && response.getStatus() <= 407) {
                         int errors = getClientErrorRateLimiter(client).incrementAndGet();
                         if (errors >= maxErrorsPerClient) {
+                            LOGGER.warn("Blocking client due to too many auth errors or bad requests: {}", client);
                             blockedClients.put(client, BLOCK);
                             // as client blocked, no reason to keep track of further errors
                             perClientErrorCount.invalidate(client);

skel/share/defaults/frontend.properties

@@ -655,6 +655,58 @@ frontend.srr.shares =
 #
 frontend.stage.supported-sitenames=
 
+
+
+#  ---- Rate limiting and blocking of clients
+#
+# The rate limiting and blocking of clients is a mechanism to protect dCache from misbehaving clients.
+#
+# There are three limits that apply to clients (in defined order):
+#
+# - The maximum number of auth or permission errors a client may issue within a defined time window.
+# - The maximum number of the requests that a single client may issue.
+# - The maximum number of overall requests per second from all clients.
+#
+# If any of these limits is exceeded then the client will get 429 (Too Many Requests) responses for all subsequent requests
+#
+# The maximum number of requests from a single client is defined as a percentage of the overall maximum number of requests.
+# For example, if the overall maximum number of requests is 1000 request per second and the per-client fraction is 25%,
+# then a single client may issue up to 250 requests per second.
+#
+# Clients that exceed the rate limit, will be unblocked after webdav.limits.per-client.block.window.time.
+# Clients that exceed the error limit, will be unblocked after webdav.limits.block.window.time.
+# If clients blocked due to global rate limit, they will be unblocked as soon as the overall request rate drops below the limit.
+#
+
+# The maximum number of clients that can be blocked at any one time. Preferably a power of 2.
+frontend.limits.max-blocked-clients =  1048576
+
+# The maximum number of requests per second allowed from all clients.
+frontend.limits.rate.overall = 1000
+
+# The maximum share of requests a single client is allowed, expressed as a percentage of the total requests across all clients.
+# For example, if this value is 25, and limit of overall requests is 1000 rps, then a single client may issue up to 25% of the
+# overall requests, which is 250 requests per second.
+frontend.limits.rate.per-client.fractions = 25
+
+# The maximum number of errors allowed within a time window before client is blocked. If more than this number of auth
+# or permission errors are received from a client within the time window, the client is blocked.
+frontend.limits.error.max-allowed = 3
+frontend.limits.error.block.window.time = 20
+(one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)\
+frontend.limits.error.block.window.time.units = SECONDS
+
+# The time window to block clients that exceed their request rate.
+frontend.limits.rate.per-client.block.window.time = 10
+(one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)\
+frontend.limits.rate.per-client.block.window.time.units = SECONDS
+
+# The idle time window to reset limit or error counters. If a client does not issue
+# any requests within this time window, the counters are reset.
+frontend.limits.blocked-clients.idle-time = 10
+(one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)\
+frontend.limits.blocked-clients.idle-time.units = SECONDS
+
 (deprecated)frontend.wellknown!wlcg-tape-rest-api.path = Use dcache.wellknown!wlcg-tape-rest-api.path instead
 
 (obsolete)frontend.dcache-view.endpoints.webapi = Use frontend.static!dcache-view.endpoints.webapi instead